osborn 0.5.3 → 0.8.0

package/dist/voice-io.d.ts

@@ -12,12 +12,16 @@ import * as openai from '@livekit/agents-plugin-openai';
 import * as silero from '@livekit/agents-plugin-silero';
 import type { RealtimeConfig } from './config.js';
 export interface STTConfig {
-    provider: 'deepgram' | 'groq-whisper' | 'openai-whisper';
+    provider: 'deepgram' | 'deepgram-flux' | 'groq-whisper' | 'openai-whisper';
     model?: string;
     language?: string;
+    /** Deepgram Flux: end-of-turn confidence threshold (0.0-1.0, default 0.7) */
+    eotThreshold?: number;
+    /** Deepgram Flux: max ms to wait before forcing turn end (default 3000) */
+    eotTimeoutMs?: number;
 }
 export interface TTSConfig {
-    provider: 'gemini' | 'openai' | 'elevenlabs' | 'deepgram';
+    provider: 'gemini' | 'openai' | 'elevenlabs' | 'deepgram' | 'groq-orpheus';
     voice?: string;
     model?: string;
 }
@@ -29,7 +33,7 @@ export interface VoiceIOConfig {
  * Create STT (Speech-to-Text) instance based on config
  * Note: Gemini STT is not available in Node.js, using Deepgram as default
  */
-export declare function createSTT(config: STTConfig): deepgram.STT | openai.STT;
+export declare function createSTT(config: STTConfig): deepgram.STT | deepgram.STTv2 | openai.STT;
 /**
  * Create TTS (Text-to-Speech) instance based on config
  * Using Gemini TTS as default (cheaper, good quality)
@@ -45,15 +49,21 @@ export declare function createTTS(config: TTSConfig): any;
  */
 export declare function createVAD(): Promise<silero.VAD>;
 /**
- * Default voice I/O configuration
+ * Default voice I/O configuration (used by realtime mode fallback)
  * Uses Deepgram STT (fast, accurate) + Deepgram TTS (fast, good)
  */
 export declare const DEFAULT_VOICE_IO_CONFIG: VoiceIOConfig;
+/**
+ * Direct mode voice config — centralized here for easy provider swapping.
+ * To switch providers: comment out the active line, uncomment the alternative.
+ */
+export declare const DIRECT_MODE_STT: STTConfig;
+export declare const DIRECT_MODE_TTS: TTSConfig;
 export interface RealtimeModelConfig {
     provider: 'openai' | 'gemini';
     openaiVoice?: 'alloy' | 'echo' | 'fable' | 'onyx' | 'nova' | 'shimmer';
     openaiModel?: string;
-    geminiVoice?: 'Puck' | 'Charon' | 'Kore' | 'Fenrir' | 'Aoede';
+    geminiVoice?: 'Charon' | 'Puck' | 'Kore' | 'Fenrir' | 'Aoede';
     geminiModel?: string;
     instructions?: string;
 }

package/dist/voice-io.js

@@ -20,6 +20,19 @@ export function createSTT(config) {
             return new deepgram.STT({
                 model: (config.model || 'nova-3'),
                 language: config.language || 'en',
+                endpointing: 550, // Wait 550ms of silence before committing final transcript (default 25ms causes mid-sentence fragments)
+            });
+        case 'deepgram-flux':
+            // Deepgram Flux (V2 API) — semantic turn detection via ML model
+            // Uses TurnInfo events: StartOfTurn, Update, EagerEndOfTurn, TurnResumed, EndOfTurn
+            // TurnResumed prevents premature commits when user pauses mid-sentence
+            // All processing is server-side (Deepgram), zero local CPU
+            console.log(`🎙️ Using Deepgram Flux STT (semantic turn detection, eotThreshold=${config.eotThreshold ?? 0.85}, eotTimeoutMs=${config.eotTimeoutMs ?? 3000})`);
+            return new deepgram.STTv2({
+                model: config.model || 'flux-general-en',
+                language: config.language || 'en',
+                eotThreshold: config.eotThreshold ?? 0.85, // 0.0-1.0 — lower = more aggressive turn detection (more false positives), higher = more lenient (more false negatives)
+                eotTimeoutMs: config.eotTimeoutMs ?? 3000, // Max ms to wait before forcing end of turn — helps when user pauses for a long time mid-sentence (default 3000ms)
             });
         case 'groq-whisper':
             return openai.STT.withGroq({
@@ -55,7 +68,17 @@ export function createTTS(config) {
             break;
         case 'deepgram':
             tts = new deepgram.TTS({
-                model: (config.model || 'aura-asteria-en'),
+                model: (config.model || 'aura-2-asteria-en'),
+            });
+            break;
+        case 'groq-orpheus':
+            // Groq Orpheus TTS via OpenAI-compatible API ($22/M chars)
+            // Voices: autumn, diana, hannah, austin, daniel, troy
+            tts = new openai.TTS({
+                model: config.model || 'canopylabs/orpheus-v1-english',
+                voice: config.voice || 'autumn',
+                apiKey: process.env.GROQ_API_KEY,
+                baseURL: 'https://api.groq.com/openai/v1',
             });
             break;
         default:
@@ -64,7 +87,7 @@ export function createTTS(config) {
     // Increase max listeners to prevent memory leak warnings
     // TTS instances can have many concurrent listeners during active conversations
     if (tts && typeof tts.setMaxListeners === 'function') {
-        tts.setMaxListeners(50);
+        tts.setMaxListeners(100);
     }
     return tts;
 }
@@ -77,23 +100,17 @@ export function createTTS(config) {
  * - False triggers from ambient noise
  */
 export async function createVAD() {
+    // VAD now only handles interruption detection — turn detection moved to Deepgram STT (server-side).
+    // Lighter settings = less local CPU from ONNX inference.
     return silero.VAD.load({
-        // Minimum 0.5s speech before triggering - prevents noise/short sounds
-        // Higher value = more complete utterances before processing
-        minSpeechDuration: 0.5,
-        // Wait 1.2s of silence before considering speech "done"
-        // Allows natural pauses mid-sentence without triggering STT
-        // (increased from 0.8s to reduce sentence splitting)
-        minSilenceDuration: 1.2,
-        // Add 0.2s padding to start of speech chunks for cleaner audio
-        prefixPaddingDuration: 0.2,
-        // Higher threshold = less sensitive to quiet sounds/noise
-        // Default is 0.5, using 0.65 to reduce false positives
-        activationThreshold: 0.65,
+        minSpeechDuration: 0.4, // 400ms — quick interruption detection
+        minSilenceDuration: 1.2, // 1200ms — responsive
+        prefixPaddingDuration: 0.1,
+        activationThreshold: 0.85, // default — balanced for interruptions only
     });
 }
 /**
- * Default voice I/O configuration
+ * Default voice I/O configuration (used by realtime mode fallback)
  * Uses Deepgram STT (fast, accurate) + Deepgram TTS (fast, good)
  */
 export const DEFAULT_VOICE_IO_CONFIG = {
@@ -104,9 +121,25 @@ export const DEFAULT_VOICE_IO_CONFIG = {
     },
     tts: {
         provider: 'deepgram',
-        voice: 'aura-asteria-en',
+        voice: 'aura-2-asteria-en',
     },
 };
+/**
+ * Direct mode voice config — centralized here for easy provider swapping.
+ * To switch providers: comment out the active line, uncomment the alternative.
+ */
+export const DIRECT_MODE_STT = {
+    // provider: 'groq-whisper', model: 'whisper-large-v3-turbo',          // Batch — needs VAD
+    // provider: 'openai-whisper', model: 'whisper-1',                     // Batch — needs VAD
+    // provider: 'deepgram', model: 'nova-3', language: 'en',             // Streaming, silence-based endpointing
+    provider: 'deepgram-flux', model: 'flux-general-en', language: 'en', // Streaming, ML-based turn detection (requires Deepgram V2 access)
+};
+export const DIRECT_MODE_TTS = {
+    // provider: 'deepgram', model: 'aura-2-asteria-en',  // WebSocket-based: handles TTS abort cleanly (no unrecoverable crash on interruption)
+    // provider: 'gemini', model: 'gemini-2.5-flash-preview-tts', voice: 'apollo',
+    provider: 'openai', model: 'tts-1', voice: 'fable', // HTTP streaming: throws APIUserAbortError on interrupt → unrecoverable session crash
+    // provider: 'groq-orpheus', model: 'canopylabs/orpheus-v1-english', voice: 'autumn',  // $22/M chars — voices: autumn, diana, hannah, austin, daniel, troy
+};
 /**
  * Create Realtime Model for native speech-to-speech
  * Supports OpenAI Realtime API and Gemini Live API
@@ -116,11 +149,10 @@ export const DEFAULT_VOICE_IO_CONFIG = {
 export function createRealtimeModel(config) {
     if (config.provider === 'gemini') {
         console.log('📱 Using Gemini Live API (realtime)');
-        // Note: 12-2025 model has a known bug causing code 1008 crashes during user interruptions
-        // with tool calls. No newer model available yet — auto-recovery in index.ts handles this.
+        // Using 'latest' alias — 12-2025 had a known 1008 crash bug during interruptions + tool calls
         return new google.beta.realtime.RealtimeModel({
-            model: config.geminiModel || 'gemini-2.5-flash-native-audio-preview-12-2025',
-            voice: config.geminiVoice || 'Puck',
+            model: config.geminiModel || 'gemini-2.5-flash-native-audio-latest',
+            voice: config.geminiVoice || 'Charon',
             // Gemini supports instructions at model level
             instructions: config.instructions,
             // Enable transcription so we get text of what the agent says

package/fly.toml

@@ -0,0 +1,30 @@
+# Osborn Agent — Fly.io Configuration
+# Change app name per user: osborn-agent-<username>
+# Deploy: fly deploy
+
+app = "osborn-agent"
+primary_region = "ewr"
+
+[build]
+
+[env]
+  NODE_ENV = "production"
+  PORT = "8741"
+  HOST = "0.0.0.0"
+  OSBORN_CWD = "/workspace"
+
+[http_service]
+  internal_port = 8741
+  force_https = true
+  auto_stop_machines = "off"
+  auto_start_machines = true
+  min_machines_running = 1
+
+[mounts]
+  source = "workspace"
+  destination = "/workspace"
+
+[[vm]]
+  memory = "1gb"
+  cpu_kind = "shared"
+  cpus = 1

package/package.json

@@ -1,10 +1,10 @@
 {
   "name": "osborn",
-  "version": "0.5.3",
+  "version": "0.8.0",
   "description": "Voice AI coding assistant - local agent that connects to Osborn frontend",
   "type": "module",
   "bin": {
-    "osborn": "./bin/cli.js"
+    "osborn": "bin/cli.js"
   },
   "scripts": {
     "dev": "tsx src/index.ts",
@@ -22,31 +22,36 @@
   ],
   "repository": {
     "type": "git",
-    "url": "https://github.com/Erriccc/osborn.git"
+    "url": "git+https://github.com/Erriccc/osborn.git"
   },
   "author": "Osborn Ojure",
   "license": "MIT",
   "dependencies": {
-    "@anthropic-ai/claude-agent-sdk": "^0.1.74",
-    "@anthropic-ai/sdk": "^0.52.0",
+    "@anthropic-ai/claude-agent-sdk": "^0.2.91",
+    "@anthropic-ai/sdk": "^0.80.0",
     "@google/genai": "^1.0.0",
-    "@livekit/agents": "^1.0.45",
-    "@livekit/agents-plugin-deepgram": "^1.0.45",
-    "@livekit/agents-plugin-elevenlabs": "^1.0.45",
-    "@livekit/agents-plugin-google": "^1.0.45",
-    "@livekit/agents-plugin-openai": "^1.0.45",
-    "@livekit/agents-plugin-silero": "^1.0.45",
+    "@livekit/agents": "^1.2.1",
+    "@livekit/agents-plugin-deepgram": "^1.2.1",
+    "@livekit/agents-plugin-elevenlabs": "^1.2.1",
+    "@livekit/agents-plugin-google": "^1.2.1",
+    "@livekit/agents-plugin-livekit": "^1.2.1",
+    "@livekit/agents-plugin-openai": "^1.2.1",
+    "@livekit/agents-plugin-silero": "^1.2.1",
     "@livekit/rtc-node": "^0.13.24",
-    "@modelcontextprotocol/sdk": "^1.26.0",
+    "@modelcontextprotocol/sdk": "^1.29.0",
     "@openai/codex-sdk": "^0.77.0",
     "@smithery/api": "^0.48.0",
+    "@types/diff": "^8.0.0",
+    "@vscode/ripgrep": "^1.17.1",
+    "diff": "^8.0.4",
     "dotenv": "^16.4.0",
     "livekit-server-sdk": "^2.15.0",
+    "minisearch": "^7.2.0",
     "node-pty": "^1.1.0",
     "tsx": "^4.0.0",
     "ws": "^8.19.0",
     "yaml": "^2.3.0",
-    "zod": "^3.23.0"
+    "zod": "^4.3.6"
   },
   "devDependencies": {
     "@types/node": "^20.0.0",