osborn 0.9.197 → 0.9.199

package/dist/claude-llm.js

@@ -31,6 +31,12 @@ const TOOL_CALL_BUDGET = 3;
  * Strip markdown formatting for TTS (text-to-speech)
  * Removes **bold**, ##headers, ```code```, etc. so TTS doesn't read them literally
  */
+function extractSpeedMarker(text) {
+    const m = text.match(/^\[SPEED:(0\.[5-9]\d*|1\.[0-3]\d*)\]/);
+    if (!m)
+        return { text };
+    return { text: text.slice(m[0].length), speed: parseFloat(m[1]) };
+}
 function stripMarkdownForTTS(text) {
     return text
         // Remove code blocks (``` ... ```)
@@ -677,6 +683,7 @@ export class ClaudeLLM extends llm.LLM {
     #currentTurnMessageId = null;
     #currentTurnChunkIndex = 0;
     #currentTurnChunks = [];
+    #currentTurnSpeed;
     #backgroundConsumerRunning = false;
     // Active queries — multiple can be running (SDK queues them internally).
     // We keep ALL references so interrupt() can stop whatever is currently executing.
@@ -1249,18 +1256,22 @@ export class ClaudeLLM extends llm.LLM {
                         this.#currentTurnMessageId = crypto.randomUUID();
                         this.#currentTurnChunkIndex = 0;
                         this.#currentTurnChunks = [];
+                        this.#currentTurnSpeed = undefined;
                     }
                     const turnMessageId = this.#currentTurnMessageId;
                     for (const block of msg.message.content) {
                         if (block.type === 'text' && block.text) {
                             const chunkIndex = this.#currentTurnChunkIndex;
-                            callbacks.eventEmitter.emit('assistant_text', { text: block.text, messageId: turnMessageId, chunkIndex });
-                            const ttsChunk = stripMarkdownForTTS(block.text);
+                            const { text: cleanText, speed } = extractSpeedMarker(block.text);
+                            if (speed !== undefined)
+                                this.#currentTurnSpeed = speed;
+                            callbacks.eventEmitter.emit('assistant_text', { text: cleanText, messageId: turnMessageId, chunkIndex });
+                            const ttsChunk = stripMarkdownForTTS(cleanText);
                             if (ttsChunk.trim()) {
                                 this.#currentTurnChunks.push(ttsChunk);
                                 this.#currentTurnChunkIndex++;
                                 console.log(`🔊 TTS say (${ttsChunk.length} chars): "${ttsChunk}"`);
-                                callbacks.eventEmitter.emit('tts_say', { text: ttsChunk, messageId: turnMessageId, chunkIndex });
+                                callbacks.eventEmitter.emit('tts_say', { text: ttsChunk, messageId: turnMessageId, chunkIndex, speed: this.#currentTurnSpeed });
                             }
                         }
                     }
@@ -1279,6 +1290,7 @@ export class ClaudeLLM extends llm.LLM {
                     this.#currentTurnMessageId = null;
                     this.#currentTurnChunkIndex = 0;
                     this.#currentTurnChunks = [];
+                    this.#currentTurnSpeed = undefined;
                     console.log('✅ Claude turn complete (persistent session stays alive)');
                 }
             }
@@ -2136,6 +2148,7 @@ class ClaudeLLMStream extends llm.LLMStream {
             let streamTurnMessageId = null;
             let streamTurnChunkIndex = 0;
             let streamTurnChunks = [];
+            let streamTurnSpeed;
             // DIRECT MODE OPTIMIZATION: When skipTTSQueue is true, we run the Claude query
             // in the background and return from run() immediately. This is critical because:
             //
@@ -2227,11 +2240,16 @@ class ClaudeLLMStream extends llm.LLMStream {
                         streamTurnMessageId = crypto.randomUUID();
                         streamTurnChunkIndex = 0;
                         streamTurnChunks = [];
+                        streamTurnSpeed = undefined;
                     }
                     for (const block of message.message.content) {
                         if (block.type === 'text' && block.text) {
                             hasOutput = true;
-                            const rawText = block.text;
+                            // Strip [SPEED:X.X] marker before emitting to frontend or TTS
+                            const { text: cleanText, speed } = extractSpeedMarker(block.text);
+                            if (speed !== undefined)
+                                streamTurnSpeed = speed;
+                            const rawText = cleanText;
                             const chunkIndex = streamTurnChunkIndex;
                             // Emit RAW text to frontend (for chat bubbles with full formatting)
                             this.#eventEmitter.emit('assistant_text', { text: rawText, messageId: streamTurnMessageId, chunkIndex });
@@ -2244,7 +2262,7 @@ class ClaudeLLMStream extends llm.LLMStream {
                                     // Direct mode: emit event for session.say() — bypasses LiveKit's
                                     // BufferedTokenStream which causes stuck/delayed/out-of-order audio
                                     console.log(`🔊 TTS say (${ttsChunk.length} chars): "${ttsChunk}"`);
-                                    this.#eventEmitter.emit('tts_say', { text: ttsChunk, messageId: streamTurnMessageId, chunkIndex });
+                                    this.#eventEmitter.emit('tts_say', { text: ttsChunk, messageId: streamTurnMessageId, chunkIndex, speed: streamTurnSpeed });
                                 }
                                 else {
                                     // Realtime mode: use LLM stream queue (framework handles TTS)

package/dist/config.js

@@ -75,18 +75,17 @@ const DEFAULT_CONFIG = {
         },
         tts: {
             // ── Active ─────────────────────────────────────────────────────────────
-            provider: 'openai',
-            model: 'tts-1-hd',
-            voice: 'fable',
-            // Soniox TTS requires @livekit/agents-plugin-soniox@1.8.1 which needs agents@1.8.1.
-            // Pending full LiveKit 1.4→1.8 upgrade. Swap back once agents are upgraded.
-            // provider: 'soniox', model: 'tts-rt-v1', voice: 'Victoria'
-            //   WebSocket streaming, clean abort, speed control, ~$4–16/M. Needs agents@1.8.1.
+            provider: 'soniox',
+            model: 'tts-rt-v1',
+            voice: 'Victoria',
+            // WebSocket streaming, clean abort, speed control 0.7–1.3x, ~$4–16/M chars.
+            // Victoria = en-GB female, refined. Also: Isla (en-GB, lively), Maya (en-US female).
             //
             // ── Alternatives ───────────────────────────────────────────────────────
+            // provider: 'openai', model: 'tts-1-hd', voice: 'fable'
+            //   $30/M chars, ~500ms TTFB, HTTP streaming. 6 voices.
             // provider: 'openai', model: 'tts-1', voice: 'fable'
             //   $15/M chars, slightly lower quality, same voices.
-            //   $15/M chars, slightly lower quality, same voices.
             // provider: 'deepgram', model: 'aura-2-asteria-en'
             //   $15/M chars, ~100ms TTFB, WebSocket. Voices: asteria luna stella hera orion arcas perseus angus orpheus.
             // provider: 'rime', model: 'mistv3', voice: 'cove'

package/dist/index.js

@@ -3441,6 +3441,13 @@ async function main() {
             }
             const sayId = Date.now(); // simple ID to correlate start/end logs
             console.log(`🗣️ [${sayId}] session.say START (${data.text.length} chars): "${data.text}"`);
+            // Apply agent-requested speed (e.g. [SPEED:0.85] stripped from response text)
+            if (data.speed !== undefined) {
+                try {
+                    tts.updateOptions?.({ speed: data.speed });
+                }
+                catch { }
+            }
             try {
                 const handle = currentSession.say(data.text);
                 if (handle && typeof handle.addDoneCallback === 'function') {
@@ -3647,9 +3654,9 @@ async function main() {
                 // a full 3s window to keep talking before deciding it was false and
                 // resuming. Other two knobs left at SDK defaults.
                 interruption: {
-                    minDuration: 1500, // default 500  — require 1.5s sustained speech (faster barge-in than 2500)
-                    minWords: 2, // default 0    — require ≥2 transcript words
-                    falseInterruptionTimeout: 3500, // default 2000 — 3.5s false-interrupt window (belt-and-suspenders since minDuration was loosened)
+                    minDuration: 800, // default 500  — require 800ms sustained speech (tightened from 1500; Soniox semantic STT reduces false positives)
+                    minWords: 1, // default 0    — require ≥1 word ("stop", "wait", "no" now count)
+                    falseInterruptionTimeout: 3500, // default 2000 — 3.5s false-interrupt window
                     // resumeFalseInterruption: true,      // default true  (unchanged)
                     // discardAudioIfUninterruptible: true,// default true  (unchanged)
                 },
@@ -4125,11 +4132,6 @@ async function main() {
                     sendAgentTranscript(text, 'conv_item');
                 }
             });
-            // FALLBACK: user_speech_committed
-            sess.on('user_speech_committed', (ev) => {
-                const transcript = ev.transcript || ev.text || '';
-                sendUserTranscript(transcript, 'committed');
-            });
             // Agent state tracking
             sess.on('agent_state_changed', (ev) => {
                 agentState = ev.newState;

package/dist/voice-io.d.ts

@@ -4,7 +4,6 @@
  */
 import * as deepgram from '@livekit/agents-plugin-deepgram';
 import * as openai from '@livekit/agents-plugin-openai';
-import * as silero from '@livekit/agents-plugin-silero';
 import * as soniox from '@livekit/agents-plugin-soniox';
 export interface STTConfig {
     provider: 'soniox' | 'deepgram' | 'deepgram-flux' | 'groq-whisper' | 'openai-whisper';
@@ -41,4 +40,4 @@ export declare function createTTS(config: TTSConfig): any;
  * - Split sentences when user pauses briefly mid-speech
  * - False triggers from ambient noise
  */
-export declare function createVAD(): Promise<silero.VAD>;
+export declare function createVAD(): Promise<import("@livekit/agents").VAD>;

package/dist/voice-io.js

@@ -23,7 +23,7 @@ export function createSTT(config) {
                 model: (config.model || 'stt-rt-v4'),
                 languageHints: config.language ? [config.language] : ['en'],
                 maxEndpointDelayMs: 1200, // give model room to decide on mid-thought pauses
-                endpointLatencyAdjustmentLevel: 2, // aggressive but not max — good for voice assistant
+                endpointLatencyAdjustmentLevel: 3, // max aggression — semantic model handles accuracy
                 context: {
                     terms: ['Claude', 'TypeScript', 'LiveKit', 'Deepgram', 'npm', 'Railway', 'Fly.io'],
                 },
@@ -76,7 +76,7 @@ export function createTTS(config) {
             tts = new soniox.TTS({
                 model: (config.model || 'tts-rt-v1'),
                 voice: config.voice || 'Maya',
-                speed: 1.0,
+                speed: 0.9,
             });
             break;
         case 'openai':

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "osborn",
-  "version": "0.9.197",
+  "version": "0.9.199",
   "description": "Voice AI coding assistant - local agent that connects to Osborn frontend",
   "type": "module",
   "bin": {
@@ -32,15 +32,15 @@
   "dependencies": {
     "@anthropic-ai/claude-agent-sdk": "^0.2.91",
     "@anthropic-ai/sdk": "^0.80.0",
-    "@livekit/agents": "1.4.6",
-    "@livekit/agents-plugin-deepgram": "1.4.6",
-    "@livekit/agents-plugin-fishaudio": "1.4.6",
-    "@livekit/agents-plugin-livekit": "1.4.6",
-    "@livekit/agents-plugin-openai": "1.4.6",
-    "@livekit/agents-plugin-rime": "1.4.6",
-    "@livekit/agents-plugin-silero": "1.4.6",
-    "@livekit/agents-plugin-soniox": "1.4.6",
-    "@livekit/rtc-node": "0.13.29",
+    "@livekit/agents": "1.8.1",
+    "@livekit/agents-plugin-deepgram": "1.8.1",
+    "@livekit/agents-plugin-fishaudio": "1.8.1",
+    "@livekit/agents-plugin-livekit": "1.8.1",
+    "@livekit/agents-plugin-openai": "1.8.1",
+    "@livekit/agents-plugin-rime": "1.8.1",
+    "@livekit/agents-plugin-silero": "1.8.1",
+    "@livekit/agents-plugin-soniox": "1.8.1",
+    "@livekit/rtc-node": "0.13.35",
     "@modelcontextprotocol/sdk": "^1.29.0",
     "@smithery/api": "^0.48.0",
     "@types/diff": "^8.0.0",