osborn 0.9.180 → 0.9.182

package/dist/turn-detector-shim.js

@@ -21,9 +21,13 @@ export class CloudTurnDetector {
     model = 'lk_end_of_utterance_multilingual';
     provider = 'livekit';
     constructor() {
-        this.#remoteUrl = process.env.LIVEKIT_REMOTE_EOT_URL;
+        const raw = process.env.LIVEKIT_REMOTE_EOT_URL;
+        // EOT endpoint is HTTP — convert wss:// → https:// if the env var uses WebSocket scheme
+        this.#remoteUrl = raw
+            ? raw.replace(/^wss:\/\//, 'https://').replace(/^ws:\/\//, 'http://')
+            : undefined;
         if (this.#remoteUrl) {
-            console.log(`🧠 Turn detector: LiveKit Cloud remote inference`);
+            console.log(`🧠 Turn detector: LiveKit Cloud remote inference (${this.#remoteUrl})`);
         }
         else {
             console.log('🧠 Turn detector: No LIVEKIT_REMOTE_EOT_URL — STT endpointing fallback');

package/dist/voice-io.d.ts

@@ -1,16 +1,10 @@
 /**
  * Voice I/O Module
- * Handles STT (Speech-to-Text), TTS (Text-to-Speech), and Realtime model creation
- *
- * Supports two modes:
- * - Direct mode: STT (Deepgram) → Claude Agent SDK → TTS (Deepgram)
- * - Realtime mode: OpenAI/Gemini native speech-to-speech models
+ * Handles STT (Speech-to-Text) and TTS (Text-to-Speech) for pipeline mode.
  */
 import * as deepgram from '@livekit/agents-plugin-deepgram';
-import * as google from '@livekit/agents-plugin-google';
 import * as openai from '@livekit/agents-plugin-openai';
 import * as silero from '@livekit/agents-plugin-silero';
-import type { RealtimeConfig } from './config.js';
 export interface STTConfig {
     provider: 'deepgram' | 'deepgram-flux' | 'groq-whisper' | 'openai-whisper';
     model?: string;
@@ -21,7 +15,7 @@ export interface STTConfig {
     eotTimeoutMs?: number;
 }
 export interface TTSConfig {
-    provider: 'gemini' | 'openai' | 'elevenlabs' | 'deepgram' | 'groq-orpheus';
+    provider: 'openai' | 'deepgram' | 'groq-orpheus' | 'fishaudio' | 'rime';
     voice?: string;
     model?: string;
 }
@@ -36,7 +30,6 @@ export interface VoiceIOConfig {
 export declare function createSTT(config: STTConfig): deepgram.STT | deepgram.STTv2 | openai.STT;
 /**
  * Create TTS (Text-to-Speech) instance based on config
- * Using Gemini TTS as default (cheaper, good quality)
  */
 export declare function createTTS(config: TTSConfig): any;
 /**
@@ -49,32 +42,8 @@ export declare function createTTS(config: TTSConfig): any;
  */
 export declare function createVAD(): Promise<silero.VAD>;
 /**
- * Default voice I/O configuration (used by realtime mode fallback)
- * Uses Deepgram STT (fast, accurate) + Deepgram TTS (fast, good)
- */
-export declare const DEFAULT_VOICE_IO_CONFIG: VoiceIOConfig;
-/**
- * Direct mode voice config — centralized here for easy provider swapping.
+ * Pipeline mode voice config — centralized here for easy provider swapping.
  * To switch providers: comment out the active line, uncomment the alternative.
  */
 export declare const DIRECT_MODE_STT: STTConfig;
 export declare const DIRECT_MODE_TTS: TTSConfig;
-export interface RealtimeModelConfig {
-    provider: 'openai' | 'gemini';
-    openaiVoice?: 'alloy' | 'echo' | 'fable' | 'onyx' | 'nova' | 'shimmer';
-    openaiModel?: string;
-    geminiVoice?: 'Charon' | 'Puck' | 'Kore' | 'Fenrir' | 'Aoede';
-    geminiModel?: string;
-    instructions?: string;
-}
-/**
- * Create Realtime Model for native speech-to-speech
- * Supports OpenAI Realtime API and Gemini Live API
- *
- * Note: Instructions are passed to voice.Agent, not to the RealtimeModel
- */
-export declare function createRealtimeModel(config: RealtimeModelConfig): google.beta.realtime.RealtimeModel | openai.realtime.RealtimeModel;
-/**
- * Create realtime model from config
- */
-export declare function createRealtimeModelFromConfig(realtimeConfig: RealtimeConfig, instructions?: string): google.beta.realtime.RealtimeModel | openai.realtime.RealtimeModel;

package/dist/voice-io.js

@@ -1,14 +1,11 @@
 /**
  * Voice I/O Module
- * Handles STT (Speech-to-Text), TTS (Text-to-Speech), and Realtime model creation
- *
- * Supports two modes:
- * - Direct mode: STT (Deepgram) → Claude Agent SDK → TTS (Deepgram)
- * - Realtime mode: OpenAI/Gemini native speech-to-speech models
+ * Handles STT (Speech-to-Text) and TTS (Text-to-Speech) for pipeline mode.
  */
 import * as deepgram from '@livekit/agents-plugin-deepgram';
-import * as google from '@livekit/agents-plugin-google';
+import * as fishaudio from '@livekit/agents-plugin-fishaudio';
 import * as openai from '@livekit/agents-plugin-openai';
+import * as rime from '@livekit/agents-plugin-rime';
 import * as silero from '@livekit/agents-plugin-silero';
 /**
  * Create STT (Speech-to-Text) instance based on config
@@ -48,18 +45,10 @@ export function createSTT(config) {
 }
 /**
  * Create TTS (Text-to-Speech) instance based on config
- * Using Gemini TTS as default (cheaper, good quality)
  */
 export function createTTS(config) {
     let tts;
     switch (config.provider) {
-        case 'gemini':
-            // Gemini TTS via google plugin
-            tts = new google.beta.TTS({
-                model: config.model || 'gemini-2.5-flash-preview-tts',
-                voice: config.voice || 'apollo',
-            });
-            break;
         case 'openai':
             tts = new openai.TTS({
                 voice: config.voice || 'alloy',
@@ -81,6 +70,27 @@ export function createTTS(config) {
                 baseURL: 'https://api.groq.com/openai/v1',
             });
             break;
+        case 'fishaudio':
+            // Fish Audio s2-pro ($15/M chars) — blind test winner, half price of OpenAI tts-1-hd
+            // voiceId: pick from Fish Audio voice library (leave undefined for system default)
+            // Requires FISH_AUDIO_API_KEY env var
+            tts = new fishaudio.TTS({
+                model: (config.model || 's2-pro'),
+                voiceId: config.voice,
+                latencyMode: 'low',
+            });
+            break;
+        case 'rime':
+            // Rime Mist v3 ($30/M chars, 37ms TTFB) — fastest commercial TTS, conversation-trained
+            // useWebsocket: true → WebSocket streaming = clean abort on interruption (like Deepgram)
+            // speaker: pick from Rime voice library, e.g. 'aurora', 'ember', 'cove'
+            // Requires RIME_API_KEY env var
+            tts = new rime.TTS({
+                modelId: (config.model || 'mistv3'),
+                speaker: config.voice || 'cove',
+                useWebsocket: true,
+            });
+            break;
         default:
             throw new Error(`Unknown TTS provider: ${config.provider}`);
     }
@@ -110,76 +120,20 @@ export async function createVAD() {
     });
 }
 /**
- * Default voice I/O configuration (used by realtime mode fallback)
- * Uses Deepgram STT (fast, accurate) + Deepgram TTS (fast, good)
- */
-export const DEFAULT_VOICE_IO_CONFIG = {
-    stt: {
-        provider: 'deepgram',
-        model: 'nova-3',
-        language: 'en',
-    },
-    tts: {
-        provider: 'deepgram',
-        voice: 'aura-2-asteria-en',
-    },
-};
-/**
- * Direct mode voice config — centralized here for easy provider swapping.
+ * Pipeline mode voice config — centralized here for easy provider swapping.
  * To switch providers: comment out the active line, uncomment the alternative.
  */
 export const DIRECT_MODE_STT = {
     // provider: 'groq-whisper', model: 'whisper-large-v3-turbo',          // Batch — needs VAD
     // provider: 'openai-whisper', model: 'whisper-1',                     // Batch — needs VAD
-    // provider: 'deepgram', model: 'nova-3', language: 'en',             // Streaming, silence-based endpointing
-    provider: 'deepgram-flux', model: 'flux-general-en', language: 'en', // Streaming, ML-based turn detection (requires Deepgram V2 access)
+    // provider: 'deepgram-flux', model: 'flux-general-en', language: 'en',  // Streaming, ML-based turn detection — Flux V2 has silent keepalive timeout bug (~30s silence kills connection)
+    provider: 'deepgram', model: 'nova-3', language: 'en', // Streaming, silence-based endpointing (550ms)
 };
 export const DIRECT_MODE_TTS = {
-    // provider: 'deepgram', model: 'aura-2-asteria-en',  // WebSocket-based: handles TTS abort cleanly (no unrecoverable crash on interruption)
-    // provider: 'gemini', model: 'gemini-2.5-flash-preview-tts', voice: 'apollo',
+    // provider: 'deepgram', model: 'aura-2-asteria-en',  // WebSocket-based: handles TTS abort cleanly — but quality rejected (run-on sentences)
     // provider: 'openai', model: 'tts-1', voice: 'fable',  // HTTP streaming: throws APIUserAbortError on interrupt → unrecoverable session crash
     provider: 'openai', model: 'tts-1-hd', voice: 'fable', // 0.9.70: test tts-1-hd — tts-1 had chronic per-sentence HTTP hangs (40s SDK watchdog → APIUserAbortError mid-message)
     // provider: 'groq-orpheus', model: 'canopylabs/orpheus-v1-english', voice: 'autumn',  // $22/M chars — voices: autumn, diana, hannah, austin, daniel, troy
+    // provider: 'fishaudio', model: 's2-pro', voice: '<voice-id>',  // $15/M chars — blind test winner, HTTP streaming (test abort behavior)
+    // provider: 'rime', model: 'mistv3', voice: 'cove',  // $30/M chars, 37ms TTFB — WebSocket (safe on interruption); voices: aurora, ember, cove
 };
-/**
- * Create Realtime Model for native speech-to-speech
- * Supports OpenAI Realtime API and Gemini Live API
- *
- * Note: Instructions are passed to voice.Agent, not to the RealtimeModel
- */
-export function createRealtimeModel(config) {
-    if (config.provider === 'gemini') {
-        console.log('📱 Using Gemini Live API (realtime)');
-        // Using 'latest' alias — 12-2025 had a known 1008 crash bug during interruptions + tool calls
-        return new google.beta.realtime.RealtimeModel({
-            model: config.geminiModel || 'gemini-2.5-flash-native-audio-latest',
-            voice: config.geminiVoice || 'Charon',
-            // Gemini supports instructions at model level
-            instructions: config.instructions,
-            // Enable transcription so we get text of what the agent says
-            inputAudioTranscription: {},
-            outputAudioTranscription: {},
-        });
-    }
-    else {
-        console.log('📱 Using OpenAI Realtime API');
-        // OpenAI RealtimeModel - instructions go to voice.Agent instead
-        return new openai.realtime.RealtimeModel({
-            model: config.openaiModel || 'gpt-4o-realtime-preview',
-            voice: config.openaiVoice || 'alloy',
-        });
-    }
-}
-/**
- * Create realtime model from config
- */
-export function createRealtimeModelFromConfig(realtimeConfig, instructions) {
-    return createRealtimeModel({
-        provider: realtimeConfig.provider || 'openai',
-        openaiVoice: realtimeConfig.openaiVoice,
-        openaiModel: realtimeConfig.openaiModel,
-        geminiVoice: realtimeConfig.geminiVoice,
-        geminiModel: realtimeConfig.geminiModel,
-        instructions,
-    });
-}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "osborn",
-  "version": "0.9.180",
+  "version": "0.9.182",
   "description": "Voice AI coding assistant - local agent that connects to Osborn frontend",
   "type": "module",
   "bin": {
@@ -32,17 +32,15 @@
   "dependencies": {
     "@anthropic-ai/claude-agent-sdk": "^0.2.91",
     "@anthropic-ai/sdk": "^0.80.0",
-    "@google/genai": "^1.0.0",
     "@livekit/agents": "1.4.6",
     "@livekit/agents-plugin-deepgram": "1.4.6",
-    "@livekit/agents-plugin-elevenlabs": "1.4.6",
-    "@livekit/agents-plugin-google": "1.4.6",
+    "@livekit/agents-plugin-fishaudio": "1.4.6",
     "@livekit/agents-plugin-livekit": "1.4.6",
     "@livekit/agents-plugin-openai": "1.4.6",
+    "@livekit/agents-plugin-rime": "1.4.6",
     "@livekit/agents-plugin-silero": "1.4.6",
     "@livekit/rtc-node": "0.13.29",
     "@modelcontextprotocol/sdk": "^1.29.0",
-    "@openai/codex-sdk": "^0.77.0",
     "@smithery/api": "^0.48.0",
     "@types/diff": "^8.0.0",
     "@vscode/ripgrep": "^1.17.1",

package/tests/autocompact-pct.test.ts

@@ -1,77 +0,0 @@
-/**
- * Regression tests for CLAUDE_AUTOCOMPACT_PCT_OVERRIDE in claude-llm.ts
- *
- * Scope: verifies that the compaction threshold is set to a value that
- * leaves meaningful context headroom before compaction fires.
- *
- * Derived from: requirements documented in CLAUDE.md (ENABLE_1M_CONTEXT,
- * compaction rationale comments in claude-llm.ts) and the 0.9.177 → 0.9.178
- * change that lowered the threshold from 92 → 60.
- * NOT derived from the implementation diff.
- */
-
-import { strict as assert } from 'node:assert'
-import { readFileSync } from 'node:fs'
-import { dirname, join } from 'node:path'
-import { fileURLToPath } from 'node:url'
-
-const here = dirname(fileURLToPath(import.meta.url))
-const src = readFileSync(join(here, '../src/claude-llm.ts'), 'utf8')
-
-let passed = 0
-let failed = 0
-
-function test(name: string, fn: () => void) {
-  try {
-    fn()
-    console.log(`  ✅ ${name}`)
-    passed++
-  } catch (err: any) {
-    console.error(`  ❌ ${name}`)
-    console.error(`     ${err.message}`)
-    failed++
-  }
-}
-
-// Extract the numeric value of CLAUDE_AUTOCOMPACT_PCT_OVERRIDE from source
-const match = src.match(/process\.env\.CLAUDE_AUTOCOMPACT_PCT_OVERRIDE\s*=\s*'(\d+)'/)
-const pctValue = match ? parseInt(match[1], 10) : NaN
-
-console.log('\n=== autocompact PCT override regression tests ===\n')
-console.log(`  (detected value: CLAUDE_AUTOCOMPACT_PCT_OVERRIDE = '${pctValue}')`)
-console.log()
-
-test('CLAUDE_AUTOCOMPACT_PCT_OVERRIDE is set in claude-llm.ts', () => {
-  assert.ok(
-    !isNaN(pctValue),
-    'CLAUDE_AUTOCOMPACT_PCT_OVERRIDE must be explicitly set in claude-llm.ts'
-  )
-})
-
-test('CLAUDE_AUTOCOMPACT_PCT_OVERRIDE is a valid percentage (1–99)', () => {
-  assert.ok(
-    pctValue >= 1 && pctValue <= 99,
-    `CLAUDE_AUTOCOMPACT_PCT_OVERRIDE must be between 1 and 99, got ${pctValue}`
-  )
-})
-
-test('CLAUDE_AUTOCOMPACT_PCT_OVERRIDE is ≤ 92 (prevents compaction at extreme tail)', () => {
-  // Values above 92 leave almost no headroom before hitting the token limit.
-  // The long-standing default was 92; anything higher than that risks the
-  // "compacting every message" bug described in memory/osborn-1m-context-and-worker-restart.md
-  assert.ok(
-    pctValue <= 92,
-    `CLAUDE_AUTOCOMPACT_PCT_OVERRIDE is ${pctValue} which is above 92 — ` +
-    'values this high risk compaction firing on every message at the context ceiling'
-  )
-})
-
-test('ENABLE_1M_CONTEXT is also set in claude-llm.ts', () => {
-  assert.ok(
-    src.includes("process.env.ENABLE_1M_CONTEXT = '1'"),
-    "ENABLE_1M_CONTEXT = '1' must be set alongside CLAUDE_AUTOCOMPACT_PCT_OVERRIDE"
-  )
-})
-
-console.log(`\n--- ${passed + failed} tests: ${passed} passed, ${failed} failed ---\n`)
-if (failed > 0) process.exit(1)

package/tests/voice-io-stt-config.test.ts

@@ -1,177 +0,0 @@
-/**
- * Regression tests for voice-io.ts STT configuration
- *
- * Scope: covers the 0.9.175 → 0.9.177 change that switches DIRECT_MODE_STT
- * from deepgram-flux (ML turn detection) back to deepgram (silence-based
- * endpointing), due to Flux V2 keepalive timeout bug (~30s of silence kills
- * the connection).
- *
- * Derived from: requirements in CLAUDE.md (Three Voice Modes), voice-io.ts
- * interface definitions, and the documented bug. NOT derived from the diff.
- */
-
-import { strict as assert } from 'node:assert'
-
-// ──────────────────────────────────────────────────────────────────────────────
-// Inline the minimal types — avoid importing from the module which requires
-// live LK plugins (can't instantiate in a unit test without credentials).
-// ──────────────────────────────────────────────────────────────────────────────
-interface STTConfig {
-  provider: 'deepgram' | 'deepgram-flux' | 'groq-whisper' | 'openai-whisper'
-  model?: string
-  language?: string
-  eotThreshold?: number
-  eotTimeoutMs?: number
-}
-
-// ──────────────────────────────────────────────────────────────────────────────
-// Read the compiled output so we get the real runtime values, not just TS types
-// ──────────────────────────────────────────────────────────────────────────────
-// We import from the transpiled dist to avoid plugin side-effects at import time.
-// If dist is missing, fall back to a direct static-analysis of the source file.
-
-let DIRECT_MODE_STT_PROVIDER: string | undefined
-let DIRECT_MODE_STT_MODEL: string | undefined
-let DIRECT_MODE_STT_LANG: string | undefined
-
-try {
-  // dist/voice-io.js is the compiled output of the build step
-  const mod = await import('../dist/voice-io.js')
-  const cfg: STTConfig = mod.DIRECT_MODE_STT
-  DIRECT_MODE_STT_PROVIDER = cfg.provider
-  DIRECT_MODE_STT_MODEL = cfg.model
-  DIRECT_MODE_STT_LANG = cfg.language
-} catch (e) {
-  // Fallback: parse source file statically to extract the active provider line
-  const { readFileSync } = await import('node:fs')
-  const { fileURLToPath } = await import('node:url')
-  const { dirname, join } = await import('node:path')
-  const here = dirname(fileURLToPath(import.meta.url))
-  const src = readFileSync(join(here, '../src/voice-io.ts'), 'utf8')
-
-  // Find the non-commented active provider line in DIRECT_MODE_STT
-  const providerMatch = src.match(
-    /export const DIRECT_MODE_STT[\s\S]*?^\s+provider:\s*'([^']+)'/m
-  )
-  const modelMatch = src.match(
-    /export const DIRECT_MODE_STT[\s\S]*?model:\s*'([^']+)'/m
-  )
-  const langMatch = src.match(
-    /export const DIRECT_MODE_STT[\s\S]*?language:\s*'([^']+)'/m
-  )
-  DIRECT_MODE_STT_PROVIDER = providerMatch?.[1]
-  DIRECT_MODE_STT_MODEL = modelMatch?.[1]
-  DIRECT_MODE_STT_LANG = langMatch?.[1]
-}
-
-// ──────────────────────────────────────────────────────────────────────────────
-// Tests
-// ──────────────────────────────────────────────────────────────────────────────
-
-let passed = 0
-let failed = 0
-
-function test(name: string, fn: () => void) {
-  try {
-    fn()
-    console.log(`  ✅ ${name}`)
-    passed++
-  } catch (err: any) {
-    console.error(`  ❌ ${name}`)
-    console.error(`     ${err.message}`)
-    failed++
-  }
-}
-
-console.log('\n=== voice-io STT config regression tests ===\n')
-
-// ── 1. Provider must be deepgram (not deepgram-flux) ──────────────────────────
-test('DIRECT_MODE_STT.provider is "deepgram" (not deepgram-flux)', () => {
-  assert.equal(
-    DIRECT_MODE_STT_PROVIDER,
-    'deepgram',
-    `Expected provider "deepgram" but got "${DIRECT_MODE_STT_PROVIDER}". ` +
-    'deepgram-flux has a known keepalive timeout bug (silent ~30s kills the connection).'
-  )
-})
-
-test('DIRECT_MODE_STT.provider is NOT "deepgram-flux"', () => {
-  assert.notEqual(
-    DIRECT_MODE_STT_PROVIDER,
-    'deepgram-flux',
-    'deepgram-flux must not be the active provider — ' +
-    'Flux V2 has silent keepalive timeout bug (~30s silence kills connection).'
-  )
-})
-
-// ── 2. Model and language ──────────────────────────────────────────────────────
-test('DIRECT_MODE_STT.model is "nova-3"', () => {
-  assert.equal(
-    DIRECT_MODE_STT_MODEL,
-    'nova-3',
-    `Expected model "nova-3" but got "${DIRECT_MODE_STT_MODEL}".`
-  )
-})
-
-test('DIRECT_MODE_STT.language is "en"', () => {
-  assert.equal(
-    DIRECT_MODE_STT_LANG,
-    'en',
-    `Expected language "en" but got "${DIRECT_MODE_STT_LANG}".`
-  )
-})
-
-// ── 3. STTConfig type-level: provider values are the expected set ─────────────
-// (These are compile-time guarantees verified at build time, but we assert
-//  them at runtime too so future changes surface here.)
-test('DIRECT_MODE_STT provider is one of the valid union members', () => {
-  const validProviders = ['deepgram', 'deepgram-flux', 'groq-whisper', 'openai-whisper']
-  assert.ok(
-    validProviders.includes(DIRECT_MODE_STT_PROVIDER!),
-    `provider "${DIRECT_MODE_STT_PROVIDER}" is not in valid set: ${validProviders.join(', ')}`
-  )
-})
-
-// ── 4. Backward-compat: createSTT deepgram path shape ────────────────────────
-// We verify the source still has the deepgram case in createSTT so the switch
-// won't hit the `default: throw` path at runtime.
-test('createSTT source still contains deepgram case (backward compat)', async () => {
-  const { readFileSync } = await import('node:fs')
-  const { fileURLToPath } = await import('node:url')
-  const { dirname, join } = await import('node:path')
-  const here = dirname(fileURLToPath(import.meta.url))
-  const src = readFileSync(join(here, '../src/voice-io.ts'), 'utf8')
-  assert.ok(
-    src.includes("case 'deepgram':"),
-    "createSTT source must still contain case 'deepgram' — removing it would throw at runtime"
-  )
-})
-
-test('createSTT source still contains deepgram-flux case (backward compat)', async () => {
-  const { readFileSync } = await import('node:fs')
-  const { fileURLToPath } = await import('node:url')
-  const { dirname, join } = await import('node:path')
-  const here = dirname(fileURLToPath(import.meta.url))
-  const src = readFileSync(join(here, '../src/voice-io.ts'), 'utf8')
-  assert.ok(
-    src.includes("case 'deepgram-flux':"),
-    "createSTT source must still contain case 'deepgram-flux' — it must remain selectable via config"
-  )
-})
-
-// ── 5. deepgram endpointing value in createSTT ────────────────────────────────
-test('createSTT deepgram case uses 550ms endpointing (mid-sentence fragment prevention)', async () => {
-  const { readFileSync } = await import('node:fs')
-  const { fileURLToPath } = await import('node:url')
-  const { dirname, join } = await import('node:path')
-  const here = dirname(fileURLToPath(import.meta.url))
-  const src = readFileSync(join(here, '../src/voice-io.ts'), 'utf8')
-  assert.ok(
-    src.includes('endpointing: 550'),
-    'createSTT deepgram case must use endpointing: 550ms to prevent mid-sentence transcript fragments'
-  )
-})
-
-// ── Summary ───────────────────────────────────────────────────────────────────
-console.log(`\n--- ${passed + failed} tests: ${passed} passed, ${failed} failed ---\n`)
-if (failed > 0) process.exit(1)