osborn 0.9.190 → 0.9.192

package/dist/claude-llm.js

@@ -62,6 +62,11 @@ function stripMarkdownForTTS(text) {
         .replace(/  +/g, ' ')
         .trim();
 }
+function getSubagentsDir(workingDir) {
+    const dir = join(workingDir, 'subagents');
+    mkdirSync(dir, { recursive: true });
+    return dir;
+}
 /**
  * Load skill files from agent/.claude/skills/{name}/SKILL.md
  * Injects into system prompt so Claude sees them as available capabilities.
@@ -1332,7 +1337,7 @@ export class ClaudeLLM extends llm.LLM {
             // agents roster prevents any SubagentStop(agent_type==='writer') from
             // firing inside this one-shot query and re-arming the backstop loop.
             const reviewerOptions = {
-                cwd: this.#opts.workingDirectory,
+                cwd: getSubagentsDir(this.#opts.workingDirectory),
                 permissionMode: 'default',
                 systemPrompt: NAMED_AGENTS.reviewer.prompt,
                 allowedTools: ['Read', 'Glob', 'Grep', 'Bash', 'Write', 'Edit'],
@@ -1441,7 +1446,7 @@ export class ClaudeLLM extends llm.LLM {
                 gitDiff,
             ].join('\n');
             const testerOptions = {
-                cwd: this.#opts.workingDirectory,
+                cwd: getSubagentsDir(this.#opts.workingDirectory),
                 permissionMode: 'default',
                 systemPrompt: NAMED_AGENTS.tester.prompt,
                 allowedTools: ['Read', 'Glob', 'Grep', 'Bash', 'Write', 'Edit'],
@@ -1528,7 +1533,7 @@ export class ClaudeLLM extends llm.LLM {
             // an agents roster would allow delegation back to the writer, which would
             // fire SubagentStop(agent_type==='writer') and re-arm the backstop.
             const gateOptions = {
-                cwd: this.#opts.workingDirectory,
+                cwd: getSubagentsDir(this.#opts.workingDirectory),
                 permissionMode: 'default',
                 systemPrompt: NAMED_AGENTS.reasoner.prompt,
                 hooks: {

package/dist/config.d.ts

@@ -2,8 +2,8 @@ import type { McpServerConfig } from './claude-handler.js';
 export type VoiceMode = 'pipeline';
 export type EditMode = 'read-only' | 'edit';
 export type AgentMode = 'plan' | 'execute' | 'research';
-export type STTProvider = 'deepgram' | 'groq-whisper' | 'openai-whisper';
-export type TTSProvider = 'openai' | 'deepgram';
+export type STTProvider = 'soniox' | 'deepgram' | 'groq-whisper' | 'openai-whisper';
+export type TTSProvider = 'soniox' | 'openai' | 'deepgram';
 export interface DirectConfig {
     stt?: {
         provider?: STTProvider;
@@ -146,7 +146,7 @@ export declare function invalidateSessionListCache(): void;
  *
  * @param limit - Max sessions to return (default 100, sorted by recency)
  */
-export declare function listAllClaudeSessions(limit?: number): Promise<ClaudeSessionEntry[]>;
+export declare function listAllClaudeSessions(limit?: number, includeSubagents?: boolean): Promise<ClaudeSessionEntry[]>;
 /**
  * Session summary for context briefing when switching sessions
  */

package/dist/config.js

@@ -47,13 +47,29 @@ const DEFAULT_CONFIG = {
     voiceMode: 'pipeline',
     direct: {
         stt: {
-            provider: 'deepgram',
-            model: 'nova-3',
+            // Soniox stt-rt-v4 — semantic endpointing (ML, not silence), word timestamps,
+            // custom vocabulary via context.terms. ~60% cheaper than nova-3 ($0.0017 vs $0.0043/min).
+            // Needs SONIOX_API_KEY.
+            provider: 'soniox',
+            model: 'stt-rt-v4',
+            // Previous: Deepgram nova-3, silence-based endpointing (550ms configured in voice-io.ts).
+            // Switch back: provider: 'deepgram', model: 'nova-3'
         },
         tts: {
-            provider: 'openai',
-            model: 'tts-1-hd',
-            voice: 'fable',
+            // Soniox tts-rt-v1 — real-time WebSocket streaming, speed control (0.7–1.3x),
+            // clean abort on interruption. Estimated ~$4–16/M chars vs OpenAI tts-1-hd $30/M.
+            // Pricing: $0.70/hr of generated speech (preview). Needs SONIOX_API_KEY.
+            provider: 'soniox',
+            model: 'tts-rt-v1',
+            voice: 'Maya',
+            // Previous: OpenAI tts-1-hd, voice fable — high quality, $30/M chars, ~500ms TTFB.
+            // Switch back: provider: 'openai', model: 'tts-1-hd', voice: 'fable'
+            // Other options already wired in voice-io.ts:
+            //   Rime Mist v3:     provider: 'rime',      voice: 'cove'  — 37ms TTFB, $30/M, WebSocket
+            //   Fish Audio s2-pro: provider: 'fishaudio', voice: '<id>'  — $15/M, voice cloning
+            //   Groq Orpheus:     provider: 'groq-orpheus', voice: 'autumn' — fast Groq chips, $22/M
+            //   Deepgram Aura-2:  provider: 'deepgram',  model: 'aura-2-asteria-en' — $15/M, ~100ms TTFB
+            //   OpenAI tts-1:     provider: 'openai',    model: 'tts-1', voice: 'fable' — $15/M
         },
     },
     mcpServers: {
@@ -555,15 +571,23 @@ export function invalidateSessionListCache() {
  *
  * @param limit - Max sessions to return (default 100, sorted by recency)
  */
-export async function listAllClaudeSessions(limit = 1000) {
+export async function listAllClaudeSessions(limit = 1000, includeSubagents = false) {
     if (_sessionListCache && Date.now() < _sessionListCache.expiresAt) {
-        return _sessionListCache.data.slice(0, limit);
+        const cached = includeSubagents
+            ? _sessionListCache.data
+            : _sessionListCache.data.filter(s => !s.projectSlug.endsWith('-subagents'));
+        return cached.slice(0, limit);
     }
     const projectsDir = getClaudeProjectsDir();
     if (!existsSync(projectsDir))
         return [];
-    // 1. Discover all project folders
+    // 1. Discover all project folders — skip subagent slugs unless explicitly requested.
+    //    Subagent sessions live in {workingDir}/subagents, which slugifies to a folder
+    //    ending in "-subagents". Filtering here keeps the session limit budget for
+    //    real user sessions and prevents subagent noise from crowding out other projects.
     const projectFolders = readdirSync(projectsDir).filter(name => {
+        if (!includeSubagents && name.endsWith('-subagents'))
+            return false;
         const fullPath = join(projectsDir, name);
         try {
             return statSync(fullPath).isDirectory();

package/dist/index.js

@@ -3603,9 +3603,19 @@ async function main() {
         // discardAudioIfUninterruptible: true, ttsReadIdleTimeout: 10000,
         // maxUnrecoverableErrors: 3) are what was silently running via caret-resolved
         // 1.4.5 throughout the user's working month. Restoring them.
-        const turnDetector = process.env.LIVEKIT_REMOTE_EOT_URL ? new CloudTurnDetector() : undefined;
+        // STT endpointing is the default (nova-3, 25ms silence-based VAD — reliable, fast).
+        // CloudTurnDetector (ML semantic EOT) is used ONLY if LIVEKIT_REMOTE_EOT_URL is set
+        // AND the endpoint passes a startup probe (returns valid JSON probability).
+        // If the probe fails (wrong URL, no auth, non-JSON response), STT is kept.
+        let turnDetection = 'stt';
+        if (process.env.LIVEKIT_REMOTE_EOT_URL) {
+            const detector = new CloudTurnDetector();
+            const eotLive = await detector.probe();
+            if (eotLive)
+                turnDetection = detector;
+        }
         const session = new voice.AgentSession({
-            turnDetection: (turnDetector ?? 'stt'),
+            turnDetection,
             preemptiveGeneration: false, // Only fire LLM on final committed transcript, not partial preemptives
             // Commented out — kept for reference. These were added across 0.9.60/0.9.61
             // to try to harden interrupt + TTS handling, but evidence (osbornojure

package/dist/turn-detector-shim.d.ts

@@ -5,14 +5,14 @@
  * because it calls getJobContext(). This shim implements the same _TurnDetector
  * interface but makes the remote HTTP call directly — no worker framework needed.
  *
+ * Auth: LiveKit Cloud inference requires a signed JWT (same format as room access
+ * tokens). We cache the token and refresh it before expiry so we're not signing
+ * on every turn.
+ *
  * On LiveKit Cloud (LIVEKIT_REMOTE_EOT_URL set): HTTP call to inference gateway.
  * Without the URL: Returns 1.0 (always end of turn — STT endpointing handles it).
  */
 import type { llm } from '@livekit/agents';
-/**
- * Implements _TurnDetector interface for LiveKit Cloud remote inference
- * without requiring JobContext / worker framework.
- */
 export declare class CloudTurnDetector {
     #private;
     readonly model = "lk_end_of_utterance_multilingual";
@@ -20,5 +20,11 @@ export declare class CloudTurnDetector {
     constructor();
     unlikelyThreshold(_language?: string): Promise<number | undefined>;
     supportsLanguage(_language?: string): Promise<boolean>;
+    /**
+     * Startup probe — send a minimal request to confirm the endpoint returns
+     * valid JSON probability. Returns true if the EOT service is live and real.
+     * Called once at session init; if false, index.ts falls back to 'stt'.
+     */
+    probe(): Promise<boolean>;
     predictEndOfTurn(chatCtx: llm.ChatContext, _timeout?: number): Promise<number>;
 }

package/dist/turn-detector-shim.js

@@ -5,19 +5,24 @@
  * because it calls getJobContext(). This shim implements the same _TurnDetector
  * interface but makes the remote HTTP call directly — no worker framework needed.
  *
+ * Auth: LiveKit Cloud inference requires a signed JWT (same format as room access
+ * tokens). We cache the token and refresh it before expiry so we're not signing
+ * on every turn.
+ *
  * On LiveKit Cloud (LIVEKIT_REMOTE_EOT_URL set): HTTP call to inference gateway.
  * Without the URL: Returns 1.0 (always end of turn — STT endpointing handles it).
  */
 import { log } from '@livekit/agents';
+import { AccessToken } from 'livekit-server-sdk';
 const REMOTE_INFERENCE_TIMEOUT = 2000;
 const MAX_HISTORY_TURNS = 15;
-/**
- * Implements _TurnDetector interface for LiveKit Cloud remote inference
- * without requiring JobContext / worker framework.
- */
+const TOKEN_TTL_SECONDS = 600; // 10-minute JWT
+const TOKEN_REFRESH_BUFFER = 60; // refresh 60s before expiry
 export class CloudTurnDetector {
     #remoteUrl;
     #logger = log();
+    #cachedToken;
+    #tokenExpiresAt = 0;
     model = 'lk_end_of_utterance_multilingual';
     provider = 'livekit';
     constructor() {
@@ -30,18 +35,78 @@ export class CloudTurnDetector {
             console.log(`🧠 Turn detector: LiveKit Cloud remote inference (${this.#remoteUrl})`);
         }
         else {
-            console.log('🧠 Turn detector: No LIVEKIT_REMOTE_EOT_URL — STT endpointing fallback');
+            console.log('🧠 Turn detector: No LIVEKIT_REMOTE_EOT_URL — STT endpointing only');
+        }
+    }
+    async #getAuthToken() {
+        const apiKey = process.env.LIVEKIT_API_KEY;
+        const apiSecret = process.env.LIVEKIT_API_SECRET;
+        if (!apiKey || !apiSecret)
+            return undefined;
+        const nowSec = Math.floor(Date.now() / 1000);
+        if (this.#cachedToken && nowSec < this.#tokenExpiresAt - TOKEN_REFRESH_BUFFER) {
+            return this.#cachedToken;
         }
+        // Sign a fresh token — identity is the agent, no room grants needed for inference
+        const at = new AccessToken(apiKey, apiSecret, {
+            identity: 'osborn-eot-agent',
+            ttl: `${TOKEN_TTL_SECONDS}s`,
+        });
+        this.#cachedToken = await at.toJwt();
+        this.#tokenExpiresAt = nowSec + TOKEN_TTL_SECONDS;
+        return this.#cachedToken;
     }
     async unlikelyThreshold(_language) {
-        return undefined; // Let the framework use defaults
+        return undefined;
     }
     async supportsLanguage(_language) {
-        return true; // Multilingual model supports all languages
+        return true;
+    }
+    /**
+     * Startup probe — send a minimal request to confirm the endpoint returns
+     * valid JSON probability. Returns true if the EOT service is live and real.
+     * Called once at session init; if false, index.ts falls back to 'stt'.
+     */
+    async probe() {
+        if (!this.#remoteUrl)
+            return false;
+        try {
+            const token = await this.#getAuthToken();
+            const headers = { 'Content-Type': 'application/json' };
+            if (token)
+                headers['Authorization'] = `Bearer ${token}`;
+            const resp = await fetch(`${this.#remoteUrl}/eot/multi`, {
+                method: 'POST',
+                body: JSON.stringify({
+                    messages: [{ role: 'user', content: 'test' }],
+                    jobId: 'osborn-probe',
+                    workerId: 'osborn-direct',
+                }),
+                headers,
+                signal: AbortSignal.timeout(3000),
+            });
+            if (!resp.ok)
+                return false;
+            const text = await resp.text();
+            try {
+                const data = JSON.parse(text);
+                const ok = typeof data.probability === 'number';
+                console.log(`🧠 EOT probe: ${ok ? '✅ live' : '❌ non-JSON ("' + text.slice(0, 30) + '")'} — ${ok ? 'using CloudTurnDetector' : 'falling back to STT'}`);
+                return ok;
+            }
+            catch {
+                console.log(`🧠 EOT probe: ❌ non-JSON response ("${text.slice(0, 40)}") — falling back to STT endpointing`);
+                return false;
+            }
+        }
+        catch (err) {
+            console.log(`🧠 EOT probe: ❌ unreachable — ${err instanceof Error ? err.message : err} — falling back to STT`);
+            return false;
+        }
     }
     async predictEndOfTurn(chatCtx, _timeout) {
         if (!this.#remoteUrl) {
-            return 1.0; // No remote URL = always end of turn (STT handles it)
+            return 1.0;
         }
         try {
             const messages = chatCtx
@@ -57,31 +122,41 @@ export class CloudTurnDetector {
                     excludeAudio: true,
                     excludeTimestamp: true,
                 }),
-                // Dummy IDs — LiveKit Cloud uses these for routing/logging, not auth
                 jobId: `osborn-${Date.now()}`,
                 workerId: 'osborn-direct',
             };
             const agentId = process.env.LIVEKIT_AGENT_ID;
-            if (agentId) {
+            if (agentId)
                 request.agentId = agentId;
-            }
+            const token = await this.#getAuthToken();
+            const headers = { 'Content-Type': 'application/json' };
+            if (token)
+                headers['Authorization'] = `Bearer ${token}`;
             const resp = await fetch(`${this.#remoteUrl}/eot/multi`, {
                 method: 'POST',
                 body: JSON.stringify(request),
-                headers: { 'Content-Type': 'application/json' },
+                headers,
                 signal: AbortSignal.timeout(REMOTE_INFERENCE_TIMEOUT),
             });
             if (!resp.ok) {
-                return 1.0; // Failed — default to end of turn
+                this.#logger.warn(`EOT inference returned ${resp.status} — falling back to STT`);
+                return 1.0;
             }
-            const data = (await resp.json());
-            if (typeof data.probability === 'number' && data.probability >= 0) {
-                return data.probability;
+            const text = await resp.text();
+            try {
+                const data = JSON.parse(text);
+                if (typeof data.probability === 'number' && data.probability >= 0) {
+                    return data.probability;
+                }
+            }
+            catch {
+                // Non-JSON response (e.g. "OK") — log once then fall through
+                this.#logger.warn(`EOT inference returned non-JSON: "${text.slice(0, 40)}" — auth may be wrong`);
             }
             return 1.0;
         }
         catch {
-            return 1.0; // Timeout/error — default to end of turn
+            return 1.0;
         }
     }
 }

package/dist/voice-io.d.ts

@@ -5,8 +5,9 @@
 import * as deepgram from '@livekit/agents-plugin-deepgram';
 import * as openai from '@livekit/agents-plugin-openai';
 import * as silero from '@livekit/agents-plugin-silero';
+import * as soniox from '@livekit/agents-plugin-soniox';
 export interface STTConfig {
-    provider: 'deepgram' | 'deepgram-flux' | 'groq-whisper' | 'openai-whisper';
+    provider: 'soniox' | 'deepgram' | 'deepgram-flux' | 'groq-whisper' | 'openai-whisper';
     model?: string;
     language?: string;
     /** Deepgram Flux: end-of-turn confidence threshold (0.0-1.0, default 0.7) */
@@ -15,7 +16,7 @@ export interface STTConfig {
     eotTimeoutMs?: number;
 }
 export interface TTSConfig {
-    provider: 'openai' | 'deepgram' | 'groq-orpheus' | 'fishaudio' | 'rime';
+    provider: 'soniox' | 'openai' | 'deepgram' | 'groq-orpheus' | 'fishaudio' | 'rime';
     voice?: string;
     model?: string;
 }
@@ -27,7 +28,7 @@ export interface VoiceIOConfig {
  * Create STT (Speech-to-Text) instance based on config
  * Note: Gemini STT is not available in Node.js, using Deepgram as default
  */
-export declare function createSTT(config: STTConfig): deepgram.STT | deepgram.STTv2 | openai.STT;
+export declare function createSTT(config: STTConfig): soniox.STT | deepgram.STT | deepgram.STTv2 | openai.STT;
 /**
  * Create TTS (Text-to-Speech) instance based on config
  */

package/dist/voice-io.js

@@ -7,13 +7,31 @@ import * as fishaudio from '@livekit/agents-plugin-fishaudio';
 import * as openai from '@livekit/agents-plugin-openai';
 import * as rime from '@livekit/agents-plugin-rime';
 import * as silero from '@livekit/agents-plugin-silero';
+import * as soniox from '@livekit/agents-plugin-soniox';
 /**
  * Create STT (Speech-to-Text) instance based on config
  * Note: Gemini STT is not available in Node.js, using Deepgram as default
  */
 export function createSTT(config) {
     switch (config.provider) {
+        case 'soniox':
+            // Soniox stt-rt-v4 — semantic endpointing: ML model holds on incomplete thoughts,
+            // commits on natural sentence ends. maxEndpointDelayMs 500–3000ms (minimum = fastest).
+            // endpointLatencyAdjustmentLevel 0–3: higher = more aggressive latency reduction
+            // while keeping semantic smarts. context.terms biases recognition toward code vocab.
+            return new soniox.STT({
+                model: (config.model || 'stt-rt-v4'),
+                languageHints: config.language ? [config.language] : ['en'],
+                maxEndpointDelayMs: 1200, // give model room to decide on mid-thought pauses
+                endpointLatencyAdjustmentLevel: 2, // aggressive but not max — good for voice assistant
+                context: {
+                    terms: ['Claude', 'TypeScript', 'LiveKit', 'Deepgram', 'npm', 'Railway', 'Fly.io'],
+                },
+            });
         case 'deepgram':
+            // Previous default. Silence-based endpointing (550ms configured = wait for 550ms quiet).
+            // Fast and reliable, but commits on any pause — doesn't understand mid-thought hesitations.
+            // Switch back: provider: 'deepgram', model: 'nova-3'
             return new deepgram.STT({
                 model: (config.model || 'nova-3'),
                 language: config.language || 'en',
@@ -49,13 +67,30 @@ export function createSTT(config) {
 export function createTTS(config) {
     let tts;
     switch (config.provider) {
+        case 'soniox':
+            // Soniox tts-rt-v1 — real-time WebSocket streaming, clean abort on interruption.
+            // Estimated ~$4–16/M chars ($0.70/hr of generated speech, preview pricing).
+            // speed: 0.7–1.3x. voices: Maya (female), others at soniox.com/docs/tts/voices.
+            // Previous TTS: OpenAI tts-1-hd (fable) — $30/M chars, ~500ms TTFB, HTTP chunked.
+            // Switch back: provider: 'openai', model: 'tts-1-hd', voice: 'fable'
+            tts = new soniox.TTS({
+                model: (config.model || 'tts-rt-v1'),
+                voice: config.voice || 'Maya',
+                speed: 1.0,
+            });
+            break;
         case 'openai':
+            // tts-1-hd: $30/M chars, ~500ms TTFB, 6 voices: alloy echo fable onyx nova shimmer.
+            // tts-1 (cheaper): $15/M chars, slightly lower quality, same voices.
             tts = new openai.TTS({
                 voice: config.voice || 'alloy',
                 model: config.model || 'tts-1',
             });
             break;
         case 'deepgram':
+            // Aura-2 voices: aura-2-asteria-en, aura-2-luna-en, aura-2-stella-en, aura-2-hera-en
+            // aura-2-orion-en, aura-2-arcas-en, aura-2-perseus-en, aura-2-angus-en, aura-2-orpheus-en
+            // ~$15/M chars, ~100ms TTFB.
             tts = new deepgram.TTS({
                 model: (config.model || 'aura-2-asteria-en'),
             });

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "osborn",
-  "version": "0.9.190",
+  "version": "0.9.192",
   "description": "Voice AI coding assistant - local agent that connects to Osborn frontend",
   "type": "module",
   "bin": {
@@ -39,6 +39,7 @@
     "@livekit/agents-plugin-openai": "1.4.6",
     "@livekit/agents-plugin-rime": "1.4.6",
     "@livekit/agents-plugin-silero": "1.4.6",
+    "@livekit/agents-plugin-soniox": "^1.8.1",
     "@livekit/rtc-node": "0.13.29",
     "@modelcontextprotocol/sdk": "^1.29.0",
     "@smithery/api": "^0.48.0",