@absolutejs/voice 0.0.22-beta.620 → 0.0.22-beta.622

package/dist/core/types.d.ts

@@ -800,6 +800,9 @@ export type VoicePluginConfig<TContext = unknown, TSession extends VoiceSessionR
     resumeGreeting?: string | ((input: {
         session: TSession;
     }) => string | Promise<string>);
+    sttRecoveryLine?: string | ((input: {
+        session: TSession;
+    }) => string | Promise<string>);
     languageStrategy?: VoiceLanguageStrategy;
     lexicon?: VoiceLexiconEntry[] | VoiceLexiconResolver<TContext>;
     phraseHints?: VoicePhraseHint[] | VoicePhraseHintResolver<TContext>;
@@ -946,6 +949,15 @@ export type CreateVoiceSessionOptions<TContext = unknown, TSession extends Voice
     resumeGreeting?: string | ((input: {
         session: TSession;
     }) => string | Promise<string>);
+    /** Spoken when the STT-health watchdog detects the stream has gone DEAF
+     *  mid-call (continuous speech energy, no transcripts landing — see
+     *  STT_HEALTH_STALE_MS). A short re-prompt ("Sorry, I think I missed that — go
+     *  ahead?") so the caller repeats into the freshly reconnected stream instead
+     *  of talking into a silently dead call. Cooldown-guarded to fire at most once
+     *  per stale episode. Receives the session. Unset = silent reconnect only. */
+    sttRecoveryLine?: string | ((input: {
+        session: TSession;
+    }) => string | Promise<string>);
     stt?: STTAdapter;
     realtime?: RealtimeAdapter;
     realtimeInputFormat?: AudioFormat;

package/dist/index.js

@@ -3816,6 +3816,7 @@ var STT_RECONNECT_FLAP_WINDOW_MS = 4000;
 var MAX_STT_RECONNECTS_IN_FLAP_WINDOW = 3;
 var STT_HEALTH_STALE_MS = 6000;
 var STT_HEALTH_SPEECH_GAP_MS = 2000;
+var STT_RECOVERY_COOLDOWN_MS = 15000;
 var DEFAULT_FORMAT = {
   channels: 1,
   container: "raw",
@@ -4082,6 +4083,7 @@ var createVoiceSession = (options) => {
   let lastSttReconnectAt = 0;
   let lastSpeechEnergyAt = 0;
   let sttHealthPhaseStart = 0;
+  let lastSttRecoverySpokenAt = 0;
   let activeTTSTurnId;
   let assistantSpeechEndsAt = 0;
   let lastAssistantAudioAt = 0;
@@ -6311,6 +6313,40 @@ var createVoiceSession = (options) => {
     }
     await completeTurn(updatedSession, turn);
   };
+  const speakAssistantLine = async (text) => {
+    if (!text.trim()) {
+      return;
+    }
+    const lineTurnId = createId();
+    await send({ text, turnId: lineTurnId, type: "assistant" });
+    try {
+      const lineTTSSession = await ensureTTSSession();
+      if (lineTTSSession) {
+        activeTTSTurnId = lineTurnId;
+        await lineTTSSession.send(text);
+        lastTtsSendAt = Date.now();
+      } else if (options.realtime) {
+        const lineRealtimeSession = await ensureAdapter();
+        activeTTSTurnId = lineTurnId;
+        await lineRealtimeSession.send(text);
+        lastTtsSendAt = Date.now();
+      }
+    } catch {}
+  };
+  const resolveSessionLine = async (line, sessionForLine) => typeof line === "function" ? line({ session: sessionForLine }) : line;
+  const speakResolvedLine = async (line, sessionForLine) => {
+    try {
+      await speakAssistantLine(await resolveSessionLine(line, sessionForLine));
+    } catch {}
+  };
+  const maybeSpeakSttRecovery = (nowMs, sessionForLine) => {
+    if (!options.sttRecoveryLine)
+      return;
+    if (nowMs - lastSttRecoverySpokenAt < STT_RECOVERY_COOLDOWN_MS)
+      return;
+    lastSttRecoverySpokenAt = nowMs;
+    speakResolvedLine(options.sttRecoveryLine, sessionForLine);
+  };
   const connectInternal = async (nextSocket) => {
     socket = nextSocket;
     const existingSession = await options.store.get(options.id);
@@ -6408,31 +6444,10 @@ var createVoiceSession = (options) => {
     warmTTSSession();
     kickCallSilenceWatchdog();
     startAmdEvaluationTimer();
-    const speakAssistantLine = async (text) => {
-      if (!text.trim()) {
-        return;
-      }
-      const lineTurnId = createId();
-      await send({ text, turnId: lineTurnId, type: "assistant" });
-      try {
-        const lineTTSSession = await ensureTTSSession();
-        if (lineTTSSession) {
-          activeTTSTurnId = lineTurnId;
-          await lineTTSSession.send(text);
-          lastTtsSendAt = Date.now();
-        } else if (options.realtime) {
-          const lineRealtimeSession = await ensureAdapter();
-          activeTTSTurnId = lineTurnId;
-          await lineRealtimeSession.send(text);
-          lastTtsSendAt = Date.now();
-        }
-      } catch {}
-    };
-    const resolveLine = async (line) => typeof line === "function" ? line({ session }) : line;
     if (options.greeting && session.turns.length === 0) {
-      await speakAssistantLine(await resolveLine(options.greeting));
+      await speakResolvedLine(options.greeting, session);
     } else if (isResume && options.resumeGreeting && session.turns.length > 0) {
-      await speakAssistantLine(await resolveLine(options.resumeGreeting));
+      await speakResolvedLine(options.resumeGreeting, session);
     }
   };
   const disconnectInternal = async (event) => {
@@ -6531,6 +6546,7 @@ var createVoiceSession = (options) => {
         sttReconnectCount = nowMs - lastSttReconnectAt < STT_RECONNECT_FLAP_WINDOW_MS ? sttReconnectCount + 1 : 1;
         lastSttReconnectAt = nowMs;
         sttHealthPhaseStart = nowMs;
+        maybeSpeakSttRecovery(nowMs, latest);
         if (sttReconnectCount <= MAX_STT_RECONNECTS_IN_FLAP_WINDOW) {
           await appendTrace({
             payload: {
@@ -39688,6 +39704,7 @@ var voice = (config) => {
       id: sessionId,
       greeting: config.greeting,
       resumeGreeting: config.resumeGreeting,
+      sttRecoveryLine: config.sttRecoveryLine,
       handoff: config.handoff,
       languageStrategy: config.languageStrategy,
       lexicon,
@@ -46252,9 +46269,15 @@ var createAnthropicVoiceAssistantModel = (options) => {
           messages: input.messages.map(messageToAnthropicMessage).filter(Boolean),
           model,
           stream: true,
-          system: [input.system, VOICE_SYSTEM_INSTRUCTIONS].filter(Boolean).join(`
+          system: [
+            {
+              cache_control: { type: "ephemeral" },
+              text: [input.system, VOICE_SYSTEM_INSTRUCTIONS].filter(Boolean).join(`
 
 `),
+              type: "text"
+            }
+          ],
           temperature: options.temperature,
           tool_choice: input.tools.length ? { type: "auto" } : { type: "none" },
           tools: input.tools.map((tool) => ({

package/dist/testing/index.js

@@ -5209,9 +5209,15 @@ var createAnthropicVoiceAssistantModel = (options) => {
           messages: input.messages.map(messageToAnthropicMessage).filter(Boolean),
           model,
           stream: true,
-          system: [input.system, VOICE_SYSTEM_INSTRUCTIONS].filter(Boolean).join(`
+          system: [
+            {
+              cache_control: { type: "ephemeral" },
+              text: [input.system, VOICE_SYSTEM_INSTRUCTIONS].filter(Boolean).join(`
 
 `),
+              type: "text"
+            }
+          ],
           temperature: options.temperature,
           tool_choice: input.tools.length ? { type: "auto" } : { type: "none" },
           tools: input.tools.map((tool) => ({
@@ -6130,6 +6136,7 @@ var STT_RECONNECT_FLAP_WINDOW_MS = 4000;
 var MAX_STT_RECONNECTS_IN_FLAP_WINDOW = 3;
 var STT_HEALTH_STALE_MS = 6000;
 var STT_HEALTH_SPEECH_GAP_MS = 2000;
+var STT_RECOVERY_COOLDOWN_MS = 15000;
 var DEFAULT_FORMAT = {
   channels: 1,
   container: "raw",
@@ -6396,6 +6403,7 @@ var createVoiceSession = (options) => {
   let lastSttReconnectAt = 0;
   let lastSpeechEnergyAt = 0;
   let sttHealthPhaseStart = 0;
+  let lastSttRecoverySpokenAt = 0;
   let activeTTSTurnId;
   let assistantSpeechEndsAt = 0;
   let lastAssistantAudioAt = 0;
@@ -8625,6 +8633,40 @@ var createVoiceSession = (options) => {
     }
     await completeTurn(updatedSession, turn);
   };
+  const speakAssistantLine = async (text) => {
+    if (!text.trim()) {
+      return;
+    }
+    const lineTurnId = createId();
+    await send({ text, turnId: lineTurnId, type: "assistant" });
+    try {
+      const lineTTSSession = await ensureTTSSession();
+      if (lineTTSSession) {
+        activeTTSTurnId = lineTurnId;
+        await lineTTSSession.send(text);
+        lastTtsSendAt = Date.now();
+      } else if (options.realtime) {
+        const lineRealtimeSession = await ensureAdapter();
+        activeTTSTurnId = lineTurnId;
+        await lineRealtimeSession.send(text);
+        lastTtsSendAt = Date.now();
+      }
+    } catch {}
+  };
+  const resolveSessionLine = async (line, sessionForLine) => typeof line === "function" ? line({ session: sessionForLine }) : line;
+  const speakResolvedLine = async (line, sessionForLine) => {
+    try {
+      await speakAssistantLine(await resolveSessionLine(line, sessionForLine));
+    } catch {}
+  };
+  const maybeSpeakSttRecovery = (nowMs, sessionForLine) => {
+    if (!options.sttRecoveryLine)
+      return;
+    if (nowMs - lastSttRecoverySpokenAt < STT_RECOVERY_COOLDOWN_MS)
+      return;
+    lastSttRecoverySpokenAt = nowMs;
+    speakResolvedLine(options.sttRecoveryLine, sessionForLine);
+  };
   const connectInternal = async (nextSocket) => {
     socket = nextSocket;
     const existingSession = await options.store.get(options.id);
@@ -8722,31 +8764,10 @@ var createVoiceSession = (options) => {
     warmTTSSession();
     kickCallSilenceWatchdog();
     startAmdEvaluationTimer();
-    const speakAssistantLine = async (text) => {
-      if (!text.trim()) {
-        return;
-      }
-      const lineTurnId = createId();
-      await send({ text, turnId: lineTurnId, type: "assistant" });
-      try {
-        const lineTTSSession = await ensureTTSSession();
-        if (lineTTSSession) {
-          activeTTSTurnId = lineTurnId;
-          await lineTTSSession.send(text);
-          lastTtsSendAt = Date.now();
-        } else if (options.realtime) {
-          const lineRealtimeSession = await ensureAdapter();
-          activeTTSTurnId = lineTurnId;
-          await lineRealtimeSession.send(text);
-          lastTtsSendAt = Date.now();
-        }
-      } catch {}
-    };
-    const resolveLine = async (line) => typeof line === "function" ? line({ session }) : line;
     if (options.greeting && session.turns.length === 0) {
-      await speakAssistantLine(await resolveLine(options.greeting));
+      await speakResolvedLine(options.greeting, session);
     } else if (isResume && options.resumeGreeting && session.turns.length > 0) {
-      await speakAssistantLine(await resolveLine(options.resumeGreeting));
+      await speakResolvedLine(options.resumeGreeting, session);
     }
   };
   const disconnectInternal = async (event) => {
@@ -8845,6 +8866,7 @@ var createVoiceSession = (options) => {
         sttReconnectCount = nowMs - lastSttReconnectAt < STT_RECONNECT_FLAP_WINDOW_MS ? sttReconnectCount + 1 : 1;
         lastSttReconnectAt = nowMs;
         sttHealthPhaseStart = nowMs;
+        maybeSpeakSttRecovery(nowMs, latest);
         if (sttReconnectCount <= MAX_STT_RECONNECTS_IN_FLAP_WINDOW) {
           await appendTrace({
             payload: {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@absolutejs/voice",
-  "version": "0.0.22-beta.620",
+  "version": "0.0.22-beta.622",
   "description": "Voice primitives and Elysia plugin for AbsoluteJS",
   "repository": {
     "type": "git",