@absolutejs/voice 0.0.22-beta.621 → 0.0.22-beta.623

package/dist/core/types.d.ts

@@ -800,6 +800,16 @@ export type VoicePluginConfig<TContext = unknown, TSession extends VoiceSessionR
     resumeGreeting?: string | ((input: {
         session: TSession;
     }) => string | Promise<string>);
+    sttRecoveryLine?: string | ((input: {
+        session: TSession;
+    }) => string | Promise<string>);
+    stuckCallClose?: {
+        afterMs: number;
+        line?: string | ((input: {
+            session: TSession;
+        }) => string | Promise<string>);
+        reason?: string;
+    };
     languageStrategy?: VoiceLanguageStrategy;
     lexicon?: VoiceLexiconEntry[] | VoiceLexiconResolver<TContext>;
     phraseHints?: VoicePhraseHint[] | VoicePhraseHintResolver<TContext>;
@@ -946,6 +956,29 @@ export type CreateVoiceSessionOptions<TContext = unknown, TSession extends Voice
     resumeGreeting?: string | ((input: {
         session: TSession;
     }) => string | Promise<string>);
+    /** Spoken when the STT-health watchdog detects the stream has gone DEAF
+     *  mid-call (continuous speech energy, no transcripts landing — see
+     *  STT_HEALTH_STALE_MS). A short re-prompt ("Sorry, I think I missed that — go
+     *  ahead?") so the caller repeats into the freshly reconnected stream instead
+     *  of talking into a silently dead call. Cooldown-guarded to fire at most once
+     *  per stale episode. Receives the session. Unset = silent reconnect only. */
+    sttRecoveryLine?: string | ((input: {
+        session: TSession;
+    }) => string | Promise<string>);
+    /** Last-resort GRACEFUL terminal close for a wedged call. If no caller-side
+     *  progress (committed turn / user partial) lands for `afterMs` on a live call
+     *  — STT permanently deaf, or the caller left — the assistant speaks `line` and
+     *  the session COMPLETES (disposition "completed") so onComplete still saves and
+     *  the call ends with a real goodbye instead of dead air + "abandoned". Reset by
+     *  real progress (committed turn / user partial / (re)connect), NOT by the
+     *  assistant's own speech, so STT recovery re-prompts can't defer it forever. */
+    stuckCallClose?: {
+        afterMs: number;
+        line?: string | ((input: {
+            session: TSession;
+        }) => string | Promise<string>);
+        reason?: string;
+    };
     stt?: STTAdapter;
     realtime?: RealtimeAdapter;
     realtimeInputFormat?: AudioFormat;

package/dist/index.js

@@ -3816,6 +3816,7 @@ var STT_RECONNECT_FLAP_WINDOW_MS = 4000;
 var MAX_STT_RECONNECTS_IN_FLAP_WINDOW = 3;
 var STT_HEALTH_STALE_MS = 6000;
 var STT_HEALTH_SPEECH_GAP_MS = 2000;
+var STT_RECOVERY_COOLDOWN_MS = 15000;
 var DEFAULT_FORMAT = {
   channels: 1,
   container: "raw",
@@ -4082,6 +4083,7 @@ var createVoiceSession = (options) => {
   let lastSttReconnectAt = 0;
   let lastSpeechEnergyAt = 0;
   let sttHealthPhaseStart = 0;
+  let lastSttRecoverySpokenAt = 0;
   let activeTTSTurnId;
   let assistantSpeechEndsAt = 0;
   let lastAssistantAudioAt = 0;
@@ -4184,6 +4186,51 @@ var createVoiceSession = (options) => {
     clearCallSilenceWatchdog();
     callSilenceWatchdog = setTimeout(fireCallSilenceTimeout, callSilenceTimeoutMs);
   };
+  const stuckCloseConfig = options.stuckCallClose;
+  const stuckCloseAfterMs = stuckCloseConfig && stuckCloseConfig.afterMs > 0 ? stuckCloseConfig.afterMs : undefined;
+  let stuckCloseWatchdog = null;
+  let stuckCloseFired = false;
+  const clearStuckCloseWatchdog = () => {
+    if (stuckCloseWatchdog) {
+      clearTimeout(stuckCloseWatchdog);
+      stuckCloseWatchdog = null;
+    }
+  };
+  const fireStuckClose = () => {
+    stuckCloseWatchdog = null;
+    if (stuckCloseFired) {
+      return;
+    }
+    stuckCloseFired = true;
+    runSerial("stuck-call-close", async () => {
+      const snapshot = await readSession();
+      if (snapshot.status === "completed" || snapshot.status === "failed" || snapshot.call?.endedAt) {
+        return;
+      }
+      await appendTrace({
+        payload: {
+          action: "stuck-call-close",
+          reason: `no caller progress for ${stuckCloseAfterMs}ms`
+        },
+        session: snapshot,
+        type: "session.error"
+      });
+      if (stuckCloseConfig?.line) {
+        await speakResolvedLine(stuckCloseConfig.line, snapshot);
+      }
+      await completeInternal(undefined, {
+        disposition: "completed",
+        reason: stuckCloseConfig?.reason ?? "stuck-call-close"
+      });
+    });
+  };
+  const kickStuckCloseWatchdog = () => {
+    if (stuckCloseAfterMs === undefined || stuckCloseFired) {
+      return;
+    }
+    clearStuckCloseWatchdog();
+    stuckCloseWatchdog = setTimeout(fireStuckClose, stuckCloseAfterMs);
+  };
   const recordingConfig = options.recording;
   const recordingChannels = new Set(recordingConfig?.channels ?? ["assistant", "user"]);
   const recordingMaxBytes = recordingConfig?.maxBytesPerChannel ?? 50 * 1024 * 1024;
@@ -4680,6 +4727,7 @@ var createVoiceSession = (options) => {
       type: "error"
     });
     clearCallSilenceWatchdog();
+    clearStuckCloseWatchdog();
     clearAmdEvaluationTimer();
     await closeTTSSession("failed");
     await closeAdapter("failed");
@@ -4789,6 +4837,7 @@ var createVoiceSession = (options) => {
       type: "complete"
     });
     clearCallSilenceWatchdog();
+    clearStuckCloseWatchdog();
     clearAmdEvaluationTimer();
     await closeTTSSession("complete");
     await closeAdapter("complete");
@@ -5266,6 +5315,9 @@ var createVoiceSession = (options) => {
     };
   };
   const handlePartial = async (transcript) => {
+    if (transcript.text.trim()) {
+      kickStuckCloseWatchdog();
+    }
     if (activeTTSTurnId !== undefined) {
       const triggeringText = transcript.text.trim();
       if (triggeringText) {
@@ -5740,6 +5792,7 @@ var createVoiceSession = (options) => {
   };
   const completeTurn = async (session, turn) => {
     console.error(`[voice] completeTurn ENTER session=${options.id} turn=${turn.id} textLen=${turn.text?.length ?? 0}`);
+    kickStuckCloseWatchdog();
     const liveOpsControl = await options.liveOps?.getControl(options.id);
     if (liveOpsControl?.assistantPaused || liveOpsControl?.operatorTakeover) {
       await appendTrace({
@@ -6311,6 +6364,40 @@ var createVoiceSession = (options) => {
     }
     await completeTurn(updatedSession, turn);
   };
+  const speakAssistantLine = async (text) => {
+    if (!text.trim()) {
+      return;
+    }
+    const lineTurnId = createId();
+    await send({ text, turnId: lineTurnId, type: "assistant" });
+    try {
+      const lineTTSSession = await ensureTTSSession();
+      if (lineTTSSession) {
+        activeTTSTurnId = lineTurnId;
+        await lineTTSSession.send(text);
+        lastTtsSendAt = Date.now();
+      } else if (options.realtime) {
+        const lineRealtimeSession = await ensureAdapter();
+        activeTTSTurnId = lineTurnId;
+        await lineRealtimeSession.send(text);
+        lastTtsSendAt = Date.now();
+      }
+    } catch {}
+  };
+  const resolveSessionLine = async (line, sessionForLine) => typeof line === "function" ? line({ session: sessionForLine }) : line;
+  const speakResolvedLine = async (line, sessionForLine) => {
+    try {
+      await speakAssistantLine(await resolveSessionLine(line, sessionForLine));
+    } catch {}
+  };
+  const maybeSpeakSttRecovery = (nowMs, sessionForLine) => {
+    if (!options.sttRecoveryLine)
+      return;
+    if (nowMs - lastSttRecoverySpokenAt < STT_RECOVERY_COOLDOWN_MS)
+      return;
+    lastSttRecoverySpokenAt = nowMs;
+    speakResolvedLine(options.sttRecoveryLine, sessionForLine);
+  };
   const connectInternal = async (nextSocket) => {
     socket = nextSocket;
     const existingSession = await options.store.get(options.id);
@@ -6407,32 +6494,12 @@ var createVoiceSession = (options) => {
     await ensureAdapter();
     warmTTSSession();
     kickCallSilenceWatchdog();
+    kickStuckCloseWatchdog();
     startAmdEvaluationTimer();
-    const speakAssistantLine = async (text) => {
-      if (!text.trim()) {
-        return;
-      }
-      const lineTurnId = createId();
-      await send({ text, turnId: lineTurnId, type: "assistant" });
-      try {
-        const lineTTSSession = await ensureTTSSession();
-        if (lineTTSSession) {
-          activeTTSTurnId = lineTurnId;
-          await lineTTSSession.send(text);
-          lastTtsSendAt = Date.now();
-        } else if (options.realtime) {
-          const lineRealtimeSession = await ensureAdapter();
-          activeTTSTurnId = lineTurnId;
-          await lineRealtimeSession.send(text);
-          lastTtsSendAt = Date.now();
-        }
-      } catch {}
-    };
-    const resolveLine = async (line) => typeof line === "function" ? line({ session }) : line;
     if (options.greeting && session.turns.length === 0) {
-      await speakAssistantLine(await resolveLine(options.greeting));
+      await speakResolvedLine(options.greeting, session);
     } else if (isResume && options.resumeGreeting && session.turns.length > 0) {
-      await speakAssistantLine(await resolveLine(options.resumeGreeting));
+      await speakResolvedLine(options.resumeGreeting, session);
     }
   };
   const disconnectInternal = async (event) => {
@@ -6531,6 +6598,7 @@ var createVoiceSession = (options) => {
         sttReconnectCount = nowMs - lastSttReconnectAt < STT_RECONNECT_FLAP_WINDOW_MS ? sttReconnectCount + 1 : 1;
         lastSttReconnectAt = nowMs;
         sttHealthPhaseStart = nowMs;
+        maybeSpeakSttRecovery(nowMs, latest);
         if (sttReconnectCount <= MAX_STT_RECONNECTS_IN_FLAP_WINDOW) {
           await appendTrace({
             payload: {
@@ -6572,6 +6640,7 @@ var createVoiceSession = (options) => {
     });
     clearSilenceTimer();
     clearCallSilenceWatchdog();
+    clearStuckCloseWatchdog();
     clearAmdEvaluationTimer();
     if (options.noiseSuppressor?.close) {
       try {
@@ -39688,6 +39757,8 @@ var voice = (config) => {
       id: sessionId,
       greeting: config.greeting,
       resumeGreeting: config.resumeGreeting,
+      sttRecoveryLine: config.sttRecoveryLine,
+      stuckCallClose: config.stuckCallClose,
       handoff: config.handoff,
       languageStrategy: config.languageStrategy,
       lexicon,

package/dist/testing/index.js

@@ -6136,6 +6136,7 @@ var STT_RECONNECT_FLAP_WINDOW_MS = 4000;
 var MAX_STT_RECONNECTS_IN_FLAP_WINDOW = 3;
 var STT_HEALTH_STALE_MS = 6000;
 var STT_HEALTH_SPEECH_GAP_MS = 2000;
+var STT_RECOVERY_COOLDOWN_MS = 15000;
 var DEFAULT_FORMAT = {
   channels: 1,
   container: "raw",
@@ -6402,6 +6403,7 @@ var createVoiceSession = (options) => {
   let lastSttReconnectAt = 0;
   let lastSpeechEnergyAt = 0;
   let sttHealthPhaseStart = 0;
+  let lastSttRecoverySpokenAt = 0;
   let activeTTSTurnId;
   let assistantSpeechEndsAt = 0;
   let lastAssistantAudioAt = 0;
@@ -6504,6 +6506,51 @@ var createVoiceSession = (options) => {
     clearCallSilenceWatchdog();
     callSilenceWatchdog = setTimeout(fireCallSilenceTimeout, callSilenceTimeoutMs);
   };
+  const stuckCloseConfig = options.stuckCallClose;
+  const stuckCloseAfterMs = stuckCloseConfig && stuckCloseConfig.afterMs > 0 ? stuckCloseConfig.afterMs : undefined;
+  let stuckCloseWatchdog = null;
+  let stuckCloseFired = false;
+  const clearStuckCloseWatchdog = () => {
+    if (stuckCloseWatchdog) {
+      clearTimeout(stuckCloseWatchdog);
+      stuckCloseWatchdog = null;
+    }
+  };
+  const fireStuckClose = () => {
+    stuckCloseWatchdog = null;
+    if (stuckCloseFired) {
+      return;
+    }
+    stuckCloseFired = true;
+    runSerial("stuck-call-close", async () => {
+      const snapshot = await readSession();
+      if (snapshot.status === "completed" || snapshot.status === "failed" || snapshot.call?.endedAt) {
+        return;
+      }
+      await appendTrace({
+        payload: {
+          action: "stuck-call-close",
+          reason: `no caller progress for ${stuckCloseAfterMs}ms`
+        },
+        session: snapshot,
+        type: "session.error"
+      });
+      if (stuckCloseConfig?.line) {
+        await speakResolvedLine(stuckCloseConfig.line, snapshot);
+      }
+      await completeInternal(undefined, {
+        disposition: "completed",
+        reason: stuckCloseConfig?.reason ?? "stuck-call-close"
+      });
+    });
+  };
+  const kickStuckCloseWatchdog = () => {
+    if (stuckCloseAfterMs === undefined || stuckCloseFired) {
+      return;
+    }
+    clearStuckCloseWatchdog();
+    stuckCloseWatchdog = setTimeout(fireStuckClose, stuckCloseAfterMs);
+  };
   const recordingConfig = options.recording;
   const recordingChannels = new Set(recordingConfig?.channels ?? ["assistant", "user"]);
   const recordingMaxBytes = recordingConfig?.maxBytesPerChannel ?? 50 * 1024 * 1024;
@@ -7000,6 +7047,7 @@ var createVoiceSession = (options) => {
       type: "error"
     });
     clearCallSilenceWatchdog();
+    clearStuckCloseWatchdog();
     clearAmdEvaluationTimer();
     await closeTTSSession("failed");
     await closeAdapter("failed");
@@ -7109,6 +7157,7 @@ var createVoiceSession = (options) => {
       type: "complete"
     });
     clearCallSilenceWatchdog();
+    clearStuckCloseWatchdog();
     clearAmdEvaluationTimer();
     await closeTTSSession("complete");
     await closeAdapter("complete");
@@ -7586,6 +7635,9 @@ var createVoiceSession = (options) => {
     };
   };
   const handlePartial = async (transcript) => {
+    if (transcript.text.trim()) {
+      kickStuckCloseWatchdog();
+    }
     if (activeTTSTurnId !== undefined) {
       const triggeringText = transcript.text.trim();
       if (triggeringText) {
@@ -8060,6 +8112,7 @@ var createVoiceSession = (options) => {
   };
   const completeTurn = async (session, turn) => {
     console.error(`[voice] completeTurn ENTER session=${options.id} turn=${turn.id} textLen=${turn.text?.length ?? 0}`);
+    kickStuckCloseWatchdog();
     const liveOpsControl = await options.liveOps?.getControl(options.id);
     if (liveOpsControl?.assistantPaused || liveOpsControl?.operatorTakeover) {
       await appendTrace({
@@ -8631,6 +8684,40 @@ var createVoiceSession = (options) => {
     }
     await completeTurn(updatedSession, turn);
   };
+  const speakAssistantLine = async (text) => {
+    if (!text.trim()) {
+      return;
+    }
+    const lineTurnId = createId();
+    await send({ text, turnId: lineTurnId, type: "assistant" });
+    try {
+      const lineTTSSession = await ensureTTSSession();
+      if (lineTTSSession) {
+        activeTTSTurnId = lineTurnId;
+        await lineTTSSession.send(text);
+        lastTtsSendAt = Date.now();
+      } else if (options.realtime) {
+        const lineRealtimeSession = await ensureAdapter();
+        activeTTSTurnId = lineTurnId;
+        await lineRealtimeSession.send(text);
+        lastTtsSendAt = Date.now();
+      }
+    } catch {}
+  };
+  const resolveSessionLine = async (line, sessionForLine) => typeof line === "function" ? line({ session: sessionForLine }) : line;
+  const speakResolvedLine = async (line, sessionForLine) => {
+    try {
+      await speakAssistantLine(await resolveSessionLine(line, sessionForLine));
+    } catch {}
+  };
+  const maybeSpeakSttRecovery = (nowMs, sessionForLine) => {
+    if (!options.sttRecoveryLine)
+      return;
+    if (nowMs - lastSttRecoverySpokenAt < STT_RECOVERY_COOLDOWN_MS)
+      return;
+    lastSttRecoverySpokenAt = nowMs;
+    speakResolvedLine(options.sttRecoveryLine, sessionForLine);
+  };
   const connectInternal = async (nextSocket) => {
     socket = nextSocket;
     const existingSession = await options.store.get(options.id);
@@ -8727,32 +8814,12 @@ var createVoiceSession = (options) => {
     await ensureAdapter();
     warmTTSSession();
     kickCallSilenceWatchdog();
+    kickStuckCloseWatchdog();
     startAmdEvaluationTimer();
-    const speakAssistantLine = async (text) => {
-      if (!text.trim()) {
-        return;
-      }
-      const lineTurnId = createId();
-      await send({ text, turnId: lineTurnId, type: "assistant" });
-      try {
-        const lineTTSSession = await ensureTTSSession();
-        if (lineTTSSession) {
-          activeTTSTurnId = lineTurnId;
-          await lineTTSSession.send(text);
-          lastTtsSendAt = Date.now();
-        } else if (options.realtime) {
-          const lineRealtimeSession = await ensureAdapter();
-          activeTTSTurnId = lineTurnId;
-          await lineRealtimeSession.send(text);
-          lastTtsSendAt = Date.now();
-        }
-      } catch {}
-    };
-    const resolveLine = async (line) => typeof line === "function" ? line({ session }) : line;
     if (options.greeting && session.turns.length === 0) {
-      await speakAssistantLine(await resolveLine(options.greeting));
+      await speakResolvedLine(options.greeting, session);
     } else if (isResume && options.resumeGreeting && session.turns.length > 0) {
-      await speakAssistantLine(await resolveLine(options.resumeGreeting));
+      await speakResolvedLine(options.resumeGreeting, session);
     }
   };
   const disconnectInternal = async (event) => {
@@ -8851,6 +8918,7 @@ var createVoiceSession = (options) => {
         sttReconnectCount = nowMs - lastSttReconnectAt < STT_RECONNECT_FLAP_WINDOW_MS ? sttReconnectCount + 1 : 1;
         lastSttReconnectAt = nowMs;
         sttHealthPhaseStart = nowMs;
+        maybeSpeakSttRecovery(nowMs, latest);
         if (sttReconnectCount <= MAX_STT_RECONNECTS_IN_FLAP_WINDOW) {
           await appendTrace({
             payload: {
@@ -8892,6 +8960,7 @@ var createVoiceSession = (options) => {
     });
     clearSilenceTimer();
     clearCallSilenceWatchdog();
+    clearStuckCloseWatchdog();
     clearAmdEvaluationTimer();
     if (options.noiseSuppressor?.close) {
       try {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@absolutejs/voice",
-  "version": "0.0.22-beta.621",
+  "version": "0.0.22-beta.623",
   "description": "Voice primitives and Elysia plugin for AbsoluteJS",
   "repository": {
     "type": "git",