@absolutejs/voice 0.0.22-beta.482 → 0.0.22-beta.484

package/dist/assistantMode.d.ts

@@ -0,0 +1,22 @@
+import type { CreateVoiceSessionOptions, VoiceSessionRecord } from "./types";
+export type VoiceAssistantMode = "cascade" | "s2s";
+export type VoiceSemanticVADConfig = {
+    createResponseAutomatically?: boolean;
+    eagerness?: "auto" | "high" | "low" | "medium";
+    silenceDurationMs?: number;
+};
+export type VoiceAssistantModality = "audio" | "text";
+export declare const resolveVoiceAssistantMode: <TContext = unknown, TSession extends VoiceSessionRecord = VoiceSessionRecord, TResult = unknown>(options: Pick<CreateVoiceSessionOptions<TContext, TSession, TResult>, "realtime" | "stt" | "tts"> & {
+    assistantMode?: VoiceAssistantMode;
+}) => VoiceAssistantMode;
+export type VoiceAssistantModeDescriptor = {
+    hasRealtime: boolean;
+    hasSTT: boolean;
+    hasTTS: boolean;
+    mode: VoiceAssistantMode;
+    modalities: VoiceAssistantModality[];
+};
+export declare const describeVoiceAssistantMode: <TContext = unknown, TSession extends VoiceSessionRecord = VoiceSessionRecord, TResult = unknown>(options: Pick<CreateVoiceSessionOptions<TContext, TSession, TResult>, "realtime" | "stt" | "tts"> & {
+    assistantMode?: VoiceAssistantMode;
+    modalities?: ReadonlyArray<VoiceAssistantModality>;
+}) => VoiceAssistantModeDescriptor;

package/dist/index.d.ts

@@ -77,6 +77,10 @@ export { DEFAULT_VOICE_REDACTION_PATTERNS, createVoiceTranscriptRedactor, redact
 export type { CreateVoiceTranscriptRedactorOptions, VoiceRedactionPattern, VoiceTranscriptRedactor, } from "./redaction";
 export { DEFAULT_VOICE_PRICE_BOOK, createVoiceCostAccountant, } from "./costAccounting";
 export type { CreateVoiceCostAccountantOptions, VoiceCostAccountant, VoiceCostBreakdown, VoiceCostLLMRecord, VoiceCostSTTRecord, VoiceCostTTSRecord, VoiceCostTelephonyRecord, VoicePriceBook, VoiceProviderRates, } from "./costAccounting";
+export { describeVoiceAssistantMode, resolveVoiceAssistantMode, } from "./assistantMode";
+export type { VoiceAssistantMode, VoiceAssistantModality, VoiceAssistantModeDescriptor, VoiceSemanticVADConfig, } from "./assistantMode";
+export { createPunctuationSemanticTurnDetector, createRegexSemanticTurnDetector, } from "./semanticTurn";
+export type { CreatePunctuationSemanticTurnDetectorOptions, CreateRegexSemanticTurnDetectorOptions, VoiceSemanticTurnDetector, VoiceSemanticTurnInput, VoiceSemanticTurnVerdict, } from "./semanticTurn";
 export { createMonologueAMDDetector } from "./amdDetector";
 export type { MonologueAMDDetectorOptions, VoiceAMDDetector, VoiceAMDDetectorInput, VoiceAMDVerdict, } from "./amdDetector";
 export { createVoiceRAGTool } from "./ragTool";

package/dist/index.js

@@ -3440,6 +3440,28 @@ var createVoiceMemoryRecordingStore = () => {
   };
 };
 
+// src/assistantMode.ts
+var resolveVoiceAssistantMode = (options) => {
+  if (options.assistantMode) {
+    return options.assistantMode;
+  }
+  if (options.realtime) {
+    return "s2s";
+  }
+  return "cascade";
+};
+var describeVoiceAssistantMode = (options) => {
+  const mode = resolveVoiceAssistantMode(options);
+  const modalities = options.modalities ? Array.from(new Set(options.modalities)) : ["audio"];
+  return {
+    hasRealtime: Boolean(options.realtime),
+    hasSTT: Boolean(options.stt),
+    hasTTS: Boolean(options.tts),
+    modalities,
+    mode
+  };
+};
+
 // src/session.ts
 var DEFAULT_RECONNECT_TIMEOUT = 30000;
 var DEFAULT_MAX_RECONNECT_ATTEMPTS = 10;
@@ -4786,6 +4808,18 @@ var createVoiceSession = (options) => {
       session,
       type: "turn.transcript"
     });
+    if (options.semanticTurnDetector) {
+      const verdict = await Promise.resolve(options.semanticTurnDetector.evaluate({
+        lastFinalTranscript: transcript,
+        partialText: session.currentTurn.partialText,
+        silenceMs: session.currentTurn.silenceStartedAt !== undefined ? Date.now() - session.currentTurn.silenceStartedAt : 0,
+        transcripts: session.currentTurn.transcripts
+      }));
+      if (verdict.endOfTurn) {
+        clearSilenceTimer();
+        await requestTurnCommit("vendor");
+      }
+    }
   };
   const resumePendingTurnCommit = (session) => {
     const pendingText = buildTurnText(session.currentTurn.transcripts, session.currentTurn.partialText, {
@@ -4810,13 +4844,20 @@ var createVoiceSession = (options) => {
     if (!inputAdapter) {
       throw new Error("Voice session requires either an stt or realtime adapter.");
     }
-    const openedSession = await inputAdapter.open({
-      format: options.realtime ? options.realtimeInputFormat ?? DEFAULT_REALTIME_FORMAT : DEFAULT_FORMAT,
+    const openedSession = await (options.realtime ? options.realtime.open({
+      format: options.realtimeInputFormat ?? DEFAULT_REALTIME_FORMAT,
       languageStrategy: options.languageStrategy,
       lexicon,
+      modalities: options.modalities,
       phraseHints,
       sessionId: options.id
-    });
+    }) : inputAdapter.open({
+      format: DEFAULT_FORMAT,
+      languageStrategy: options.languageStrategy,
+      lexicon,
+      phraseHints,
+      sessionId: options.id
+    }));
     const generation = ++adapterGenerationCounter;
     sttSession = openedSession;
     activeAdapterGeneration = generation;
@@ -4986,9 +5027,10 @@ var createVoiceSession = (options) => {
       });
       await appendTrace({
         payload: {
+          assistantMode: resolveVoiceAssistantMode(options),
+          realtimeConfigured: Boolean(options.realtime),
           text: output.assistantText,
-          ttsConfigured: Boolean(options.tts),
-          realtimeConfigured: Boolean(options.realtime)
+          ttsConfigured: Boolean(options.tts)
         },
         session,
         turnId: turn.id,
@@ -35292,6 +35334,76 @@ var createVoiceCostAccountant = (options = {}) => {
     })
   };
 };
+// src/semanticTurn.ts
+var DEFAULT_END_PUNCTUATION = [".", "?", "!"];
+var DEFAULT_FILLER_WORDS = [
+  "uh",
+  "um",
+  "er",
+  "ah",
+  "like",
+  "you know",
+  "i mean",
+  "well",
+  "so"
+];
+var stripTerminalPunctuation = (text) => text.replace(/[\s.?!]+$/u, "").trim();
+var createPunctuationSemanticTurnDetector = (options = {}) => {
+  const endPunctuation = options.endPunctuation ?? DEFAULT_END_PUNCTUATION;
+  const fillerWords = (options.fillerWords ?? DEFAULT_FILLER_WORDS).map((word) => word.toLowerCase());
+  const minPartialWords = options.minPartialWords ?? 2;
+  return {
+    evaluate: ({ lastFinalTranscript, partialText }) => {
+      const candidate = partialText.trim().length > 0 ? partialText : lastFinalTranscript?.text ?? "";
+      const trimmed = candidate.trim();
+      if (!trimmed) {
+        return { endOfTurn: false, reason: "empty" };
+      }
+      const wordCount = trimmed.split(/\s+/u).filter(Boolean).length;
+      if (wordCount < minPartialWords) {
+        return { endOfTurn: false, reason: "below-min-words" };
+      }
+      const lastChar = trimmed.at(-1);
+      const endsWithTerminal = typeof lastChar === "string" && endPunctuation.includes(lastChar);
+      if (!endsWithTerminal) {
+        return { endOfTurn: false, reason: "no-terminal-punctuation" };
+      }
+      const lastWord = stripTerminalPunctuation(trimmed).split(/\s+/u).at(-1)?.toLowerCase();
+      if (lastWord && fillerWords.includes(lastWord)) {
+        return { endOfTurn: false, reason: "trailing-filler" };
+      }
+      return {
+        confidence: 0.9,
+        endOfTurn: true,
+        reason: "terminal-punctuation"
+      };
+    }
+  };
+};
+var createRegexSemanticTurnDetector = (options) => {
+  const minPartialWords = options.minPartialWords ?? 2;
+  return {
+    evaluate: ({ lastFinalTranscript, partialText }) => {
+      const candidate = partialText.trim().length > 0 ? partialText : lastFinalTranscript?.text ?? "";
+      const trimmed = candidate.trim();
+      if (!trimmed) {
+        return { endOfTurn: false, reason: "empty" };
+      }
+      const wordCount = trimmed.split(/\s+/u).filter(Boolean).length;
+      if (wordCount < minPartialWords) {
+        return { endOfTurn: false, reason: "below-min-words" };
+      }
+      const match = options.endPattern.exec(trimmed);
+      if (!match) {
+        return { endOfTurn: false, reason: "pattern-miss" };
+      }
+      return {
+        endOfTurn: true,
+        reason: "pattern-match"
+      };
+    }
+  };
+};
 // src/amdDetector.ts
 var createMonologueAMDDetector = (options = {}) => {
   const minMonologueMs = options.minMonologueMs ?? 8000;
@@ -45952,6 +46064,7 @@ export {
   resolveVoiceDiagnosticsTraceFilter,
   resolveVoiceAuditTrailFilter,
   resolveVoiceAuditDeliveryFilter,
+  resolveVoiceAssistantMode,
   resolveVoiceAssistantMemoryNamespace,
   resolveTurnDetectionConfig,
   resolveLatestVoiceCallDebuggerSessionId,
@@ -46146,6 +46259,7 @@ export {
   evaluateVoiceAgentSquadContractEvidence,
   encodeTwilioMulawBase64,
   encodePcmAsWav,
+  describeVoiceAssistantMode,
   deliverVoiceTraceEventsToSinks,
   deliverVoiceObservabilityExport,
   deliverVoiceMonitorIssueNotifications,
@@ -46489,6 +46603,8 @@ export {
   createStoredVoiceExternalObjectMap,
   createStoredVoiceCallReviewArtifact,
   createRiskyTurnCorrectionHandler,
+  createRegexSemanticTurnDetector,
+  createPunctuationSemanticTurnDetector,
   createPlivoVoiceRoutes,
   createPlivoVoiceResponse,
   createPlivoMediaStreamBridge,

package/dist/semanticTurn.d.ts

@@ -0,0 +1,27 @@
+import type { Transcript } from "./types";
+export type VoiceSemanticTurnInput = {
+    audioLevel?: number;
+    lastFinalTranscript?: Transcript;
+    partialText: string;
+    silenceMs: number;
+    transcripts: Transcript[];
+};
+export type VoiceSemanticTurnVerdict = {
+    confidence?: number;
+    endOfTurn: boolean;
+    reason?: string;
+};
+export type VoiceSemanticTurnDetector = {
+    evaluate: (input: VoiceSemanticTurnInput) => Promise<VoiceSemanticTurnVerdict> | VoiceSemanticTurnVerdict;
+};
+export type CreatePunctuationSemanticTurnDetectorOptions = {
+    endPunctuation?: ReadonlyArray<string>;
+    fillerWords?: ReadonlyArray<string>;
+    minPartialWords?: number;
+};
+export declare const createPunctuationSemanticTurnDetector: (options?: CreatePunctuationSemanticTurnDetectorOptions) => VoiceSemanticTurnDetector;
+export type CreateRegexSemanticTurnDetectorOptions = {
+    endPattern: RegExp;
+    minPartialWords?: number;
+};
+export declare const createRegexSemanticTurnDetector: (options: CreateRegexSemanticTurnDetectorOptions) => VoiceSemanticTurnDetector;

package/dist/testing/index.js

@@ -5408,6 +5408,28 @@ var createVoiceMemoryRecordingStore = () => {
   };
 };
 
+// src/assistantMode.ts
+var resolveVoiceAssistantMode = (options) => {
+  if (options.assistantMode) {
+    return options.assistantMode;
+  }
+  if (options.realtime) {
+    return "s2s";
+  }
+  return "cascade";
+};
+var describeVoiceAssistantMode = (options) => {
+  const mode = resolveVoiceAssistantMode(options);
+  const modalities = options.modalities ? Array.from(new Set(options.modalities)) : ["audio"];
+  return {
+    hasRealtime: Boolean(options.realtime),
+    hasSTT: Boolean(options.stt),
+    hasTTS: Boolean(options.tts),
+    modalities,
+    mode
+  };
+};
+
 // src/session.ts
 var DEFAULT_RECONNECT_TIMEOUT = 30000;
 var DEFAULT_MAX_RECONNECT_ATTEMPTS2 = 10;
@@ -6754,6 +6776,18 @@ var createVoiceSession = (options) => {
       session,
       type: "turn.transcript"
     });
+    if (options.semanticTurnDetector) {
+      const verdict = await Promise.resolve(options.semanticTurnDetector.evaluate({
+        lastFinalTranscript: transcript,
+        partialText: session.currentTurn.partialText,
+        silenceMs: session.currentTurn.silenceStartedAt !== undefined ? Date.now() - session.currentTurn.silenceStartedAt : 0,
+        transcripts: session.currentTurn.transcripts
+      }));
+      if (verdict.endOfTurn) {
+        clearSilenceTimer();
+        await requestTurnCommit("vendor");
+      }
+    }
   };
   const resumePendingTurnCommit = (session) => {
     const pendingText = buildTurnText(session.currentTurn.transcripts, session.currentTurn.partialText, {
@@ -6778,13 +6812,20 @@ var createVoiceSession = (options) => {
     if (!inputAdapter) {
       throw new Error("Voice session requires either an stt or realtime adapter.");
     }
-    const openedSession = await inputAdapter.open({
-      format: options.realtime ? options.realtimeInputFormat ?? DEFAULT_REALTIME_FORMAT : DEFAULT_FORMAT,
+    const openedSession = await (options.realtime ? options.realtime.open({
+      format: options.realtimeInputFormat ?? DEFAULT_REALTIME_FORMAT,
       languageStrategy: options.languageStrategy,
       lexicon,
+      modalities: options.modalities,
       phraseHints,
       sessionId: options.id
-    });
+    }) : inputAdapter.open({
+      format: DEFAULT_FORMAT,
+      languageStrategy: options.languageStrategy,
+      lexicon,
+      phraseHints,
+      sessionId: options.id
+    }));
     const generation = ++adapterGenerationCounter;
     sttSession = openedSession;
     activeAdapterGeneration = generation;
@@ -6954,9 +6995,10 @@ var createVoiceSession = (options) => {
       });
       await appendTrace({
         payload: {
+          assistantMode: resolveVoiceAssistantMode(options),
+          realtimeConfigured: Boolean(options.realtime),
           text: output.assistantText,
-          ttsConfigured: Boolean(options.tts),
-          realtimeConfigured: Boolean(options.realtime)
+          ttsConfigured: Boolean(options.tts)
         },
         session,
         turnId: turn.id,

package/dist/types.d.ts

@@ -194,7 +194,10 @@ export type RealtimeAdapterOpenOptions = {
     format: AudioFormat;
     languageStrategy?: VoiceLanguageStrategy;
     lexicon?: VoiceLexiconEntry[];
+    modalities?: ReadonlyArray<"audio" | "text">;
     phraseHints?: VoicePhraseHint[];
+    promptCacheKey?: string;
+    semanticVAD?: import("./assistantMode").VoiceSemanticVADConfig;
     signal?: AbortSignal;
 };
 export type RealtimeAdapter<TOptions extends RealtimeAdapterOpenOptions = RealtimeAdapterOpenOptions> = {
@@ -731,6 +734,9 @@ export type CreateVoiceSessionOptions<TContext = unknown, TSession extends Voice
         provider?: string;
     };
     redact?: import("./redaction").VoiceTranscriptRedactor;
+    semanticTurnDetector?: import("./semanticTurn").VoiceSemanticTurnDetector;
+    assistantMode?: import("./assistantMode").VoiceAssistantMode;
+    modalities?: ReadonlyArray<"audio" | "text">;
     reconnect: Required<VoiceReconnectConfig>;
     phraseHints?: VoicePhraseHint[];
     sessionMetadata?: Record<string, unknown>;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@absolutejs/voice",
-  "version": "0.0.22-beta.482",
+  "version": "0.0.22-beta.484",
   "description": "Voice primitives and Elysia plugin for AbsoluteJS",
   "repository": {
     "type": "git",