@absolutejs/voice 0.0.22-beta.633 → 0.0.22-beta.635

package/dist/core/types.d.ts

@@ -78,6 +78,18 @@ export type VoiceTranscriptSentiment = {
     metadata?: Record<string, unknown>;
     score?: number;
 };
+/** A provider-normalized word hypothesis. Keeping this evidence attached to the
+ * transcript lets applications detect a single risky name or number that would
+ * otherwise disappear inside a high turn-level average confidence. */
+export type TranscriptWord = {
+    confidence?: number;
+    endedAtMs?: number;
+    language?: string;
+    punctuatedText?: string;
+    speaker?: string | number;
+    startedAtMs?: number;
+    text: string;
+};
 export type Transcript = {
     id: string;
     text: string;
@@ -89,6 +101,7 @@ export type Transcript = {
     startedAtMs?: number;
     endedAtMs?: number;
     vendor?: string;
+    words?: TranscriptWord[];
 };
 export type VoiceTranscriptQuality = {
     averageConfidence?: number;
@@ -101,6 +114,8 @@ export type VoiceTranscriptQuality = {
     partialTranscriptCount: number;
     selectedTranscriptCount: number;
     source: "fallback" | "primary";
+    lowestWordConfidence?: number;
+    wordConfidenceSampleCount?: number;
 };
 export type VoiceTurnCorrectionDiagnostics = {
     attempted: boolean;
@@ -118,7 +133,7 @@ export type VoiceTurnCostEstimate = {
     primaryAudioMs: number;
     totalBillableAudioMs: number;
 };
-export type VoiceFallbackSelectionReason = "fallback-empty" | "primary-empty" | "word-count-margin" | "confidence-margin" | "word-count-tiebreak" | "kept-primary";
+export type VoiceFallbackSelectionReason = "fallback-empty" | "primary-empty" | "word-count-margin" | "confidence-margin" | "word-count-tiebreak" | "policy-preference" | "kept-primary";
 export type VoiceFallbackDiagnostics = {
     attempted: boolean;
     fallbackConfidence?: number;
@@ -130,6 +145,7 @@ export type VoiceFallbackDiagnostics = {
     selected: boolean;
     selectionReason: VoiceFallbackSelectionReason;
     trigger: "empty-turn" | "low-confidence" | "empty-or-low-confidence" | "always";
+    triggerReason?: VoiceFallbackTriggerReason;
 };
 export type VoicePartialEvent = {
     type: "partial";
@@ -417,6 +433,16 @@ export type VoiceSTTLifecycle = "continuous" | "turn-scoped";
 export type VoiceTurnProfile = "fast" | "balanced" | "long-form";
 export type VoiceTurnQualityProfile = "general" | "accent-heavy" | "noisy-room" | "short-command";
 export type VoiceTurnFallbackTrigger = "empty-turn" | "low-confidence" | "empty-or-low-confidence" | "always";
+export type VoiceFallbackTriggerReason = "always" | "empty-turn" | "risk-policy" | "transcript-confidence" | "word-confidence";
+export type VoiceSTTFallbackCandidate = {
+    averageConfidence: number;
+    lowestWordConfidence?: number;
+    text: string;
+    transcripts: Transcript[];
+    wordCount: number;
+    words: TranscriptWord[];
+};
+export type VoiceSTTFallbackRiskPolicy = (candidate: VoiceSTTFallbackCandidate) => boolean;
 export type VoiceSTTFallbackConfig = {
     adapter: STTAdapter;
     trigger?: VoiceTurnFallbackTrigger;
@@ -426,6 +452,15 @@ export type VoiceSTTFallbackConfig = {
     settleMs?: number;
     completionTimeoutMs?: number;
     maxAttemptsPerTurn?: number;
+    /** Run the fallback when any provider word falls below this value. */
+    wordConfidenceThreshold?: number;
+    /** Application-specific routing for high-value turns, entities, or correction
+     * language that should receive a second transcription pass regardless of the
+     * aggregate confidence. */
+    riskPolicy?: VoiceSTTFallbackRiskPolicy;
+    /** Prefer a non-empty independent fallback for these audited trigger reasons,
+     * including providers that do not expose comparable confidence scores. */
+    preferFallbackOn?: VoiceFallbackTriggerReason[];
 };
 export type VoiceResolvedSTTFallbackConfig = {
     adapter: STTAdapter;
@@ -436,6 +471,9 @@ export type VoiceResolvedSTTFallbackConfig = {
     settleMs: number;
     completionTimeoutMs: number;
     maxAttemptsPerTurn: number;
+    wordConfidenceThreshold?: number;
+    riskPolicy?: VoiceSTTFallbackRiskPolicy;
+    preferFallbackOn?: VoiceFallbackTriggerReason[];
 };
 export type VoiceTurnDetectionConfig = {
     profile?: VoiceTurnProfile;

package/dist/index.js

@@ -4025,7 +4025,10 @@ var createEmptyCurrentTurn = () => ({
   silenceStartedAt: undefined,
   transcripts: []
 });
-var cloneTranscript = (transcript) => ({ ...transcript });
+var cloneTranscript = (transcript) => ({
+  ...transcript,
+  words: transcript.words?.map((word) => ({ ...word }))
+});
 var encodeBase64 = (chunk) => Buffer2.from(chunk).toString("base64");
 var countWords2 = (text) => text.trim().split(/\s+/).filter(Boolean).length;
 var normalizeText2 = (text) => text.trim().replace(/\s+/g, " ");
@@ -4070,9 +4073,12 @@ var calculateMeanConfidence = (transcripts) => {
   }
   return sum / total;
 };
+var collectTranscriptWords = (transcripts) => transcripts.flatMap((transcript) => transcript.words ?? []);
+var collectWordConfidences = (transcripts) => collectTranscriptWords(transcripts).map((word) => word.confidence).filter((confidence) => typeof confidence === "number" && Number.isFinite(confidence));
 var createTurnQuality = (transcripts, source, fallbackUsed, fallbackDiagnostics, correctionDiagnostics, costEstimate) => {
   const sampledTranscripts = transcripts.filter((transcript) => typeof transcript.confidence === "number");
   const confidenceSampleCount = sampledTranscripts.length;
+  const wordConfidences = collectWordConfidences(transcripts);
   return {
     averageConfidence: confidenceSampleCount > 0 ? sampledTranscripts.reduce((sum, transcript) => sum + transcript.confidence, 0) / confidenceSampleCount : undefined,
     confidenceSampleCount,
@@ -4083,7 +4089,9 @@ var createTurnQuality = (transcripts, source, fallbackUsed, fallbackDiagnostics,
     finalTranscriptCount: transcripts.filter((transcript) => transcript.isFinal).length,
     partialTranscriptCount: transcripts.filter((transcript) => !transcript.isFinal).length,
     selectedTranscriptCount: transcripts.length,
-    source
+    source,
+    lowestWordConfidence: wordConfidences.length > 0 ? Math.min(...wordConfidences) : undefined,
+    wordConfidenceSampleCount: wordConfidences.length
   };
 };
 var createTurnCostEstimate = (input) => {
@@ -4100,20 +4108,38 @@ var createTurnCostEstimate = (input) => {
   };
 };
 var normalizeCorrectionText = (text) => normalizeText2(text);
-var isFallbackNeeded = (candidate, config) => {
+var evaluateFallbackNeed = (candidate, config) => {
   const trimmed = normalizeText2(candidate.text);
   const wordCount = countWords2(trimmed);
+  const averageConfidence = calculateMeanConfidence(candidate.transcripts);
+  const words = collectTranscriptWords(candidate.transcripts);
+  const wordConfidences = collectWordConfidences(candidate.transcripts);
+  const lowestWordConfidence = wordConfidences.length > 0 ? Math.min(...wordConfidences) : undefined;
+  const resolvedCandidate = {
+    averageConfidence,
+    lowestWordConfidence,
+    text: candidate.text,
+    transcripts: candidate.transcripts,
+    wordCount,
+    words
+  };
   if (config.trigger === "always") {
-    return true;
+    return { needed: true, reason: "always" };
   }
-  if (config.trigger === "empty-turn") {
-    return wordCount < config.minTextLength;
+  if (wordCount < config.minTextLength && (config.trigger === "empty-turn" || config.trigger === "empty-or-low-confidence")) {
+    return { needed: true, reason: "empty-turn" };
   }
-  const averageConfidence = calculateMeanConfidence(candidate.transcripts);
-  if (config.trigger === "low-confidence") {
-    return averageConfidence > 0 && averageConfidence < config.confidenceThreshold;
+  if (config.riskPolicy?.(resolvedCandidate)) {
+    return { needed: true, reason: "risk-policy" };
+  }
+  if (config.wordConfidenceThreshold !== undefined && lowestWordConfidence !== undefined && lowestWordConfidence < config.wordConfidenceThreshold) {
+    return { needed: true, reason: "word-confidence" };
   }
-  return averageConfidence > 0 && averageConfidence < config.confidenceThreshold || wordCount < config.minTextLength;
+  const checksConfidence = config.trigger === "low-confidence" || config.trigger === "empty-or-low-confidence";
+  if (checksConfidence && averageConfidence > 0 && averageConfidence < config.confidenceThreshold) {
+    return { needed: true, reason: "transcript-confidence" };
+  }
+  return { needed: false, reason: undefined };
 };
 var selectBetterTurnText = (candidate, fallback) => {
   if (!fallback.text) {
@@ -4232,7 +4258,10 @@ var createVoiceSession = (options) => {
     minTextLength: options.sttFallback.minTextLength ?? DEFAULT_FALLBACK_MIN_TEXT_LENGTH,
     replayWindowMs: options.sttFallback.replayWindowMs ?? DEFAULT_FALLBACK_REPLAY_MS,
     settleMs: options.sttFallback.settleMs ?? DEFAULT_FALLBACK_SETTLE_MS,
-    trigger: options.sttFallback.trigger ?? "empty-or-low-confidence"
+    trigger: options.sttFallback.trigger ?? "empty-or-low-confidence",
+    wordConfidenceThreshold: options.sttFallback.wordConfidenceThreshold,
+    riskPolicy: options.sttFallback.riskPolicy,
+    preferFallbackOn: options.sttFallback.preferFallbackOn
   } : undefined;
   const appendTrace = async (input) => {
     await options.trace?.append({
@@ -5379,7 +5408,8 @@ var createVoiceSession = (options) => {
       text: primaryText,
       transcripts: primaryTranscripts
     };
-    if (!isFallbackNeeded(candidate, sttFallback)) {
+    const fallbackNeed = evaluateFallbackNeed(candidate, sttFallback);
+    if (!fallbackNeed.needed) {
       return null;
     }
     fallbackAttemptsForCurrentTurn += 1;
@@ -5495,7 +5525,11 @@ var createVoiceSession = (options) => {
       text: primaryText,
       wordCount: countWords2(normalizeText2(primaryText))
     };
-    const selection = selectBetterTurnText(primaryCandidate, fallbackCandidate);
+    const policyPrefersFallback = fallbackCandidate.text.length > 0 && fallbackNeed.reason !== undefined && sttFallback.preferFallbackOn?.includes(fallbackNeed.reason);
+    const selection = policyPrefersFallback ? {
+      reason: "policy-preference",
+      winner: fallbackCandidate
+    } : selectBetterTurnText(primaryCandidate, fallbackCandidate);
     const diagnostics = {
       attempted: true,
       fallbackConfidence: fallbackCandidate.confidence,
@@ -5506,7 +5540,8 @@ var createVoiceSession = (options) => {
       primaryWordCount: primaryCandidate.wordCount,
       selected: selection.winner.text === fallbackCandidate.text,
       selectionReason: selection.reason,
-      trigger: sttFallback.trigger
+      trigger: sttFallback.trigger,
+      triggerReason: fallbackNeed.reason
     };
     if (selection.winner.text === primaryCandidate.text) {
       return {
@@ -25397,7 +25432,9 @@ var resolveSTTFallbackConfig = (config) => {
     minTextLength: config.minTextLength ?? 2,
     replayWindowMs: config.replayWindowMs ?? 8000,
     settleMs: config.settleMs ?? 220,
-    trigger: config.trigger ?? "empty-or-low-confidence"
+    trigger: config.trigger ?? "empty-or-low-confidence",
+    wordConfidenceThreshold: config.wordConfidenceThreshold,
+    riskPolicy: config.riskPolicy
   };
 };
 var normalizePhraseHints = (hints) => (hints ?? []).map((hint) => ({

package/dist/testing/index.js

@@ -6408,7 +6408,10 @@ var createEmptyCurrentTurn = () => ({
   silenceStartedAt: undefined,
   transcripts: []
 });
-var cloneTranscript = (transcript) => ({ ...transcript });
+var cloneTranscript = (transcript) => ({
+  ...transcript,
+  words: transcript.words?.map((word) => ({ ...word }))
+});
 var encodeBase64 = (chunk) => Buffer2.from(chunk).toString("base64");
 var countWords2 = (text) => text.trim().split(/\s+/).filter(Boolean).length;
 var normalizeText2 = (text) => text.trim().replace(/\s+/g, " ");
@@ -6453,9 +6456,12 @@ var calculateMeanConfidence = (transcripts) => {
   }
   return sum / total;
 };
+var collectTranscriptWords = (transcripts) => transcripts.flatMap((transcript) => transcript.words ?? []);
+var collectWordConfidences = (transcripts) => collectTranscriptWords(transcripts).map((word) => word.confidence).filter((confidence) => typeof confidence === "number" && Number.isFinite(confidence));
 var createTurnQuality = (transcripts, source, fallbackUsed, fallbackDiagnostics, correctionDiagnostics, costEstimate) => {
   const sampledTranscripts = transcripts.filter((transcript) => typeof transcript.confidence === "number");
   const confidenceSampleCount = sampledTranscripts.length;
+  const wordConfidences = collectWordConfidences(transcripts);
   return {
     averageConfidence: confidenceSampleCount > 0 ? sampledTranscripts.reduce((sum, transcript) => sum + transcript.confidence, 0) / confidenceSampleCount : undefined,
     confidenceSampleCount,
@@ -6466,7 +6472,9 @@ var createTurnQuality = (transcripts, source, fallbackUsed, fallbackDiagnostics,
     finalTranscriptCount: transcripts.filter((transcript) => transcript.isFinal).length,
     partialTranscriptCount: transcripts.filter((transcript) => !transcript.isFinal).length,
     selectedTranscriptCount: transcripts.length,
-    source
+    source,
+    lowestWordConfidence: wordConfidences.length > 0 ? Math.min(...wordConfidences) : undefined,
+    wordConfidenceSampleCount: wordConfidences.length
   };
 };
 var createTurnCostEstimate = (input) => {
@@ -6483,20 +6491,38 @@ var createTurnCostEstimate = (input) => {
   };
 };
 var normalizeCorrectionText = (text) => normalizeText2(text);
-var isFallbackNeeded = (candidate, config) => {
+var evaluateFallbackNeed = (candidate, config) => {
   const trimmed = normalizeText2(candidate.text);
   const wordCount = countWords2(trimmed);
+  const averageConfidence = calculateMeanConfidence(candidate.transcripts);
+  const words = collectTranscriptWords(candidate.transcripts);
+  const wordConfidences = collectWordConfidences(candidate.transcripts);
+  const lowestWordConfidence = wordConfidences.length > 0 ? Math.min(...wordConfidences) : undefined;
+  const resolvedCandidate = {
+    averageConfidence,
+    lowestWordConfidence,
+    text: candidate.text,
+    transcripts: candidate.transcripts,
+    wordCount,
+    words
+  };
   if (config.trigger === "always") {
-    return true;
+    return { needed: true, reason: "always" };
   }
-  if (config.trigger === "empty-turn") {
-    return wordCount < config.minTextLength;
+  if (wordCount < config.minTextLength && (config.trigger === "empty-turn" || config.trigger === "empty-or-low-confidence")) {
+    return { needed: true, reason: "empty-turn" };
   }
-  const averageConfidence = calculateMeanConfidence(candidate.transcripts);
-  if (config.trigger === "low-confidence") {
-    return averageConfidence > 0 && averageConfidence < config.confidenceThreshold;
+  if (config.riskPolicy?.(resolvedCandidate)) {
+    return { needed: true, reason: "risk-policy" };
+  }
+  if (config.wordConfidenceThreshold !== undefined && lowestWordConfidence !== undefined && lowestWordConfidence < config.wordConfidenceThreshold) {
+    return { needed: true, reason: "word-confidence" };
+  }
+  const checksConfidence = config.trigger === "low-confidence" || config.trigger === "empty-or-low-confidence";
+  if (checksConfidence && averageConfidence > 0 && averageConfidence < config.confidenceThreshold) {
+    return { needed: true, reason: "transcript-confidence" };
   }
-  return averageConfidence > 0 && averageConfidence < config.confidenceThreshold || wordCount < config.minTextLength;
+  return { needed: false, reason: undefined };
 };
 var selectBetterTurnText = (candidate, fallback) => {
   if (!fallback.text) {
@@ -6615,7 +6641,10 @@ var createVoiceSession = (options) => {
     minTextLength: options.sttFallback.minTextLength ?? DEFAULT_FALLBACK_MIN_TEXT_LENGTH,
     replayWindowMs: options.sttFallback.replayWindowMs ?? DEFAULT_FALLBACK_REPLAY_MS,
     settleMs: options.sttFallback.settleMs ?? DEFAULT_FALLBACK_SETTLE_MS,
-    trigger: options.sttFallback.trigger ?? "empty-or-low-confidence"
+    trigger: options.sttFallback.trigger ?? "empty-or-low-confidence",
+    wordConfidenceThreshold: options.sttFallback.wordConfidenceThreshold,
+    riskPolicy: options.sttFallback.riskPolicy,
+    preferFallbackOn: options.sttFallback.preferFallbackOn
   } : undefined;
   const appendTrace = async (input) => {
     await options.trace?.append({
@@ -7762,7 +7791,8 @@ var createVoiceSession = (options) => {
       text: primaryText,
       transcripts: primaryTranscripts
     };
-    if (!isFallbackNeeded(candidate, sttFallback)) {
+    const fallbackNeed = evaluateFallbackNeed(candidate, sttFallback);
+    if (!fallbackNeed.needed) {
       return null;
     }
     fallbackAttemptsForCurrentTurn += 1;
@@ -7878,7 +7908,11 @@ var createVoiceSession = (options) => {
       text: primaryText,
       wordCount: countWords2(normalizeText2(primaryText))
     };
-    const selection = selectBetterTurnText(primaryCandidate, fallbackCandidate);
+    const policyPrefersFallback = fallbackCandidate.text.length > 0 && fallbackNeed.reason !== undefined && sttFallback.preferFallbackOn?.includes(fallbackNeed.reason);
+    const selection = policyPrefersFallback ? {
+      reason: "policy-preference",
+      winner: fallbackCandidate
+    } : selectBetterTurnText(primaryCandidate, fallbackCandidate);
     const diagnostics = {
       attempted: true,
       fallbackConfidence: fallbackCandidate.confidence,
@@ -7889,7 +7923,8 @@ var createVoiceSession = (options) => {
       primaryWordCount: primaryCandidate.wordCount,
       selected: selection.winner.text === fallbackCandidate.text,
       selectionReason: selection.reason,
-      trigger: sttFallback.trigger
+      trigger: sttFallback.trigger,
+      triggerReason: fallbackNeed.reason
     };
     if (selection.winner.text === primaryCandidate.text) {
       return {
@@ -10251,7 +10286,9 @@ var resolveBenchmarkFallbackConfig = (config) => {
     minTextLength: config.minTextLength ?? 2,
     replayWindowMs: config.replayWindowMs ?? 8000,
     settleMs: config.settleMs ?? 220,
-    trigger: config.trigger ?? "empty-or-low-confidence"
+    trigger: config.trigger ?? "empty-or-low-confidence",
+    wordConfidenceThreshold: config.wordConfidenceThreshold,
+    riskPolicy: config.riskPolicy
   };
 };
 var chunkAudio2 = (audio, bytesPerChunk) => {
@@ -14426,7 +14463,9 @@ var resolveSTTFallbackConfig = (config) => {
     minTextLength: config.minTextLength ?? 2,
     replayWindowMs: config.replayWindowMs ?? 8000,
     settleMs: config.settleMs ?? 220,
-    trigger: config.trigger ?? "empty-or-low-confidence"
+    trigger: config.trigger ?? "empty-or-low-confidence",
+    wordConfidenceThreshold: config.wordConfidenceThreshold,
+    riskPolicy: config.riskPolicy
   };
 };
 var normalizePhraseHints = (hints) => (hints ?? []).map((hint) => ({

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@absolutejs/voice",
-  "version": "0.0.22-beta.633",
+  "version": "0.0.22-beta.635",
   "description": "Voice primitives and Elysia plugin for AbsoluteJS",
   "repository": {
     "type": "git",