@juspay/neurolink 13.6.5 → 13.8.0

package/dist/voice/streaming/chunkedStreamAdapter.js

@@ -1,7 +1,8 @@
 /**
  * Client-side streaming for batch STT engines.
  *
- * PCM16LE frames → energy-gated utterances (see `endpointer.ts`) → the
+ * PCM16LE frames → utterances found by a speech detector (Silero VAD by
+ * default) or the energy gate (see `endpointer.ts`) → the
  * current utterance is re-transcribed every `intervalMs` → LocalAgreement
  * commits the prefix two passes agree on → a final pass when the utterance
  * ends. The adapter knows nothing about providers or correction: it calls
@@ -26,10 +27,37 @@
  */
 import { logger } from "../../utils/logger.js";
 import { STTError } from "../errors.js";
+import { SpeechFrameScorer } from "../vad/frameScorer.js";
+import { loadSileroDetector, resolveVadEngine, resolveVadThresholds, } from "../vad/sileroVad.js";
 import { Endpointer, ENGINE_SAMPLE_RATE, resolveEndpointerConfig, } from "./endpointer.js";
 import { createLocalAgreement, resetLocalAgreement, splitWords, updateLocalAgreement, } from "./localAgreement.js";
 /** Default cadence of interim passes. */
 const DEFAULT_INTERVAL_MS = 900;
+/**
+ * With a speech detector, audio after the last speech block beyond this
+ * margin is not sent to the engine: the closing pause (750 ms of it on every
+ * utterance that ends in silence) is where a batch model invents endings.
+ */
+const TRIM_TAIL_MS = 300;
+/**
+ * Digital silence appended to every chunk an engine gets, so it always ends
+ * in a clean pause: a chunk cut off right after speech sent Qwen3-ASR into a
+ * decoding loop until its timeout (180 s on a file, the utterance lost);
+ * 200 ms of zeros was enough to stop it, this is the margin.
+ */
+export const ENGINE_TAIL_PAD_MS = 400;
+/**
+ * A short utterance whose text would mean more words per second of detected
+ * speech than anyone can say (fast speech is 5–6) is an engine filling a
+ * pause, not a sentence. Only judged while the detector counted under 1.5 s
+ * of speech, where the rate is meaningful and the loss, if ever wrong, is a
+ * word or two.
+ */
+const MAX_WORDS_PER_SPEECH_SECOND = 12;
+const WORD_RATE_GUARD_MAX_SPEECH_MS = 1500;
+/** Logged once per process: streams fall back quietly after that. */
+let detectorUnavailableLogged = false;
+let autoDetectorRetryAt = 0;
 /** An interim pass needs at least this much audio the previous pass did not see. */
 const MIN_NEW_AUDIO_INTERIM_S = 0.8;
 /** Below this much new audio since the last pass, the final reuses the interim text. */
@@ -45,6 +73,98 @@ const MIN_NEW_AUDIO_FINAL_S = 0.4;
 const TAKEOVER_MIN_AUDIO_S = 3;
 const TAKEOVER_MIN_SPEECH_MS = 1500;
 const TAKEOVER_UNSURE_PASSES = 2;
+/**
+ * Probe thresholds. The first pass of an utterance runs the primary and the
+ * fallback in parallel once there is this much audio and speech, and the
+ * utterance's engine is chosen from that pass: the fallback leads when the
+ * primary is unsure or empty and the fallback produced Latin-script words.
+ * Under these amounts even a parallel pass tells nothing (a language score
+ * is noise below ~1 s), so the probe waits for the next pass instead.
+ */
+const PROBE_MIN_AUDIO_S = 0.8;
+const PROBE_MIN_SPEECH_MS = 500;
+/**
+ * Two verdicts on less audio than this only pick who leads for now: the
+ * Indic engine is unsure of nearly everything under ~2.5 s (a Hinglish
+ * sentence it later scores −0.10 reads as unsure at 1 s), so an early
+ * "English heard, primary unsure" must not lock or pin the utterance.
+ */
+const AGREE_MIN_AUDIO_S = 2.5;
+/**
+ * A primary that is "sure" while the fallback heard English shows its own
+ * text only on a strong score. Measured on the IndicConformer wrapper: Indian
+ * languages −0.01…−0.10, Hinglish −0.09…−0.14, Indian-accented English over
+ * the −0.18 cutoff on a short stream pass at −0.156 (and −0.35…−0.58 once
+ * the utterance is whole). Between the two, the fallback's text is shown and
+ * the final decides on the whole utterance.
+ */
+const MIXED_PRIMARY_MIN_SCORE = -0.12;
+/**
+ * "Unsure" is only evidence against an Indian language well below the
+ * cutoff: Telugu read −0.181 on a 2.3 s stream pass (the cutoff is −0.18)
+ * while English sits at −0.35 and lower. Between the two, the fallback's
+ * "English" (which Qwen3-ASR also says of Telugu, a language it does not
+ * know) neither settles nor pins the utterance.
+ */
+const CLEAR_UNSURE_MAX_SCORE = -0.25;
+/**
+ * Probes shorter than this do not count toward the utterance's score: a
+ * second of audio reads −0.3 for Hinglish and −0.9 for English alike.
+ * Longer ones are averaged, weighted by length, so one pass that dips (−0.153
+ * among Hinglish passes at −0.06…−0.11) cannot flip the utterance.
+ */
+const SCORE_MIN_AUDIO_S = 1.5;
+/** Indian languages: what an Indic primary engine is for; any other language belongs to the fallback. */
+const INDIC_LANGUAGES = new Set([
+    "as",
+    "bn",
+    "brx",
+    "doi",
+    "gu",
+    "hi",
+    "kn",
+    "kok",
+    "ks",
+    "mai",
+    "ml",
+    "mni",
+    "mr",
+    "ne",
+    "or",
+    "pa",
+    "sa",
+    "sat",
+    "sd",
+    "ta",
+    "te",
+    "ur",
+]);
+/** The ISO 639 part of an engine's language report, lower case; `undefined` for none or "und". */
+function langCode(value) {
+    const code = value?.trim().toLowerCase().split(/[-_]/u)[0];
+    return code && code !== "und" && code !== "auto" ? code : undefined;
+}
+/** Share of the text's letters in an Indic script. */
+function indicShare(text) {
+    return 1 - latinShare(text);
+}
+/** Letters outside the Latin blocks (Indic, Arabic, Thai, Burmese, CJK, Hangul). */
+const NON_LATIN_LETTER = /[\u0900-\u0DFF\u0600-\u06FF\u0E00-\u0E7F\u1000-\u109F\u3040-\u30FF\u4E00-\u9FFF\uAC00-\uD7AF]/gu;
+const LATIN_LETTER = /[A-Za-z\u00C0-\u024F]/gu;
+/** Share of the text's letters that are Latin; 1 for text with no letters at all. */
+function latinShare(text) {
+    const latin = text.match(LATIN_LETTER)?.length ?? 0;
+    const other = text.match(NON_LATIN_LETTER)?.length ?? 0;
+    return latin + other === 0 ? 1 : latin / (latin + other);
+}
+/** Probe passes before an utterance's engine locks when the caller sets none. */
+const DEFAULT_LOCK_AFTER_PASSES = 2;
+/** How long auto mode uses the energy gate after the speech detector failed to load. */
+const AUTO_DETECTOR_RETRY_MS = 10 * 60_000;
+/** Below this share of speech blocks an utterance is mostly pause. */
+const NEAR_SILENT_SPEECH_SHARE = 0.25;
+/** An output this short on a mostly-pause utterance is a filler, not speech. */
+const NEAR_SILENT_MAX_WORDS = 3;
 const DEFAULT_FALLBACK_WHEN = [
     "unsure",
     "empty",
@@ -101,10 +221,38 @@ class UtteranceRun {
     shown = "";
     switched = false;
     confidentOnce = false;
+    /** Passes on which the primary was confident; two veto the takeover for good, one only delays it. */
+    confidentPasses = 0;
     unsureN = 0;
     languageDetected;
     language;
     lastLanguageKey = "";
+    /** Probe passes run so far; the engine locks after `routing.lockAfterPasses`. */
+    probePasses = 0;
+    /** Probe passes on which the primary was confident / unsure, for the lock rule. */
+    confidentProbes = 0;
+    unsureProbes = 0;
+    locked = false;
+    /** The engines disagreed on the language: the final runs both on the whole utterance. */
+    lookAgain = false;
+    /** Consecutive probes (with enough audio) on which both verdicts agreed. */
+    agreedProbes = 0;
+    /** The fallback transcribes in this language from now on (the utterance was settled as it). */
+    fallbackPin;
+    /** The reading the probe did not show, from the latest probe: kept for the final when the engines disagreed. */
+    alternative;
+    /** The primary's language score on each probe of at least `SCORE_MIN_AUDIO_S`, with its audio length. */
+    primaryScores = [];
+    /** The primary's score over the utterance so far, weighted by audio; `undefined` before a long enough probe. */
+    get primaryScore() {
+        let sum = 0;
+        let weight = 0;
+        for (const { score, seconds } of this.primaryScores) {
+            sum += score * seconds;
+            weight += seconds;
+        }
+        return weight > 0 ? sum / weight : undefined;
+    }
     constructor(u) {
         this.u = u;
     }
@@ -139,6 +287,7 @@ function liftLanguage(res) {
     return {
         language: language ?? "und",
         detected: flag ?? true,
+        flagged: flag !== undefined,
         ...(scores.length ? { scores } : {}),
     };
 }
@@ -168,9 +317,28 @@ class ChunkedStreamSession {
     queue = new StreamEventQueue();
     runs = new Map();
     fallbackWhen;
+    probe;
+    /** Set once the primary has failed and the fallback took over, so the warning is logged once per stream. */
+    primaryFailureWarned = false;
+    lockAfterPasses;
+    carryPrior;
+    /** The last language an engine was confident about, carried into later utterances. */
+    sessionPrior;
+    /** Latin share of the last final with more than three words; the script the session speaks. */
+    sessionLatinShare;
     engineTail = Promise.resolve();
     busy = 0;
+    inFlight = 0;
+    slotWaiters = [];
+    /** Passes queued or running, settled ones removed; the end-of-input wait drains it. */
+    pendingPasses = new Set();
+    concurrency;
+    finalRetries;
     aborted = false;
+    /** Scores the stream for the endpointer when a speech detector is in use. */
+    scorer = null;
+    /** What found the utterances, as reported on events. */
+    vadName = "energy";
     constructor(run, options) {
         this.run = run;
         this.options = options;
@@ -186,6 +354,11 @@ class ChunkedStreamSession {
         this.fallbackWhen = options.fallback?.when?.length
             ? options.fallback.when
             : DEFAULT_FALLBACK_WHEN;
+        this.probe = !!options.fallback && options.routing?.probe !== false;
+        this.lockAfterPasses = Math.max(1, Math.floor(options.routing?.lockAfterPasses ?? DEFAULT_LOCK_AFTER_PASSES));
+        this.carryPrior = options.routing?.prior !== "none";
+        this.concurrency = Math.max(1, Math.floor(options.concurrency ?? 1));
+        this.finalRetries = Math.max(0, Math.floor(options.finalRetries ?? 0));
     }
     async *events(frames) {
         const iterator = frames[Symbol.asyncIterator]();
@@ -201,6 +374,27 @@ class ChunkedStreamSession {
             }
         }
     }
+    /**
+     * A mostly-pause utterance whose output is a word or two in a script the
+     * session has not spoken (a lone "不。" or "हाँ" on an English meeting):
+     * discarded as noise. Text in the session's own script is always kept.
+     */
+    nearSilentNoise(r, text, seconds) {
+        if (seconds <= 0 ||
+            r.u.speechMs / (seconds * 1000) >= NEAR_SILENT_SPEECH_SHARE) {
+            return false;
+        }
+        if (text.split(/\s+/u).filter(Boolean).length > NEAR_SILENT_MAX_WORDS) {
+            return false;
+        }
+        const share = latinShare(text);
+        const sessionLatin = this.sessionLatinShare;
+        // Unknown session script: only non-Latin fillers are suspect.
+        if (sessionLatin === undefined) {
+            return share < 0.5;
+        }
+        return Math.abs(share - sessionLatin) >= 0.5;
+    }
     label(role) {
         return role === "fallback"
             ? (this.options.fallback?.label ?? "fallback")
@@ -208,6 +402,13 @@ class ChunkedStreamSession {
     }
     /** Reads frames until the source ends, then finalizes and closes the queue. */
     async pump(iterator) {
+        const fatal = await this.initDetector();
+        if (fatal !== undefined) {
+            this.queue.push({ type: "error", message: fatal, recoverable: false });
+            this.queue.close();
+            void Promise.resolve(iterator.return?.()).catch(() => undefined);
+            return;
+        }
         let carry = null;
         let sourceError;
         try {
@@ -234,6 +435,9 @@ class ChunkedStreamSession {
                     samples[s] = (v >= 0x8000 ? v - 0x10000 : v) / 0x8000;
                 }
                 carry = total % 2 === 1 ? byteAt(total - 1) : null;
+                if (this.scorer) {
+                    await this.scoreFrames(samples);
+                }
                 this.handle(this.endpointer.push(samples));
                 this.maybeInterim();
             }
@@ -244,12 +448,12 @@ class ChunkedStreamSession {
         // Whatever was being said when the source ended (or failed) is still finalized.
         this.handle(this.endpointer.flush());
         try {
-            // Every final is chained on the engine queue; wait until the chain is empty.
-            let tail;
-            do {
-                tail = this.engineTail;
-                await tail;
-            } while (tail !== this.engineTail);
+            // Every final is on the engine queue; wait until nothing is queued or
+            // in flight. With several slots the passes are not a chain, so the
+            // set of pending passes is what is waited on, not the last one started.
+            while (this.pendingPasses.size > 0) {
+                await Promise.allSettled([...this.pendingPasses]);
+            }
         }
         catch {
             // Individual passes report their own errors; nothing to add here.
@@ -263,6 +467,90 @@ class ChunkedStreamSession {
         }
         this.queue.close();
     }
+    /**
+     * Chooses the speech detector for this stream: the caller's own, or Silero
+     * per `vad` / `NEUROLINK_STT_VAD`, or none (the energy gate). Returns the
+     * message of a fatal problem — a detector asked for by name that cannot
+     * load — or `undefined`.
+     */
+    async initDetector() {
+        const injected = this.options.speechDetector;
+        let detector = null;
+        if (injected) {
+            detector = injected;
+        }
+        else if (injected === undefined) {
+            const engine = resolveVadEngine(this.options.vad);
+            // In auto mode a failed load is not retried by every new stream (a
+            // model download can take minutes to time out); it is tried again
+            // after AUTO_DETECTOR_RETRY_MS.
+            const skipAuto = engine !== "silero" && Date.now() < autoDetectorRetryAt;
+            if (engine !== "energy" && !skipAuto) {
+                try {
+                    detector = await loadSileroDetector(this.options.vad);
+                    autoDetectorRetryAt = 0;
+                }
+                catch (err) {
+                    if (engine === "silero") {
+                        return errorMessage(err);
+                    }
+                    autoDetectorRetryAt = Date.now() + AUTO_DETECTOR_RETRY_MS;
+                    if (!detectorUnavailableLogged) {
+                        detectorUnavailableLogged = true;
+                        logger.warn(`[transcribeStream] speech detector unavailable, streams use the energy gate: ${errorMessage(err)}`);
+                    }
+                }
+            }
+        }
+        if (!detector) {
+            return undefined;
+        }
+        const scorer = new SpeechFrameScorer(detector, this.inputRate);
+        this.scorer = scorer;
+        this.vadName = detector.name;
+        this.endpointer.attachSpeech({
+            ...resolveVadThresholds(this.options.vad),
+            probabilityAt: (seconds) => scorer.probabilityAt(seconds),
+        });
+        return undefined;
+    }
+    /** Scores the frames these samples complete; a detector that breaks hands the stream to the energy gate. */
+    async scoreFrames(samples) {
+        const scorer = this.scorer;
+        if (!scorer) {
+            return;
+        }
+        try {
+            await scorer.push(samples);
+        }
+        catch (err) {
+            logger.warn(`[transcribeStream] speech detector failed (${errorMessage(err)}); the energy gate takes over`);
+            this.scorer = null;
+            this.vadName = "energy";
+            this.endpointer.attachSpeech(null);
+        }
+    }
+    /** How an utterance's audio is cut for the engine: the closing pause trimmed when a detector marks the speech, a silence pad always. */
+    engineCut() {
+        return this.scorer
+            ? { trimTailMs: TRIM_TAIL_MS, padMs: ENGINE_TAIL_PAD_MS }
+            : { padMs: ENGINE_TAIL_PAD_MS };
+    }
+    /**
+     * More words per second of detected speech than anyone can say, on a short
+     * utterance: the engine filled a pause with a stock phrase.
+     */
+    impossibleWordRate(r, text) {
+        if (!this.scorer || !r.u.speech) {
+            return false;
+        }
+        const speechMs = r.u.speechMs;
+        if (speechMs <= 0 || speechMs > WORD_RATE_GUARD_MAX_SPEECH_MS) {
+            return false;
+        }
+        const n = text.split(/\s+/u).filter(Boolean).length;
+        return n / (speechMs / 1000) > MAX_WORDS_PER_SPEECH_SECOND;
+    }
     handle(events) {
         for (const ev of events) {
             if (ev.type === "start") {
@@ -282,13 +570,33 @@ class ChunkedStreamSession {
             }
         }
     }
-    /** Runs `fn` after every queued engine call; at most one is in flight. */
+    /**
+     * Runs `fn` once a slot is free. Live streaming has one slot, so passes
+     * run in order and a caption never shows a stale pass; the file path sets
+     * `concurrency` higher and finals of different utterances overlap.
+     */
     withEngine(fn) {
         this.busy++;
-        const next = this.engineTail.then(fn).finally(() => {
-            this.busy--;
-        });
+        const slots = this.concurrency;
+        const run = async () => {
+            while (this.inFlight >= slots) {
+                await new Promise((resolve) => this.slotWaiters.push(resolve));
+            }
+            this.inFlight++;
+            try {
+                await fn();
+            }
+            finally {
+                this.inFlight--;
+                this.busy--;
+                this.slotWaiters.shift()?.();
+            }
+        };
+        const next = slots > 1 ? run() : this.engineTail.then(run);
         this.engineTail = next.catch(() => undefined);
+        const tracked = next.catch(() => undefined);
+        this.pendingPasses.add(tracked);
+        void tracked.finally(() => this.pendingPasses.delete(tracked));
         return next;
     }
     maybeInterim() {
@@ -302,8 +610,10 @@ class ChunkedStreamSession {
         }
         // The cadence runs on the audio clock: a live source advances it in real
         // time, and a file replayed faster than real time gets the same passes
-        // per second of audio, limited only by the one-in-flight rule.
-        if (this.endpointer.audioSeconds - r.lastReqAt < this.intervalS) {
+        // per second of audio, limited only by the one-in-flight rule. An
+        // infinite cadence means no interim passes at all (the file path).
+        if (!Number.isFinite(this.intervalS) ||
+            this.endpointer.audioSeconds - r.lastReqAt < this.intervalS) {
             return;
         }
         if (u.length - r.sentLen < MIN_NEW_AUDIO_INTERIM_S * this.inputRate) {
@@ -315,7 +625,9 @@ class ChunkedStreamSession {
             }
         });
     }
-    emitLanguage(r, lang) {
+    emitLanguage(r, lang, 
+    /** Whether this verdict may become the stream's language (only settled ones may, in the fused path). */
+    carry = true) {
         if (!lang) {
             return;
         }
@@ -323,6 +635,10 @@ class ChunkedStreamSession {
         r.languageDetected = lang.detected;
         if (lang.detected) {
             r.confidentOnce = true;
+            r.confidentPasses++;
+            if (carry && lang.language && lang.language !== "und") {
+                this.sessionPrior = lang.language;
+            }
         }
         const key = `${lang.language}|${lang.detected}`;
         if (key === r.lastLanguageKey) {
@@ -337,13 +653,17 @@ class ChunkedStreamSession {
             ...(lang.scores ? { scores: lang.scores } : {}),
         });
     }
-    async callEngine(r, pcm, final) {
+    async callEngine(r, pcm, final, engine = r.engine, pin = r.fallbackPin) {
         try {
             const res = await this.run(pcm, {
                 utterance: r.u.id,
                 final,
-                engine: r.engine,
+                engine,
                 seconds: r.u.length / this.inputRate,
+                ...(this.carryPrior && engine === "primary" && this.sessionPrior
+                    ? { languagePrior: this.sessionPrior }
+                    : {}),
+                ...(engine === "fallback" && pin ? { languageHint: pin } : {}),
             });
             return { res };
         }
@@ -351,6 +671,286 @@ class ChunkedStreamSession {
             return { err };
         }
     }
+    /**
+     * The utterance's engine and language from both engines' verdicts:
+     *
+     * - the fallback heard English (or any language outside India) and the
+     *   primary is unsure: the fallback leads, its language is the label, and
+     *   it is pinned to that language for the rest of the utterance;
+     * - the fallback heard English and the primary is confident: code-mixed
+     *   speech (or an accent the primary over-reads); the primary leads, and
+     *   the final looks again with both engines on the whole utterance;
+     * - the fallback heard an Indian language and the primary is confident:
+     *   the primary leads, both agreed;
+     * - the primary is unsure, the stream has been speaking a language outside
+     *   India, and the fallback names another one: the fallback is asked again
+     *   in the stream's language (a short pass of accented English is where it
+     *   drifts into Devanagari); still another script means a real switch;
+     * - the fallback heard an Indian language and the primary is unsure: the
+     *   primary leads when it has words.
+     */
+    fuse(r, s) {
+        // IndicConformer's Hindi and Punjabi (or Marathi, Nepali) scores are
+        // near-ties on Hindi speech: Hindi within 0.05 of the top score is Hindi,
+        // which keeps Hinglish labelled Hindi and open to the fallback's verdict.
+        const topScore = s.primary?.scores?.[0]?.score;
+        const hindiScore = s.primary?.scores?.find((x) => langCode(x.language) === "hi")?.score;
+        const primaryLang = topScore !== undefined &&
+            hindiScore !== undefined &&
+            topScore - hindiScore <= 0.05
+            ? "hi"
+            : langCode(s.primary?.language);
+        // Sure means the engine said so: a result that carries no verdict at all
+        // (a gateway dropped the field) is not a confident one.
+        const primarySure = s.primary?.detected === true && s.primary.flagged && !!s.primaryText;
+        const passScore = s.primary?.scores?.[0]?.score;
+        if (passScore !== undefined &&
+            r.u.length / this.inputRate >= SCORE_MIN_AUDIO_S) {
+            r.primaryScores.push({
+                score: passScore,
+                seconds: r.u.length / this.inputRate,
+            });
+        }
+        const primaryScore = r.primaryScore ?? passScore;
+        // Clearly not an Indian language: no words at all, or a score far below
+        // the cutoff (an engine without scores keeps its own unsure verdict).
+        const clearlyUnsure = !s.primaryText ||
+            (!primarySure &&
+                (primaryScore === undefined || primaryScore <= CLEAR_UNSURE_MAX_SCORE));
+        const heardIndic = INDIC_LANGUAGES.has(s.heard);
+        // Before any confident verdict, a provisional pass treats the stream as
+        // English: a one-second answer in an Indic script from the fallback on
+        // audio the primary cannot place is a guess, and showing it is worse
+        // than romanized text the primary replaces once it is sure.
+        const provisional = r.u.length / this.inputRate < AGREE_MIN_AUDIO_S;
+        // Only for an Indian-language primary (the accent case this exists for):
+        // a Spanish utterance on Whisper must never be re-asked in English.
+        const indicPrimary = primaryLang !== undefined && INDIC_LANGUAGES.has(primaryLang);
+        const stream = this.sessionPrior ?? (provisional && indicPrimary ? "en" : undefined);
+        // A stream that has been speaking a language outside India, a primary
+        // that cannot place this audio, and a fallback naming another language:
+        // a short pass of accented speech is exactly where the fallback guesses
+        // (Hindi, even Chinese, for one second of Indian English). Ask it again
+        // in the stream's language; an answer still in another script means the
+        // speaker really switched.
+        // Only English: the accent drift this exists for. Re-asking in any other
+        // stream language would turn one wrong guess into a whole session's.
+        if (clearlyUnsure &&
+            stream === "en" &&
+            s.heard !== stream &&
+            s.fallbackText) {
+            return {
+                engine: "fallback",
+                language: stream,
+                agreed: false,
+                repin: stream,
+                why: `fallback heard ${s.heard} in a ${stream} stream, primary unsure: asked again in ${stream}`,
+            };
+        }
+        // Qwen3-ASR knows Hindi (and Urdu) among the Indian languages, nothing
+        // else: on Tamil it says English or even Malay. Its verdict is no
+        // evidence against a primary strongly sure of a language it cannot
+        // hear; the second opinion still reconciles any English in it.
+        const fallbackBlind = primarySure &&
+            primaryLang !== undefined &&
+            INDIC_LANGUAGES.has(primaryLang) &&
+            primaryLang !== "hi" &&
+            primaryLang !== "ur" &&
+            (primaryScore === undefined || primaryScore >= MIXED_PRIMARY_MIN_SCORE);
+        if (fallbackBlind) {
+            return {
+                engine: "primary",
+                language: primaryLang,
+                agreed: true,
+                why: `primary strongly sure of ${primaryLang}, a language the fallback cannot name (it said ${s.heard})`,
+            };
+        }
+        if (!heardIndic) {
+            if (!primarySure) {
+                return {
+                    engine: s.fallbackText ? "fallback" : "primary",
+                    language: s.heard,
+                    agreed: clearlyUnsure && !!s.fallbackText,
+                    ...(clearlyUnsure ? { pin: s.heard } : {}),
+                    why: `fallback heard ${s.heard}, primary ${clearlyUnsure ? "clearly unsure" : "marginal"}`,
+                };
+            }
+            const strong = primaryScore === undefined || primaryScore >= MIXED_PRIMARY_MIN_SCORE;
+            return {
+                engine: strong ? "primary" : "fallback",
+                language: strong ? primaryLang : s.heard,
+                agreed: false,
+                why: `primary sure of ${primaryLang}, fallback heard ${s.heard}: mixed, ${strong ? "primary strong" : "primary marginal"} (utterance ${primaryScore?.toFixed(3)})`,
+            };
+        }
+        if (primarySure) {
+            return {
+                engine: "primary",
+                language: primaryLang,
+                agreed: true,
+                why: `both heard an Indian language (${primaryLang}/${s.heard})`,
+            };
+        }
+        return {
+            engine: s.primaryText ? "primary" : "fallback",
+            language: s.heard,
+            agreed: false,
+            why: `fallback heard ${s.heard}, primary unsure`,
+        };
+    }
+    /**
+     * The probe: both engines on the same audio, the utterance's engine chosen
+     * from what came back. Returns the result the caption should use.
+     */
+    async probePass(r, pcm, isFinal) {
+        const [p, f] = await Promise.all([
+            this.callEngine(r, pcm, isFinal, "primary"),
+            this.callEngine(r, pcm, isFinal, "fallback"),
+        ]);
+        r.probePasses++;
+        const lang = liftLanguage(p.res);
+        const primaryText = p.res?.text?.trim() ?? "";
+        const fallbackText = f.res?.text?.trim() ?? "";
+        const when = this.fallbackWhen;
+        const primaryFailed = p.err !== undefined || (when.includes("empty") && !primaryText);
+        const unsure = when.includes("unsure") && lang?.detected === false;
+        const confident = lang?.detected === true;
+        if (confident) {
+            r.confidentProbes++;
+        }
+        else if (unsure) {
+            r.unsureProbes++;
+        }
+        // Verdicts that disagreed buy one more probe, so a single call on two or
+        // three seconds of audio never settles an utterance by itself.
+        const disagreed = r.confidentProbes > 0 && r.unsureProbes > 0;
+        const lastProbe = r.probePasses >= this.lockAfterPasses + (disagreed ? 1 : 0);
+        // The fallback's words count when there are any; whether they are mostly
+        // Latin decides an "unsure" call (an English sentence with one Hindi
+        // word is still English), never an "empty" one: words always beat none.
+        const fallbackHasWords = fallbackText.length > 0;
+        const fallbackMostlyLatin = fallbackHasWords && latinShare(fallbackText) >= 0.7;
+        const lead = (engine, why) => {
+            if (r.engine !== engine) {
+                r.engine = engine;
+                r.switched = engine === "fallback";
+                resetLocalAgreement(r.la);
+            }
+            if (logger.shouldLog("debug")) {
+                logger.debug(`[transcribeStream] utterance ${r.u.id}: probe ${r.probePasses} → ${engine} leads (${why})`);
+            }
+        };
+        // Two verdicts. A fallback that names the language it heard (Qwen3-ASR
+        // through stt-aux does; a pinned pass does not count) settles what the
+        // primary's own score cannot: an Indian accent pulls an Indic engine's
+        // score toward "confident", never the other engine's language call.
+        const pinnedPass = r.fallbackPin !== undefined;
+        const heard = pinnedPass ? undefined : langCode(f.res?.language);
+        // Only an engine that states its own confidence can be weighed against the
+        // fallback's verdict; one that reports none (Deepgram, Whisper) keeps the
+        // rules below, where the fallback takes over only when the primary is
+        // empty or flags itself unsure.
+        if (heard && !p.err && f.res && lang?.flagged) {
+            const verdict = this.fuse(r, {
+                primary: lang,
+                primaryText,
+                fallbackText,
+                heard,
+            });
+            if (verdict.engine === "fallback" && verdict.repin && fallbackText) {
+                // The fallback named an Indic language in a stream that speaks
+                // English, on audio the primary cannot place: ask it again in the
+                // stream's language rather than show the wrong script. If it still
+                // answers in an Indic script, the speech really is Indic.
+                const again = await this.callEngine(r, pcm, isFinal, "fallback", verdict.repin);
+                const againText = again.res?.text?.trim() ?? "";
+                if (againText && indicShare(againText) <= 0.5) {
+                    f.res = again.res;
+                }
+                else {
+                    verdict.engine =
+                        primaryText && INDIC_LANGUAGES.has(heard) ? "primary" : "fallback";
+                    verdict.language = heard;
+                    verdict.why += `; it insisted, so the speaker switched to ${heard}`;
+                }
+            }
+            const enough = r.u.length / this.inputRate >= AGREE_MIN_AUDIO_S;
+            r.agreedProbes = verdict.agreed && enough ? r.agreedProbes + 1 : 0;
+            // Two agreeing probes, not one: under load the first probe can land
+            // late on a single weak reading, and one call never settles an utterance.
+            const settled = r.agreedProbes >= 2;
+            const score = lang?.scores?.[0]?.score;
+            lead(verdict.engine, `${verdict.why}${score !== undefined ? ` [primary ${lang?.language} ${score}${lang?.flagged ? "" : ", no flag"}]` : ""}${enough ? "" : " (provisional)"}`);
+            const otherText = verdict.engine === "fallback"
+                ? primaryText
+                : (f.res?.text?.trim() ?? "");
+            r.alternative = otherText
+                ? {
+                    engine: verdict.engine === "fallback" ? "primary" : "fallback",
+                    text: otherText,
+                }
+                : undefined;
+            // Only agreement locks: a mixed or unsure utterance keeps both engines
+            // running, and its final decides on the whole utterance.
+            r.locked = settled;
+            r.lookAgain = !settled;
+            if (verdict.pin && settled) {
+                r.fallbackPin = verdict.pin;
+            }
+            // A weak guess (Qwen3-ASR called a Tamil utterance Malay) must not
+            // become the stream's language: only a settled verdict carries over.
+            this.emitLanguage(r, {
+                language: verdict.language ?? "und",
+                detected: verdict.language !== undefined,
+                flagged: true,
+                ...(lang?.scores ? { scores: lang.scores } : {}),
+            }, settled);
+            return verdict.engine === "fallback" ? f : p;
+        }
+        if (primaryFailed && fallbackHasWords) {
+            lead("fallback", p.err !== undefined ? "primary failed" : "primary empty");
+            // A failed call (a timeout on a busy gateway) says nothing about the
+            // language: try the primary again on the next pass. An empty answer does.
+            r.locked = p.err === undefined;
+            r.lookAgain = p.err !== undefined;
+            this.emitLanguage(r, lang);
+            this.emitLanguage(r, liftLanguage(f.res));
+            return f;
+        }
+        if (confident) {
+            // The primary knows this language. One verdict does not settle it,
+            // though: a language score on a few seconds of audio jitters (an
+            // English clip scored −0.46, −0.16 and −0.44 on three consecutive
+            // passes against a −0.18 threshold), so the lock waits for a second
+            // confident probe — unless the fallback has nothing to offer anyway,
+            // or this was the last probe.
+            const settled = r.confidentProbes >= 2 || !fallbackHasWords || lastProbe;
+            lead("primary", settled ? "primary confident" : "primary confident, one more probe");
+            r.locked = settled;
+            this.emitLanguage(r, lang);
+            return p;
+        }
+        if (unsure && fallbackMostlyLatin) {
+            // Lead with the fallback now; a later probe pass may still hand the
+            // utterance back if the primary turns confident with more audio.
+            lead("fallback", "primary unsure");
+            r.locked = lastProbe;
+            this.emitLanguage(r, lang);
+            this.emitLanguage(r, liftLanguage(f.res));
+            return f;
+        }
+        // No verdict either way: whoever has words leads, the primary on a tie.
+        if (!primaryText && fallbackHasWords) {
+            lead("fallback", "only the fallback had words");
+            r.locked = lastProbe;
+            this.emitLanguage(r, liftLanguage(f.res));
+            return f;
+        }
+        lead("primary", lang ? "primary not unsure" : "no language verdict");
+        r.locked = lastProbe;
+        this.emitLanguage(r, lang);
+        return p;
+    }
     /**
      * One pass over the whole utterance so far. Returns the text to show, the
      * result it came from (when the engine answered), and whether the pass failed.
@@ -360,19 +960,50 @@ class ChunkedStreamSession {
         r.lastReqAt = this.endpointer.audioSeconds;
         r.passes++;
         const t0 = Date.now();
-        const pcm = pcmIn ?? this.endpointer.pcm16k(r.u);
+        const pcm = pcmIn ?? this.endpointer.pcm16k(r.u, this.engineCut());
         const seconds = r.u.length / this.inputRate;
-        let { res, err } = await this.callEngine(r, pcm, isFinal);
+        // The final looks again with both engines unless they already agreed:
+        // the whole utterance is the best evidence there will be.
+        const probing = this.probe &&
+            (!r.locked || (isFinal && r.lookAgain)) &&
+            seconds >= PROBE_MIN_AUDIO_S &&
+            r.u.speechMs >= PROBE_MIN_SPEECH_MS;
+        // Outside the probe, a fallback leading an utterance already labelled
+        // with a confident language outside India transcribes in it: an unpinned
+        // pass of accented English can still drift into Devanagari.
+        const pin = r.fallbackPin ??
+            (r.engine === "fallback" &&
+                r.languageDetected === true &&
+                r.language &&
+                !INDIC_LANGUAGES.has(langCode(r.language) ?? "")
+                ? langCode(r.language)
+                : undefined);
+        let { res, err } = probing
+            ? await this.probePass(r, pcm, isFinal)
+            : await this.callEngine(r, pcm, isFinal, r.engine, pin);
+        // A file is not live: a final that failed (a timeout on a busy gateway)
+        // is tried again rather than reported as silence.
+        for (let retry = 0; err !== undefined && isFinal && retry < this.finalRetries; retry++) {
+            if (logger.shouldLog("debug")) {
+                logger.debug(`[transcribeStream] utterance ${r.u.id}: final pass failed (${errorMessage(err)}), retry ${retry + 1}`);
+            }
+            ({ res, err } = await this.callEngine(r, pcm, isFinal, r.engine, pin));
+        }
         let text = res?.text?.trim() ?? "";
         let lang = liftLanguage(res);
-        this.emitLanguage(r, lang);
-        if (this.options.fallback && r.engine === "primary" && !r.switched) {
+        if (!probing) {
+            this.emitLanguage(r, lang);
+        }
+        if (this.options.fallback &&
+            r.engine === "primary" &&
+            !r.switched &&
+            !probing) {
             const when = this.fallbackWhen;
             const unsure = (when.includes("unsure") && lang?.detected === false) ||
                 (when.includes("empty") && !err && !text) ||
                 (when.includes("error") && err !== undefined);
             r.unsureN = unsure ? r.unsureN + 1 : 0;
-            if (!r.confidentOnce &&
+            if (r.confidentPasses < 2 &&
                 seconds >= TAKEOVER_MIN_AUDIO_S &&
                 r.u.speechMs >= TAKEOVER_MIN_SPEECH_MS &&
                 r.unsureN >= TAKEOVER_UNSURE_PASSES) {
@@ -381,6 +1012,13 @@ class ChunkedStreamSession {
                 r.engine = "fallback";
                 r.switched = true;
                 resetLocalAgreement(r.la);
+                // A primary that errors (a wrong model name, a refused key) would
+                // otherwise be hidden by the fallback's captions. Logged as an error
+                // because the logger drops warnings unless debugging.
+                if (err !== undefined && !this.primaryFailureWarned) {
+                    this.primaryFailureWarned = true;
+                    logger.error(`[transcribeStream] primary engine ${this.label("primary")} failed (${errorMessage(err)}); fallback ${this.label("fallback")} is transcribing instead`);
+                }
                 if (logger.shouldLog("debug")) {
                     logger.debug(`[transcribeStream] utterance ${r.u.id}: primary unsure on ${r.unsureN} passes, fallback takes over`);
                 }
@@ -433,7 +1071,7 @@ class ChunkedStreamSession {
     }
     finalize(r, reason) {
         r.final = true;
-        const pcm = this.endpointer.pcm16k(r.u);
+        const pcm = this.endpointer.pcm16k(r.u, this.engineCut());
         try {
             this.options.onUtteranceEnd?.(pcm, r.u.id, r.engine);
         }
@@ -447,8 +1085,11 @@ class ChunkedStreamSession {
                 return;
             }
             const seconds = r.u.length / this.inputRate;
+            // Engines that disagreed on the language get the final look on the
+            // whole utterance even when the last interim covered all of it.
             const needFinal = r.u.length - r.sentLen >= MIN_NEW_AUDIO_FINAL_S * this.inputRate ||
-                r.passes === 0;
+                r.passes === 0 ||
+                r.lookAgain;
             let text = r.shown;
             let res;
             let keptInterim = false;
@@ -467,10 +1108,34 @@ class ChunkedStreamSession {
                 }
             }
             this.runs.delete(r.u.id);
+            if (text && this.nearSilentNoise(r, text, seconds)) {
+                // Little speech energy and a word or two in a script the session
+                // does not speak: the engine filled a pause, not a sentence.
+                if (logger.shouldLog("debug")) {
+                    logger.debug(`[transcribeStream] utterance ${r.u.id}: near-silent utterance, output discarded`);
+                }
+                text = "";
+            }
+            if (text && this.impossibleWordRate(r, text)) {
+                if (logger.shouldLog("debug")) {
+                    logger.debug(`[transcribeStream] utterance ${r.u.id}: ${r.u.speechMs} ms of speech cannot hold the engine's words, output discarded`);
+                }
+                text = "";
+            }
+            const speechSeconds = Math.round(r.u.speechMs) / 1000;
             if (!text) {
-                this.queue.push({ type: "silence", utterance: r.u.id, seconds });
+                this.queue.push({
+                    type: "silence",
+                    utterance: r.u.id,
+                    seconds,
+                    vad: this.vadName,
+                    speechSeconds,
+                });
                 return;
             }
+            if (text.split(/\s+/u).filter(Boolean).length > NEAR_SILENT_MAX_WORDS) {
+                this.sessionLatinShare = latinShare(text);
+            }
             const start = r.u.startSeconds;
             const end = start + seconds;
             const words = res && !keptInterim ? shiftWords(res.words, start) : undefined;
@@ -494,12 +1159,30 @@ class ChunkedStreamSession {
                 },
                 ...(words ? { words } : {}),
                 ...(r.language ? { language: r.language } : {}),
-                ...(r.languageDetected !== undefined
-                    ? { languageDetected: r.languageDetected }
-                    : {}),
-                engine: this.label(r.engine),
+                // An utterance the two engines never agreed on (code-mixed speech,
+                // most often) has no confident language, whatever its last label.
+                ...(r.lookAgain && r.alternative
+                    ? { languageDetected: false }
+                    : r.languageDetected !== undefined
+                        ? { languageDetected: r.languageDetected }
+                        : {}),
+                engine: res &&
+                    !keptInterim &&
+                    typeof res.metadata?.routedProvider === "string"
+                    ? res.metadata.routedProvider
+                    : this.label(r.engine),
                 ...(r.switched ? { fallbackUsed: true } : {}),
+                ...(r.lookAgain && r.alternative && r.alternative.text !== text
+                    ? {
+                        alternative: {
+                            engine: this.label(r.alternative.engine),
+                            text: r.alternative.text,
+                        },
+                    }
+                    : {}),
                 seconds,
+                vad: this.vadName,
+                speechSeconds,
             });
             if (logger.shouldLog("debug")) {
                 logger.debug(`[transcribeStream] utterance ${r.u.id} final (${reason}${keptInterim ? ", kept live text" : ""}${needFinal ? "" : ", reused interim"})`);