@juspay/neurolink 13.6.5 → 13.8.0

package/dist/voice/transcribeStream.js

@@ -23,9 +23,13 @@
  * @module voice/transcribeStream
  */
 import { logger } from "../utils/logger.js";
+import { collapseRepetitions, isPromptEcho, loopShare, } from "./correction/guards.js";
 import { SpanSerializer, SpanStatus, SpanType, getMetricsAggregator, } from "../observability/index.js";
 import { createWavFile } from "./audio-utils.js";
-import { correctTranscript, isCorrectionEnabled } from "./correction/index.js";
+import { normalizeSpeechLevel } from "./audioLevel.js";
+import { normalizeLanguageCode } from "./languages.js";
+import { outputLanguagesOf, renderOutputs } from "./output/render.js";
+import { correctTranscript, correctTranscriptSentences, isCorrectionEnabled, } from "./correction/index.js";
 import { StreamEventQueue, chunkedTranscribeStream, } from "./streaming/chunkedStreamAdapter.js";
 import { ENDPOINTER_DEFAULTS, ENGINE_SAMPLE_RATE, } from "./streaming/endpointer.js";
 import { createLocalAgreement, resetLocalAgreement, splitWords, updateLocalAgreement, } from "./streaming/localAgreement.js";
@@ -40,6 +44,9 @@ const NON_STT_KEYS = new Set([
     "dictionary",
     "correction",
     "fallback",
+    "routing",
+    "chunking",
+    "output",
     "streaming",
     "credentials",
 ]);
@@ -59,6 +66,100 @@ export function encodeWav16k(pcm) {
  * dictionary terms as vocabulary and appended to the prompt, `"auto"`
  * language left to the engine.
  */
+/** The words an engine is prompted with (dictionary terms, the caller's vocabulary, the fixed phrasing). */
+export function promptTermsOf(options) {
+    return [
+        ...(options.vocabulary ?? []),
+        ...(options.dictionary ?? []).flatMap((d) => [
+            d.term,
+            ...(d.heardAs ?? []),
+        ]),
+        "Names and terms used",
+    ].filter((t) => typeof t === "string" && t.trim().length > 0);
+}
+function wordCount(text) {
+    return text.split(/\s+/u).filter(Boolean).length;
+}
+/**
+ * Engine output through the guards: a looping phrase is collapsed and the
+ * share of words it took is recorded on `metadata.loopShare`; an output
+ * that is only the prompt echoed back becomes empty. `note` receives a
+ * human-readable line for each guard that fired.
+ */
+export function guardEngineResult(res, promptTerms, note) {
+    const original = res.text ?? "";
+    if (!original.trim()) {
+        return res;
+    }
+    if (isPromptEcho(original, promptTerms)) {
+        note("prompt echo discarded");
+        return {
+            ...res,
+            text: "",
+            words: undefined,
+            segments: undefined,
+            metadata: {
+                ...(res.metadata ?? { latency: 0 }),
+                hallucination: "prompt-echo",
+            },
+        };
+    }
+    const collapsed = collapseRepetitions(original);
+    if (collapsed.removed === 0) {
+        return res;
+    }
+    const share = loopShare(collapsed.removed, wordCount(original));
+    note(`repetition collapsed (${collapsed.removed} words)`);
+    const metadata = {
+        ...(res.metadata ?? { latency: 0 }),
+        loopShare: share,
+    };
+    if (!res.segments || res.segments.length <= 1) {
+        return {
+            ...res,
+            text: collapsed.text,
+            // Word timings no longer line up with the collapsed text.
+            words: undefined,
+            segments: res.segments?.length
+                ? [{ ...res.segments[0], text: collapsed.text }]
+                : undefined,
+            metadata,
+        };
+    }
+    // Many segments (a long, diarized file): collapse each segment on its own,
+    // so speaker turns and timings survive, and drop word timings only inside
+    // the segments whose text changed.
+    const changed = [];
+    const segments = res.segments.map((segment) => {
+        const own = collapseRepetitions(segment.text);
+        if (own.removed === 0) {
+            return segment;
+        }
+        changed.push({
+            start: segment.start ?? Number.NEGATIVE_INFINITY,
+            end: segment.end ?? Number.POSITIVE_INFINITY,
+        });
+        return { ...segment, text: own.text };
+    });
+    if (changed.length === 0) {
+        // The loop ran across segment boundaries: keep the segments as the
+        // engine wrote them, and only the joined text collapsed.
+        return { ...res, text: collapsed.text, metadata };
+    }
+    const words = res.words?.filter((w) => {
+        const { start, end } = w;
+        return (start === undefined ||
+            end === undefined ||
+            !changed.some((c) => start >= c.start && end <= c.end));
+    });
+    return {
+        ...res,
+        text: segments.map((s) => s.text).join(" "),
+        segments,
+        words,
+        metadata,
+    };
+}
 function buildEngineOptions(options, model, audio) {
     const passthrough = {};
     for (const [key, value] of Object.entries(options)) {
@@ -87,6 +188,14 @@ function buildEngineOptions(options, model, audio) {
         sampleRate: audio.sampleRate,
     };
 }
+/**
+ * A final's language when it can be trusted: an unsure verdict ("hi?", the
+ * Indic engine's fallback label on speech it could not place) must never
+ * decide a rewrite's rules or make a rendering look like the same language.
+ */
+function confidentLanguage(final) {
+    return final.languageDetected === false ? undefined : final.language;
+}
 /** Uint8Array frames → Buffers (no copy), reporting each frame as it is handed over. */
 async function* asBuffers(frames, onFrame) {
     for await (const f of frames) {
@@ -365,7 +474,16 @@ export async function* runTranscribeStream(options, deps) {
         ? resolveOptional(options.fallback.provider, "fallback")
         : undefined;
     const correcting = isCorrectionEnabled(options.dictionary, options.correction);
-    const secondSpec = correcting ? options.correction?.secondOpinion : undefined;
+    const secondSetting = correcting
+        ? options.correction?.secondOpinion
+        : undefined;
+    // "auto": the fallback engine reads every utterance the primary leads, so
+    // the rewrite can reconcile both readings of code-mixed speech.
+    const secondSpec = secondSetting === "auto"
+        ? options.fallback
+            ? { provider: options.fallback.provider, model: options.fallback.model }
+            : undefined
+        : secondSetting;
     const secondProvider = !useNative && secondSpec
         ? resolveOptional(secondSpec.provider, "second-opinion")
         : undefined;
@@ -373,16 +491,174 @@ export async function* runTranscribeStream(options, deps) {
     const primaryOpts = buildEngineOptions(options, options.model, wavAudio);
     const fallbackOpts = buildEngineOptions(options, options.fallback?.model, wavAudio);
     const secondOpts = buildEngineOptions(options, secondSpec?.model, wavAudio);
+    const otherOpts = buildEngineOptions(options, undefined, wavAudio);
+    // Language routing: which provider transcribes which language.
+    const routes = new Map();
+    const pinnedLanguage = normalizeLanguageCode(options.language) !== undefined;
+    if (!useNative && !pinnedLanguage) {
+        for (const [lang, name] of Object.entries(options.routing?.engines ?? {})) {
+            const key = lang.trim() === "*" ? "*" : normalizeLanguageCode(lang);
+            const resolved = key && typeof name === "string" && name.trim()
+                ? resolveOptional(name, `routing (${lang})`)
+                : undefined;
+            if (key && resolved) {
+                routes.set(key, resolved);
+            }
+        }
+    }
+    const optionsFor = (name) => name === provider
+        ? primaryOpts
+        : name === fallbackProvider
+            ? fallbackOpts
+            : otherOpts;
+    const normalize = options.streaming?.normalize !== false;
+    const levelled = (pcm) => normalize ? normalizeSpeechLevel(pcm) : pcm;
     // Per-utterance bookkeeping shared by the adapter hooks and the correction tasks.
     const finalPassMs = new Map();
     const endedAt = new Map();
     const secondOpinions = new Map();
-    const run = async (pcm, pass) => {
-        const wav = encodeWav16k(pcm);
+    // Words the engines were prompted with: an output made only of these is
+    // the prompt echoed back on a pause, not speech.
+    const promptTerms = promptTermsOf(options);
+    const guardNotes = new Map();
+    const noteGuard = (utterance, note) => {
+        const notes = guardNotes.get(utterance) ?? [];
+        if (!notes.includes(note)) {
+            notes.push(note);
+        }
+        guardNotes.set(utterance, notes);
+    };
+    // `fileModeStreamOptions` turns interim passes off with an infinite cadence;
+    // a live caller cannot send one (JSON has no Infinity).
+    const fileMode = options.streaming?.intervalMs === Number.POSITIVE_INFINITY;
+    const callEngine = (wav, engine, prior, hint) => engine === "fallback" && fallbackProvider
+        ? deps.transcribe(wav, fallbackProvider, hint ? { ...fallbackOpts, language: hint } : fallbackOpts)
+        : deps.transcribe(wav, provider, prior ? { ...primaryOpts, languagePrior: prior } : primaryOpts);
+    /** The utterance as two halves, joined: what an engine that choked on the whole gets instead. */
+    const runHalves = async (pcm, pass) => {
+        const mid = Math.floor(pcm.length / 2);
+        const halves = await Promise.all([
+            callEngine(encodeWav16k(pcm.subarray(0, mid)), pass.engine, pass.languagePrior, pass.languageHint),
+            callEngine(encodeWav16k(pcm.subarray(mid)), pass.engine, pass.languagePrior, pass.languageHint),
+        ]);
+        return guardEngineResult({
+            ...halves[0],
+            text: [halves[0].text, halves[1].text]
+                .filter((t) => t.trim())
+                .join(" "),
+            words: undefined,
+            segments: undefined,
+        }, promptTerms, () => undefined);
+    };
+    /** Utterances handed to another engine by routing: later passes go straight to it. */
+    const routedUtterances = new Map();
+    /** One pass on a routed engine, tagged with it; `undefined` when it heard nothing or failed. */
+    const runRouted = async (pcm, pass, target, lang, fallbackLanguage) => {
+        try {
+            const alt = guardEngineResult(await deps.transcribe(encodeWav16k(pcm), target, {
+                ...optionsFor(target),
+                languagePrior: lang,
+            }), promptTerms, (note) => noteGuard(pass.utterance, note));
+            if (alt.text.trim() && alt.metadata?.languageDetected !== false) {
+                return {
+                    ...alt,
+                    language: alt.language ?? fallbackLanguage,
+                    metadata: {
+                        ...(alt.metadata ?? { latency: 0 }),
+                        routedProvider: target,
+                    },
+                };
+            }
+            noteGuard(pass.utterance, `routing ${lang} → ${target} heard nothing`);
+        }
+        catch (error) {
+            noteGuard(pass.utterance, `routing ${lang} → ${target} failed (${error instanceof Error ? error.message : String(error)})`);
+        }
+        return undefined;
+    };
+    /**
+     * A pass whose detected language belongs to another engine is read again
+     * by that engine, and the rest of the utterance goes to it; its text is
+     * used when it heard words and reads the language. Any failure keeps the
+     * text already in hand.
+     */
+    const routeFinal = async (res, pcm, pass) => {
+        if (routes.size === 0) {
+            return res;
+        }
+        const lang = normalizeLanguageCode(res.language);
+        if (!lang) {
+            return res;
+        }
+        const target = routes.get(lang) ?? routes.get("*");
+        const used = pass.engine === "fallback" && fallbackProvider
+            ? fallbackProvider
+            : provider;
+        if (!target || target === used) {
+            return res;
+        }
+        const alt = await runRouted(pcm, pass, target, lang, res.language);
+        if (!alt) {
+            return res;
+        }
+        noteGuard(pass.utterance, `routed ${lang} → ${target}`);
+        routedUtterances.set(pass.utterance, { target, lang });
+        return alt;
+    };
+    const run = async (rawPcm, pass) => {
         const t0 = Date.now();
-        const res = pass.engine === "fallback" && fallbackProvider
-            ? await deps.transcribe(wav, fallbackProvider, fallbackOpts)
-            : await deps.transcribe(wav, provider, primaryOpts);
+        const pcm = levelled(rawPcm);
+        const routed = routedUtterances.get(pass.utterance);
+        if (routed) {
+            const direct = await runRouted(pcm, pass, routed.target, routed.lang, routed.lang);
+            if (direct) {
+                if (pass.final) {
+                    finalPassMs.set(pass.utterance, Date.now() - t0);
+                }
+                return direct;
+            }
+        }
+        let res;
+        try {
+            res = await callEngine(encodeWav16k(pcm), pass.engine, pass.languagePrior, pass.languageHint);
+        }
+        catch (error) {
+            // A final the engine could not finish (a decoding loop that ran into
+            // the timeout, most often) is tried as two halves before it is given
+            // up: a loop is a property of the chunk, not of the speech, and a
+            // retry of the same audio would only stall as long again.
+            if (!pass.final || pcm.length < ENGINE_SAMPLE_RATE * 4) {
+                throw error;
+            }
+            let halves;
+            try {
+                halves = await runHalves(pcm, pass);
+            }
+            catch {
+                throw error;
+            }
+            noteGuard(pass.utterance, `engine failed on the whole utterance (${error instanceof Error ? error.message : String(error)}) · re-run as two halves`);
+            if (pass.final) {
+                finalPassMs.set(pass.utterance, Date.now() - t0);
+            }
+            return halves;
+        }
+        res = guardEngineResult(res, promptTerms, (note) => noteGuard(pass.utterance, note));
+        // A final that was mostly a loop is, on a file, worth a second attempt
+        // as two halves: the loop is a property of the chunk, not the speech.
+        const loop = res.metadata?.loopShare;
+        if (fileMode &&
+            pass.final &&
+            typeof loop === "number" &&
+            loop >= 0.5 &&
+            pcm.length >= ENGINE_SAMPLE_RATE * 4) {
+            const joined = await runHalves(pcm, pass).catch(() => undefined);
+            if (joined && wordCount(joined.text) > wordCount(res.text)) {
+                noteGuard(pass.utterance, "loop: utterance re-run as two halves");
+                res = joined;
+            }
+        }
+        res = await routeFinal(res, pcm, pass);
         if (pass.final) {
             finalPassMs.set(pass.utterance, Date.now() - t0);
         }
@@ -398,6 +674,7 @@ export async function* runTranscribeStream(options, deps) {
             ...options.streaming,
             sampleRate: inputRate,
             engineLabel: provider,
+            ...(options.routing ? { routing: options.routing } : {}),
             ...(fallbackProvider
                 ? {
                     fallback: {
@@ -415,7 +692,7 @@ export async function* runTranscribeStream(options, deps) {
                 if (!secondProvider || engine !== "primary") {
                     return;
                 }
-                secondOpinions.set(utterance, runSecondOpinion(deps, pcm, secondProvider, secondOpts, utterance));
+                secondOpinions.set(utterance, runSecondOpinion(deps, levelled(pcm), secondProvider, secondOpts, utterance));
             },
         });
     const takeSecondOpinion = async (utterance) => {
@@ -423,6 +700,72 @@ export async function* runTranscribeStream(options, deps) {
         secondOpinions.delete(utterance);
         return p ? await p : undefined;
     };
+    // Corrected utterances, oldest first: the rewrite's read-only context.
+    const contextUtterances = Math.max(0, Math.floor(options.correction?.contextUtterances ?? 0));
+    const recentCorrected = [];
+    const previousFor = (utterance) => contextUtterances > 0
+        ? recentCorrected
+            .filter((r) => r.utterance < utterance)
+            .slice(-contextUtterances)
+            .map((r) => r.text)
+        : undefined;
+    // Output languages: every utterance rendered once into the languages in
+    // effect when it settles, with earlier utterances as context.
+    const outputContext = Math.max(0, Math.floor(options.output?.contextUtterances ?? 3));
+    const outputHistory = [];
+    const emitOutputs = async (utterance, source, corrected, language, basedOn) => {
+        const languages = outputLanguagesOf(options.output);
+        if (languages.length === 0 || aborted || !source.trim()) {
+            return;
+        }
+        const rendered = await renderOutputs({
+            source: [source],
+            ...(corrected && corrected !== source
+                ? { corrected: [corrected] }
+                : {}),
+            ...(language ? { language } : {}),
+            languages,
+            previous: outputContext > 0
+                ? outputHistory
+                    .filter((h) => h.utterance < utterance)
+                    .slice(-outputContext)
+                    .map((h) => ({ text: h.text, outputs: h.outputs }))
+                : [],
+            ...(options.dictionary ? { dictionary: options.dictionary } : {}),
+            ...(options.output ? { options: options.output } : {}),
+        }, deps);
+        if (aborted) {
+            return;
+        }
+        const texts = {};
+        for (const tag of languages) {
+            const r = rendered[tag];
+            if (!r) {
+                continue;
+            }
+            if (r.text) {
+                texts[tag] = r.text;
+            }
+            out.push({
+                type: "output",
+                utterance,
+                language: tag,
+                text: r.text,
+                status: r.status,
+                basedOn,
+                ms: r.ms,
+                ...(r.reason ? { reason: r.reason } : {}),
+            });
+        }
+        outputHistory.push({
+            utterance,
+            text: corrected ?? source,
+            outputs: texts,
+        });
+        if (outputHistory.length > 20) {
+            outputHistory.shift();
+        }
+    };
     /** Correction of one final; pushes `correcting` partials and one `corrected`. */
     const correct = async (final, preSteps, alt) => {
         const utterance = final.utterance;
@@ -431,17 +774,39 @@ export async function* runTranscribeStream(options, deps) {
         finalPassMs.delete(utterance);
         endedAt.delete(utterance);
         const steps = [...preSteps];
-        const secondOpinion = !final.fallbackUsed && alt?.text ? alt.text : undefined;
-        if (secondSpec && !final.fallbackUsed && !useNative && !alt?.text) {
+        // The final's own alternative (both engines ran on an utterance whose
+        // language they disagreed on) is the better second reading: it is the
+        // engine that lost, whichever one led.
+        const alternative = secondSetting === "auto" && final.alternative?.text.trim()
+            ? final.alternative
+            : undefined;
+        const secondOpinion = alternative
+            ? alternative.text
+            : !final.fallbackUsed && alt?.text
+                ? alt.text
+                : undefined;
+        if (alternative) {
+            steps.push(`reconciled with ${alternative.engine}'s reading`);
+        }
+        if (alternative) {
+            // The step above already says which reading was reconciled.
+        }
+        else if (secondSpec && !final.fallbackUsed && !useNative && !alt?.text) {
             steps.push("second opinion returned nothing");
         }
+        else if (secondOpinion && secondProvider) {
+            steps.push(`second opinion ${secondProvider}`);
+        }
         // Nothing after the consumer has gone: no second opinion is reconciled
         // and no rewrite is paid for that nobody will read.
         if (aborted) {
             return;
         }
         try {
-            const res = await correctTranscript({
+            const correctFn = options.correction?.perSentence === false
+                ? correctTranscript
+                : correctTranscriptSentences;
+            const res = await correctFn({
                 text: final.text,
                 ...(secondOpinion ? { secondOpinion } : {}),
                 ...(options.dictionary ? { dictionary: options.dictionary } : {}),
@@ -449,6 +814,12 @@ export async function* runTranscribeStream(options, deps) {
                 ...(final.languageDetected !== undefined
                     ? { languageDetected: final.languageDetected }
                     : {}),
+                ...(confidentLanguage(final)
+                    ? { language: confidentLanguage(final) }
+                    : {}),
+                ...(previousFor(utterance)?.length
+                    ? { previous: previousFor(utterance) }
+                    : {}),
                 onPartial: (text) => {
                     if (text) {
                         out.push({ type: "correcting", utterance, text });
@@ -469,6 +840,12 @@ export async function* runTranscribeStream(options, deps) {
                     totalMs: Date.now() - started,
                 },
             });
+            const settled = res.text || final.text;
+            recentCorrected.push({ utterance, text: settled });
+            if (recentCorrected.length > 20) {
+                recentCorrected.shift();
+            }
+            await emitOutputs(utterance, final.text, settled, confidentLanguage(final), "corrected");
         }
         catch (err) {
             // correctTranscript keeps its own timeouts and fails open; this is the
@@ -484,6 +861,7 @@ export async function* runTranscribeStream(options, deps) {
                 steps: [...steps, `correction failed: ${message} · raw text kept`],
                 timings: { transcribeMs, totalMs: Date.now() - started },
             });
+            await emitOutputs(utterance, final.text, undefined, confidentLanguage(final), "final");
         }
     };
     const pending = new Set();
@@ -501,21 +879,40 @@ export async function* runTranscribeStream(options, deps) {
     const producer = (async () => {
         try {
             for (;;) {
-                const { value: ev, done } = await iterator.next();
-                if (done || aborted) {
+                const next = await iterator.next();
+                if (next.done || aborted) {
                     break;
                 }
+                const raw = next.value;
+                if (raw.type === "silence") {
+                    routedUtterances.delete(raw.utterance);
+                }
+                // A final reused from a routed interim names the engine that wrote it.
+                const routedTo = raw.type === "final"
+                    ? routedUtterances.get(raw.utterance)
+                    : undefined;
+                const ev = raw.type === "final" && routedTo && !raw.fallbackUsed
+                    ? { ...raw, engine: routedTo.target }
+                    : raw;
                 if (ev.type === "final") {
                     utterances++;
+                    routedUtterances.delete(ev.utterance);
                     out.push(ev);
-                    const preSteps = ev.fallbackUsed
-                        ? [`${provider} unsure → ${ev.engine} took over`]
-                        : [];
+                    const preSteps = [
+                        ...(ev.fallbackUsed
+                            ? [`${provider} unsure → ${ev.engine} took over`]
+                            : []),
+                        ...(guardNotes.get(ev.utterance) ?? []),
+                    ];
+                    guardNotes.delete(ev.utterance);
                     if (correcting && !aborted) {
                         track(takeSecondOpinion(ev.utterance).then((alt) => aborted ? undefined : correct(ev, preSteps, alt)));
                     }
                     else {
                         secondOpinions.delete(ev.utterance);
+                        if (!aborted) {
+                            track(emitOutputs(ev.utterance, ev.text, undefined, confidentLanguage(ev), "final"));
+                        }
                     }
                 }
                 else if (ev.type === "silence" && secondOpinions.has(ev.utterance)) {

package/dist/voice/vad/assets.d.ts

@@ -0,0 +1,36 @@
+/**
+ * The Silero VAD model file: the pinned revision, where it lives and the
+ * first-use download. The same rules as Whistle's assets, for one file
+ * instead of three: an explicit directory, then `NEUROLINK_VAD_DIR`, then
+ * `NEUROLINK_MODEL_DIR/silero-vad`, then the copy shipped with the package
+ * (`models/silero-vad`), then `~/.neurolink/models/silero-vad`, the only
+ * directory a download ever writes to.
+ *
+ * Upstream: `snakers4/silero-vad`, MIT.
+ *
+ * @module voice/vad/assets
+ */
+import type { LocalModelAsset, STTVadOptions } from "../../types/index.js";
+/** The pinned model: Silero VAD v6.2, 2.3 MB, 8 and 16 kHz. */
+export declare const SILERO_VAD_ASSET: LocalModelAsset;
+/**
+ * Where the model lives: `options.modelDir` → `NEUROLINK_VAD_DIR` →
+ * `NEUROLINK_MODEL_DIR/silero-vad` → the copy shipped with the package →
+ * `~/.neurolink/models/silero-vad`.
+ */
+export declare function resolveVadDir(options?: STTVadOptions): string;
+/**
+ * Whether a missing model may be fetched: on unless `options.autoDownload`
+ * is `false` or `NEUROLINK_VAD_AUTO_DOWNLOAD` is `0` / `false` / `off` / `no`.
+ */
+export declare function vadAutoDownloadEnabled(options?: STTVadOptions): boolean;
+/** Cheap presence check (no hashing). */
+export declare function vadModelPresent(dir: string): boolean;
+/**
+ * Makes sure the model is in `dir` and matches its pinned hash, downloading
+ * it when missing and allowed. Resolves to the model's path. Concurrent
+ * calls for the same directory share one preparation.
+ */
+export declare function ensureVadModel(dir: string, autoDownload: boolean): Promise<string>;
+/** The "here is exactly what to put where" body of every "unusable" message. */
+export declare function describeVadModel(dir: string, reason: string): string;