@juspay/neurolink 13.6.5 → 13.8.0

package/dist/voice/eval/sttEvalStore.js

@@ -0,0 +1,397 @@
+/**
+ * STT evaluation collector: keeps what two STTs wrote for the same call, side
+ * by side, and measures how far apart they are per speaker.
+ *
+ * A sender (the call agent) posts batches of finalized segments, one STT
+ * labelled `primary` (shown to people) and the other `shadow` (run only to be
+ * compared). Segment ids are the sender's and stable, so a resent batch
+ * replaces what it sent before rather than doubling it.
+ *
+ * With a directory, every accepted batch is appended to `<dir>/<call>.jsonl`
+ * and replayed on start, so comparisons survive a restart; without one, calls
+ * live in memory only. With `NEUROLINK_STT_EVAL_LANGFUSE_*` set, every batch
+ * and the call's comparison are also exported to that Langfuse project (see
+ * sttEvalLangfuse.ts); only the environment decides that, never a request.
+ */
+import { createHash } from "node:crypto";
+import { appendFile, mkdir, readFile, readdir } from "node:fs/promises";
+import { join } from "node:path";
+import { logger } from "../../utils/logger.js";
+import { phoneticKey } from "../codeMix.js";
+import { STTEvalLangfuseExporter, sttEvalLangfuseConfigFromEnv, } from "./sttEvalLangfuse.js";
+const DEFAULT_MAX_CALLS = 1000;
+const DEFAULT_MAX_SEGMENTS = 50_000;
+/** Words per side above which a speaker is aligned in time windows instead of whole. */
+const FULL_ALIGN_MAX_WORDS = 4000;
+const WINDOW_S = 300;
+export class STTEvalStore {
+    _dir;
+    _maxCalls;
+    _maxSegments;
+    _calls = new Map();
+    /** Per call: the tail of its accept queue, so batches apply and append in arrival order. */
+    _queues = new Map();
+    _ready;
+    _langfuse;
+    constructor(options = {}) {
+        const dir = options.dir ?? process.env.NEUROLINK_STT_EVAL_DIR?.trim();
+        this._dir = dir ? dir : undefined;
+        this._maxCalls = Math.max(1, options.maxCalls ?? DEFAULT_MAX_CALLS);
+        this._maxSegments = Math.max(1, options.maxSegmentsPerCall ?? DEFAULT_MAX_SEGMENTS);
+        const langfuse = sttEvalLangfuseConfigFromEnv();
+        this._langfuse = langfuse
+            ? new STTEvalLangfuseExporter(langfuse)
+            : undefined;
+    }
+    async accept(batch) {
+        await this._load();
+        return this._serial(batch.call_id, async () => {
+            const call = (await this._callFor(batch.call_id)) ?? this._create(batch);
+            const counts = this._apply(call, batch);
+            if (this._dir) {
+                try {
+                    await appendFile(this._fileFor(batch.call_id), `${JSON.stringify(batch)}\n`, "utf8");
+                }
+                catch (error) {
+                    logger.warn("[stt-eval] could not persist batch", {
+                        callId: batch.call_id,
+                        error: error instanceof Error ? error.message : String(error),
+                    });
+                }
+            }
+            if (this._langfuse) {
+                // Background: Langfuse never delays or fails the sender's batch.
+                void this._langfuse.export(batch, this._reportOf(call, false));
+            }
+            this._evict();
+            return {
+                call_id: call.callId,
+                batch: batch.batch,
+                final: call.final,
+                ...counts,
+                segments: countBySource(call),
+            };
+        });
+    }
+    async list() {
+        await this._load();
+        return [...this._calls.values()]
+            .sort((a, b) => b.updatedAt - a.updatedAt)
+            .map(summarize);
+    }
+    async report(callId, includeSegments = false) {
+        await this._load();
+        const call = await this._callFor(callId);
+        return call ? this._reportOf(call, includeSegments) : undefined;
+    }
+    _reportOf(call, includeSegments) {
+        const ordered = [...call.segments.values()].sort((a, b) => timeOf(a) - timeOf(b));
+        const bySpeaker = new Map();
+        for (const segment of ordered) {
+            const list = bySpeaker.get(segment.participant_identity) ?? [];
+            list.push(segment);
+            bySpeaker.set(segment.participant_identity, list);
+        }
+        let distance = 0;
+        let refWords = 0;
+        let phoneticDistance = 0;
+        const speakers = [];
+        for (const [identity, segments] of bySpeaker) {
+            const compared = compareSpeaker(identity, segments, call.models);
+            speakers.push(compared.comparison);
+            distance += compared.plain.distance;
+            phoneticDistance += compared.phonetic.distance;
+            refWords += compared.plain.refWords;
+        }
+        return {
+            ...summarize(call),
+            overall: {
+                disagreement: refWords ? round(distance / refWords) : null,
+                phonetic_disagreement: refWords
+                    ? round(phoneticDistance / refWords)
+                    : null,
+            },
+            speakers_compared: speakers,
+            ...(includeSegments ? { segment_list: ordered } : {}),
+        };
+    }
+    /** Runs `task` after every earlier task for the same call has settled. */
+    _serial(callId, task) {
+        const previous = this._queues.get(callId) ?? Promise.resolve();
+        const run = previous.then(task, task);
+        const tail = run.catch(() => undefined);
+        this._queues.set(callId, tail);
+        void tail.then(() => {
+            if (this._queues.get(callId) === tail) {
+                this._queues.delete(callId);
+            }
+        });
+        return run;
+    }
+    _create(batch) {
+        const call = {
+            callId: batch.call_id,
+            room: batch.room ?? null,
+            models: { ...batch.models },
+            final: false,
+            batches: new Set(),
+            segments: new Map(),
+            updatedAt: Date.now(),
+        };
+        this._calls.set(batch.call_id, call);
+        return call;
+    }
+    _apply(call, batch) {
+        let inserted = 0;
+        let updated = 0;
+        let ignored = 0;
+        for (const segment of batch.segments) {
+            if (call.segments.has(segment.id)) {
+                updated++;
+            }
+            else if (call.segments.size >= this._maxSegments) {
+                ignored++;
+                continue;
+            }
+            else {
+                inserted++;
+            }
+            call.segments.set(segment.id, { ...segment });
+        }
+        call.batches.add(batch.batch);
+        call.final = call.final || batch.final === true;
+        call.room = batch.room ?? call.room;
+        call.models = { ...batch.models };
+        call.updatedAt = Date.now();
+        // Re-insert so Map order tracks recency for eviction.
+        this._calls.delete(call.callId);
+        this._calls.set(call.callId, call);
+        return { inserted, updated, ignored };
+    }
+    async _callFor(callId) {
+        const held = this._calls.get(callId);
+        if (held || !this._dir) {
+            return held;
+        }
+        // Evicted from memory but still on disk.
+        const text = await readFile(this._fileFor(callId), "utf8").catch(() => undefined);
+        return text === undefined ? undefined : this._replay(text);
+    }
+    _evict() {
+        while (this._calls.size > this._maxCalls) {
+            const oldest = this._calls.keys().next().value;
+            if (oldest === undefined) {
+                return;
+            }
+            this._calls.delete(oldest);
+        }
+    }
+    _fileFor(callId) {
+        const safe = callId.replace(/[^A-Za-z0-9._-]/gu, "_").slice(0, 80);
+        const hash = createHash("sha256").update(callId).digest("hex").slice(0, 10);
+        return join(this._dir ?? "", `${safe}-${hash}.jsonl`);
+    }
+    /** Replays the directory once; every caller awaits the same load. */
+    _load() {
+        this._ready ??= this._loadDir();
+        return this._ready;
+    }
+    async _loadDir() {
+        const dir = this._dir;
+        if (!dir) {
+            return;
+        }
+        try {
+            await mkdir(dir, { recursive: true });
+            const files = (await readdir(dir)).filter((f) => f.endsWith(".jsonl"));
+            for (const file of files) {
+                this._replay(await readFile(join(dir, file), "utf8"));
+            }
+            // Replay order is filesystem order; keep the most recent calls.
+            const recent = [...this._calls.values()].sort((a, b) => a.updatedAt - b.updatedAt);
+            this._calls.clear();
+            for (const call of recent) {
+                this._calls.set(call.callId, call);
+            }
+            this._evict();
+        }
+        catch (error) {
+            logger.warn("[stt-eval] could not load stored calls", {
+                dir,
+                error: error instanceof Error ? error.message : String(error),
+            });
+        }
+    }
+    _replay(text) {
+        let call;
+        let lastSeen = 0;
+        for (const line of text.split("\n")) {
+            if (!line.trim()) {
+                continue;
+            }
+            try {
+                const batch = JSON.parse(line);
+                call = call ?? this._create(batch);
+                this._apply(call, batch);
+                for (const segment of batch.segments) {
+                    lastSeen = Math.max(lastSeen, segment.received_at * 1000);
+                }
+            }
+            catch {
+                // A torn last line from a crash mid-write: skip it.
+            }
+        }
+        if (call && lastSeen > 0) {
+            call.updatedAt = lastSeen;
+        }
+        return call;
+    }
+}
+function summarize(call) {
+    let first = null;
+    let last = null;
+    const speakers = new Set();
+    for (const segment of call.segments.values()) {
+        speakers.add(segment.participant_identity);
+        first =
+            first === null
+                ? segment.received_at
+                : Math.min(first, segment.received_at);
+        last =
+            last === null ? segment.received_at : Math.max(last, segment.received_at);
+    }
+    return {
+        call_id: call.callId,
+        room: call.room,
+        models: call.models,
+        final: call.final,
+        batches: call.batches.size,
+        segments: countBySource(call),
+        speakers: speakers.size,
+        first_received_at: first,
+        last_received_at: last,
+        updated_at: call.updatedAt,
+    };
+}
+function countBySource(call) {
+    let primary = 0;
+    let shadow = 0;
+    for (const segment of call.segments.values()) {
+        if (segment.source === "primary") {
+            primary++;
+        }
+        else {
+            shadow++;
+        }
+    }
+    return { primary, shadow };
+}
+function timeOf(segment) {
+    return segment.start_time ?? segment.end_time ?? segment.received_at;
+}
+function compareSpeaker(identity, segments, models) {
+    const primary = segments.filter((s) => s.source === "primary");
+    const shadow = segments.filter((s) => s.source === "shadow");
+    const name = segments.find((s) => s.participant_name)?.participant_name ?? identity;
+    const words = (list) => list.flatMap((s) => wordsOf(s.text));
+    const primaryWords = words(primary);
+    const shadowWords = words(shadow);
+    const windowed = primaryWords.length > FULL_ALIGN_MAX_WORDS ||
+        shadowWords.length > FULL_ALIGN_MAX_WORDS;
+    const plain = windowed
+        ? alignWindows(primary, shadow, (t) => t)
+        : align(primaryWords, shadowWords);
+    const phonetic = windowed
+        ? alignWindows(primary, shadow, phoneticKey)
+        : align(primaryWords.map(phoneticKey), shadowWords.map(phoneticKey));
+    return {
+        comparison: {
+            participant_identity: identity,
+            participant_name: name,
+            primary: sideOf(primary, models.primary, primaryWords.length),
+            shadow: sideOf(shadow, models.shadow, shadowWords.length),
+            disagreement: plain.refWords
+                ? round(plain.distance / plain.refWords)
+                : null,
+            phonetic_disagreement: phonetic.refWords
+                ? round(phonetic.distance / phonetic.refWords)
+                : null,
+            method: windowed ? "windowed" : "full",
+        },
+        plain,
+        phonetic,
+    };
+}
+function sideOf(segments, fallbackModel, words) {
+    const languages = new Map();
+    let latencySum = 0;
+    let latencyCount = 0;
+    for (const segment of segments) {
+        if (segment.language) {
+            languages.set(segment.language, (languages.get(segment.language) ?? 0) + 1);
+        }
+        if (typeof segment.end_time === "number") {
+            const latency = segment.received_at - segment.end_time;
+            if (latency >= 0 && latency < 600) {
+                latencySum += latency;
+                latencyCount++;
+            }
+        }
+    }
+    return {
+        model: segments[0]?.model ?? fallbackModel,
+        segments: segments.length,
+        words,
+        languages: [...languages.entries()]
+            .sort((a, b) => b[1] - a[1])
+            .map(([language]) => language),
+        mean_latency_s: latencyCount ? round(latencySum / latencyCount) : null,
+    };
+}
+/** Long speakers: align consecutive 5-minute windows, so cost stays bounded. */
+function alignWindows(primary, shadow, key) {
+    const bucket = (list) => {
+        const out = new Map();
+        for (const segment of list) {
+            const w = Math.floor(timeOf(segment) / WINDOW_S);
+            const words = out.get(w) ?? [];
+            words.push(...wordsOf(segment.text).map(key));
+            out.set(w, words);
+        }
+        return out;
+    };
+    const p = bucket(primary);
+    const s = bucket(shadow);
+    let distance = 0;
+    let refWords = 0;
+    for (const w of new Set([...p.keys(), ...s.keys()])) {
+        const result = align(p.get(w) ?? [], s.get(w) ?? []);
+        distance += result.distance;
+        refWords += result.refWords;
+    }
+    return { distance, refWords };
+}
+function align(ref, hyp) {
+    const row = Array.from({ length: hyp.length + 1 }, (_, j) => j);
+    for (let i = 1; i <= ref.length; i++) {
+        let previous = row[0];
+        row[0] = i;
+        for (let j = 1; j <= hyp.length; j++) {
+            const above = row[j];
+            row[j] = Math.min(row[j] + 1, row[j - 1] + 1, previous + (ref[i - 1] === hyp[j - 1] ? 0 : 1));
+            previous = above;
+        }
+    }
+    return { distance: row[hyp.length], refWords: ref.length };
+}
+function wordsOf(text) {
+    return text
+        .toLowerCase()
+        .normalize("NFKC")
+        .replace(/[^\p{L}\p{M}\p{N}\s']/gu, " ")
+        .split(/\s+/u)
+        .filter(Boolean);
+}
+function round(value) {
+    return Math.round(value * 1000) / 1000;
+}

package/dist/voice/index.js

@@ -246,6 +246,9 @@ function endpointHandler(endpoint) {
         baseURL: endpoint.baseURL,
         timeoutMs: endpoint.timeoutMs,
         model: endpoint.model,
+        priorLanguageForm: endpoint.priorLanguageForm,
+        lidModel: endpoint.lidModel,
+        vocabularyPrompt: endpoint.vocabularyPrompt,
     });
 }
 /**

package/dist/voice/languages.d.ts

@@ -0,0 +1,34 @@
+/**
+ * Language codes and writing systems for the transcribe pipeline.
+ *
+ * Engines report languages in different shapes ("hi", "hin", "Hindi",
+ * "en-US"); routing, correction and output rendering need one: a lower-case
+ * ISO 639 code. Output targets are BCP-47 tags whose script subtag says how
+ * the text is written (`hi-Latn` is Hindi in Latin letters), and the guards
+ * on a rendering check the text against that script.
+ *
+ * @module voice/languages
+ */
+import type { ParsedOutputTag, ScriptCode } from "../types/index.js";
+/**
+ * The language part of whatever an engine reported, as a lower-case ISO 639
+ * code: "en-US" → "en", "Hindi" → "hi", "auto-ta" → "ta". `undefined` for
+ * nothing, "auto" and "und".
+ */
+export declare function normalizeLanguageCode(value: string | undefined): string | undefined;
+export declare function isIndicLanguage(language: string | undefined): boolean;
+/** Letters of each script in `text`; scripts with none are absent. */
+export declare function scriptCounts(text: string): Map<ScriptCode, number>;
+/** The script most letters of `text` are in, `undefined` for text without letters. */
+export declare function dominantScript(text: string): ScriptCode | undefined;
+/** Whether `text` holds letters of an Indic script (Devanagari through Sinhala). */
+export declare function hasIndicScript(text: string): boolean;
+/**
+ * A BCP-47 output tag: language, optional script, optional region
+ * (`es`, `fr-CA`, `hi-Latn`, `zh-Hant`). `undefined` for anything else.
+ */
+export declare function parseOutputTag(tag: string): ParsedOutputTag | undefined;
+/** Share of `text`'s letters written in `script` (Japanese accepts kanji as well as kana). */
+export declare function scriptShare(text: string, script: ScriptCode): number;
+/** "es" → "Spanish", "hi-Latn" → "Hindi (Latin script)"; the tag itself when the runtime cannot name it. */
+export declare function describeOutputTag(tag: string): string;