@maheidem/pi-audio-transcribe 0.2.0

package/validate.ts

@@ -0,0 +1,388 @@
+/**
+ * Transcript validation.
+ *
+ * Two layers, both deterministic (no LLM):
+ *
+ *  STRUCTURAL — inspects one ASR response for the failure modes that
+ *  actually occur on oMLX/Qwen3-ASR:
+ *    empty        silence (verified: real silence returns text="" ,
+ *               language=null, segments[0].language="None")
+ *    repetition  n-gram loops, the classic ASR hallucination
+ *    garbled     replacement/control characters
+ *    duration    segments[].end vs ffprobe ground truth
+ *    rate        words-per-second outside plausible human speech
+ *    language    reported label vs content heuristics (server answered
+ *               "English" for a Portuguese clip on synthetic audio)
+ *
+ *  CROSS-CHECK — re-transcribes with the language the server itself
+ *  reported and compares. On real audio auto and explicit hints agree
+ *  byte-for-byte, so divergence is a genuine instability signal rather
+ *  than noise. That agreement is what makes "always auto-detect" safe.
+ */
+
+import type { AsrResponse, AsrSegment } from "./omlx.ts";
+import { audioDurationFromSegments, normalizeLanguage } from "./omlx.ts";
+
+export type CheckStatus = "pass" | "warn" | "fail" | "info";
+
+export interface Check {
+	id: string;
+	status: CheckStatus;
+	detail: string;
+}
+
+export type Verdict = "ok" | "review" | "no_speech" | "failed";
+
+export interface CrossCheckResult {
+	performed: boolean;
+	skippedReason?: string;
+	languageUsed?: string | null;
+	similarity?: number;
+	textChanged?: boolean;
+	/** Text from the second pass, kept when it differs so it is not lost. */
+	altText?: string;
+	latencySeconds?: number;
+}
+
+export interface ValidationReport {
+	verdict: Verdict;
+	confidence: number;
+	checks: Check[];
+	recommendations: string[];
+	language: {
+		requested: string | null;
+		reported: string | null;
+		normalized: string | null;
+		heuristic: string | null;
+		consistent: boolean;
+	};
+	timings: { audioSeconds: number | null; asrLatencySeconds: number | null };
+	crossCheck: CrossCheckResult;
+}
+
+const WORD_RE = /[\p{L}\p{N}'’-]+/gu;
+
+export function countWords(text: string): number {
+	return (text.match(WORD_RE) ?? []).length;
+}
+
+/** Stability threshold for the cross-check. */
+export const SIMILARITY_WARN = 0.9;
+export const SIMILARITY_FAIL = 0.7;
+
+/** Normalize for comparison: case, accents, whitespace, punctuation. */
+export function normalizeForCompare(text: string): string {
+	return text
+		.toLowerCase()
+		.normalize("NFD")
+		.replace(/[\u0300-\u036f]/g, "")
+		.replace(/[^\p{L}\p{N}\s]/gu, " ")
+		.replace(/\s+/g, " ")
+		.trim();
+}
+
+/** Sørensen–Dice coefficient over word bigrams; falls back to unigrams. */
+export function similarity(a: string, b: string): number {
+	const na = normalizeForCompare(a);
+	const nb = normalizeForCompare(b);
+	if (na === nb) return 1;
+	const grams = (s: string): string[] => {
+		const w = s.split(" ").filter(Boolean);
+		if (w.length < 2) return w;
+		const out: string[] = [];
+		for (let i = 0; i < w.length - 1; i++) out.push(`${w[i]} ${w[i + 1]}`);
+		return out;
+	};
+	const A = grams(na);
+	const B = grams(nb);
+	if (!A.length || !B.length) return 0;
+	const counts = new Map<string, number>();
+	for (const g of A) counts.set(g, (counts.get(g) ?? 0) + 1);
+	let hits = 0;
+	for (const g of B) {
+		const c = counts.get(g) ?? 0;
+		if (c > 0) {
+			counts.set(g, c - 1);
+			hits++;
+		}
+	}
+	return (2 * hits) / (A.length + B.length);
+}
+
+/**
+ * Detect consecutive n-gram repetition (ASR loops).
+ * Tries phrase lengths 1..N so both single-word stuttering
+ * ("sim sim sim sim") and longer loops are caught.
+ * Returns the repeated phrase when it repeats back-to-back `minRepeats`+.
+ */
+export function detectRepetitionLoop(text: string, maxN = 4, minRepeats = 3): string | null {
+	const words = normalizeForCompare(text).split(" ").filter(Boolean);
+	for (let n = 1; n <= maxN; n++) {
+		if (words.length < n * minRepeats) continue;
+		for (let i = 0; i + n * minRepeats <= words.length; i++) {
+			const phrase = words.slice(i, i + n).join(" ");
+			let repeats = 1;
+			while (words.slice(i + repeats * n, i + (repeats + 1) * n).join(" ") === phrase) repeats++;
+			if (repeats >= minRepeats) return phrase;
+		}
+	}
+	return null;
+}
+
+/** Fraction of duplicate word bigrams — catches interleaved "sim sim sim". */
+export function repetitionRatio(text: string): number {
+	const words = normalizeForCompare(text).split(" ").filter(Boolean);
+	if (words.length < 6) return 0;
+	const seen = new Set<string>();
+	let dup = 0;
+	for (let i = 0; i < words.length - 1; i++) {
+		const g = `${words[i]} ${words[i + 1]}`;
+		if (seen.has(g)) dup++;
+		else seen.add(g);
+	}
+	return dup / (words.length - 1);
+}
+
+const PT_MARKERS = [
+	"não", "nao", "está", "esta", "são", "sao", "gaveta", "medida", "talheres", "centímetro", "centimetro",
+	"milímetro", "milimetro", "vinte", "quarenta", "né", "nè", "também", "tambem", "muito", "obrigado",
+	"obrigada", "vocês", "voce", "pra", "pro", "aí", "ai", "então", "entao", "depois", "agora", "aqui",
+];
+const EN_MARKERS = ["the", "and", "this", "that", "with", "have", "would", "measure", "drawer", "about", "here"];
+const ES_MARKERS = ["ñ", "está", "muy", "gracias", "también", "porque", "esto", "ese", "esa"];
+
+/** Word-boundary containment, so "the" does not match "theme". */
+function wordAt(haystack: string, word: string): boolean {
+	return new RegExp("\\b" + word.replace(/[.*+?^${}()|[\]\\]/g, "\\$&") + "\\b", "i").test(haystack);
+}
+
+/** Cheap language guess from content. Only distinguishes the languages in play here. */
+export function heuristicLanguage(text: string): string | null {
+	const low = text.toLowerCase();
+	const hits: Array<[string, number]> = [
+		["pt", PT_MARKERS.reduce((a, m) => a + (low.includes(m) ? 1 : 0), 0)],
+		["en", EN_MARKERS.reduce((a, m) => a + (wordAt(low, m) ? 1 : 0), 0)],
+		["es", ES_MARKERS.reduce((a, m) => a + (low.includes(m) ? 1 : 0), 0)],
+	];
+	if (/[\u4e00-\u9fff]/.test(text)) return "zh";
+	if (/[\u3040-\u30ff]/.test(text)) return "ja";
+	if (/[\uac00-\ud7af]/.test(text)) return "ko";
+	if (/[\u0400-\u04ff]/.test(text)) return "ru";
+	hits.sort((a, b) => b[1] - a[1]);
+	return hits[0][1] > 0 ? hits[0][0] : null;
+}
+
+/** Characters that indicate a decoding/encoding problem rather than speech. */
+export function garbleRatio(text: string): number {
+	if (!text.length) return 0;
+	let bad = 0;
+	for (const ch of text) {
+		const cp = ch.codePointAt(0) ?? 0;
+		if (cp === 0xfffd) bad++;
+		else if (cp < 32 && cp !== 9 && cp !== 10 && cp !== 13) bad++;
+		else if (cp >= 0xe000 && cp <= 0xf8ff) bad++; // private use area
+	}
+	return bad / [...text].length;
+}
+
+export interface ProbeInfo {
+	durationSeconds?: number | null;
+	sampleRateHz?: number | null;
+	channels?: number | null;
+	codec?: string | null;
+	available?: boolean;
+	error?: string;
+}
+
+export interface ValidateInput {
+	response: AsrResponse;
+	requestedLanguage?: string | null;
+	probe?: ProbeInfo;
+	crossCheck?: CrossCheckResult;
+	minWordsForReview?: number;
+}
+
+/** Run every structural check (plus the supplied cross-check result). */
+export function validateSpeech(input: ValidateInput): ValidationReport {
+	const { response, requestedLanguage = null, probe, crossCheck } = input;
+	const text = typeof response.text === "string" ? response.text : "";
+	const trimmed = text.trim();
+	const checks: Check[] = [];
+	const recommendations: string[] = [];
+
+	const reported = normalizeLanguage(response.language);
+	const reportedRaw = response.language ?? null;
+	const heuristic = heuristicLanguage(trimmed);
+	const segDuration = audioDurationFromSegments(response.segments as AsrSegment[] | undefined);
+	const audioSeconds = probe?.durationSeconds ?? segDuration ?? null;
+	const asrLatencySeconds = typeof response.duration === "number" ? response.duration : null;
+
+	// 1. Empty / silence
+	if (!trimmed) {
+		const silenceEvidence = (response.segments ?? []).every((s) => normalizeLanguage(s.language) === null);
+		checks.push({
+			id: "speech_present",
+			status: silenceEvidence ? "warn" : "fail",
+			detail: silenceEvidence
+				? "No speech detected (server reported language 'None'); audio appears silent"
+				: "Empty transcript but language was reported — server returned nothing usable",
+		});
+		return {
+			verdict: "no_speech",
+			confidence: 0,
+			checks,
+			recommendations: ["Audio contains no recognizable speech. Check the recording level or re-record."],
+			language: { requested: requestedLanguage, reported: reportedRaw, normalized: null, heuristic: null, consistent: true },
+			timings: { audioSeconds, asrLatencySeconds },
+			crossCheck: crossCheck ?? { performed: false, skippedReason: "not run (no speech)" },
+		};
+	}
+	checks.push({ id: "speech_present", status: "pass", detail: `${countWords(trimmed)} words, ${trimmed.length} characters` });
+
+	// 2. Repetition loops
+	const loop = detectRepetitionLoop(trimmed);
+	if (loop) {
+		checks.push({ id: "repetition_loop", status: "fail", detail: `Repeated phrase detected: "${loop}" — likely ASR hallucination` });
+		recommendations.push("Transcript contains a repetition loop; treat the tail of the text as unreliable.");
+	} else {
+		const ratio = repetitionRatio(trimmed);
+		if (ratio > 0.35) checks.push({ id: "repetition_loop", status: "warn", detail: `High duplicate-bigram ratio (${(ratio * 100).toFixed(0)}%)` });
+		else checks.push({ id: "repetition_loop", status: "pass", detail: `No repetition loop (duplicate bigram ratio ${(ratio * 100).toFixed(0)}%)` });
+	}
+
+	// 3. Garbled characters
+	const garble = garbleRatio(trimmed);
+	if (garble > 0.02) {
+		checks.push({ id: "encoding", status: "fail", detail: `${(garble * 100).toFixed(1)}% invalid characters — decoding problem` });
+		recommendations.push("Transcript has invalid characters; the audio may be corrupted or in an unsupported encoding.");
+	} else {
+		checks.push({ id: "encoding", status: "pass", detail: "Clean UTF-8 text" });
+	}
+
+	// 4. Duration: ffprobe is ground truth; segments[].end is the server's own claim.
+	if (probe?.durationSeconds && segDuration != null) {
+		const delta = Math.abs(probe.durationSeconds - segDuration);
+		const pct = probe.durationSeconds > 0 ? delta / probe.durationSeconds : 0;
+		if (pct > 0.2) checks.push({ id: "duration", status: "warn", detail: `segments end at ${segDuration.toFixed(2)}s but file is ${probe.durationSeconds.toFixed(2)}s (${(pct * 100).toFixed(0)}% off) — audio after ${segDuration.toFixed(0)}s may be untranscribed` });
+		else checks.push({ id: "duration", status: "pass", detail: `Transcript covers ${segDuration.toFixed(2)}s of ${probe.durationSeconds.toFixed(2)}s` });
+	} else if (probe?.durationSeconds) {
+		checks.push({ id: "duration", status: "info", detail: `File is ${probe.durationSeconds.toFixed(2)}s; server returned no segment timing` });
+	} else {
+		checks.push({ id: "duration", status: "info", detail: "ffprobe unavailable; duration taken from segments[].end" });
+	}
+
+	// 5. Speech rate
+	if (audioSeconds && audioSeconds > 0.5) {
+		const wps = countWords(trimmed) / audioSeconds;
+		if (wps < 0.5) {
+			checks.push({ id: "speech_rate", status: "warn", detail: `${wps.toFixed(2)} words/s — very sparse, long silence or missed speech` });
+		} else if (wps > 25) {
+			checks.push({ id: "speech_rate", status: "warn", detail: `${wps.toFixed(2)} words/s — implausibly fast, likely repetition loop` });
+		} else {
+			checks.push({ id: "speech_rate", status: "pass", detail: `${wps.toFixed(2)} words/s` });
+		}
+	}
+
+	// 6. Language consistency (server has been known to label Portuguese as English)
+	const reportedNorm = reported;
+	let consistent = true;
+	if (requestedLanguage) {
+		const want = normalizeLanguage(requestedLanguage);
+		if (want && reportedNorm && want !== reportedNorm) {
+			consistent = false;
+			checks.push({ id: "language", status: "warn", detail: `Requested "${requestedLanguage}" but server reported "${reportedRaw}"` });
+		} else if (want && !reportedNorm) {
+			consistent = false;
+			checks.push({ id: "language", status: "warn", detail: `Requested "${requestedLanguage}" but server reported no language` });
+		} else {
+			checks.push({ id: "language", status: "pass", detail: `Language "${reportedRaw}" matches request` });
+		}
+	} else if (reportedNorm && heuristic && reportedNorm !== heuristic) {
+		consistent = false;
+		checks.push({ id: "language", status: "warn", detail: `Server reported "${reportedRaw}" but content looks ${heuristic} — auto-detect is unreliable here, retry with an explicit language` });
+		recommendations.push(`Auto-detected language disagrees with content. Re-run with language=${heuristic} to confirm.`);
+	} else if (reportedNorm && heuristic) {
+		checks.push({ id: "language", status: "pass", detail: `Auto-detected "${reportedRaw}" agrees with content` });
+	} else {
+		checks.push({ id: "language", status: "info", detail: `Server reported "${reportedRaw ?? "null"}"; content heuristic inconclusive` });
+	}
+
+	// 7. Cross-check
+	if (crossCheck?.performed) {
+		const sim = crossCheck.similarity ?? 0;
+		if (sim >= SIMILARITY_WARN) checks.push({ id: "cross_check", status: "pass", detail: `Re-transcription with language=${crossCheck.languageUsed ?? "auto"} matches ${(sim * 100).toFixed(1)}%` });
+		else if (sim >= SIMILARITY_FAIL) checks.push({ id: "cross_check", status: "warn", detail: `Passes differ by ${((1 - sim) * 100).toFixed(1)}% — wording is not stable` });
+		else checks.push({ id: "cross_check", status: "fail", detail: `Passes diverge (${(sim * 100).toFixed(1)}% match) — low confidence, needs human review` });
+		if (sim < SIMILARITY_WARN) recommendations.push("Transcription is unstable across passes; verify against the audio or pin an explicit language.");
+	} else if (crossCheck && !crossCheck.performed) {
+		checks.push({ id: "cross_check", status: "info", detail: `Not run${crossCheck.skippedReason ? `: ${crossCheck.skippedReason}` : ""}` });
+	}
+
+	// Score
+	let score = 1;
+	for (const c of checks) {
+		if (c.status === "fail") score -= 0.45;
+		else if (c.status === "warn") score -= 0.12;
+	}
+	const confidence = Math.max(0, Math.min(1, Number(score.toFixed(3))));
+	const hasFail = checks.some((c) => c.status === "fail");
+	const hasWarn = checks.some((c) => c.status === "warn");
+	const verdict: Verdict = hasFail ? "failed" : hasWarn ? "review" : "ok";
+
+	return {
+		verdict,
+		confidence,
+		checks,
+		recommendations,
+		language: { requested: requestedLanguage, reported: reportedRaw, normalized: reportedNorm, heuristic, consistent },
+		timings: { audioSeconds, asrLatencySeconds },
+		crossCheck: crossCheck ?? { performed: false, skippedReason: "disabled" },
+	};
+}
+
+/**
+ * Extract spelled-out measurements as digits.
+ * The transcript keeps its original wording; this is additive derived data
+ * so nothing the user dictated is silently rewritten.
+ */
+export function extractMeasurements(text: string): Array<{ raw: string; value: number; unit: string }> {
+	const UNIT: Record<string, string> = { centimetro: "cm", centímetro: "cm", centimetros: "cm", centímetros: "cm", milimetro: "mm", milímetro: "mm", milimetros: "mm", milímetros: "mm", metro: "m", metros: "m", polegada: "in", polegadas: "in" };
+	const out: Array<{ raw: string; value: number; unit: string }> = [];
+	const low = text.toLowerCase();
+	const unitRe = "cent[ií]metros?|mil[ií]metros?|metros?|polegadas?";
+
+	// Numeric form, including the spoken decimal "26 ponto 2 centímetros".
+	const numRe = new RegExp(`(\\d+(?:[.,]\\d+)?)\\s*(?:ponto\\s+(\\d+))?\\s*(${unitRe})\\b`, "g");
+	let m: RegExpExecArray | null;
+	while ((m = numRe.exec(low))) {
+		const whole = Number(m[1]!.replace(",", "."));
+		const frac = m[2] ? Number(`0.${m[2]}`) : 0;
+		const value = whole + frac;
+		if (Number.isFinite(value)) out.push({ raw: m[0]!.trim(), value: Number(value.toFixed(3)), unit: UNIT[m[3]!] ?? m[3]! });
+	}
+
+	// Spoken form: "vinte seis centímetros", "quarenta e um centímetros",
+	// "vinte seis centímetros ponto dois milímetros". Written Brazilian
+	// Portuguese inserts "e"; spoken usually does not, so it is optional.
+	const tensRe = "vinte|trinta|quarenta|cinquenta|sessenta|setenta|oitenta|noventa";
+	const onesRe = "dezasseis|dezesseis|dezessete|dezoito|dezenove|quinze|quatorze|catorze|treze|doze|onze|dez|nove|oito|sete|seis|cinco|quatro|tr[eê]s|duas|dois|uma|um";
+	const spelledRe = new RegExp(`\\b(?:(?<tens>${tensRe})(?:\\s+e)?(?:\\s+(?<ones>${onesRe}))?|(?<single>${onesRe}))\\s*(${unitRe})(?:\\s+ponto\\s+(?<frac>\\d+))?\\b`, "g");
+	while ((m = spelledRe.exec(low))) {
+		const g = m.groups ?? {};
+		const value = (tensValue(g.tens) + onesValue(g.ones)) || onesValue(g.single);
+		if (!value) continue;
+		const unit = UNIT[m[4]!] ?? m[4]!;
+		out.push({ raw: m[0]!.trim(), value, unit });
+	}
+	return out;
+}
+
+const TENS: Record<string, number> = { vinte: 20, trinta: 30, quarenta: 40, cinquenta: 50, sessenta: 60, setenta: 70, oitenta: 80, noventa: 90 };
+const ONES: Record<string, number> = { um: 1, uma: 1, dois: 2, duas: 2, tres: 3, três: 3, quatro: 4, cinco: 5, seis: 6, sete: 7, oito: 8, nove: 9, dez: 10, onze: 11, doze: 12, treze: 13, catorze: 14, quatorze: 14, quinze: 15, dezasseis: 16, dezesseis: 16, dezessete: 17, dezoito: 18, dezenove: 19 };
+
+function tensValue(w?: string): number {
+	return w ? TENS[w] ?? 0 : 0;
+}
+function onesValue(w?: string): number {
+	return w ? ONES[w] ?? 0 : 0;
+}