@ossclip/core 0.1.24 → 0.1.26

package/src/phonetics.ts

@@ -12,6 +12,49 @@
  * "unrelated", and it must stay dependency-free and deterministic.
  */
 
+/**
+ * Marks Arabic-script text carries that two transcribers disagree about
+ * without disagreeing about the WORD: harakat/vowel diacritics
+ * (U+064B–U+065F, U+0670), tatweel (U+0640, a pure typographic stretch), and
+ * the zero-width joiners (U+200C/U+200D). Whisper emits them inconsistently
+ * and an LLM writing a correction rarely reproduces them, so leaving them in
+ * makes a correct repair either miss `locate()` outright or read as
+ * "different from what was heard" purely on invisible marks. Only tatweel
+ * survives the `\p{L}\p{N}` filter below (it is Lm, a letter); the rest are
+ * stripped here so the intent is legible rather than an accident of Unicode
+ * categories.
+ */
+// Escaped, not literal: three of these code points are invisible in an editor.
+const ARABIC_NOISE = /[\u064B-\u065F\u0670\u0640\u200C\u200D]/g;
+
+/**
+ * Comparable form for two pieces of text: case-, punctuation- and
+ * whitespace-insensitive, in ANY script.
+ *
+ * Shared by `phonetics.ts` and `producer/repair.ts` on purpose — they used to
+ * hold two copies and the copy in `repair.ts` was `[^a-z0-9\s]`, i.e.
+ * Latin-only. Field case (2026-08-18): every one of 11 recorded Urdu repairs
+ * normalized to the empty string, so `norm(heard) === norm(correction)` was
+ * `"" === ""` and ALL 11 were refused as "identical to what was heard" —
+ * including `پرسٹ` → `فرسٹ`, which shares no letters with what it replaced.
+ *
+ * Keeping letters and digits of every script also KEEPS accented Latin
+ * ("café", "über") where the old expression deleted it. That is the same bug
+ * in miniature — a French word normalized to "caf" — so it is a fix, not a
+ * regression. Pure-ASCII input is byte-identical to the old behaviour, which
+ * is pinned by a test.
+ */
+export function normalizeForCompare(s: string): string {
+  return s
+    .normalize("NFC")
+    .toLowerCase()
+    .replace(ARABIC_NOISE, "")
+    .replace(/[^\p{L}\p{N}\s]/gu, "")
+    .split(/\s+/)
+    .filter(Boolean)
+    .join(" ");
+}
+
 /**
  * Digraphs collapsed before single letters, longest first. The ch/sh and th
  * sounds get DIGIT placeholders on purpose: a letter placeholder would be
@@ -106,6 +149,47 @@ export function soundsLike(a: string, b: string): number {
   return Math.max(0, 1 - dist / Math.max(ka.length, kb.length));
 }
 
+/**
+ * 0..1 similarity of the TEXT itself, for scripts the phonetic key cannot
+ * represent (`phoneticKey` is defined over a-z, so anything non-Latin keys to
+ * ""). Same shape as `soundsLike` — normalized edit distance over the longer
+ * string — but run on the normalized text rather than a consonant skeleton.
+ *
+ * This is a weaker signal than a phonetic key and it is meant to be: an
+ * Urdu-script mishearing differs from the truth by a letter or two of the same
+ * script, so edit distance still separates it from an unrelated phrase. What
+ * it cannot do is fold vowels, which is why the floor below is calibrated
+ * against real data instead of borrowing SOUNDS_LIKE_FLOOR.
+ */
+export function textSimilarity(a: string, b: string): number {
+  const na = normalizeForCompare(a);
+  const nb = normalizeForCompare(b);
+  if (na.length === 0 && nb.length === 0) return 1;
+  if (na.length === 0 || nb.length === 0) return 0;
+  return Math.max(0, 1 - levenshtein(na, nb) / Math.max(na.length, nb.length));
+}
+
+/**
+ * Floor for the non-Latin fallback, MEASURED rather than guessed.
+ *
+ * The 11 Urdu repairs recorded in a real production.json (2026-08-18) score,
+ * sorted: 0.333, 0.400, 0.500, 0.500, 0.545, 0.583, 0.636, 0.750, 0.750,
+ * 0.800, 0.800. The 0.333 is `حقیقہ ٹون` → `ہیکاتھون` ("hackathon"), a genuine
+ * repair and the worst of the set because the recognizer both re-segmented the
+ * word and changed its opening letter. Admitting it sets the ceiling on the
+ * floor; 0.33 is the largest value that does.
+ *
+ * Against that, unrelated four-word spans lifted from the same transcript
+ * score 0.167–0.250 and are refused. The band is narrow, and it is narrow for
+ * the same reason the Latin one is (see SOUNDS_LIKE_FLOOR): two SHORT
+ * unrelated Urdu spans can still land above it — measured, `پرسٹ ہیک` vs
+ * `ٹرس می` scores 0.500. There is no onset test here to catch that, because
+ * two of the 11 genuine repairs change their first letter. So this gate is
+ * real but shallow; the span, token-count and length guards in
+ * `applyRepairs` are what keep it from being a rewrite licence.
+ */
+export const TEXT_SIMILARITY_FLOOR = 0.33;
+
 /**
  * Default floor for "this is a repair, not a rewrite". Deliberately low,
  * because a real mishearing can move word boundaries ("code churn" → "coach
@@ -124,11 +208,27 @@ export const SOUNDS_LIKE_FLOOR = 0.34;
  *    a phrase, essentially never its onset. This is what rejects a rewrite:
  *    "revenue" for "churn" and "monetization" for "agents" both score in the
  *    same range as a true repair, and both fail the onset test.
+ *
+ * When either side has no Latin letters there is no key to compare, and this
+ * used to answer `ka === kb` — `"" === ""`, i.e. YES for any two non-Latin
+ * strings however unrelated. That is no gate at all for an Urdu transcript, so
+ * those pairs route to `textSimilarity` instead (2026-08-18 field case).
+ * Latin-to-Latin comparisons never reach that branch and are unchanged.
  */
 export function soundsSimilar(a: string, b: string, floor = SOUNDS_LIKE_FLOOR): boolean {
   const ka = phraseKey(a);
   const kb = phraseKey(b);
-  if (ka.length === 0 || kb.length === 0) return ka === kb;
+  if (ka.length === 0 || kb.length === 0) {
+    // `floor` is deliberately NOT reused here: it is calibrated against
+    // consonant skeletons, which are shorter and coarser than the text this
+    // branch compares, so the same number means something else. Taking the
+    // larger of the two was tried and is wrong — the default 0.34 alone
+    // rejects a measured genuine repair scoring 0.333. The only caller that
+    // raises the floor (reconcileCopy, 0.6) cannot reach this branch anyway:
+    // its candidate tokens are stripped to `[A-Za-z]`, so its key is never
+    // empty and a non-Latin spoken word scores ~0 against it regardless.
+    return textSimilarity(a, b) >= TEXT_SIMILARITY_FLOOR;
+  }
   if (ka[0] !== kb[0]) return false;
   return soundsLike(a, b) >= floor;
 }

package/src/producer/index.ts

@@ -25,6 +25,7 @@ import {
 export * from "./provider";
 export * from "./usage";
 export * from "./beats";
+export * from "./youtube";
 export * from "./scene-props";
 export * from "./repair";
 export { AnthropicProvider, DEFAULT_CLAUDE_MODEL } from "./anthropic";

package/src/producer/repair.ts

@@ -1,7 +1,7 @@
 import { z } from "zod/v4";
 import type { Transcript, Word } from "../schema";
 import type { Scene } from "../scene-schema";
-import { soundsSimilar } from "../phonetics";
+import { normalizeForCompare, soundsSimilar } from "../phonetics";
 import type { LlmProvider } from "./provider";
 
 /**
@@ -76,7 +76,11 @@ A correction must sound essentially identical to what was heard. If a span is no
 
 For each fix give the word-index span, the exact text you are replacing (\`heard\`), and the corrected text.`;
 
-export function buildRepairUserPrompt(transcript: Transcript, speaker?: string): string {
+export function buildRepairUserPrompt(
+  transcript: Transcript,
+  speaker?: string,
+  dictionary?: readonly string[],
+): string {
   const words = transcript.words.map((w, i) => `[${i}]${w.text}`).join(" ");
   return (
     (speaker
@@ -89,20 +93,30 @@ export function buildRepairUserPrompt(transcript: Transcript, speaker?: string):
         `About the speaker (use this to recognise names the recognizer mangled, ` +
         `never to introduce facts): ${speaker}\n\n`
       : "") +
+    (dictionary && dictionary.length > 0
+      ? // The user's dictionary (F4, 2026-08-16): same failure class as the
+        // speaker hint — "Jason" for JSON is a lookup once the model knows
+        // the term, a guess otherwise.
+        `Vouched terms the speaker uses — prefer these spellings when the ` +
+        `audio matches: ${dictionary.join(", ")}\n\n`
+      : "") +
     `Word-indexed transcript (indices refer to THIS list):\n${words}\n\n` +
     `Report only spans that are clearly mishearings.`
   );
 }
 
-/** Comparable form: case- and punctuation-insensitive. */
-function norm(s: string): string {
-  return s
-    .toLowerCase()
-    .replace(/[^a-z0-9\s]/g, "")
-    .split(/\s+/)
-    .filter(Boolean)
-    .join(" ");
-}
+/**
+ * Comparable form: case- and punctuation-insensitive, in any script.
+ *
+ * This was a second, Latin-only copy (`[^a-z0-9\s]`) of what is now
+ * `normalizeForCompare`. The two drifted in the worst possible way: on an Urdu
+ * transcript every string normalized to "", so `norm(actual) ===
+ * norm(r.correction)` was true for every proposal and all 11 repairs in a real
+ * run were refused as "identical to what was heard" (2026-08-18) — while
+ * `locate()`, comparing "" to "", "matched" the first span it tried without
+ * verifying anything. One shared helper so they cannot drift again.
+ */
+const norm = normalizeForCompare;
 
 function spanText(transcript: Transcript, startWord: number, endWord: number): string {
   return transcript.words
@@ -171,6 +185,13 @@ export interface ApplyRepairsOptions {
    * phonetic gate. Everything else about it is still checked.
    */
   speaker?: string;
+  /**
+   * The user's dictionary (F4, 2026-08-16) — vouched the same way the
+   * speaker hint is: terms the user typed themselves join the vouched set,
+   * so a fully-vouched correction ("Jason" → "JSON") may pass the phonetic
+   * gate. Every other guard still applies.
+   */
+  dictionary?: readonly string[];
 }
 
 export function applyRepairs(
@@ -180,18 +201,23 @@ export function applyRepairs(
 ): { transcript: Transcript; applied: AppliedRepair[] } {
   const maxIndex = transcript.words.length - 1;
   /**
-   * Names the user vouched for via `--speaker`. A recognizer that turned
-   * "Ahsan" into the initialism "SM" produces a correction no phonetic
-   * measure will accept — and refusing it leaves the wrong name in the
-   * captions, which is the failure the hint exists to prevent. So a correction
-   * built ENTIRELY from words the user supplied is exempt from that one gate.
-   * Every other guard still applies, and the exemption can only ever
-   * substitute text the user typed themselves.
+   * Words the user vouched for — via `--speaker` and, since F4 (2026-08-16),
+   * via the dictionary. A recognizer that turned "Ahsan" into the initialism
+   * "SM" produces a correction no phonetic measure will accept — and refusing
+   * it leaves the wrong name in the captions, which is the failure the hint
+   * exists to prevent. So a correction built ENTIRELY from words the user
+   * supplied is exempt from that one gate. Every other guard still applies,
+   * and the exemption can only ever substitute text the user typed themselves.
    */
-  const speakerWords = new Set(norm(opts.speaker ?? "").split(" ").filter(Boolean));
-  const speakerVouched = (correction: string): boolean => {
+  const vouchedWords = new Set(
+    [
+      ...norm(opts.speaker ?? "").split(" "),
+      ...(opts.dictionary ?? []).flatMap((term) => norm(term).split(" ")),
+    ].filter(Boolean),
+  );
+  const userVouched = (correction: string): boolean => {
     const tokens = norm(correction).split(" ").filter(Boolean);
-    return tokens.length > 0 && tokens.every((t) => speakerWords.has(t));
+    return tokens.length > 0 && tokens.every((t) => vouchedWords.has(t));
   };
   const results: AppliedRepair[] = [];
   const accepted: Array<{ startWord: number; endWord: number; tokens: string[] }> = [];
@@ -205,6 +231,12 @@ export function applyRepairs(
    */
   const locate = (r: TranscriptRepair): { startWord: number; endWord: number } | null => {
     const want = norm(r.heard);
+    // A quote that normalizes to nothing is not an anchor: "" compares equal
+    // to the first span whose own normalisation is empty, so the search would
+    // "find" a span it never verified. That was live for every non-Latin
+    // transcript until norm() was fixed above; refuse it explicitly so it
+    // cannot come back through some other all-punctuation quote.
+    if (want.length === 0) return null;
     // Widths to try, in order of trust. The QUOTED TEXT is the reliable part
     // of a proposal, so its own token count leads; the claimed span is a
     // fallback for a quote whose normalisation splits differently. Trusting
@@ -285,7 +317,7 @@ export function applyRepairs(
       record(`"${r.correction}" is too different in length from "${actual}"`);
       continue;
     }
-    if (!soundsSimilar(actual, r.correction) && !speakerVouched(r.correction)) {
+    if (!soundsSimilar(actual, r.correction) && !userVouched(r.correction)) {
       // The gate that keeps this a repair pass and not a rewrite pass.
       record(`"${r.correction}" does not sound like "${actual}" — rewrite, not a repair`);
       continue;
@@ -340,7 +372,7 @@ export async function repairTranscript(
     try {
       const result = await provider.complete({
         system: REPAIR_SYSTEM,
-        user: buildRepairUserPrompt(transcript, opts.speaker),
+        user: buildRepairUserPrompt(transcript, opts.speaker, opts.dictionary),
         schema: TranscriptRepairSchema,
         schemaName: "transcript_repair",
         // EDITORIAL on purpose, despite looking mechanical. Measured on the real