@ossclip/core 0.1.25 → 0.1.27

package/src/phonetics.ts

@@ -12,6 +12,49 @@
  * "unrelated", and it must stay dependency-free and deterministic.
  */
 
+/**
+ * Marks Arabic-script text carries that two transcribers disagree about
+ * without disagreeing about the WORD: harakat/vowel diacritics
+ * (U+064B–U+065F, U+0670), tatweel (U+0640, a pure typographic stretch), and
+ * the zero-width joiners (U+200C/U+200D). Whisper emits them inconsistently
+ * and an LLM writing a correction rarely reproduces them, so leaving them in
+ * makes a correct repair either miss `locate()` outright or read as
+ * "different from what was heard" purely on invisible marks. Only tatweel
+ * survives the `\p{L}\p{N}` filter below (it is Lm, a letter); the rest are
+ * stripped here so the intent is legible rather than an accident of Unicode
+ * categories.
+ */
+// Escaped, not literal: three of these code points are invisible in an editor.
+const ARABIC_NOISE = /[\u064B-\u065F\u0670\u0640\u200C\u200D]/g;
+
+/**
+ * Comparable form for two pieces of text: case-, punctuation- and
+ * whitespace-insensitive, in ANY script.
+ *
+ * Shared by `phonetics.ts` and `producer/repair.ts` on purpose — they used to
+ * hold two copies and the copy in `repair.ts` was `[^a-z0-9\s]`, i.e.
+ * Latin-only. Field case (2026-08-18): every one of 11 recorded Urdu repairs
+ * normalized to the empty string, so `norm(heard) === norm(correction)` was
+ * `"" === ""` and ALL 11 were refused as "identical to what was heard" —
+ * including `پرسٹ` → `فرسٹ`, which shares no letters with what it replaced.
+ *
+ * Keeping letters and digits of every script also KEEPS accented Latin
+ * ("café", "über") where the old expression deleted it. That is the same bug
+ * in miniature — a French word normalized to "caf" — so it is a fix, not a
+ * regression. Pure-ASCII input is byte-identical to the old behaviour, which
+ * is pinned by a test.
+ */
+export function normalizeForCompare(s: string): string {
+  return s
+    .normalize("NFC")
+    .toLowerCase()
+    .replace(ARABIC_NOISE, "")
+    .replace(/[^\p{L}\p{N}\s]/gu, "")
+    .split(/\s+/)
+    .filter(Boolean)
+    .join(" ");
+}
+
 /**
  * Digraphs collapsed before single letters, longest first. The ch/sh and th
  * sounds get DIGIT placeholders on purpose: a letter placeholder would be
@@ -106,6 +149,47 @@ export function soundsLike(a: string, b: string): number {
   return Math.max(0, 1 - dist / Math.max(ka.length, kb.length));
 }
 
+/**
+ * 0..1 similarity of the TEXT itself, for scripts the phonetic key cannot
+ * represent (`phoneticKey` is defined over a-z, so anything non-Latin keys to
+ * ""). Same shape as `soundsLike` — normalized edit distance over the longer
+ * string — but run on the normalized text rather than a consonant skeleton.
+ *
+ * This is a weaker signal than a phonetic key and it is meant to be: an
+ * Urdu-script mishearing differs from the truth by a letter or two of the same
+ * script, so edit distance still separates it from an unrelated phrase. What
+ * it cannot do is fold vowels, which is why the floor below is calibrated
+ * against real data instead of borrowing SOUNDS_LIKE_FLOOR.
+ */
+export function textSimilarity(a: string, b: string): number {
+  const na = normalizeForCompare(a);
+  const nb = normalizeForCompare(b);
+  if (na.length === 0 && nb.length === 0) return 1;
+  if (na.length === 0 || nb.length === 0) return 0;
+  return Math.max(0, 1 - levenshtein(na, nb) / Math.max(na.length, nb.length));
+}
+
+/**
+ * Floor for the non-Latin fallback, MEASURED rather than guessed.
+ *
+ * The 11 Urdu repairs recorded in a real production.json (2026-08-18) score,
+ * sorted: 0.333, 0.400, 0.500, 0.500, 0.545, 0.583, 0.636, 0.750, 0.750,
+ * 0.800, 0.800. The 0.333 is `حقیقہ ٹون` → `ہیکاتھون` ("hackathon"), a genuine
+ * repair and the worst of the set because the recognizer both re-segmented the
+ * word and changed its opening letter. Admitting it sets the ceiling on the
+ * floor; 0.33 is the largest value that does.
+ *
+ * Against that, unrelated four-word spans lifted from the same transcript
+ * score 0.167–0.250 and are refused. The band is narrow, and it is narrow for
+ * the same reason the Latin one is (see SOUNDS_LIKE_FLOOR): two SHORT
+ * unrelated Urdu spans can still land above it — measured, `پرسٹ ہیک` vs
+ * `ٹرس می` scores 0.500. There is no onset test here to catch that, because
+ * two of the 11 genuine repairs change their first letter. So this gate is
+ * real but shallow; the span, token-count and length guards in
+ * `applyRepairs` are what keep it from being a rewrite licence.
+ */
+export const TEXT_SIMILARITY_FLOOR = 0.33;
+
 /**
  * Default floor for "this is a repair, not a rewrite". Deliberately low,
  * because a real mishearing can move word boundaries ("code churn" → "coach
@@ -124,11 +208,27 @@ export const SOUNDS_LIKE_FLOOR = 0.34;
  *    a phrase, essentially never its onset. This is what rejects a rewrite:
  *    "revenue" for "churn" and "monetization" for "agents" both score in the
  *    same range as a true repair, and both fail the onset test.
+ *
+ * When either side has no Latin letters there is no key to compare, and this
+ * used to answer `ka === kb` — `"" === ""`, i.e. YES for any two non-Latin
+ * strings however unrelated. That is no gate at all for an Urdu transcript, so
+ * those pairs route to `textSimilarity` instead (2026-08-18 field case).
+ * Latin-to-Latin comparisons never reach that branch and are unchanged.
  */
 export function soundsSimilar(a: string, b: string, floor = SOUNDS_LIKE_FLOOR): boolean {
   const ka = phraseKey(a);
   const kb = phraseKey(b);
-  if (ka.length === 0 || kb.length === 0) return ka === kb;
+  if (ka.length === 0 || kb.length === 0) {
+    // `floor` is deliberately NOT reused here: it is calibrated against
+    // consonant skeletons, which are shorter and coarser than the text this
+    // branch compares, so the same number means something else. Taking the
+    // larger of the two was tried and is wrong — the default 0.34 alone
+    // rejects a measured genuine repair scoring 0.333. The only caller that
+    // raises the floor (reconcileCopy, 0.6) cannot reach this branch anyway:
+    // its candidate tokens are stripped to `[A-Za-z]`, so its key is never
+    // empty and a non-Latin spoken word scores ~0 against it regardless.
+    return textSimilarity(a, b) >= TEXT_SIMILARITY_FLOOR;
+  }
   if (ka[0] !== kb[0]) return false;
   return soundsLike(a, b) >= floor;
 }

package/src/producer/repair.ts

@@ -1,7 +1,7 @@
 import { z } from "zod/v4";
 import type { Transcript, Word } from "../schema";
 import type { Scene } from "../scene-schema";
-import { soundsSimilar } from "../phonetics";
+import { normalizeForCompare, soundsSimilar } from "../phonetics";
 import type { LlmProvider } from "./provider";
 
 /**
@@ -105,15 +105,18 @@ export function buildRepairUserPrompt(
   );
 }
 
-/** Comparable form: case- and punctuation-insensitive. */
-function norm(s: string): string {
-  return s
-    .toLowerCase()
-    .replace(/[^a-z0-9\s]/g, "")
-    .split(/\s+/)
-    .filter(Boolean)
-    .join(" ");
-}
+/**
+ * Comparable form: case- and punctuation-insensitive, in any script.
+ *
+ * This was a second, Latin-only copy (`[^a-z0-9\s]`) of what is now
+ * `normalizeForCompare`. The two drifted in the worst possible way: on an Urdu
+ * transcript every string normalized to "", so `norm(actual) ===
+ * norm(r.correction)` was true for every proposal and all 11 repairs in a real
+ * run were refused as "identical to what was heard" (2026-08-18) — while
+ * `locate()`, comparing "" to "", "matched" the first span it tried without
+ * verifying anything. One shared helper so they cannot drift again.
+ */
+const norm = normalizeForCompare;
 
 function spanText(transcript: Transcript, startWord: number, endWord: number): string {
   return transcript.words
@@ -228,6 +231,12 @@ export function applyRepairs(
    */
   const locate = (r: TranscriptRepair): { startWord: number; endWord: number } | null => {
     const want = norm(r.heard);
+    // A quote that normalizes to nothing is not an anchor: "" compares equal
+    // to the first span whose own normalisation is empty, so the search would
+    // "find" a span it never verified. That was live for every non-Latin
+    // transcript until norm() was fixed above; refuse it explicitly so it
+    // cannot come back through some other all-punctuation quote.
+    if (want.length === 0) return null;
     // Widths to try, in order of trust. The QUOTED TEXT is the reliable part
     // of a proposal, so its own token count leads; the claimed span is a
     // fallback for a quote whose normalisation splits differently. Trusting

package/src/recut.ts

@@ -391,3 +391,64 @@ export function applyUserCuts(
     removedSec,
   };
 }
+
+/** What `pruneHidesInsideCuts` hands back: the doc (same reference when
+ * nothing was pruned — the caller's changed-gate reads `pruned.length`), and
+ * the retired keys so produce can SAY what it retired. */
+export interface PrunedHides {
+  doc: OverrideDoc;
+  pruned: string[];
+}
+
+/**
+ * Retire `captionWordsHidden` entries whose word the final cutlist REMOVES
+ * (§59b revisited 2026-08-18 — the "captions + video" delete gesture writes
+ * both a hide and a cut in one commit).
+ *
+ * Once the cut lands, `buildCaptionLines` drops the word before the hide
+ * layer ever sees it, so the hide key would report `found: null` ("the cut
+ * removed it", `captionHideDropLine`) on every subsequent run forever — the
+ * cut SUPERSEDES the hide, the same superseded philosophy `overrides.ts`'s
+ * caption-key migration applies. Hides whose source instant is OUTSIDE every
+ * removed segment are kept verbatim — as are keys that are not §137 `w<ms>`
+ * anchors at all, which name no instant this can test (see the guard below).
+ *
+ * HALF-OPEN interval (`srcIn <= src < srcOut`), on purpose — the two edges
+ * are NOT symmetric. A word starting exactly at `srcIn` IS cut: that is
+ * precisely where the FIRST word of a captions+video delete lands (its
+ * srcStart round-trips through the TimeMap to the resolved cut's own srcIn),
+ * and `mapWord` clamps that instant into the removal and drops the word — a
+ * strictly-inside test never retired the gesture's own first hide, leaving
+ * it a permanent `found: null` drop report. A word starting exactly at
+ * `srcOut` belongs to the NEXT kept span (`buildCaptionLines` still emits it
+ * — a seam instant has a kept-side preimage, `timemap.ts`), so its hide is
+ * still doing work and must survive.
+ */
+export function pruneHidesInsideCuts(doc: OverrideDoc, cutlist: readonly Segment[]): PrunedHides {
+  const pruned: string[] = [];
+  const kept: OverrideDoc["captionWordsHidden"] = {};
+  for (const [key, entry] of Object.entries(doc.captionWordsHidden)) {
+    // SOURCE-KEYED ONLY, parsed and not coerced. `captionWordsHidden` is an
+    // unpinned `z.record` (unlike `CaptionRangeEditSchema`'s `/^w\d+$/`), so a
+    // hand-edited or legacy-keyed doc reaches here: a POSITIONAL key like "17"
+    // would slice to "7", parse as 7ms, land inside any early cut and be
+    // deleted with nothing said. The editor guards the identical case and
+    // states the rule (`apps/editor/src/useEdits.ts:587-600`): only §137
+    // `w<ms>` keys carry an interval-testable instant, and an entry this
+    // function cannot honestly locate is KEPT.
+    if (!/^w\d+$/.test(key)) {
+      kept[key] = entry;
+      continue;
+    }
+    // `captionKeyFor`'s quantization inverted (`w${Math.round(sec * 1000)}`,
+    // overrides.ts): the key IS the word's source instant, ms-quantized.
+    const srcSec = parseInt(key.slice(1), 10) / 1000;
+    const removed = cutlist.some(
+      (seg) => seg.kind === "remove" && srcSec >= seg.srcIn && srcSec < seg.srcOut,
+    );
+    if (removed) pruned.push(key);
+    else kept[key] = entry;
+  }
+  if (pruned.length === 0) return { doc, pruned };
+  return { doc: { ...doc, captionWordsHidden: kept }, pruned };
+}

package/src/transcribe.ts

@@ -78,6 +78,51 @@ function repairSplitSegments(json: WhisperJson): WhisperJson {
   };
 }
 
+/**
+ * Run length at which a stack of zero-length words at ONE instant stops being
+ * a rounding artifact and becomes a repetition-loop hallucination. Real speech
+ * never emits 8 tokens at a single instant; the field case emitted 118.
+ */
+export const REPETITION_BURST_MIN = 8;
+
+/**
+ * Drop whisper repetition-loop bursts (field case 2026-08-18): an Urdu take
+ * re-decoded a whole phrase as 118 CONSECUTIVE tokens all stamped
+ * `from === to === 31040` — zero length, at one instant. The stamp repair
+ * below then fans such a burst out into 118 fabricated 50ms words marching
+ * forward from 31.04s, so the phrase ships TWICE in the captions (31.04s and
+ * 33.54s) and a fifth of the transcript carries the tell-tale exactly-0.05s
+ * duration. `-mc 0` in whisperArgs is the decoder-side mitigation for the same
+ * failure; it did not prevent this occurrence, and it can never repair an
+ * already-cached transcript.json — hence a parse-side guard too.
+ *
+ * A burst is a MAXIMAL run of consecutive zero-length/inverted words sharing
+ * one `start`. Equality is exact, not epsilon: these stamps are integer
+ * milliseconds divided by 1000, so members of one burst are the same double
+ * bit-for-bit, and a tolerance would only start swallowing real neighbors.
+ * Runs shorter than REPETITION_BURST_MIN fall through untouched — a lone
+ * zero-length stamp is a rounding artifact, not a hallucination. The drop is
+ * silent by design: this function is pure and total, and there is no logging
+ * channel in the parse path to warn on.
+ */
+export function dropRepetitionBursts(words: readonly Word[]): Word[] {
+  const out: Word[] = [];
+  let i = 0;
+  while (i < words.length) {
+    const w = words[i]!;
+    if (w.end > w.start) {
+      out.push(w);
+      i++;
+      continue;
+    }
+    let j = i + 1;
+    while (j < words.length && words[j]!.end <= words[j]!.start && words[j]!.start === w.start) j++;
+    if (j - i < REPETITION_BURST_MIN) for (let k = i; k < j; k++) out.push(words[k]!);
+    i = j;
+  }
+  return out;
+}
+
 const STRICT_UTF8 = new TextDecoder("utf-8", { fatal: true });
 
 /**
@@ -115,11 +160,19 @@ export function parseWhisperJson(json: WhisperJson): Transcript {
     if (!raw || !raw.trim()) continue;
     const text = raw.trim();
     if (NOISE_TOKEN.test(text)) continue;
+    // A word already CLOSED by Arabic-script sentence punctuation refuses
+    // continuations (field case 2026-08-18): whisper emits `۔` and the next
+    // sentence's first token with no leading whitespace, and the plain
+    // whitespace rule fused them into one unsplittable word ("ہوں۔اس").
+    // Deliberately NOT the Latin `.`/`!`/`?` — whisper tokenizes decimals
+    // ("3", ".", "5") and abbreviations as bare continuations too, and
+    // splitting those would shred "3.5" into two words. ۔ (U+06D4) and
+    // ؟ (U+061F) have no such second job.
     const startsWord = /^\s/.test(raw) || words.length === 0;
     const start = seg.offsets.from / 1000;
     const end = seg.offsets.to / 1000;
     const last = words[words.length - 1];
-    if (!startsWord && last) {
+    if (!startsWord && last && !/[۔؟]$/.test(last.text)) {
       last.text += text;
       last.end = Math.max(last.end, end);
     } else {
@@ -146,14 +199,18 @@ export function parseWhisperJson(json: WhisperJson): Transcript {
     else if (next) next.start = Math.min(next.start, w.start);
     words.splice(i, 1);
   }
+  // BEFORE the repair loop, never after: the repair rewrites every burst
+  // member into a distinct monotone stamp, so once it has run the shared
+  // timestamp — the only evidence a burst existed — is gone.
+  const kept = dropRepetitionBursts(words);
   // Whisper occasionally emits zero-length or inverted stamps; repair minimally.
-  for (let i = 0; i < words.length; i++) {
-    const w = words[i]!;
+  for (let i = 0; i < kept.length; i++) {
+    const w = kept[i]!;
     if (w.end <= w.start) w.end = w.start + 0.05;
-    const next = words[i + 1];
+    const next = kept[i + 1];
     if (next && next.start < w.end) next.start = w.end;
   }
-  return { language: json.result?.language ?? "en", words };
+  return { language: json.result?.language ?? "en", words: kept };
 }
 
 export interface WhisperOptions {
@@ -192,6 +249,15 @@ export function whisperArgs(opts: WhisperOptions, wavPath: string): string[] {
     "-oj",
     "-of", opts.outBase,
     "-ml", "1",
+    // No text context across 30s decode windows (field case 2026-08-18): an
+    // Urdu take hit whisper's repetition loop — a whole sentence re-decoded
+    // as 261 zero-duration tokens — and carrying the previous window's text
+    // into the decoder is the known trigger. `-mc 0` is the standard
+    // mitigation and leaves `--prompt` (the dictionary bias) untouched.
+    // Cached transcript.json files decoded without it are knowingly still
+    // reused (transcriptCacheReusable's no-spurious-retranscribe rule);
+    // delete a workdir's transcript.json to re-decode with it.
+    "-mc", "0",
     "--no-prints",
   ];
   if (opts.language !== undefined) args.push("-l", opts.language);