@tandryukha/aidemo 0.8.0 → 0.15.0

package/src/captions.ts

@@ -11,6 +11,7 @@ import {
   VoiceManifestSchema,
   CaptionsManifestSchema,
   type CaptionsManifest,
+  type CaptionsConfig,
   type Storyboard,
 } from "./types.js";
 import { resolveNarrationLanguage } from "./i18n.js";
@@ -38,8 +39,52 @@ interface Word {
   end: number; // seconds
 }
 
-const MAX_WORDS = 6;
-const MAX_CUE_MS = 3000;
+/**
+ * Cue-segmentation defaults (issue #48). The old 6-word / 3 s hard cap split
+ * half of all cue breaks mid-clause ("…nobody has / filled in a form, and no /
+ * account exists yet"), which is harder to read than the narration. Broadcast
+ * practice is ~2 lines x ~37 characters, 1–6 s per cue, broken at punctuation
+ * or a conjunction. Override per storyboard via the `captions` block.
+ */
+const DEFAULT_SEG: Required<
+  Pick<CaptionsConfig, "maxWords" | "maxCueMs" | "minCueMs" | "maxLines" | "maxCharsPerLine">
+> = {
+  maxWords: 12,
+  maxCueMs: 6000,
+  minCueMs: 1000,
+  maxLines: 2,
+  maxCharsPerLine: 38,
+};
+
+/** Conjunctions/discourse markers that make a decent break point, by language. */
+const CONJUNCTIONS = new Set([
+  // en
+  "and", "but", "so", "or", "then", "because", "while", "when", "which", "that", "if",
+  // et
+  "ja", "aga", "või", "siis", "kui", "sest", "mis", "ning", "kuid",
+  // de / nl / sv
+  "und", "aber", "oder", "dann", "weil", "wenn", "och", "men", "eller", "en", "maar",
+  // fr / es / it / pt
+  "et", "mais", "ou", "donc", "parce", "quand", "y", "pero", "o", "entonces", "porque",
+  "e", "ma", "oppure", "quindi", "perché", "mas", "então",
+]);
+
+type SegConfig = typeof DEFAULT_SEG;
+
+function segConfig(storyboard?: Storyboard): SegConfig {
+  return { ...DEFAULT_SEG, ...cleanSeg(storyboard?.captions) };
+}
+
+function cleanSeg(c: CaptionsConfig | undefined): Partial<SegConfig> {
+  if (!c) return {};
+  const out: Partial<SegConfig> = {};
+  if (c.maxWords != null) out.maxWords = c.maxWords;
+  if (c.maxCueMs != null) out.maxCueMs = c.maxCueMs;
+  if (c.minCueMs != null) out.minCueMs = c.minCueMs;
+  if (c.maxLines != null) out.maxLines = c.maxLines;
+  if (c.maxCharsPerLine != null) out.maxCharsPerLine = c.maxCharsPerLine;
+  return out;
+}
 
 /** Whisper's `prompt` is only used to bias roughly its first ~224 tokens of
  *  context; cap what we send so a long demo's script doesn't balloon the
@@ -93,7 +138,8 @@ export async function generateCaptions(
     for (let i = 0; i < total; i++) opts.onSceneComplete?.(ids[i] ?? "", i, total);
   };
   if (total > 0) opts.onSceneStart?.(ids[0] ?? "", 0, total);
-  const config = captionConfig(await loadGapMs(project));
+  const seg = segConfig(storyboard);
+  const config = captionConfig(await loadGapMs(project), seg);
 
   const prompt = storyboard ? buildSttPrompt(storyboard) : undefined;
   const language = opts.language ?? resolveNarrationLanguage(storyboard, project.lang);
@@ -151,24 +197,88 @@ export async function generateCaptions(
   if (words.length === 0) {
     log("no word timestamps returned; captions may be empty");
   }
-  const cues = groupWords(words, sceneEnds);
+  const cues = groupWords(words, sceneEnds, seg);
   await writeCaptionFiles(project, cues);
-  await writeCaptionsManifest(project, { mode: "stt", inputHash, config, cues });
+  await writeCaptionsManifest(project, {
+    mode: "stt",
+    inputHash,
+    config,
+    scenes: splitWordsByScene(words, ids, sceneEnds, config.gapMs),
+    cues,
+  });
   reportDone();
 }
 
 /**
- * Whisper prompt bias: the known narration script, in scene order, so the
- * transcript converges on the correctly-spelled words instead of guessing
- * from audio phonetics alone. Capped to MAX_PROMPT_CHARS (see above).
+ * Split an absolute transcript into scene-relative words (for narration
+ * anchors and per-scene reuse). Scene i spans [prevEnd + gap, sceneEnds[i]];
+ * a word is assigned by its start time.
+ */
+function splitWordsByScene(
+  words: Word[],
+  ids: string[],
+  sceneEnds: number[],
+  gapMs: number
+): NonNullable<CaptionsManifest["scenes"]> {
+  const out: NonNullable<CaptionsManifest["scenes"]> = [];
+  let startMs = 0;
+  for (let i = 0; i < sceneEnds.length; i++) {
+    const endMs = sceneEnds[i];
+    const mine = words
+      .filter((w) => {
+        const t = w.start * 1000;
+        return t >= startMs - 1 && (i === sceneEnds.length - 1 || t < endMs + gapMs / 2);
+      })
+      .map((w) => ({
+        word: w.word.trim(),
+        start: Math.max(0, w.start - startMs / 1000),
+        end: Math.max(0, w.end - startMs / 1000),
+      }));
+    out.push({
+      id: ids[i] ?? String(i),
+      hash: hashOf({ stt: true, startMs, endMs }),
+      durationMs: endMs - startMs,
+      words: mine,
+    });
+    startMs = endMs + gapMs;
+  }
+  return out;
+}
+
+/** Scene-relative words for a scene; script-timed fallback when the manifest has none. */
+export async function sceneWordsFor(
+  project: Project,
+  sceneId: string,
+  narration: string,
+  durationMs: number
+): Promise<Word[]> {
+  const manifest = await readCaptionsManifest(project);
+  const stored = manifest?.scenes?.find((s) => s.id === sceneId);
+  if (stored && stored.words.length) return stored.words;
+  return deriveSceneWords(narration, durationMs);
+}
+
+/**
+ * Whisper prompt bias: any `knownTerms` glossary first (product names and
+ * jargon Whisper mis-hears — it only biases on roughly its first 224 tokens,
+ * so the terms must lead), then the known narration script in scene order, so
+ * the transcript converges on correctly-spelled words instead of guessing from
+ * audio phonetics alone. Capped to MAX_PROMPT_CHARS (see above).
  */
 function buildSttPrompt(storyboard: Storyboard): string | undefined {
+  const terms = (storyboard.knownTerms ?? [])
+    .map((t) => t.trim())
+    .filter(Boolean);
+  const glossary = terms.length ? `${terms.join(", ")}. ` : "";
   const text = storyboard.scenes
     .map((s) => s.narration.trim())
     .filter(Boolean)
     .join(" ");
-  if (!text) return undefined;
-  return text.length > MAX_PROMPT_CHARS ? text.slice(0, MAX_PROMPT_CHARS) : text;
+  const prompt = glossary + text;
+  if (!prompt.trim()) return undefined;
+  return prompt.length > MAX_PROMPT_CHARS
+    ? prompt.slice(0, MAX_PROMPT_CHARS)
+    : prompt;
 }
 
 /**
@@ -188,7 +298,8 @@ export async function generateCaptionsOffline(
     await readJson(project.voiceManifestPath)
   );
   const durById = new Map(voice.scenes.map((s) => [s.id, s.durationMs]));
-  const config = captionConfig(voice.gapMs);
+  const seg = segConfig(storyboard);
+  const config = captionConfig(voice.gapMs, seg);
 
   // Per-scene caption identity = narration + this scene's measured duration (+
   // grouping config). The scene-relative word timings depend on nothing else, so
@@ -269,7 +380,7 @@ export async function generateCaptionsOffline(
     cursor += voice.gapMs;
   }
 
-  const cues = groupWords(words, sceneEnds);
+  const cues = groupWords(words, sceneEnds, seg);
   await writeCaptionFiles(project, cues);
   await writeCaptionsManifest(project, {
     mode: "offline",
@@ -305,6 +416,12 @@ async function writeCaptionFiles(project: Project, cues: Cue[]): Promise<void> {
   await fs.writeFile(project.captionsSrtPath, toSrt(cues));
   await fs.writeFile(project.captionsVttPath, toVtt(cues));
   await writeJson(project.captionsCuesPath, cues);
+  // Readability, so an agent sees it without opening the SRT (issue #48).
+  const stats = cueStats(cues);
+  log(
+    `cues: ${cues.length}, ${stats.midClausePct}% mid-clause breaks, ` +
+      `${stats.short} shorter than 1s`
+  );
   ok(`captions → ${project.captionsSrtPath} (${cues.length} cues)`);
   ok(`captions → ${project.captionsCuesPath}`);
 }
@@ -317,8 +434,8 @@ async function writeCaptionFiles(project: Project, cues: Cue[]): Promise<void> {
 // ---------------------------------------------------------------------------
 
 /** Grouping parameters that affect cue segmentation — part of the cache key. */
-function captionConfig(gapMs: number): CaptionsManifest["config"] {
-  return { gapMs, maxWords: MAX_WORDS, maxCueMs: MAX_CUE_MS };
+function captionConfig(gapMs: number, cfg: SegConfig = DEFAULT_SEG): CaptionsManifest["config"] {
+  return { gapMs, maxWords: cfg.maxWords, maxCueMs: cfg.maxCueMs };
 }
 
 /** Inter-scene gap from voice.json (0 when absent — captions need voice.json). */
@@ -387,38 +504,129 @@ async function sceneIdsFor(project: Project): Promise<string[]> {
   return voice.success ? voice.data.scenes.map((s) => s.id) : [];
 }
 
-function groupWords(words: Word[], sceneEnds: number[]): Cue[] {
-  const cues: Cue[] = [];
+/** Text of a run of words, with punctuation re-attached. */
+function cueText(ws: Word[]): string {
+  // trim each word: faster-whisper servers pad words with leading spaces
+  return ws
+    .map((w) => w.word.trim())
+    .join(" ")
+    .replace(/\s+([,.!?;:])/g, "$1")
+    .trim();
+}
+
+/** A word that ends a clause (punctuation) — the best place to break. */
+function endsClause(w: Word): boolean {
+  return /[,;:—–]$/.test(w.word.trim());
+}
+
+function endsSentence(w: Word): boolean {
+  return /[.!?]["'”’)]?$/.test(w.word.trim());
+}
+
+/**
+ * Split a run of words into cues at the most readable boundaries: sentence
+ * ends first, then clause punctuation, then a conjunction, and only then on
+ * pure length. Never leaves a 2–3 word tail dangling when the previous
+ * boundary can absorb it.
+ */
+function splitRun(ws: Word[], cfg: SegConfig): Word[][] {
+  const capChars = cfg.maxLines * cfg.maxCharsPerLine;
+  const fits = (run: Word[]): boolean =>
+    run.length <= cfg.maxWords &&
+    cueText(run).length <= capChars &&
+    (run[run.length - 1].end - run[0].start) * 1000 <= cfg.maxCueMs;
+  if (ws.length === 0) return [];
+  if (fits(ws)) return [ws];
+  // The furthest index that still fits, then walk back to a real boundary.
+  let end = 1;
+  while (end < ws.length && fits(ws.slice(0, end + 1))) end++;
+  const floor = Math.max(1, Math.ceil(end * 0.45));
+  let cut = end;
+  for (let i = end - 1; i >= floor; i--) {
+    if (endsSentence(ws[i]) || endsClause(ws[i])) {
+      cut = i + 1;
+      break;
+    }
+  }
+  if (cut === end) {
+    for (let i = end - 1; i >= floor; i--) {
+      const next = ws[i + 1];
+      if (next && CONJUNCTIONS.has(next.word.trim().toLowerCase().replace(/[^\p{L}]/gu, ""))) {
+        cut = i + 1;
+        break;
+      }
+    }
+  }
+  return [ws.slice(0, cut), ...splitRun(ws.slice(cut), cfg)];
+}
+
+/** Merge a too-short cue into the neighbour it still fits in. */
+function mergeShort(runs: Word[][], cfg: SegConfig): Word[][] {
+  const capChars = cfg.maxLines * cfg.maxCharsPerLine;
+  const room = (a: Word[], b: Word[]): boolean =>
+    a.length + b.length <= cfg.maxWords &&
+    cueText([...a, ...b]).length <= capChars &&
+    (b[b.length - 1].end - a[0].start) * 1000 <= cfg.maxCueMs;
+  const out: Word[][] = [];
+  for (const run of runs) {
+    const durMs = (run[run.length - 1].end - run[0].start) * 1000;
+    const prev = out[out.length - 1];
+    if (durMs < cfg.minCueMs && prev && room(prev, run)) {
+      out[out.length - 1] = [...prev, ...run];
+      continue;
+    }
+    out.push(run);
+  }
+  return out;
+}
+
+function groupWords(words: Word[], sceneEnds: number[], cfg: SegConfig = DEFAULT_SEG): Cue[] {
+  // Scene boundaries and sentence ends are hard breaks; inside those runs the
+  // split is clause-aware.
+  const runs: Word[][] = [];
   let cur: Word[] = [];
   let sceneIdx = 0;
-  const flush = () => {
-    if (cur.length === 0) return;
-    cues.push({
-      index: cues.length + 1,
-      startMs: Math.round(cur[0].start * 1000),
-      endMs: Math.round(cur[cur.length - 1].end * 1000),
-      // trim each word: faster-whisper servers pad words with leading spaces
-      text: cur.map((w) => w.word.trim()).join(" ").replace(/\s+([,.!?])/g, "$1").trim(),
-    });
+  const cut = (): void => {
+    if (cur.length) runs.push(cur);
     cur = [];
   };
   for (const w of words) {
     const startMs = w.start * 1000;
-    // Crossed into a later scene? Break the cue there.
-    while (
-      sceneIdx < sceneEnds.length - 1 &&
-      startMs >= sceneEnds[sceneIdx]
-    ) {
-      flush();
+    while (sceneIdx < sceneEnds.length - 1 && startMs >= sceneEnds[sceneIdx]) {
+      cut();
       sceneIdx++;
     }
     cur.push(w);
-    const durMs = (w.end - cur[0].start) * 1000;
-    const endsSentence = /[.!?]$/.test(w.word.trim());
-    if (cur.length >= MAX_WORDS || durMs >= MAX_CUE_MS || endsSentence) flush();
+    if (endsSentence(w)) cut();
   }
-  flush();
-  return cues;
+  cut();
+  const split = runs.flatMap((r) => splitRun(r, cfg));
+  return mergeShort(split, cfg).map((ws, i) => ({
+    index: i + 1,
+    startMs: Math.round(ws[0].start * 1000),
+    endMs: Math.round(ws[ws.length - 1].end * 1000),
+    text: cueText(ws),
+  }));
+}
+
+/** Readability stats for the log: mid-clause breaks and too-short cues. */
+export function cueStats(cues: Cue[], minCueMs = DEFAULT_SEG.minCueMs): {
+  short: number;
+  midClause: number;
+  midClausePct: number;
+} {
+  let short = 0;
+  let midClause = 0;
+  cues.forEach((c, i) => {
+    if (c.endMs - c.startMs < minCueMs) short++;
+    const next = cues[i + 1];
+    if (!next) return;
+    const cleanBreak = /[.!?,;:—–]["'”’)]?$/.test(c.text.trim());
+    const startsLower = /^\p{Ll}/u.test(next.text.trim());
+    if (!cleanBreak && startsLower) midClause++;
+  });
+  const breaks = Math.max(1, cues.length - 1);
+  return { short, midClause, midClausePct: Math.round((midClause / breaks) * 100) };
 }
 
 function toSrt(cues: Cue[]): string {

package/src/capture.ts

@@ -572,3 +572,116 @@ export async function cropCaptureToViewport(
     dst,
   ]);
 }
+
+// ---------------------------------------------------------------------------
+// post-capture content verification
+// ---------------------------------------------------------------------------
+
+/** Downscaled grayscale thumbnail size for the content comparison. */
+const VERIFY_W = 64;
+const VERIFY_H = 36;
+/** Mean |Δgray| below this = same content even before correlating. */
+const VERIFY_MAD_PASS = 15;
+/** Minimum Pearson correlation of the two thumbnails to pass. */
+const VERIFY_CORR_PASS = 0.7;
+
+/** Decode one frame of `input` to a raw grayscale VERIFY_W×VERIFY_H buffer. */
+async function grayThumb(input: string, preArgs: string[]): Promise<Buffer> {
+  const out = `${input}.gray`;
+  await runFfmpeg([
+    ...preArgs,
+    "-i",
+    input,
+    "-vf",
+    `scale=${VERIFY_W}:${VERIFY_H}:flags=area,format=gray`,
+    "-frames:v",
+    "1",
+    "-f",
+    "rawvideo",
+    out,
+  ]);
+  const buf = await fs.readFile(out);
+  await fs.rm(out, { force: true });
+  return buf;
+}
+
+/**
+ * Verify that the CONTENT of the cropped capture is the page that was driven
+ * (issue #21). The geometry gates above prove the crop *rectangle* is right,
+ * but not that the right *pixels* were under it: a display-region screen grab
+ * records whatever is painted at those coordinates, so a browser window that
+ * is occluded (another app raised mid-take), on another display, or behind a
+ * notification produces a take that passes every geometry check and still
+ * shows someone else's window — silent garbage, and a privacy hazard, since
+ * the leaked pixels ship in an otherwise "publish-ready" video.
+ *
+ * The reference is a CDP screenshot of the page taken at capture-stop time
+ * (the page's own buffer — immune to occlusion). Both it and the tail frame
+ * of the cropped capture are reduced to a small grayscale thumbnail (ffmpeg
+ * rawvideo — no image deps) and compared: near-identical mean gray delta
+ * passes outright; otherwise the Pearson correlation must clear a threshold
+ * (robust to the gamma/color-profile shift between a screen grab and a CDP
+ * screenshot). A mismatch ABORTS like the geometry gates
+ * (AIDEMO_NATIVE_CROP_UNSAFE=1 downgrades to a loud warning).
+ */
+export async function verifyCaptureMatchesPage(
+  croppedVideo: string,
+  referencePng: string
+): Promise<void> {
+  // Tail frame: the reference screenshot was taken just before capture stop,
+  // and the page is static by then (the last scene has ended).
+  const vid = await grayThumb(croppedVideo, ["-sseof", "-0.3"]);
+  const ref = await grayThumb(referencePng, []);
+  const n = Math.min(vid.length, ref.length);
+  if (n === 0) {
+    log("⚠ capture verification skipped (empty thumbnail) — frame-review the take");
+    return;
+  }
+  let sumAbs = 0;
+  let sumA = 0;
+  let sumB = 0;
+  for (let i = 0; i < n; i++) {
+    sumAbs += Math.abs(vid[i] - ref[i]);
+    sumA += vid[i];
+    sumB += ref[i];
+  }
+  const mad = sumAbs / n;
+  const meanA = sumA / n;
+  const meanB = sumB / n;
+  let cov = 0;
+  let varA = 0;
+  let varB = 0;
+  for (let i = 0; i < n; i++) {
+    const da = vid[i] - meanA;
+    const db = ref[i] - meanB;
+    cov += da * db;
+    varA += da * da;
+    varB += db * db;
+  }
+  // Degenerate (near-uniform) thumbnails make correlation meaningless — the
+  // MAD pass above is the only meaningful check for e.g. a blank white page.
+  const corr =
+    varA > 1e-6 && varB > 1e-6 ? cov / Math.sqrt(varA * varB) : NaN;
+  log(
+    `capture verification: mean |Δgray| ${mad.toFixed(1)} (pass ≤ ${VERIFY_MAD_PASS}), ` +
+      `correlation ${Number.isNaN(corr) ? "n/a (uniform frame)" : corr.toFixed(3)} (pass ≥ ${VERIFY_CORR_PASS})`
+  );
+  if (mad <= VERIFY_MAD_PASS || (!Number.isNaN(corr) && corr >= VERIFY_CORR_PASS)) {
+    return;
+  }
+  gateGeometryProblems(
+    [
+      `the captured pixels do not look like the page that was driven ` +
+        `(mean gray delta ${mad.toFixed(1)}, correlation ${Number.isNaN(corr) ? "n/a" : corr.toFixed(3)}) — ` +
+        `the browser window was likely occluded by another window, on a different ` +
+        `display than the captured one, or covered by a notification during the take`,
+    ],
+    `Native capture verification failed: the recording shows something other than ` +
+      `the driven page. The cropped take (${croppedVideo}) and the page reference ` +
+      `(${referencePng}) were kept — compare them side by side, and treat the take ` +
+      `as sensitive until reviewed (it may show other windows). Re-record with the ` +
+      `browser window front-most and unoccluded on the captured display, or use the ` +
+      `default --capture playwright (records the page buffer directly — immune to ` +
+      `occlusion).`
+  );
+}

package/src/cards.ts

@@ -1,5 +1,5 @@
 import { chromium } from "playwright";
-import type { Card } from "./types.js";
+import type { Brand, Card } from "./types.js";
 
 /**
  * Rasterizes an intro/outro title card to a PNG with headless Chrome — same
@@ -13,7 +13,9 @@ export async function renderCardPng(
   outPath: string,
   logicalW: number,
   logicalH: number,
-  scale: number
+  scale: number,
+  brand?: Brand,
+  logoDataUri?: string
 ): Promise<void> {
   const browser = await chromium.launch({ channel: "chrome", headless: true });
   const context = await browser.newContext({
@@ -22,7 +24,7 @@ export async function renderCardPng(
   });
   const page = await context.newPage();
   try {
-    await page.setContent(html(card, logicalW), { waitUntil: "load" });
+    await page.setContent(html(card, logicalW, brand, logoDataUri), { waitUntil: "load" });
     await page.screenshot({ path: outPath });
   } finally {
     await browser.close();
@@ -36,21 +38,27 @@ function esc(text: string): string {
     .replace(/>/g, "&gt;");
 }
 
-function html(card: Card, logicalW: number): string {
+function html(card: Card, logicalW: number, brand?: Brand, logoDataUri?: string): string {
   const background =
     card.background ??
     "radial-gradient(120% 140% at 20% 0%, #182036 0%, #0b0d16 55%, #07080f 100%)";
-  const accent = card.accent ?? "#6c8cff";
+  const accent = card.accent ?? brand?.accent ?? "#6c8cff";
+  const font = brand?.font ?? '-apple-system,"Helvetica Neue",Arial,sans-serif';
   const titleSize = Math.round(logicalW / 17);
   const subSize = Math.round(logicalW / 40);
   const subtitle = card.subtitle
     ? `<div class="sub">${esc(card.subtitle)}</div>`
     : "";
+  const logo = logoDataUri
+    ? `<img class="logo" src="${logoDataUri}" alt="">`
+    : "";
   return `<!doctype html><html><head><meta charset="utf-8"><style>
     html,body{margin:0;padding:0;width:100%;height:100%;}
     body{background:${background};display:flex;align-items:center;justify-content:center;}
+    .logo{display:block;margin:0 auto ${Math.round(subSize * 1.4)}px;
+      max-width:${Math.round(logicalW * 0.18)}px;max-height:${Math.round(logicalW * 0.08)}px;}
     .box{max-width:78%;text-align:center;
-      font-family:-apple-system,"Helvetica Neue",Arial,sans-serif;
+      font-family:${font};
       -webkit-font-smoothing:antialiased;}
     .rule{width:${Math.round(logicalW / 22)}px;height:5px;border-radius:3px;
       background:${accent};margin:0 auto ${Math.round(subSize * 1.2)}px;}
@@ -60,6 +68,7 @@ function html(card: Card, logicalW: number): string {
     .sub{margin-top:${Math.round(subSize * 0.9)}px;font-size:${subSize}px;
       line-height:1.4;font-weight:450;color:rgba(226,231,244,.72);}
   </style></head><body><div class="box">
+    ${logo}
     <div class="rule"></div>
     <div class="title">${esc(card.title)}</div>
     ${subtitle}