@tandryukha/aidemo 0.8.0 → 0.14.1

package/src/frame.ts

@@ -0,0 +1,205 @@
+/**
+ * Produced-look frame: padding, background, rounded corners, drop shadow and
+ * an optional browser/mac chrome bar around the content — the "Screen Studio"
+ * look, done with the same portable trick as cards and captions. Headless
+ * Chrome rasterizes ONE canvas-sized PNG whose video area is a transparent
+ * rounded hole (CSS mask); compose pads the content onto a solid canvas and
+ * overlays that PNG. Two baseline filters (`pad`, `overlay`), no drawbox.
+ */
+
+import { chromium } from "playwright";
+import type { Brand, Frame } from "./types.js";
+
+export interface FrameLayout {
+  /** Canvas size in output px (even). */
+  canvasW: number;
+  canvasH: number;
+  /** Where the content lands on the canvas, output px. */
+  offsetX: number;
+  offsetY: number;
+  /** Solid fallback color used under the PNG (matches the background's base). */
+  padColor: string;
+}
+
+const CHROME_H = 40;
+/** Default `safeTop` strip: an iPhone status bar's worth of bezel. */
+const SAFE_TOP = 46;
+/** Device bezel thickness (logical px) for phone chrome. */
+const BEZEL = 14;
+const DEFAULT_BG =
+  "linear-gradient(135deg, #1b2440 0%, #0e1322 55%, #0a0d18 100%)";
+
+export function isDeviceChrome(chrome: Frame["chrome"]): boolean {
+  return chrome === "iphone" || chrome === "android";
+}
+
+/**
+ * Status-bar strip reserved above the video for device chrome, logical px.
+ * Without it the island / punch-hole is drawn OVER the recording's top pixels
+ * (fine for a page that starts with whitespace, wrong for an app that paints a
+ * header there); with it the camera cut-out sits on bezel instead.
+ */
+function safeTopPx(frame: Frame): number {
+  if (!isDeviceChrome(frame.chrome) || frame.safeTop == null || frame.safeTop === false) return 0;
+  return frame.safeTop === true ? SAFE_TOP : Math.round(frame.safeTop);
+}
+
+function esc(text: string): string {
+  return text.replace(/&/g, "&amp;").replace(/</g, "&lt;").replace(/>/g, "&gt;");
+}
+
+/** Geometry only (no browser) — compose needs it before rendering. */
+export function frameLayout(
+  frame: Frame,
+  videoW: number,
+  videoH: number,
+  pxScale: number,
+  aspect?: number
+): FrameLayout {
+  const device = isDeviceChrome(frame.chrome);
+  // A bezel needs room; the default padding already has it.
+  const pad = Math.max(frame.padding ?? 48, device ? BEZEL + 8 : 0);
+  const chromeH = frame.chrome && frame.chrome !== "none" && !device ? CHROME_H : 0;
+  // The bar and the device status strip are mutually exclusive (one is browser
+  // chrome, the other device chrome) — both push the video down the same way.
+  const topInset = chromeH + safeTopPx(frame);
+  const even = (n: number) => Math.round(n) & ~1;
+  // Logical canvas; when the output has a target aspect (preset or explicit
+  // size) grow the padding on one axis so the canvas already matches it and
+  // the final resize is a pure scale — no letterbox bars around a frame.
+  let W = videoW + 2 * pad;
+  let H = videoH + 2 * pad + topInset;
+  if (aspect && aspect > 0) {
+    if (W / H < aspect) W = H * aspect;
+    else H = W / aspect;
+  }
+  const padX = (W - videoW) / 2;
+  const padY = (H - videoH - topInset) / 2;
+  return {
+    canvasW: even(W * pxScale),
+    canvasH: even(H * pxScale),
+    offsetX: Math.round(padX * pxScale),
+    offsetY: Math.round((padY + topInset) * pxScale),
+    padColor: "#0e1322",
+  };
+}
+
+/**
+ * Rasterize the frame PNG: canvas background + shadow + chrome, with a
+ * rounded transparent hole exactly where the video sits.
+ */
+export async function renderFramePng(
+  frame: Frame,
+  brand: Brand | undefined,
+  outPath: string,
+  videoW: number,
+  videoH: number,
+  pxScale: number,
+  aspect?: number
+): Promise<FrameLayout> {
+  const layout = frameLayout(frame, videoW, videoH, pxScale, aspect);
+  const chrome = frame.chrome ?? "none";
+  const device = isDeviceChrome(chrome);
+  const radius = frame.radius ?? (device ? 40 : 14);
+  const chromeH = chrome !== "none" && !device ? CHROME_H : 0;
+  const strip = safeTopPx(frame);
+  const topInset = chromeH + strip;
+  const accent = brand?.accent ?? "#6c8cff";
+  const background =
+    frame.background ??
+    (brand?.accent
+      ? `linear-gradient(135deg, ${accent} 0%, #0e1322 70%, #0a0d18 100%)`
+      : DEFAULT_BG);
+  const font = brand?.font ?? '-apple-system,"Helvetica Neue",Arial,sans-serif';
+  const W = Math.round(layout.canvasW / pxScale);
+  const H = Math.round(layout.canvasH / pxScale);
+  // Hole = the video rect; the chrome bar sits directly above it, sharing the
+  // window's rounded top corners (the video keeps the rounded bottom ones).
+  const winX = Math.round(layout.offsetX / pxScale);
+  const winY = Math.round(layout.offsetY / pxScale) - topInset;
+  const winW = videoW;
+  const winH = videoH + topInset;
+  const holeY = winY + topInset;
+  // A browser bar owns the window's top corners, so the video below it is
+  // square there. A device status strip is screen, not chrome: keep the corners
+  // rounded — the sliver outside them is bezel-black, exactly like the real
+  // rounded display corner.
+  const rTop = chromeH ? 0 : radius;
+  // SVG mask (alpha): opaque everywhere except the rounded video rect.
+  const hole =
+    `M${winX} ${holeY + rTop} ` +
+    `a${rTop} ${rTop} 0 0 1 ${rTop} -${rTop} h${winW - 2 * rTop} ` +
+    `a${rTop} ${rTop} 0 0 1 ${rTop} ${rTop} v${videoH - rTop - radius} ` +
+    `a${radius} ${radius} 0 0 1 -${radius} ${radius} h-${winW - 2 * radius} ` +
+    `a${radius} ${radius} 0 0 1 -${radius} -${radius} z`;
+  const mask =
+    `<svg xmlns='http://www.w3.org/2000/svg' width='${W}' height='${H}'>` +
+    `<path fill='white' fill-rule='evenodd' d='M0 0h${W}v${H}H0z ${hole}'/></svg>`;
+  const maskUrl = `url("data:image/svg+xml;utf8,${encodeURIComponent(mask)}")`;
+  const shadow =
+    frame.shadow === false ? "none" : "0 24px 60px rgba(0,0,0,.55), 0 2px 8px rgba(0,0,0,.35)";
+  let chromeHtml = "";
+  let deviceHtml = "";
+  let deviceCss = "";
+  if (device) {
+    // Bezel behind the hole (masked with the canvas) + an unmasked camera
+    // cut-out drawn over the video: a dynamic island or a punch-hole.
+    const bx = winX - BEZEL;
+    const by = winY - BEZEL;
+    deviceCss =
+      `.bezel{position:absolute;left:${bx}px;top:${by}px;width:${winW + 2 * BEZEL}px;height:${winH + 2 * BEZEL}px;` +
+      `border-radius:${radius + BEZEL}px;background:#0b0b0d;border:2px solid #2b2d33;box-shadow:${shadow};}` +
+      `.cam{position:absolute;background:#000;}` +
+      // With a safe strip the cut-out is centered IN the strip (over bezel);
+      // without one it keeps its old place, drawn over the video's top pixels.
+      (chrome === "iphone"
+        ? `.island{left:${winX + winW / 2 - 55}px;top:${strip ? winY + Math.max(2, (strip - 32) / 2) : winY + 12}px;width:110px;height:32px;border-radius:16px;}`
+        : `.hole{left:${winX + winW / 2 - 7}px;top:${strip ? winY + Math.max(2, (strip - 14) / 2) : winY + 14}px;width:14px;height:14px;border-radius:50%;}`);
+    deviceHtml = `<div class="cam ${chrome === "iphone" ? "island" : "hole"}"></div>`;
+  }
+  if (chromeH) {
+    const lights =
+      chrome === "mac"
+        ? `<span class="l" style="background:#ff5f57"></span><span class="l" style="background:#febc2e"></span><span class="l" style="background:#28c840"></span>`
+        : `<span class="l" style="background:#3a4157"></span><span class="l" style="background:#3a4157"></span><span class="l" style="background:#3a4157"></span>`;
+    const label = frame.url ?? frame.title ?? "";
+    chromeHtml =
+      `<div class="bar"><div class="lights">${lights}</div>` +
+      (label ? `<div class="pill">${esc(label)}</div>` : "") +
+      `</div>`;
+  }
+  const html = `<!doctype html><html><head><meta charset="utf-8"><style>
+    html,body{margin:0;padding:0;background:transparent;width:${W}px;height:${H}px;overflow:hidden;}
+    *{box-sizing:border-box;font-family:${font};-webkit-font-smoothing:antialiased;}
+    .canvas{position:absolute;inset:0;background:${background};
+      -webkit-mask-image:${maskUrl};mask-image:${maskUrl};}
+    .win{position:absolute;left:${winX}px;top:${winY}px;width:${winW}px;height:${winH}px;
+      border-radius:${radius}px;box-shadow:${device ? "none" : shadow};background:${device ? "#0b0b0d" : "#0b0e18"};}
+    ${deviceCss}
+    .bar{position:absolute;left:${winX}px;top:${winY}px;width:${winW}px;height:${chromeH}px;
+      border-radius:${radius}px ${radius}px 0 0;background:#1a1f2e;
+      display:flex;align-items:center;padding:0 14px;gap:14px;
+      border-bottom:1px solid rgba(255,255,255,.06);}
+    .lights{display:flex;gap:8px;}
+    .l{width:12px;height:12px;border-radius:50%;display:inline-block;}
+    .pill{flex:1;max-width:60%;margin:0 auto;height:24px;line-height:24px;border-radius:7px;
+      background:rgba(255,255,255,.07);color:rgba(230,234,245,.75);font-size:13px;
+      text-align:center;overflow:hidden;white-space:nowrap;text-overflow:ellipsis;padding:0 12px;}
+  </style></head><body>
+    <div class="canvas">${device ? '<div class="bezel"></div>' : ""}<div class="win"></div>${chromeHtml}</div>${deviceHtml}
+  </body></html>`;
+
+  const browser = await chromium.launch({ channel: "chrome", headless: true });
+  const context = await browser.newContext({
+    viewport: { width: W, height: H },
+    deviceScaleFactor: pxScale,
+  });
+  const page = await context.newPage();
+  try {
+    await page.setContent(html, { waitUntil: "load" });
+    await page.screenshot({ path: outPath, omitBackground: true });
+  } finally {
+    await browser.close();
+  }
+  return layout;
+}

package/src/frames.ts

@@ -0,0 +1,159 @@
+/**
+ * `aidemo frames <dir>` — dump evenly spaced PNG frames from a take or the
+ * final video for agent-side review (issue #44). Reviewing a render used to
+ * mean hand-typed `ffmpeg -ss` per frame; this writes `output/frames/<source>-
+ * <mm-ss>.png` at a fixed cadence, sized for a vision model, and prints the
+ * paths.
+ */
+
+import { promises as fs } from "node:fs";
+import { join, resolve, relative } from "node:path";
+import { runFfmpeg, probeDurationMs } from "./ffmpeg.js";
+import type { Project } from "./project.js";
+import { ensureDir, exists, readJson } from "./util.js";
+import { TimelineSchema } from "./types.js";
+
+export interface FramesOptions {
+  /** Seconds between frames. Default 3. */
+  everySec?: number;
+  /**
+   * "final" (output/final-demo.mp4, default), "raw" (the latest raw file), or
+   * "take" (the whole recorded take — follows the timeline across the raw
+   * files a resumed take is spliced from, which `raw` cannot see).
+   */
+  source?: "final" | "raw" | "take";
+  /** Output width in px (aspect kept). Default 640 — legible, cheap to look at. */
+  width?: number;
+  /** Output directory. Default <demo>/output/frames. */
+  outDir?: string;
+}
+
+export interface FramesResult {
+  source: string;
+  durationMs: number;
+  everySec: number;
+  files: string[];
+}
+
+/** `mm-ss` (plus tenths when the cadence isn't whole seconds) — unique + sortable. */
+function stamp(sec: number, whole: boolean): string {
+  const m = Math.floor(sec / 60);
+  const s = sec - m * 60;
+  const ss = whole
+    ? String(Math.round(s)).padStart(2, "0")
+    : s.toFixed(1).padStart(4, "0");
+  return `${String(m).padStart(2, "0")}-${ss}`;
+}
+
+export async function extractFrames(
+  project: Project,
+  opts: FramesOptions = {}
+): Promise<FramesResult> {
+  const everySec = Math.max(0.2, opts.everySec ?? 3);
+  const width = Math.max(64, Math.round(opts.width ?? 640));
+  const kind = opts.source ?? "final";
+  if (kind === "take") return takeFrames(project, everySec, width, opts.outDir);
+  const source = kind === "raw" ? await project.resolveRawVideo() : project.outputPath;
+  if (!(await exists(source))) {
+    throw new Error(
+      kind === "raw"
+        ? `no take at ${source} — run 'aidemo record <dir>' (or probe) first`
+        : `no video at ${source} — run 'aidemo render <dir>' (or compose) first`
+    );
+  }
+  const outDir = opts.outDir ?? project.p("output", "frames");
+  await ensureDir(outDir);
+  // Clear this source's previous dump so stale frames don't mix in.
+  for (const f of await fs.readdir(outDir).catch(() => [] as string[])) {
+    if (f.startsWith(`${kind}-`) && f.endsWith(".png")) {
+      await fs.rm(join(outDir, f), { force: true });
+    }
+  }
+  const durationMs = await probeDurationMs(source);
+  const pattern = join(outDir, `${kind}-%04d.png`);
+  await runFfmpeg([
+    "-i",
+    source,
+    "-vf",
+    `fps=1/${everySec},scale=${width}:-2`,
+    "-vsync",
+    "vfr",
+    pattern,
+  ]);
+  // ffmpeg numbers frames 0001…; rename to the timestamp each one represents.
+  const produced = (await fs.readdir(outDir))
+    .filter((f) => new RegExp(`^${kind}-\\d{4}\\.png$`).test(f))
+    .sort();
+  const files: string[] = [];
+  const whole = Number.isInteger(everySec);
+  for (const [i, f] of produced.entries()) {
+    const at = i * everySec;
+    const name = `${kind}-${stamp(at, whole)}.png`;
+    await fs.rename(join(outDir, f), join(outDir, name));
+    files.push(join(outDir, name));
+  }
+  return { source, durationMs, everySec, files };
+}
+
+/**
+ * `--source take`: sample the RECORDED take end to end. A resumed take
+ * (`record --from-scene`) is spliced from more than one raw file, so `raw`
+ * shows only the newest tail; this walks the timeline scene by scene, seeking
+ * into whichever file each scene names, and stamps the frames with the scene
+ * they came from.
+ */
+async function takeFrames(
+  project: Project,
+  everySec: number,
+  width: number,
+  outDirOpt?: string
+): Promise<FramesResult> {
+  const timeline = TimelineSchema.parse(await readJson(project.timelinePath));
+  const rawVideo = await project.resolveRawVideo();
+  const outDir = outDirOpt ?? project.p("output", "frames");
+  await ensureDir(outDir);
+  for (const f of await fs.readdir(outDir).catch(() => [] as string[])) {
+    if (f.startsWith("take-") && f.endsWith(".png")) await fs.rm(join(outDir, f), { force: true });
+  }
+  const files: string[] = [];
+  const whole = Number.isInteger(everySec);
+  const sources = new Set<string>();
+  let elapsedMs = 0; // content time across the whole take
+  let carryMs = 0; // ms owed to the next scene so the cadence never resets
+  for (const scene of timeline.scenes) {
+    const file = scene.source ? resolve(project.dir, scene.source) : rawVideo;
+    if (!(await exists(file))) {
+      throw new Error(`take frames: scene "${scene.id}" points at ${file}, which is gone`);
+    }
+    sources.add(file);
+    const leadInMs = scene.leadInMs ?? timeline.leadInMs;
+    const sceneMs = Math.max(0, scene.endMs - scene.startMs);
+    for (let at = carryMs; at < sceneMs; at += everySec * 1000) {
+      const seek = (scene.startMs + leadInMs + at) / 1000;
+      const name = `take-${stamp((elapsedMs + at) / 1000, whole)}-${scene.id}.png`;
+      await runFfmpeg([
+        "-ss",
+        seek.toFixed(3),
+        "-i",
+        file,
+        "-frames:v",
+        "1",
+        "-vf",
+        `scale=${width}:-2`,
+        join(outDir, name),
+      ]);
+      files.push(join(outDir, name));
+    }
+    // Where the next sample falls inside the following scene.
+    const stepMs = everySec * 1000;
+    const shot = Math.max(0, Math.ceil((sceneMs - carryMs) / stepMs));
+    carryMs = Math.max(0, carryMs + shot * stepMs - sceneMs);
+    elapsedMs += sceneMs;
+  }
+  return {
+    source: [...sources].map((f) => relative(project.dir, f)).join(", "),
+    durationMs: elapsedMs,
+    everySec,
+    files,
+  };
+}

package/src/golden.ts

@@ -7,6 +7,7 @@ import {
   type Storyboard,
 } from "./types.js";
 import { writeJson, exists } from "./util.js";
+import { join } from "node:path";
 
 /**
  * Golden probe/timeline diffing — a demo doubles as a regression test (vhs's
@@ -107,3 +108,29 @@ function valuesEqual(a: unknown, b: unknown): boolean {
 function fmt(v: unknown): string {
   return v === undefined ? "(absent)" : JSON.stringify(v);
 }
+
+/**
+ * For a golden diff, the drift-suggestion files the probe just wrote for the
+ * actions whose `ok`/`found` flipped (`logs/drift-<scene>-<n>.json`, see
+ * player diagnostics) — so the fix is one edit away. Only existing files.
+ */
+export async function driftFilesForDiff(
+  project: Project,
+  golden: ProbeGolden,
+  diffs: string[]
+): Promise<Array<{ scene: string; action: number; file: string }>> {
+  const out: Array<{ scene: string; action: number; file: string }> = [];
+  const seen = new Set<string>();
+  for (const d of diffs) {
+    const m = /^\$\.scenes\[(\d+)\]\.actions\[(\d+)\]\.(ok|found):/.exec(d);
+    if (!m) continue;
+    const scene = golden.scenes[Number(m[1])];
+    if (!scene) continue;
+    const action = Number(m[2]);
+    const file = join(project.dir, "logs", `drift-${scene.id}-${action + 1}.json`);
+    if (seen.has(file) || !(await exists(file))) continue;
+    seen.add(file);
+    out.push({ scene: scene.id, action: action + 1, file });
+  }
+  return out;
+}

package/src/guide.ts

@@ -0,0 +1,102 @@
+/**
+ * Topic-scoped slices of docs/AUTHORING.md. The full guide is ~1000 lines;
+ * agents usually need one slice (schema, attention, ChatGPT-app facts…).
+ * Slicing is by H2 heading so the doc stays the single source of truth —
+ * topics are just named sets of H2 prefixes.
+ */
+
+import { readFile } from "node:fs/promises";
+import { resolve } from "node:path";
+import { ENGINE_ROOT } from "./config.js";
+
+export const GUIDE_TOPICS = {
+  core: [
+    "Interfaces",
+    "Pipeline",
+    "Steps",
+    "Demo-director principles",
+    "Storyboard schema",
+    "Action vocabulary",
+    "Verify before declaring done",
+  ],
+  schema: ["Storyboard schema", "Action vocabulary", "Narration-anchored beats"],
+  polish: [
+    "Demo-director principles",
+    "Narration-anchored beats",
+    "Transitions, output sizing",
+    "Attention",
+    "Motion blur & cursor",
+    "Produced look",
+  ],
+  attention: ["Attention", "Motion blur & cursor"],
+  "chatgpt-apps": ["ChatGPT Apps SDK recording"],
+  stills: ["Stills"],
+  variants: ["Parameterized storyboards", "Multi-language renders"],
+  captions: ["Captions:", "Multi-language renders"],
+  debug: ["Verify before declaring done", "Debugging", "Demo as regression test"],
+  ci: ["Demo as regression test", "Interfaces"],
+} as const;
+
+export type GuideTopic = keyof typeof GUIDE_TOPICS;
+export const GUIDE_TOPIC_NAMES = Object.keys(GUIDE_TOPICS) as GuideTopic[];
+
+export const guidePath = (): string => resolve(ENGINE_ROOT, "docs", "AUTHORING.md");
+
+export async function readGuide(): Promise<string> {
+  return readFile(guidePath(), "utf8");
+}
+
+interface Section {
+  heading: string;
+  body: string;
+}
+
+/** Split the guide into its H2 sections (the preamble before the first H2 is heading ""). */
+export function splitGuide(md: string): Section[] {
+  const out: Section[] = [];
+  let heading = "";
+  let buf: string[] = [];
+  let inFence = false;
+  for (const line of md.split("\n")) {
+    if (/^```/.test(line)) inFence = !inFence;
+    if (!inFence && /^## /.test(line)) {
+      out.push({ heading, body: buf.join("\n") });
+      heading = line.slice(3).trim();
+      buf = [line];
+      continue;
+    }
+    buf.push(line);
+  }
+  out.push({ heading, body: buf.join("\n") });
+  return out;
+}
+
+/** Headings in the doc, in order (for `guide --list`, MCP hints). */
+export function guideHeadings(md: string): string[] {
+  return splitGuide(md)
+    .map((s) => s.heading)
+    .filter(Boolean);
+}
+
+/**
+ * Slice the guide to a topic (or to any H2 whose heading starts with the
+ * given text, case-insensitive). Returns the sections in document order
+ * plus a one-line pointer to the other topics; unknown topic → null.
+ */
+export function sliceGuide(md: string, topic: string): string | null {
+  const sections = splitGuide(md);
+  const key = topic.trim().toLowerCase();
+  const prefixes: readonly string[] =
+    (GUIDE_TOPICS as Record<string, readonly string[]>)[key] ?? [topic.trim()];
+  const picked = sections.filter(
+    (s) => s.heading && prefixes.some((p) => s.heading.toLowerCase().startsWith(p.toLowerCase()))
+  );
+  if (!picked.length) return null;
+  const others = GUIDE_TOPIC_NAMES.filter((t) => t !== key).join(", ");
+  return (
+    `<!-- aidemo authoring guide · topic "${topic}" (${picked.length} section${picked.length === 1 ? "" : "s"}). ` +
+    `Other topics: ${others}; omit topic for the full guide. -->\n\n` +
+    picked.map((s) => s.body.trimEnd()).join("\n\n") +
+    "\n"
+  );
+}

package/src/i18n.ts

@@ -62,10 +62,13 @@ export function localizeStoryboard(sb: Storyboard, lang: string): Storyboard {
     ...sb,
     intro: localizeCard(sb.intro, lang),
     outro: localizeCard(sb.outro, lang),
-    scenes: sb.scenes.map((scene) => ({
-      ...scene,
-      narration: scene.narrations?.[lang] ?? scene.narration,
-    })),
+    scenes: sb.scenes.map((scene) => {
+      const translated = scene.narrations?.[lang];
+      if (translated == null) return scene;
+      const anchors = scene.narrationAnchors?.[lang];
+      const { anchors: _base, ...rest } = scene;
+      return { ...rest, narration: translated, ...(anchors ? { anchors } : {}) };
+    }),
   };
 }