@tandryukha/aidemo 0.14.1 → 0.15.0

package/src/ffmpeg.ts

@@ -171,6 +171,74 @@ export function probeFlatTailMs(
   });
 }
 
+/**
+ * Mirror of `probeFlatTailMs` for the HEAD of a file: how many ms at the start
+ * are a flat, contentless frame (the browser's white pre-paint after a `goto`).
+ * Used to open the video on the product instead of on a blank page, and to pick
+ * a poster frame that isn't a white rectangle (issue #47). Best-effort: any
+ * probe failure resolves to 0.
+ */
+export function probeFlatLeadMs(
+  file: string,
+  windowMs = 4000,
+  startMs = 0
+): Promise<number> {
+  return new Promise((resolvePromise) => {
+    const proc = spawn(
+      "ffmpeg",
+      [
+        "-hide_banner",
+        "-nostats",
+        ...(startMs > 0 ? ["-ss", (startMs / 1000).toFixed(3)] : []),
+        "-i",
+        file,
+        "-t",
+        (windowMs / 1000).toFixed(3),
+        "-vf",
+        "signalstats,metadata=print:file=-",
+        "-f",
+        "null",
+        "-",
+      ],
+      { stdio: ["ignore", "pipe", "ignore"] }
+    );
+    let out = "";
+    proc.stdout.on("data", (d) => (out += d.toString()));
+    proc.on("error", () => resolvePromise(0));
+    proc.on("close", () => {
+      const frames: Array<{ t: number; spread: number }> = [];
+      let t = NaN;
+      let low = NaN;
+      let high = NaN;
+      const flush = (): void => {
+        if (!Number.isNaN(t) && !Number.isNaN(low) && !Number.isNaN(high)) {
+          frames.push({ t, spread: high - low });
+        }
+        low = NaN;
+        high = NaN;
+      };
+      for (const line of out.split("\n")) {
+        const ts = /pts_time:\s*([\d.]+)/.exec(line);
+        if (ts) {
+          flush();
+          t = parseFloat(ts[1]);
+          continue;
+        }
+        const lo = /YLOW=\s*([\d.]+)/.exec(line);
+        if (lo) low = parseFloat(lo[1]);
+        const hi = /YHIGH=\s*([\d.]+)/.exec(line);
+        if (hi) high = parseFloat(hi[1]);
+      }
+      flush();
+      const FLAT_SPREAD = 4;
+      if (!frames.length || frames[0].spread > FLAT_SPREAD) return resolvePromise(0);
+      let i = 0;
+      while (i + 1 < frames.length && frames[i + 1].spread <= FLAT_SPREAD) i++;
+      resolvePromise(Math.max(0, Math.round(frames[i].t * 1000)));
+    });
+  });
+}
+
 /** A motionless span found by `freezedetect`, in ms from the file's start. */
 export interface FreezeSpan {
   startMs: number;

package/src/frames.ts

@@ -26,6 +26,14 @@ export interface FramesOptions {
   width?: number;
   /** Output directory. Default <demo>/output/frames. */
   outDir?: string;
+  /**
+   * Also tile the extracted frames into one contact sheet PNG
+   * (`output/frames/<source>-sheet.png`) — the single most useful review
+   * artifact for a finished cut (issue #52), and one image instead of 40.
+   */
+  sheet?: boolean;
+  /** Columns in the contact sheet. Default 5. */
+  sheetColumns?: number;
 }
 
 export interface FramesResult {
@@ -33,6 +41,8 @@ export interface FramesResult {
   durationMs: number;
   everySec: number;
   files: string[];
+  /** The contact sheet, when `sheet` was asked for. */
+  sheet?: string;
 }
 
 /** `mm-ss` (plus tenths when the cadence isn't whole seconds) — unique + sortable. */
@@ -52,7 +62,7 @@ export async function extractFrames(
   const everySec = Math.max(0.2, opts.everySec ?? 3);
   const width = Math.max(64, Math.round(opts.width ?? 640));
   const kind = opts.source ?? "final";
-  if (kind === "take") return takeFrames(project, everySec, width, opts.outDir);
+  if (kind === "take") return takeFrames(project, everySec, width, opts.outDir, opts);
   const source = kind === "raw" ? await project.resolveRawVideo() : project.outputPath;
   if (!(await exists(source))) {
     throw new Error(
@@ -76,8 +86,8 @@ export async function extractFrames(
     source,
     "-vf",
     `fps=1/${everySec},scale=${width}:-2`,
-    "-vsync",
-    "vfr",
+    // No `-vsync vfr` here: the `fps` filter already sets the rate, and
+    // ffmpeg 8 removed the option outright (portability baseline).
     pattern,
   ]);
   // ffmpeg numbers frames 0001…; rename to the timestamp each one represents.
@@ -92,7 +102,41 @@ export async function extractFrames(
     await fs.rename(join(outDir, f), join(outDir, name));
     files.push(join(outDir, name));
   }
-  return { source, durationMs, everySec, files };
+  const sheet = opts.sheet
+    ? await buildSheet(files, join(outDir, `${kind}-sheet.png`), opts.sheetColumns)
+    : undefined;
+  return { source, durationMs, everySec, files, ...(sheet ? { sheet } : {}) };
+}
+
+/**
+ * Tile frames into one contact sheet (ffmpeg `tile`). Uses the already-scaled
+ * PNGs as an image sequence, so no second decode of the video.
+ */
+export async function buildSheet(
+  files: string[],
+  outPath: string,
+  columns = 5
+): Promise<string | undefined> {
+  if (!files.length) return undefined;
+  const cols = Math.max(1, Math.min(10, columns));
+  const rows = Math.ceil(files.length / cols);
+  const inputs = files.flatMap((f) => ["-i", f]);
+  const chain =
+    files.map((_, i) => `[${i}:v]scale=480:-2,pad=iw+8:ih+8:4:4:color=0x11141c[t${i}]`).join(";") +
+    ";" +
+    files.map((_, i) => `[t${i}]`).join("") +
+    `concat=n=${files.length}:v=1:a=0,tile=${cols}x${rows}[sheet]`;
+  await runFfmpeg([
+    ...inputs,
+    "-filter_complex",
+    chain,
+    "-map",
+    "[sheet]",
+    "-frames:v",
+    "1",
+    outPath,
+  ]);
+  return outPath;
 }
 
 /**
@@ -106,7 +150,8 @@ async function takeFrames(
   project: Project,
   everySec: number,
   width: number,
-  outDirOpt?: string
+  outDirOpt?: string,
+  sheetOpts?: Pick<FramesOptions, "sheet" | "sheetColumns">
 ): Promise<FramesResult> {
   const timeline = TimelineSchema.parse(await readJson(project.timelinePath));
   const rawVideo = await project.resolveRawVideo();
@@ -150,10 +195,14 @@ async function takeFrames(
     carryMs = Math.max(0, carryMs + shot * stepMs - sceneMs);
     elapsedMs += sceneMs;
   }
+  const sheet = sheetOpts?.sheet
+    ? await buildSheet(files, join(outDir, "take-sheet.png"), sheetOpts.sheetColumns)
+    : undefined;
   return {
     source: [...sources].map((f) => relative(project.dir, f)).join(", "),
     durationMs: elapsedMs,
     everySec,
     files,
+    ...(sheet ? { sheet } : {}),
   };
 }

package/src/import-trace.ts

@@ -274,6 +274,17 @@ function stepsFromTrace(entries: Map<string, Buffer>, notes: string[]): Step[] {
   return steps;
 }
 
+/**
+ * Escape a literal for embedding inside a double-quoted selector value. The
+ * captured text still carries the spec's own source escapes, so unescape once
+ * first; then escape backslashes before quotes — escaping only `"` leaves a
+ * trailing `\` producing `"foo\"`, which swallows the closing quote and
+ * rewrites the selector.
+ */
+function q(v: string): string {
+  return v.replace(/\\(.)/g, "$1").replace(/\\/g, "\\\\").replace(/"/g, '\\"');
+}
+
 /** Tolerant line-based parse of a Playwright test file (page.* chains). */
 export function stepsFromTest(src: string, notes: string[]): Step[] {
   const steps: Step[] = [];
@@ -310,30 +321,30 @@ export function stepsFromTest(src: string, notes: string[]): Step[] {
       let part = "";
       switch (fn) {
         case "getByRole":
-          part = name ? `role=${arg}[name="${S(name, 1)}"${exact ? "" : "i"}]` : `role=${arg}`;
+          part = name ? `role=${arg}[name="${q(S(name, 1))}"${exact ? "" : "i"}]` : `role=${arg}`;
           break;
         case "getByTestId":
-          part = `[data-testid="${arg}"]`;
+          part = `[data-testid="${q(arg)}"]`;
           break;
         case "getByText":
-          part = exact ? `text="${arg}"` : `text=${arg}`;
+          part = exact ? `text="${q(arg)}"` : `text=${arg}`;
           break;
         case "getByLabel":
           // Same engine the trace importer emits: resolves to the control.
-          part = `internal:label="${arg.replace(/"/g, '\\"')}"${exact ? "s" : "i"}`;
+          part = `internal:label="${q(arg)}"${exact ? "s" : "i"}`;
           break;
         case "getByPlaceholder":
-          part = `[placeholder="${arg}"]`;
+          part = `[placeholder="${q(arg)}"]`;
           break;
         case "getByTitle":
-          part = `[title="${arg}"]`;
+          part = `[title="${q(arg)}"]`;
           break;
         case "getByAltText":
-          part = `[alt="${arg}"]`;
+          part = `[alt="${q(arg)}"]`;
           break;
         case "locator":
           part = arg;
-          if (hasText) part += `:has-text("${S(hasText, 1)}")`;
+          if (hasText) part += `:has-text("${q(S(hasText, 1))}")`;
           break;
         case "frameLocator":
           frameSel = arg;
@@ -426,7 +437,7 @@ export function stepsFromTest(src: string, notes: string[]): Step[] {
         if (attr && val && !regexArg) {
           steps.push({
             method: "assert",
-            selector: `${selector} >> :scope[${attr}="${val.replace(/"/g, '\\"')}"]`,
+            selector: `${selector} >> :scope[${attr}="${q(val)}"]`,
           });
         } else if (attr) {
           steps.push({ method: "assert", selector: `${selector} >> :scope[${attr}]` });

package/src/inspect.ts

@@ -13,6 +13,7 @@
 import { chromium, type Frame, type Page } from "playwright";
 import { chromeProfileDir } from "./config.js";
 import { ensureProfileUnlocked } from "./login.js";
+import { applySeeds, type Seeds } from "./setup.js";
 
 export interface InspectElement {
   /** ARIA role (explicit or implied by the tag). */
@@ -60,6 +61,13 @@ export interface InspectOptions {
   frames?: Record<string, string>;
   /** Settle time after load (ms, default 1200). */
   settleMs?: number;
+  /**
+   * Cookies / localStorage to inject before the first navigation — the same
+   * seeding `probe`/`record` do, so an authenticated page can be scanned
+   * (issue #53). Collect them with `collectSeeds` (storyboard `setup` +
+   * `--storage-state` / `--cookie`).
+   */
+  seeds?: Seeds | null;
 }
 
 const DEFAULT_LIMIT = 80;
@@ -199,6 +207,7 @@ export async function inspectPage(opts: InspectOptions): Promise<InspectResult>
   });
   try {
     const page = context.pages()[0] ?? (await context.newPage());
+    if (opts.seeds) await applySeeds(context, page, opts.seeds);
     await page.goto(opts.url, { waitUntil: "domcontentloaded" });
     await page
       .waitForLoadState("networkidle", { timeout: 4000 })

package/src/lint.ts

@@ -13,8 +13,9 @@
  * be 60 % frozen frame before TTS is paid for.
  */
 
-import type { Storyboard, Scene, Action } from "./types.js";
+import { OUTPUT_PRESETS, type Storyboard, type Scene, type Action } from "./types.js";
 import { clampScaleForWidth } from "./zoom.js";
+import { compileUserRegex } from "./safe-regex.js";
 
 export type LintSeverity = "error" | "warn" | "info";
 
@@ -163,6 +164,9 @@ export function estimateActionMs(a: Action, autoIdle = false): number {
       return 800;
     case "back":
       return 1400; // like goto
+    case "dialog":
+      // Arming a handler costs nothing; `holdMs` pauses before answering.
+      return 20 + (a.holdMs ?? 0);
     default:
       return 300;
   }
@@ -359,14 +363,14 @@ export function lintStoryboard(
         for (const re of [a.textMatches, a.url]) {
           if (re == null) continue;
           try {
-            new RegExp(re);
+            compileUserRegex(re, "", "assert");
           } catch (e) {
             push({
               severity: "error",
               code: "assert-bad-regex",
               scene: scene.id,
               action: ai,
-              message: `\`assert\` regex ${JSON.stringify(re)} is invalid: ${(e as Error).message}`,
+              message: `\`assert\` regex ${JSON.stringify(re)} is unusable: ${(e as Error).message}`,
             });
           }
         }
@@ -573,6 +577,74 @@ export function lintStoryboard(
     }
   }
 
+  // --- legibility + framing (issues #51, #58) ---
+  const dsf = sb.video.deviceScaleFactor ?? 1;
+  const framePad = sb.frame?.padding ?? (sb.frame ? 48 : 0);
+  if (sb.video.width >= 1800 && dsf === 1) {
+    const contentPct = sb.frame
+      ? Math.round(((sb.video.width - 2 * framePad) / sb.video.width) * 100)
+      : 100;
+    push({
+      severity: "warn",
+      code: "viewport-legibility",
+      message:
+        `recording at ${sb.video.width}x${sb.video.height} CSS px: a desktop site lays out its ` +
+        `content column in the middle of that viewport, so body text ends up ~14px` +
+        (sb.frame ? ` inside a frame that keeps only ${contentPct}% of the canvas` : "") +
+        " — unreadable after the output downscale",
+      fix:
+        "record at 1280–1440 CSS px with `video.deviceScaleFactor: 2` and let `output.preset` " +
+        "scale the 2x raw down; the UI is a third bigger on screen and stays crisp",
+    });
+  }
+  const preset = sb.output?.preset ? OUTPUT_PRESETS[sb.output.preset] : null;
+  const outW = sb.output?.width ?? preset?.width;
+  const outH = sb.output?.height ?? preset?.height;
+  const fit = sb.output?.fit ?? preset?.fit ?? "contain";
+  if (outW && outH) {
+    const capAspect = sb.video.width / sb.video.height;
+    const outAspect = outW / outH;
+    if (fit === "contain" && Math.abs(capAspect - outAspect) / outAspect > 0.02) {
+      const barPx =
+        capAspect > outAspect
+          ? Math.round((outH - outW / capAspect) / 2)
+          : Math.round((outW - outH * capAspect) / 2);
+      const side = capAspect > outAspect ? "top and bottom" : "left and right";
+      push({
+        severity: "warn",
+        code: "output-letterbox",
+        message:
+          `capture is ${sb.video.width}x${sb.video.height} (${capAspect.toFixed(2)}:1) but the output ` +
+          `is ${outW}x${outH} (${outAspect.toFixed(2)}:1) with fit "contain" — ${barPx}px bars on the ${side}` +
+          (sb.frame ? ", on top of the frame padding's own bars" : ""),
+        fix:
+          `either record at the output's aspect (e.g. ${outH === 1080 ? "1600x900" : `${outW / 2}x${outH / 2}`} ` +
+          `with video.deviceScaleFactor: 2) or set output.fit: "cover" for an edge-to-edge cut`,
+      });
+    }
+  }
+
+  // --- opening frame (issue #47) ---
+  const firstScene = sb.scenes[0];
+  const opener = firstScene?.actions[0];
+  if (opener?.op === "goto") {
+    const settles = firstScene.actions
+      .slice(1)
+      .findIndex((a) => a.op === "waitFor" || a.op === "pause" || a.op === "waitForChange");
+    if (settles !== 0) {
+      push({
+        severity: "info",
+        code: "blank-open",
+        scene: firstScene.id,
+        action: 0,
+        message:
+          "the demo opens on a `goto` with nothing between it and the first beat — the video " +
+          "(and the poster) starts on the browser's white pre-paint while the first caption is already up",
+        fix: "follow the goto with a waitFor on the page's own content (or set `output.trimLeadingBlank: true`)",
+      });
+    }
+  }
+
   const order: Record<LintSeverity, number> = { error: 0, warn: 1, info: 2 };
   issues.sort((a, b) => order[a.severity] - order[b.severity]);
   return { issues, estimate, narrationTotalMs, wordsPerSec: wps };

package/src/mcp/jobs.ts

@@ -27,6 +27,7 @@ export type JobKind =
   | "gif"
   | "stills"
   | "frames"
+  | "qa"
   | "inspect"
   | "walkthrough";
 

package/src/mcp/server.ts

@@ -4,7 +4,7 @@ import { z } from "zod";
 import { zodToJsonSchema } from "zod-to-json-schema";
 import { readFile } from "node:fs/promises";
 import { resolve } from "node:path";
-import { ENGINE_ROOT, engineVersion, captionsAutoOffline } from "../config.js";
+import { ENGINE_ROOT, engineVersion, captionsAutoOffline, chromeProfileDir } from "../config.js";
 import { Project, parseStoryboard } from "../project.js";
 import { StoryboardSchema, SeedCookieSchema, type SeedCookie, type Storyboard } from "../types.js";
 import { generateVoice } from "../voice.js";
@@ -23,7 +23,9 @@ import { exportGif } from "../gif.js";
 import { buildEmbed } from "../embed.js";
 import { extractStills, storyboardHasStills } from "../stills.js";
 import { extractFrames } from "../frames.js";
+import { runQa, logQa } from "../qa.js";
 import { inspectPage } from "../inspect.js";
+import { collectSeeds, runPreflight } from "../setup.js";
 import { importTrace, scaffoldImported } from "../import-trace.js";
 import { exportWalkthrough } from "../walkthrough.js";
 import { GUIDE_TOPIC_NAMES, guideHeadings, guidePath as guideFilePath, sliceGuide } from "../guide.js";
@@ -1079,7 +1081,11 @@ export function buildMcpServer(): { server: McpServer; jobs: JobManager } {
           captionsSrt: lp.captionsSrtPath,
           report: lp.reportPath,
           durationMs: report.durationMs,
+          // warningCount lets an orchestrating agent gate on the render
+          // without parsing the warning list (issue #59).
+          warningCount: report.warnings.length,
           warnings: report.warnings,
+          ...(report.blankOpenMs ? { blankOpenMs: report.blankOpenMs } : {}),
         };
       })
   );
@@ -1168,11 +1174,37 @@ export function buildMcpServer(): { server: McpServer; jobs: JobManager } {
           ...(gifPath ? { gif: gifPath } : {}),
           report: lp.reportPath,
           durationMs: report.durationMs,
+          // warningCount lets an orchestrating agent gate on the render
+          // without parsing the warning list (issue #59).
+          warningCount: report.warnings.length,
           warnings: report.warnings,
+          ...(report.blankOpenMs ? { blankOpenMs: report.blankOpenMs } : {}),
         };
       })
   );
 
+  registerJob(
+    "qa",
+    "Post-render checks on the FINISHED MP4 — blank opening, blank poster, " +
+      "master loudness, music-bed level, per-scene hold shares + compose " +
+      "warnings, caption readability, frame/chapter hygiene. Run it after " +
+      "compose/render: it sees what a viewer sees, which neither lint (pre-take) " +
+      "nor report.json (structural) can.",
+    {
+      dir: DIR_INPUT,
+      params: PARAMS_INPUT,
+      lang: LANG_INPUT,
+    },
+    (project, args) => async (job) =>
+      jobs.runStage(job, "qa", async () => {
+        const storyboard = await project.loadStoryboard({ params: args.params });
+        const sb = args.lang ? localizeStoryboard(storyboard, args.lang) : storyboard;
+        const res = await runQa(langProject(project, args.lang), sb);
+        logQa(res);
+        return res;
+      })
+  );
+
   registerJob(
     "gif",
     "Convert output/final-demo.mp4 to a README-ready GIF.",
@@ -1295,18 +1327,50 @@ export function buildMcpServer(): { server: McpServer; jobs: JobManager } {
         .record(z.string(), z.string())
         .optional()
         .describe("iframe selectors to scan too, as in the storyboard `frames` block"),
+      useSetup: z
+        .boolean()
+        .optional()
+        .describe(
+          "honour the storyboard's setup block (storageState/cookies/preflight) so an " +
+            "authenticated page can be scanned. Default true."
+        ),
+      storageState: z
+        .string()
+        .optional()
+        .describe("extra Playwright storageState JSON to seed before the scan"),
     },
     (project, args) => async (job) =>
       jobs.runStage(job, "inspect", async () => {
         const stamp = new Date().toISOString().replace(/[:.]/g, "-");
         const png = project.p("logs", `inspect-${stamp}.png`);
+        // #53: scan authenticated pages — same seeding surface as probe/record.
+        let storyboard: Storyboard | null = null;
+        if (args.useSetup !== false) {
+          storyboard = await project.loadStoryboard({ relaxed: true }).catch(() => null);
+        }
+        if (storyboard?.setup?.preflight) {
+          await runPreflight(storyboard.setup.preflight, {
+            demoDir: project.dir,
+            storyboardPath: project.storyboardPath,
+            profileDir: args.profile ?? chromeProfileDir(),
+          });
+          storyboard = await project
+            .loadStoryboard({ relaxed: true })
+            .catch(() => storyboard);
+        }
+        const seeds = await collectSeeds(
+          project.dir,
+          storyboard ?? ({ setup: undefined } as never),
+          { storageState: args.storageState }
+        );
         const res = await inspectPage({
           url: args.url,
           headless: args.headless !== false,
           profileDir: args.profile,
           limit: args.limit,
-          viewport: args.viewport,
-          frames: args.frames,
+          viewport: args.viewport ?? storyboard?.video,
+          frames: args.frames ?? storyboard?.frames,
+          seeds,
           screenshotPath: png,
         });
         const file = project.p("logs", `inspect-${stamp}.json`);