@tandryukha/aidemo 0.8.0

package/src/project.ts

@@ -0,0 +1,287 @@
+import { resolve, isAbsolute } from "node:path";
+import { StoryboardSchema, type Storyboard } from "./types.js";
+import { applyParams } from "./params.js";
+import { ensureDir, exists, readJson, writeJson, log } from "./util.js";
+
+/**
+ * Resolves and creates the standard artifact layout for one demo project.
+ *
+ *   <dir>/
+ *     input/brief.md
+ *     generated/storyboard.json  timeline.json  captions.srt  captions.vtt
+ *     recordings/raw.webm
+ *     audio/scene-*.mp3  narration.mp3  voice.json
+ *     output/final-demo.mp4
+ *     logs/
+ */
+export class Project {
+  readonly dir: string;
+  /**
+   * Optional language code for a multi-language render (see src/i18n.ts). When
+   * set, the per-LANGUAGE artifacts move under a language namespace so they
+   * don't clobber the default: audio → audio/<lang>/, captions →
+   * captions.<lang>.*, output → final-demo.<lang>.mp4. The SHARED take
+   * (recordings/raw.*, generated/timeline.json, generated/storyboard.json) is
+   * language-independent and stays put — that's what lets one recording feed N
+   * language renders. Unset → byte-for-byte the original single-language layout.
+   */
+  readonly lang?: string;
+  constructor(dir: string, lang?: string) {
+    this.dir = isAbsolute(dir) ? dir : resolve(process.cwd(), dir);
+    this.lang = lang || undefined;
+  }
+
+  p(...parts: string[]): string {
+    return resolve(this.dir, ...parts);
+  }
+
+  /** Language-scoped audio dir (audio/<lang>/ for a variant; audio/ by default). */
+  private audioDir(): string {
+    return this.lang ? this.p("audio", this.lang) : this.p("audio");
+  }
+
+  /** captions.<lang>.<ext> for a variant; captions.<ext> by default. */
+  private captionPath(ext: string): string {
+    return this.p("generated", this.lang ? `captions.${this.lang}.${ext}` : `captions.${ext}`);
+  }
+
+  get storyboardPath() {
+    return this.p("generated", "storyboard.json");
+  }
+  get timelinePath() {
+    return this.p("generated", "timeline.json");
+  }
+  /**
+   * Resolved template params of the last load are persisted here so a later
+   * stage-only re-run (e.g. `aidemo compose <dir>` after a parameterized record)
+   * reuses the SAME values — keeping narration, captions and the take in sync.
+   */
+  get paramsPath() {
+    return this.p("generated", "params.json");
+  }
+  get captionsSrtPath() {
+    return this.captionPath("srt");
+  }
+  get captionsVttPath() {
+    return this.captionPath("vtt");
+  }
+  get captionsCuesPath() {
+    return this.captionPath("cues.json");
+  }
+  /** Caption reuse manifest (per-scene input hashes + stored word timings). */
+  get captionsManifestPath() {
+    return this.p("generated", "captions.manifest.json");
+  }
+  /** Checked-in golden baseline for `aidemo probe --golden` regression checks. */
+  get goldenProbePath() {
+    return this.p("golden", "probe.json");
+  }
+  get rawVideoPath() {
+    return this.p("recordings", "raw.webm");
+  }
+  /** Native/OBS captures are transcoded here (Playwright records .webm). */
+  get rawVideoMp4Path() {
+    return this.p("recordings", "raw.mp4");
+  }
+  /**
+   * One preserved previous-take generation: `record` rotates the existing
+   * take (raw video + timeline) to these paths instead of deleting it, so a
+   * bad new take never destroys the last good one. Roll back by copying the
+   * .prev files over the current ones.
+   */
+  get rawVideoPrevPath() {
+    return this.p("recordings", "raw.prev.webm");
+  }
+  get rawVideoMp4PrevPath() {
+    return this.p("recordings", "raw.prev.mp4");
+  }
+  get timelinePrevPath() {
+    return this.p("generated", "timeline.prev.json");
+  }
+  /** The raw recording to compose from, whichever capture path produced it. */
+  async resolveRawVideo(): Promise<string> {
+    return (await exists(this.rawVideoMp4Path))
+      ? this.rawVideoMp4Path
+      : this.rawVideoPath;
+  }
+  get narrationPath() {
+    return resolve(this.audioDir(), "narration.mp3");
+  }
+  get voiceManifestPath() {
+    return resolve(this.audioDir(), "voice.json");
+  }
+  sceneAudioPath(id: string) {
+    return resolve(this.audioDir(), `scene-${id}.mp3`);
+  }
+  get outputPath() {
+    return this.p("output", this.lang ? `final-demo.${this.lang}.mp4` : "final-demo.mp4");
+  }
+  get gifPath() {
+    return this.p("output", this.lang ? `final-demo.${this.lang}.gif` : "final-demo.gif");
+  }
+  /** Directory for screenshot-mode stills (one PNG per named `still` marker). */
+  get stillsDir() {
+    return this.p("output", "stills");
+  }
+  get composeTmpDir() {
+    return this.p(this.lang ? `.compose-tmp-${this.lang}` : ".compose-tmp");
+  }
+  get gifTmpDir() {
+    return this.p(".gif-tmp");
+  }
+
+  async ensureDirs(): Promise<void> {
+    for (const d of [
+      "input",
+      "generated",
+      "recordings",
+      "audio",
+      "output",
+      "logs",
+    ]) {
+      await ensureDir(this.p(d));
+    }
+    // Language-scoped audio lives in a per-lang subdir (audio/<lang>/).
+    if (this.lang) await ensureDir(this.audioDir());
+  }
+
+  /**
+   * Load + validate the storyboard. `relaxed` (for `aidemo probe`) makes
+   * narration optional — a record-only dry run to verify selectors/timing
+   * doesn't need a script — by injecting an empty narration where missing
+   * before validating against the one schema.
+   *
+   * `params` are explicit template overrides (CLI `--param`, MCP `params`,
+   * variant params); they take precedence and are typo-guarded. When none are
+   * given, the last run's persisted params (generated/params.json) are reused
+   * so stage-only re-runs stay consistent. The resolved set is persisted here.
+   */
+  async loadStoryboard(
+    opts: { relaxed?: boolean; params?: Record<string, string> } = {}
+  ): Promise<Storyboard> {
+    const raw = await readJson<unknown>(this.storyboardPath);
+    const explicit =
+      opts.params && Object.keys(opts.params).length ? opts.params : undefined;
+    const provided = explicit ?? (await this.readPersistedParams());
+    const parsed = parseStoryboard(raw, {
+      relaxed: opts.relaxed,
+      params: provided,
+      // Typo-guard only explicit user input; persisted params are filtered.
+      strict: explicit != null,
+    });
+    if (!parsed.ok) {
+      throw new Error(
+        `Invalid storyboard.json:\n${parsed.issues
+          .map((i) => `  - ${i.path}: ${i.message}`)
+          .join("\n")}`
+      );
+    }
+    for (const w of parsed.warnings) log(w);
+    if (Object.keys(parsed.resolved).length) {
+      await this.persistParams(parsed.resolved);
+    }
+    return parsed.storyboard;
+  }
+
+  /** Last run's resolved template params, or undefined if none were persisted. */
+  async readPersistedParams(): Promise<Record<string, string> | undefined> {
+    if (!(await exists(this.paramsPath))) return undefined;
+    try {
+      const data = await readJson<{ params?: Record<string, string> }>(
+        this.paramsPath
+      );
+      return data && typeof data.params === "object" ? data.params : undefined;
+    } catch {
+      return undefined;
+    }
+  }
+
+  /** Persist the resolved template params for later stage-only re-runs. */
+  async persistParams(resolved: Record<string, string>): Promise<void> {
+    await ensureDir(this.p("generated"));
+    await writeJson(this.paramsPath, { params: resolved });
+  }
+}
+
+export interface StoryboardIssue {
+  path: string;
+  message: string;
+  code: string;
+}
+
+export type StoryboardParse =
+  | {
+      ok: true;
+      storyboard: Storyboard;
+      /** Resolved template params after applying declared defaults + overrides. */
+      resolved: Record<string, string>;
+      warnings: string[];
+    }
+  | { ok: false; issues: StoryboardIssue[] };
+
+/**
+ * Validate a raw storyboard value against the schema, returning structured
+ * issues instead of throwing. `relaxed` (probe) makes narration optional by
+ * injecting an empty one where missing before validating — a record-only dry
+ * run doesn't need a script.
+ *
+ * `params` are template overrides; substitution runs AFTER zod validation on
+ * the typed storyboard (so placeholders only reach string fields) and reports
+ * unresolved/undeclared placeholders as a `params` issue.
+ */
+export function parseStoryboard(
+  raw: unknown,
+  opts: { relaxed?: boolean; params?: Record<string, string>; strict?: boolean } = {}
+): StoryboardParse {
+  if (
+    opts.relaxed &&
+    raw &&
+    typeof raw === "object" &&
+    Array.isArray((raw as { scenes?: unknown }).scenes)
+  ) {
+    for (const s of (raw as { scenes: Array<Record<string, unknown>> }).scenes) {
+      if (s && typeof s === "object" && s.narration == null) s.narration = "";
+    }
+  }
+  const parsed = StoryboardSchema.safeParse(raw);
+  if (!parsed.success) {
+    return {
+      ok: false,
+      issues: parsed.error.issues.map((i) => ({
+        path: i.path.join("."),
+        message: i.message,
+        code: i.code,
+      })),
+    };
+  }
+  // Resolve + substitute {{template}} params on the typed storyboard.
+  const applied = applyParams(parsed.data, {
+    provided: opts.params,
+    strict: opts.strict,
+  });
+  if (!applied.ok) {
+    return {
+      ok: false,
+      issues: [{ path: "params", message: applied.message, code: "params" }],
+    };
+  }
+  return {
+    ok: true,
+    storyboard: applied.storyboard,
+    resolved: applied.resolved,
+    warnings: storyboardWarnings(applied.storyboard),
+  };
+}
+
+/** Surface silent no-ops in the storyboard so they aren't a surprise. */
+export function storyboardWarnings(sb: Storyboard): string[] {
+  const warnings: string[] = [];
+  const withCue = sb.scenes.filter((s) => s.music?.cue).map((s) => s.id);
+  if (withCue.length) {
+    warnings.push(
+      `⚠ per-scene music.cue is informational only and currently ignored ` +
+        `(scenes: ${withCue.join(", ")})`
+    );
+  }
+  return warnings;
+}

package/src/recorder.ts

@@ -0,0 +1,320 @@
+import { chromium } from "playwright";
+import { promises as fs } from "node:fs";
+import type {
+  Storyboard,
+  Timeline,
+  TimelineScene,
+  ProbeGoldenScene,
+} from "./types.js";
+import { Project } from "./project.js";
+import { cursorInitScript } from "./cursor.js";
+import { runStoryboard } from "./player.js";
+import {
+  chromeProfileDir,
+  defaultCaptureMode,
+  captureDevice,
+  obsUrl,
+  obsPassword,
+} from "./config.js";
+import {
+  type CaptureMode,
+  type CaptureProvider,
+  type ViewportGeometry,
+  AvfoundationCapture,
+  ObsCapture,
+  measureViewportGeometry,
+  cropCaptureToViewport,
+} from "./capture.js";
+import { ensureDir, exists, writeJson, log, ok, step } from "./util.js";
+import { probeDurationMs } from "./ffmpeg.js";
+import { dirname, join } from "node:path";
+
+const FPS = 30;
+
+export interface RecordOptions {
+  /** Chrome user-data dir (logged-in profile). Defaults to config profile. */
+  profileDir?: string;
+  /** Show the browser window. Default true (channel chrome needs a display). */
+  headed?: boolean;
+  /**
+   * Capture path. "playwright" (default) = built-in CDP video; "native" =
+   * ffmpeg avfoundation screen grab (macOS); "obs" = OBS via obs-websocket.
+   * The external paths record the real screen for higher fidelity, then crop
+   * to the viewport. Env default: AIDEMO_CAPTURE.
+   */
+  capture?: CaptureMode;
+  /** Progress hooks + best-effort cancellation for job runners (MCP). */
+  onSceneStart?: (sceneId: string, index: number, total: number) => void;
+  onSceneComplete?: (scene: TimelineScene, index: number, total: number) => void;
+  signal?: AbortSignal;
+  /**
+   * Golden-probe capture (see PlayerOptions.probe / src/golden.ts). When present,
+   * the run records a normalized per-action outcome into this array and doesn't
+   * abort on an action failure — for `aidemo probe --golden/--update-golden`.
+   */
+  probe?: ProbeGoldenScene[];
+}
+
+/**
+ * Drives the storyboard's actions through a real, logged-in Chrome while
+ * recording — either via Playwright's built-in video (raw.webm) or an
+ * external screen capture (raw.mp4). Also writes timeline.json.
+ */
+export async function record(
+  project: Project,
+  storyboard: Storyboard,
+  options: RecordOptions = {}
+): Promise<Timeline> {
+  step(`Recording "${storyboard.title}"`);
+  await project.ensureDirs();
+  await ensureDir(dirname(project.rawVideoPath));
+
+  const mode = (options.capture ?? defaultCaptureMode()) as CaptureMode;
+  const external = mode !== "playwright";
+  if (external && options.headed === false) {
+    throw new Error(
+      `--capture ${mode} records the real screen; run headed (drop --headless).`
+    );
+  }
+
+  const profileDir = options.profileDir ?? chromeProfileDir();
+  const { width, height } = storyboard.video;
+  const videoDir = dirname(project.rawVideoPath);
+  log(`profile: ${profileDir}`);
+  log(`viewport: ${width}x${height}`);
+  if (external) log(`capture mode: ${mode}`);
+
+  // Preserve the previous take as ONE `.prev` generation instead of deleting
+  // it: a bad new take (mid-run failure, wrong native-capture crop) must not
+  // destroy the last good recording. Roll back by copying raw.prev.* /
+  // timeline.prev.json over the current files.
+  const prevGeneration: Array<[string, string]> = [
+    [project.rawVideoPath, project.rawVideoPrevPath],
+    [project.rawVideoMp4Path, project.rawVideoMp4PrevPath],
+    [project.timelinePath, project.timelinePrevPath],
+  ];
+  for (const [, prev] of prevGeneration) await fs.rm(prev, { force: true });
+  const preserved: string[] = [];
+  for (const [current, prev] of prevGeneration) {
+    if (await exists(current)) {
+      await fs.rename(current, prev);
+      preserved.push(prev);
+    }
+  }
+  if (preserved.length) {
+    log(`previous take preserved → ${preserved.join(", ")}`);
+  }
+  // Clear stray recordings so we can unambiguously pick the new one afterwards.
+  for (const f of await fs.readdir(videoDir).catch(() => [])) {
+    if (f.endsWith(".webm") && !f.includes(".prev")) {
+      await fs.rm(join(videoDir, f), { force: true });
+    }
+  }
+  const screenCapPath = join(videoDir, "capture.mkv");
+  await fs.rm(screenCapPath, { force: true });
+
+  let context;
+  try {
+    context = await chromium.launchPersistentContext(profileDir, {
+      channel: "chrome",
+      headless: options.headed === false,
+      // Hide the automation fingerprint (navigator.webdriver=false) so
+      // Cloudflare on chatgpt.com doesn't throw a "Verify you are human"
+      // challenge mid-recording. The profile's real cf_clearance cookie is
+      // then honored because the browser matches the manual-login fingerprint.
+      args: [
+        "--disable-blink-features=AutomationControlled",
+        // Keep the window at a known spot on the primary screen for capture.
+        ...(external ? ["--window-position=40,60"] : []),
+      ],
+      // External capture records the REAL screen and needs real window
+      // geometry, so it runs UN-emulated (viewport: null — under emulation
+      // innerWidth/screen.width/devicePixelRatio all report fake values,
+      // which mis-scaled the crop to the whole screen on Retina displays,
+      // issue #13). measureViewportGeometry() below sizes the actual window
+      // to the storyboard viewport instead. The built-in recorder keeps the
+      // emulated viewport and 2x scale for crisper output.
+      ...(external
+        ? { viewport: null }
+        : {
+            viewport: { width, height },
+            deviceScaleFactor: 2,
+            recordVideo: {
+              dir: dirname(project.rawVideoPath),
+              size: { width, height },
+            },
+          }),
+    });
+  } catch (err) {
+    throw new Error(
+      `Failed to launch Chrome with profile ${profileDir}.\n` +
+        `If Chrome is already open on this profile, quit it first (Playwright ` +
+        `needs exclusive access to the profile).\nOriginal error: ${
+          (err as Error).message
+        }`
+    );
+  }
+
+  // Cursor rendering. Default: BAKE the animated cursor into every frame before
+  // any page script runs. When the storyboard opts into compose-time cursor
+  // control (a `cursor` block), skip the bake and record a clean, cursor-free
+  // take instead — the player logs the cursor path and compose draws it as an
+  // overlay, so hide/resize becomes a recompose rather than a re-record.
+  const composeCursor = !!storyboard.cursor;
+  if (!composeCursor) {
+    await context.addInitScript({ content: cursorInitScript() });
+  } else {
+    log("cursor: compose-time overlay (recording cursor-free take + path)");
+  }
+
+  const page = context.pages()[0] ?? (await context.newPage());
+
+  // Start from a clean blank so the recording's lead-in isn't the new-tab page.
+  await page.goto("about:blank");
+
+  let capture: CaptureProvider | null = null;
+  let geo: ViewportGeometry | null = null;
+  if (external) {
+    await page.bringToFront();
+    // Size the real window to the storyboard viewport and measure where it
+    // sits on screen (CDP window bounds + un-emulated page metrics, re-read
+    // until stable). Throws BEFORE any screen pixels are captured when the
+    // geometry can't be made safe to crop (see capture.ts, issue #13).
+    geo = await measureViewportGeometry(page, { width, height });
+    capture =
+      mode === "native"
+        ? new AvfoundationCapture(screenCapPath, captureDevice(), FPS)
+        : new ObsCapture(obsUrl(), obsPassword());
+    log(`starting ${capture.name}`);
+    await capture.start();
+  }
+
+  const t0 = Date.now();
+  const logsDir = project.p("logs");
+  const doneScenes: TimelineScene[] = [];
+  let timeline: Timeline;
+  let capFile: string | null = null;
+  let tailMs = 0;
+  let recordErr: unknown = null;
+  try {
+    timeline = await runStoryboard(page, storyboard, {
+      t0,
+      video: { width, height },
+      logsDir,
+      signal: options.signal,
+      probe: options.probe,
+      captureCursorPath: composeCursor,
+      onSceneStart: options.onSceneStart,
+      onSceneComplete: (s, i, total) => {
+        doneScenes.push(s);
+        options.onSceneComplete?.(s, i, total);
+      },
+    });
+    if (capture) {
+      // Stop immediately so the tail past the last scene stays small. Measure
+      // the tail at stop-initiation — the recorder stops taking frames at the
+      // stop signal, not when it finishes flushing the file.
+      const stopAt = Date.now();
+      capFile = await capture.stop();
+      tailMs = Math.max(0, stopAt - (t0 + timeline.totalMs));
+    }
+  } catch (err) {
+    // Salvage: a failure in scene 7 of 7 (a phantom click, a platform
+    // interruption) otherwise discards minutes of good recording. Keep the
+    // scenes that completed, still finalize the video below, then re-throw so
+    // the exit code stays 1. `record --from-scene <id>` resume is future work.
+    recordErr = err;
+    const lastEnd = doneScenes.length ? doneScenes[doneScenes.length - 1].endMs : 0;
+    timeline = { totalMs: lastEnd, leadInMs: 0, scenes: doneScenes };
+    if (capture && !capFile) {
+      const stopAt = Date.now();
+      capFile = await capture.stop().catch(() => null);
+      tailMs = Math.max(0, stopAt - (t0 + timeline.totalMs));
+    }
+  } finally {
+    if (capture && !capFile) await capture.stop().catch(() => {});
+    // For built-in capture, closing the context finalizes the video file.
+    await context.close();
+  }
+
+  let finalized = false;
+  try {
+  if (external) {
+    await fs.rm(project.rawVideoPath, { force: true });
+    await cropCaptureToViewport(
+      capFile!,
+      project.rawVideoMp4Path,
+      geo!,
+      width,
+      height,
+      FPS
+    );
+    await fs.rm(capFile!, { force: true });
+    const videoMs = await probeDurationMs(project.rawVideoMp4Path);
+    // Capture started before t0 and stopped ~tailMs after the last scene.
+    timeline.leadInMs = Math.max(0, videoMs - tailMs - timeline.totalMs);
+    await writeJson(project.timelinePath, timeline);
+    log(
+      `video ${videoMs}ms, content ${timeline.totalMs}ms, tail ${tailMs}ms → lead-in ${timeline.leadInMs}ms`
+    );
+    ok(`recording → ${project.rawVideoMp4Path}`);
+    finalized = true;
+  } else {
+    // saveAs() is unreliable for persistent contexts (needs the closed
+    // connection), so locate the finalized .webm on disk and move it. A demo may
+    // open a second tab (e.g. a checkout hand-off to an external site), which Playwright
+    // records as its own short .webm — so pick the LARGEST file (the full main-page
+    // recording), not just the first one, then drop the strays. The preserved
+    // previous take (raw.prev.webm) is not a candidate.
+    const produced = (await fs.readdir(videoDir)).filter(
+      (f) => f.endsWith(".webm") && !f.includes(".prev")
+    );
+    if (produced.length === 0) {
+      throw new Error(`No video produced in ${videoDir}`);
+    }
+    let mainVideo = produced[0];
+    let mainSize = -1;
+    for (const f of produced) {
+      const { size } = await fs.stat(join(videoDir, f));
+      if (size > mainSize) {
+        mainSize = size;
+        mainVideo = f;
+      }
+    }
+    for (const f of produced) {
+      if (f !== mainVideo) await fs.rm(join(videoDir, f), { force: true });
+    }
+    await fs.rename(join(videoDir, mainVideo), project.rawVideoPath);
+
+    // The front lead-in (about:blank + launch) = actual video length minus the
+    // measured content span. This is more reliable than timing the launch.
+    const videoMs = await probeDurationMs(project.rawVideoPath);
+    timeline.leadInMs = Math.max(0, videoMs - timeline.totalMs);
+    await writeJson(project.timelinePath, timeline);
+    log(
+      `video ${videoMs}ms, content ${timeline.totalMs}ms → lead-in ${timeline.leadInMs}ms`
+    );
+    ok(`recording → ${project.rawVideoPath}`);
+    finalized = true;
+  }
+  } catch (finalizeErr) {
+    // A partial take may not finalize cleanly (e.g. a truncated webm); tolerate
+    // it when we're already salvaging, but surface a real finalize failure.
+    if (!recordErr) throw finalizeErr;
+    log(`salvage: could not finalize recording (${(finalizeErr as Error).message})`);
+  }
+
+  if (finalized) {
+    ok(
+      `timeline  → ${project.timelinePath} (${timeline.scenes.length} scenes, ${timeline.totalMs}ms)`
+    );
+  }
+  if (recordErr) {
+    log(
+      `salvaged ${doneScenes.length}/${storyboard.scenes.length} scene(s) before failure; ` +
+        `see logs/ for the screenshot + frame dump, then re-run`
+    );
+    throw recordErr;
+  }
+  return timeline;
+}