@tandryukha/aidemo 0.8.0 → 0.14.1

package/src/cursor-overlay.ts

@@ -14,6 +14,8 @@
  * like the baked one.
  */
 
+import { type Key, num, piecewiseExpr, simplifyKeys } from "./expr.js";
+
 /** A cursor sample already mapped into final content time + output pixels. */
 export interface CursorPoint {
   /** Content-time seconds. */
@@ -34,121 +36,11 @@ export interface HideWindow {
  * up (ENOMEM) somewhere past ~100 flat terms — measured ~80 OK / 120 fail on a
  * stock build. X(t) and Y(t) are INDEPENDENT overlay expressions, so this budget
  * is per axis; keeping it well under the cliff (and small) also protects weaker
- * ffmpeg builds. The path is simplified to fit (simplifyAxis) — cheap, because
+ * ffmpeg builds. The path is simplified to fit (expr.ts simplifyKeys) — cheap, because
  * the player's glides are eased S-curves that RDP collapses to a few keys each.
  */
 const KEY_BUDGET = 48;
 
-interface Key {
-  t: number;
-  v: number;
-}
-
-function num(n: number): string {
-  const s = n.toFixed(3).replace(/\.?0+$/, "");
-  return s === "" || s === "-" ? "0" : s;
-}
-
-/** Linear-interpolation (vertical) error at `p` against the a→b chord. */
-function chordError(a: Key, b: Key, p: Key): number {
-  const dt = b.t - a.t;
-  const vAt = dt < 1e-9 ? a.v : a.v + ((b.v - a.v) * (p.t - a.t)) / dt;
-  return Math.abs(p.v - vAt);
-}
-
-/**
- * Ramer–Douglas–Peucker on a value-over-time polyline, using vertical (linear-
- * interpolation) error in the value's own units (pixels) — so `eps` is exactly
- * "max pixels the drawn cursor may drift from the recorded path". Keeps the
- * points where the curve actually bends; a straight or held stretch collapses to
- * its endpoints. Iterative to avoid deep recursion on long inputs.
- */
-function rdp(keys: Key[], eps: number): Key[] {
-  const n = keys.length;
-  if (n <= 2) return keys.slice();
-  const keep = new Uint8Array(n);
-  keep[0] = 1;
-  keep[n - 1] = 1;
-  const stack: Array<[number, number]> = [[0, n - 1]];
-  while (stack.length) {
-    const [lo, hi] = stack.pop()!;
-    if (hi - lo < 2) continue;
-    let maxErr = -1;
-    let idx = -1;
-    for (let i = lo + 1; i < hi; i++) {
-      const e = chordError(keys[lo], keys[hi], keys[i]);
-      if (e > maxErr) {
-        maxErr = e;
-        idx = i;
-      }
-    }
-    if (maxErr > eps && idx >= 0) {
-      keep[idx] = 1;
-      stack.push([lo, idx], [idx, hi]);
-    }
-  }
-  const out: Key[] = [];
-  for (let i = 0; i < n; i++) if (keep[i]) out.push(keys[i]);
-  return out;
-}
-
-/**
- * Simplify one axis to at most KEY_BUDGET keys: start at a sub-pixel epsilon and
- * grow it until the point count fits, so a busy path just drifts a pixel or two
- * more rather than blowing the expression budget.
- */
-function simplifyAxis(keys: Key[]): Key[] {
-  if (keys.length <= KEY_BUDGET) {
-    const r = rdp(keys, 1.2);
-    if (r.length <= KEY_BUDGET) return r;
-  }
-  let eps = 1.2;
-  for (let i = 0; i < 24; i++) {
-    const r = rdp(keys, eps);
-    if (r.length <= KEY_BUDGET) return r;
-    eps *= 1.6;
-  }
-  // Backstop: uniform thin (should never be reached for a real cursor path).
-  const step = Math.ceil(keys.length / KEY_BUDGET);
-  const out = keys.filter((_, i) => i % step === 0);
-  if (out[out.length - 1] !== keys[keys.length - 1]) out.push(keys[keys.length - 1]);
-  return out;
-}
-
-/**
- * Piecewise-linear expression over overlay's `t` (seconds): holds the first
- * value before the first key, linearly interpolates between keys, holds the last
- * value after the last.
- *
- * Emitted as a FLAT sum of half-open gated segments — NOT nested `if`s. A path
- * has dozens–hundreds of keys, and ffmpeg's expression parser blows its nesting
- * limit ("Missing ')' or too many args") a hundred `if`s deep (zoom.ts gets away
- * with nested `if` only because it has a handful of keys). Each segment
- * [t_i, t_{i+1}) contributes `gte(t,t_i)*lt(t,t_{i+1})*(v_i + slope*(t-t_i))`;
- * the gates are disjoint half-open intervals so exactly one term is live at a
- * time. Depth stays ~3 regardless of key count.
- */
-function piecewiseLinear(keys: Key[]): string {
-  if (keys.length === 1) return num(keys[0].v);
-  const first = keys[0];
-  const last = keys[keys.length - 1];
-  const terms: string[] = [`lt(t,${num(first.t)})*${num(first.v)}`];
-  for (let i = 0; i < keys.length - 1; i++) {
-    const a = keys[i];
-    const b = keys[i + 1];
-    const dt = b.t - a.t;
-    const gate = `gte(t,${num(a.t)})*lt(t,${num(b.t)})`;
-    if (a.v === b.v || dt < 1e-6) {
-      terms.push(`${gate}*${num(a.v)}`);
-    } else {
-      const slope = (b.v - a.v) / dt;
-      terms.push(`${gate}*(${num(a.v)}+(${num(slope)})*(t-${num(a.t)}))`);
-    }
-  }
-  terms.push(`gte(t,${num(last.t)})*${num(last.v)}`);
-  return terms.join("+");
-}
-
 /**
  * Build the `-filter_complex` string overlaying the cursor PNG (input 1) onto
  * the content (input 0) along `rawPts`, hidden during `hideWindows`. Returns
@@ -170,8 +62,10 @@ export function buildCursorFilter(
   }
   // Simplify each axis independently (X and Y are separate expressions) so each
   // stays under the ffmpeg expression budget.
-  const X = piecewiseLinear(simplifyAxis(mono.map((p) => ({ t: p.t, v: p.x }))));
-  const Y = piecewiseLinear(simplifyAxis(mono.map((p) => ({ t: p.t, v: p.y }))));
+  const axis = (vals: Key[]): string =>
+    piecewiseExpr(simplifyKeys(vals, KEY_BUDGET, 1.2), "t", "linear");
+  const X = axis(mono.map((p) => ({ t: p.t, v: p.x })));
+  const Y = axis(mono.map((p) => ({ t: p.t, v: p.y })));
 
   // enable is single-quoted so the commas inside between()/not() are literal,
   // not filtergraph separators (same reason zoom single-quotes its expressions).

package/src/cursor.ts

@@ -22,7 +22,9 @@ const CURSOR_STROKE = "#111111";
  */
 export async function renderCursorPng(
   outPath: string,
-  sizePx: number
+  sizePx: number,
+  style: "arrow" | "dot" = "arrow",
+  color = "rgba(255,90,95,.85)"
 ): Promise<void> {
   // Logical canvas is 32 units (24 arrow + shadow headroom); scale to sizePx.
   const LOGICAL = 32;
@@ -42,8 +44,12 @@ export async function renderCursorPng(
       </style></head><body>
         <svg width="${LOGICAL}" height="${LOGICAL}" viewBox="0 0 ${LOGICAL} ${LOGICAL}"
              xmlns="http://www.w3.org/2000/svg">
-          <path d="${CURSOR_ARROW_PATH}" fill="${CURSOR_FILL}"
-            stroke="${CURSOR_STROKE}" stroke-width="1.3" stroke-linejoin="round"/>
+          ${
+            style === "dot"
+              ? `<circle cx="16" cy="16" r="9" fill="${color}" stroke="rgba(255,255,255,.9)" stroke-width="2"/>`
+              : `<path d="${CURSOR_ARROW_PATH}" fill="${CURSOR_FILL}"
+            stroke="${CURSOR_STROKE}" stroke-width="1.3" stroke-linejoin="round"/>`
+          }
         </svg>
       </body></html>`,
       { waitUntil: "load" }

package/src/distribute.ts

@@ -20,7 +20,8 @@ import { platform, release } from "node:os";
 import { ENGINE_ROOT, engineVersion, openAiBaseUrl, ttsProvider } from "./config.js";
 import { localTtsStatus } from "./voice-local.js";
 import { Project } from "./project.js";
-import { STARTER_BRIEF, STARTER_STORYBOARD } from "./starter.js";
+import { STARTER_BRIEF, STARTER_STORYBOARD, briefFromInspect, storyboardFromInspect } from "./starter.js";
+import { inspectPage } from "./inspect.js";
 import { ensureDir, exists, ok, step, fail, log } from "./util.js";
 
 const execFileAsync = promisify(execFile);
@@ -239,7 +240,14 @@ async function registerMcpServer(
 export async function scaffoldDemo(
   baseDir: string,
   name: string,
-  opts: { force?: boolean } = {}
+  opts: {
+    force?: boolean;
+    /** Draft from a live page: inspect it and turn headings/selectors into scenes. */
+    fromUrl?: string;
+    headed?: boolean;
+    profileDir?: string;
+    viewport?: { width: number; height: number };
+  } = {}
 ): Promise<string> {
   const dir = resolve(baseDir, "demos", name);
   if ((await exists(dir)) && !opts.force) {
@@ -248,6 +256,23 @@ export async function scaffoldDemo(
   }
   const project = new Project(dir);
   await project.ensureDirs();
+  if (opts.fromUrl) {
+    step(`Inspecting ${opts.fromUrl} for a first draft`);
+    const res = await inspectPage({
+      url: opts.fromUrl,
+      headless: !opts.headed,
+      profileDir: opts.profileDir,
+      viewport: opts.viewport,
+      screenshotPath: project.p("logs", "inspect-init.png"),
+    });
+    await fs.writeFile(project.p("logs", "inspect-init.json"), JSON.stringify(res, null, 2));
+    await fs.writeFile(project.p("input", "brief.md"), briefFromInspect(name, res));
+    await fs.writeFile(project.storyboardPath, storyboardFromInspect(name, res));
+    ok(
+      `scaffolded demos/${name}/ from ${res.finalUrl || res.url} — ${res.headings.length} heading(s), ${res.elements.length} element(s) seen`
+    );
+    return dir;
+  }
   await fs.writeFile(project.p("input", "brief.md"), STARTER_BRIEF(name));
   await fs.writeFile(project.storyboardPath, STARTER_STORYBOARD(name));
   ok(`scaffolded demos/${name}/`);
@@ -493,10 +518,18 @@ export async function fileFeedback(
  */
 export async function feedback(
   dir: string | undefined,
-  opts: { web?: boolean; dryRun?: boolean } = {}
+  opts: {
+    web?: boolean;
+    dryRun?: boolean;
+    title?: string;
+    description?: string;
+  } = {}
 ): Promise<void> {
   const demoDir = dir ? targetOf(dir) : null;
-  const ctx = await buildFeedback(demoDir);
+  const ctx = await buildFeedback(demoDir, {
+    title: opts.title,
+    description: opts.description,
+  });
 
   if (opts.dryRun) {
     step(`feedback (dry run) → ${UPSTREAM_REPO}`);

package/src/expr.ts

@@ -0,0 +1,130 @@
+/**
+ * Shared helpers for building ffmpeg filter *expressions* from keyframes.
+ *
+ * ffmpeg's expression evaluator has two independent cliffs, and both bite once
+ * a demo gets long enough:
+ *
+ *  1. **Parser nesting.** A chain of nested `if(...)`s dies with
+ *     `Missing ')' or too many args` about a hundred levels deep. A 13-scene
+ *     take with ~40 focus points hit this in the zoom pass (issue #36).
+ *  2. **AST size.** Past roughly a hundred flat terms the evaluator allocates
+ *     until it fails (ENOMEM) — measured ~80 OK / 120 fail on a stock build.
+ *
+ * So every expression here is emitted as a FLAT sum of disjoint half-open
+ * gated segments (depth ~3 regardless of key count), and every channel is
+ * first simplified (Ramer–Douglas–Peucker) to a key budget that stays well
+ * under cliff 2.
+ */
+
+export interface Key {
+  /** Seconds. */
+  t: number;
+  v: number;
+}
+
+/** Format a number for an ffmpeg expression (no trailing zeros, no `NaN`). */
+export function num(n: number): string {
+  const s = n.toFixed(4).replace(/\.?0+$/, "");
+  return s === "" || s === "-" ? "0" : s;
+}
+
+/** Linear-interpolation (vertical) error at `p` against the a→b chord. */
+function chordError(a: Key, b: Key, p: Key): number {
+  const dt = b.t - a.t;
+  const vAt = dt < 1e-9 ? a.v : a.v + ((b.v - a.v) * (p.t - a.t)) / dt;
+  return Math.abs(p.v - vAt);
+}
+
+/**
+ * Ramer–Douglas–Peucker on a value-over-time polyline, using vertical error in
+ * the value's own units — so `eps` is exactly "how far the drawn channel may
+ * drift from the planned one". Keeps the points where the curve actually
+ * bends; a straight or held stretch collapses to its endpoints. Iterative to
+ * avoid deep recursion on long inputs.
+ */
+export function rdp(keys: Key[], eps: number): Key[] {
+  const n = keys.length;
+  if (n <= 2) return keys.slice();
+  const keep = new Uint8Array(n);
+  keep[0] = 1;
+  keep[n - 1] = 1;
+  const stack: Array<[number, number]> = [[0, n - 1]];
+  while (stack.length) {
+    const [lo, hi] = stack.pop()!;
+    if (hi - lo < 2) continue;
+    let maxErr = -1;
+    let idx = -1;
+    for (let i = lo + 1; i < hi; i++) {
+      const e = chordError(keys[lo], keys[hi], keys[i]);
+      if (e > maxErr) {
+        maxErr = e;
+        idx = i;
+      }
+    }
+    if (maxErr > eps && idx >= 0) {
+      keep[idx] = 1;
+      stack.push([lo, idx], [idx, hi]);
+    }
+  }
+  const out: Key[] = [];
+  for (let i = 0; i < n; i++) if (keep[i]) out.push(keys[i]);
+  return out;
+}
+
+/**
+ * Simplify one channel to at most `budget` keys: start at `eps` and grow it
+ * until the point count fits, so a busy channel just drifts slightly more
+ * rather than blowing the expression budget.
+ */
+export function simplifyKeys(keys: Key[], budget: number, eps: number): Key[] {
+  if (keys.length <= budget) {
+    const r = rdp(keys, eps);
+    if (r.length <= budget) return r;
+  }
+  let e = eps;
+  for (let i = 0; i < 24; i++) {
+    const r = rdp(keys, e);
+    if (r.length <= budget) return r;
+    e *= 1.6;
+  }
+  // Backstop: uniform thin (should never be reached for a real channel).
+  const step = Math.ceil(keys.length / budget);
+  const out = keys.filter((_, i) => i % step === 0);
+  if (out[out.length - 1] !== keys[keys.length - 1]) out.push(keys[keys.length - 1]);
+  return out;
+}
+
+type Interp = "linear" | "smooth";
+
+/**
+ * Piecewise expression over the time expression `T` (seconds): holds the first
+ * value before the first key, interpolates between keys (linearly, or with a
+ * smoothstep ease), holds the last value after the last key.
+ *
+ * Emitted as a flat sum of half-open gated terms — see the module header for
+ * why nesting `if`s is not an option.
+ */
+export function piecewiseExpr(keys: Key[], T: string, interp: Interp): string {
+  if (keys.length === 0) return "0";
+  if (keys.length === 1) return num(keys[0].v);
+  const first = keys[0];
+  const last = keys[keys.length - 1];
+  const terms: string[] = [`lt(${T},${num(first.t)})*${num(first.v)}`];
+  for (let i = 0; i < keys.length - 1; i++) {
+    const a = keys[i];
+    const b = keys[i + 1];
+    const dt = b.t - a.t;
+    const gate = `gte(${T},${num(a.t)})*lt(${T},${num(b.t)})`;
+    if (a.v === b.v || dt < 1e-6) {
+      terms.push(`${gate}*${num(a.v)}`);
+    } else if (interp === "linear") {
+      const slope = (b.v - a.v) / dt;
+      terms.push(`${gate}*(${num(a.v)}+(${num(slope)})*(${T}-${num(a.t)}))`);
+    } else {
+      const p = `((${T}-${num(a.t)})/${num(dt)})`;
+      terms.push(`${gate}*(${num(a.v)}+(${num(b.v - a.v)})*${p}*${p}*(3-2*${p}))`);
+    }
+  }
+  terms.push(`gte(${T},${num(last.t)})*${num(last.v)}`);
+  return terms.join("+");
+}

package/src/ffmpeg.ts

@@ -81,3 +81,159 @@ export function probeDurationMs(file: string): Promise<number> {
     });
   });
 }
+
+/**
+ * Length (ms) of the *flat* tail of a video — the trailing run of frames that
+ * are a single solid color.
+ *
+ * Compose freeze-holds a scene's last frame when the narration outruns the
+ * recorded action. When the scene ends right before a `goto`, that last frame
+ * is the browser's white pre-paint, and the hold clones *white* for as long as
+ * the narration runs (issue #36: 37 near-blank seconds out of 209). Trimming
+ * the flat tail first makes the hold clone the last frame that actually shows
+ * the app.
+ *
+ * Flatness is `YHIGH - YLOW` from `signalstats` — the spread between the luma
+ * 90th and 10th percentiles. Percentiles rather than min/max so a cursor arrow
+ * or a stray antialiased pixel doesn't make a blank page look like content;
+ * `YDEV` would be the natural measure but many ffmpeg builds don't emit it.
+ *
+ * Scans at most `windowMs` from the end; returns 0 when the tail has real
+ * content, or when the probe fails for any reason (best-effort polish, never a
+ * reason to fail a compose).
+ */
+export function probeFlatTailMs(
+  file: string,
+  durMs: number,
+  windowMs = 4000
+): Promise<number> {
+  const start = Math.max(0, durMs - windowMs) / 1000;
+  return new Promise((resolvePromise) => {
+    const proc = spawn(
+      "ffmpeg",
+      [
+        "-hide_banner",
+        "-nostats",
+        "-ss",
+        start.toFixed(3),
+        "-i",
+        file,
+        "-vf",
+        "signalstats,metadata=print:file=-",
+        "-f",
+        "null",
+        "-",
+      ],
+      { stdio: ["ignore", "pipe", "ignore"] }
+    );
+    let out = "";
+    proc.stdout.on("data", (d) => (out += d.toString()));
+    proc.on("error", () => resolvePromise(0));
+    proc.on("close", () => {
+      // The metadata muxer emits a header line per frame followed by one line
+      // per key:
+      //   frame:12  pts:...  pts_time:0.4
+      //   lavfi.signalstats.YLOW=16
+      //   ...
+      const frames: Array<{ t: number; spread: number }> = [];
+      let t = NaN;
+      let low = NaN;
+      let high = NaN;
+      const flush = (): void => {
+        if (!Number.isNaN(t) && !Number.isNaN(low) && !Number.isNaN(high)) {
+          frames.push({ t, spread: high - low });
+        }
+        low = NaN;
+        high = NaN;
+      };
+      for (const line of out.split("\n")) {
+        const ts = /pts_time:\s*([\d.]+)/.exec(line);
+        if (ts) {
+          flush();
+          t = parseFloat(ts[1]);
+          continue;
+        }
+        const lo = /YLOW=\s*([\d.]+)/.exec(line);
+        if (lo) low = parseFloat(lo[1]);
+        const hi = /YHIGH=\s*([\d.]+)/.exec(line);
+        if (hi) high = parseFloat(hi[1]);
+      }
+      flush();
+      if (frames.length === 0) return resolvePromise(0);
+      // Walk back while frames stay flat. A real page — even a mostly-empty
+      // one — has chrome, text and edges, so its spread is far above this.
+      const FLAT_SPREAD = 4;
+      let i = frames.length - 1;
+      if (frames[i].spread > FLAT_SPREAD) return resolvePromise(0);
+      while (i > 0 && frames[i - 1].spread <= FLAT_SPREAD) i--;
+      resolvePromise(Math.max(0, Math.round(durMs - (start + frames[i].t) * 1000)));
+    });
+  });
+}
+
+/** A motionless span found by `freezedetect`, in ms from the file's start. */
+export interface FreezeSpan {
+  startMs: number;
+  endMs: number;
+}
+
+/**
+ * Spans of `file` where nothing on screen moves for at least `minMs`
+ * (ffmpeg `freezedetect`). Used by compose's `autoIdle` to trim dead air the
+ * storyboard never annotated — a slow XHR, a fixed `pause`, a video the page
+ * hasn't started yet.
+ *
+ * Best-effort like `probeFlatTailMs`: any probe failure resolves to `[]`, so a
+ * missing filter or an odd container is a no-op, never a failed compose.
+ */
+export function probeFreezeSpans(
+  file: string,
+  minMs = 1500,
+  noise = 0.003
+): Promise<FreezeSpan[]> {
+  const d = Math.max(0.1, minMs / 1000);
+  return new Promise((resolvePromise) => {
+    const proc = spawn(
+      "ffmpeg",
+      [
+        "-hide_banner",
+        "-nostats",
+        "-i",
+        file,
+        "-vf",
+        `freezedetect=n=${noise}:d=${d.toFixed(3)},metadata=print:file=-`,
+        "-an",
+        "-f",
+        "null",
+        "-",
+      ],
+      { stdio: ["ignore", "pipe", "ignore"] }
+    );
+    let out = "";
+    proc.stdout.on("data", (d2) => (out += d2.toString()));
+    proc.on("error", () => resolvePromise([]));
+    proc.on("close", () => {
+      // freezedetect emits, per detected span:
+      //   lavfi.freezedetect.freeze_start=12.34
+      //   lavfi.freezedetect.freeze_duration=2.5
+      //   lavfi.freezedetect.freeze_end=14.84
+      // A span still frozen at EOF has a start with no end.
+      const spans: FreezeSpan[] = [];
+      let start: number | null = null;
+      for (const line of out.split("\n")) {
+        const s = /freeze_start=\s*([\d.]+)/.exec(line);
+        if (s) {
+          start = parseFloat(s[1]);
+          continue;
+        }
+        const e = /freeze_end=\s*([\d.]+)/.exec(line);
+        if (e && start != null) {
+          spans.push({ startMs: Math.round(start * 1000), endMs: Math.round(parseFloat(e[1]) * 1000) });
+          start = null;
+        }
+      }
+      if (start != null) spans.push({ startMs: Math.round(start * 1000), endMs: Number.MAX_SAFE_INTEGER });
+      resolvePromise(spans.filter((s) => s.endMs > s.startMs));
+    });
+  });
+}