@gtanczyk/diorama 0.0.0-stage → 0.2.0

package/app/src/film/clip.ts

@@ -0,0 +1,386 @@
+// The X clip's edit as pure functions: which spans of the recorded takes become shots (from the click log), how
+// each shot sits in a 16:9 and a 1080x1350 frame, and where the world's own sound files go under them. diorama clip records
+// the takes and renders what these plan (src/commands/clip.ts).
+import type { SoundData } from '../play/sound.ts';
+import { spokenLine } from '../play/sound.ts';
+import type { Action } from '../story/engine.ts';
+
+/** What happened in a recorded take, in seconds from its first video frame; x, y in page pixels. */
+export type TakeEvent =
+  /** A cursor click: on a story action's target, or (no action) on the floor to set the hero walking. */
+  | { kind: 'click'; t: number; x: number; y: number; place: string; action?: Action }
+  /** The hero arrived at a thing and uses it (the page's 'use' event): the moment the click shows its effect. */
+  | { kind: 'use'; t: number; thing: string }
+  | { kind: 'say'; t: number; text: string; who: string }
+  /** The fade out to another place starts. */
+  | { kind: 'exit'; t: number; from: string; to: string }
+  | { kind: 'enter'; t: number; place: string };
+
+export type Take = {
+  /** Seconds of video. */
+  duration: number;
+  /** The place the take starts in. */
+  place: string;
+  events: TakeEvent[];
+  /** Where the hero is on the page over time (page x of a 1920 px wide page). */
+  hero: Array<{ t: number; x: number }>;
+};
+
+/** A recorded take, or a place coming apart into its layers (the explode of the shot's place). */
+export type ShotSource = 'game' | 'end' | 'explode';
+
+export type Shot = {
+  source: ShotSource;
+  /** Span of the source video, seconds. */
+  from: number;
+  to: number;
+  /** Playback speed. */
+  speed: number;
+  place: string;
+  /** Left edge of the 1080 px wide crop of the 2400x1350 frame, for the 1080x1350 version of a --zoom clip. */
+  cropX: number;
+  caption?: string;
+  /** Seconds the last frame is held at the end. */
+  hold?: number;
+  why: string;
+};
+
+export const TALL = { scaledWidth: 2400, width: 1080 };
+/** Seconds of the opening shot before the first click lands, and of the whole opening shot (the hero walks first). */
+export const OPEN_LEAD = 0.2;
+export const OPEN_SHOT = 2.2;
+/** Seconds of each explode in the clip (the whole orbit-popout take, at speed 1), and the end card's last-frame hold. */
+export const EXPLODE = 4;
+export const END_HOLD = 0.5;
+/** Seconds of the last place before its closing explode. */
+const END_SHOT = 2.8;
+/** The least the story shots get, per place, however short --seconds is. */
+const LEAST_PER_PLACE = 2.4;
+const BEFORE = 0.5;
+const AFTER = 2.5;
+const LONGEST = 4;
+const SHORTEST = 1.2;
+/** After 'enter' the page is still fading in from black. */
+const FADE_IN = 0.45;
+
+/** Flat, the orbit-popout explode shows the place's painting this many times its frame size, in the middle of a render of EXPLODE_RENDER. */
+export const EXPLODE_FLAT_SCALE = 1.06;
+export const EXPLODE_RENDER = { width: 1920, height: 1080 };
+/** The fill round a frame that does not cover the clip: the frame itself, scaled to cover, blurred and darkened. */
+export const FILL = { sigma: 45, brightness: -0.22, saturation: 0.8 };
+
+type Size = { width: number; height: number };
+
+/**
+ * An ffmpeg filter chain (one video in, one out) that shows a `src` frame whole and centred in w x h, at the full
+ * height (16:9 round a 4:3 frame) or the full width (1080x1350: 1080x810 with bars above and below), the rest filled
+ * with the same frame scaled to cover, blurred and darkened.
+ */
+export function fillFrame(src: Size, w: number, h: number): string {
+  const fit = w / h >= src.width / src.height ? `scale=-2:${h}` : `scale=${w}:-2`;
+  return `split[fill_a][fill_b];[fill_b]scale=${w}:${h}:force_original_aspect_ratio=increase,crop=${w}:${h},gblur=sigma=${FILL.sigma},`
+    + `eq=brightness=${FILL.brightness}:saturation=${FILL.saturation}[fill_bg];[fill_a]${fit}:flags=lanczos[fill_fg];[fill_bg][fill_fg]overlay=(W-w)/2:(H-h)/2`;
+}
+
+/**
+ * An ffmpeg filter chain that cuts an explode render round its flat painting (a place `frame` big), so the painting's
+ * flat state lands where a fillFrame'd game frame does and the cut into it does not jump in scale: 16:9 crops a 16:9
+ * box as tall as the painting; 1080x1350 crops the painting's width at the render's full height, scaled to 1080 wide
+ * on the render's own dark ground.
+ */
+export function explodeFrame(frame: Size, w: number, h: number): string {
+  const R = EXPLODE_RENDER;
+  const fw = Math.min(R.width, Math.round(frame.width * EXPLODE_FLAT_SCALE)), fh = Math.min(R.height, Math.round(frame.height * EXPLODE_FLAT_SCALE));
+  if (w >= h) {
+    const cw = Math.min(R.width, Math.floor((fh * w) / h / 2) * 2), ch = Math.min(fh, Math.round((cw * h) / w));
+    return `crop=${cw}:${ch}:${Math.round((R.width - cw) / 2)}:${Math.round((R.height - ch) / 2)},scale=${w}:${h}:flags=lanczos`;
+  }
+  return `crop=${fw}:${R.height}:${Math.round((R.width - fw) / 2)}:0,scale=${w}:-2:flags=lanczos,pad=${w}:${h}:0:(oh-ih)/2:color=0x090a10`;
+}
+
+export const shotSeconds = (s: Shot) => (s.to - s.from) / s.speed + (s.hold ?? 0);
+
+/** Clip start of each shot. */
+export function shotStarts(shots: Shot[]): number[] {
+  const starts: number[] = [];
+  let t = 0;
+  for (const s of shots) {
+    starts.push(t);
+    t += shotSeconds(s);
+  }
+  return starts;
+}
+
+const targetOf = (a: Action | undefined) => (!a ? undefined : a.kind === 'go' ? a.via : a.kind === 'talk' ? a.npc : a.kind === 'do' ? a.thing : undefined);
+const round = (x: number) => Math.round(x * 1000) / 1000;
+
+/**
+ * The left edge of the 1080 px crop that keeps the hero (and the click, when both fit) in a 2400 px wide frame.
+ * `hero` and `click` are page x on a 1920 px wide page.
+ */
+export function cropFor(hero: number | undefined, click?: number): number {
+  const k = TALL.scaledWidth / 1920;
+  const max = TALL.scaledWidth - TALL.width;
+  let centre = hero === undefined ? TALL.scaledWidth / 2 : hero * k;
+  if (hero !== undefined && click !== undefined && Math.abs(click - hero) * k < TALL.width * 0.8) centre = ((hero + click) / 2) * k;
+  else if (hero === undefined && click !== undefined) centre = click * k;
+  return Math.round(Math.max(0, Math.min(max, centre - TALL.width / 2)));
+}
+
+function heroAt(take: Take, from: number, to: number): number | undefined {
+  const xs = take.hero.filter((h) => h.t >= from && h.t <= to).map((h) => h.x).sort((a, b) => a - b);
+  if (xs.length) return xs[Math.floor(xs.length / 2)];
+  const near = [...take.hero].sort((a, b) => Math.abs(a.t - (from + to) / 2) - Math.abs(b.t - (from + to) / 2))[0];
+  return near?.x;
+}
+
+/** Spans of a take where the page is black or fading: from each exit to just after the next enter. */
+function darkSpans(take: Take): Array<[number, number]> {
+  const out: Array<[number, number]> = [];
+  for (const e of take.events) {
+    if (e.kind !== 'exit') continue;
+    const enter = take.events.find((n) => n.kind === 'enter' && n.t >= e.t);
+    out.push([e.t, enter ? enter.t + FADE_IN : take.duration]);
+  }
+  return out;
+}
+
+/** Where `place` holds at time t of the take. */
+function placeAt(take: Take, t: number): string {
+  let place = take.place;
+  for (const e of take.events) if (e.kind === 'enter' && e.t <= t) place = e.place;
+  return place;
+}
+
+type Candidate = { from: number; to: number; at: number; click?: number; priority: number; place: string; why: string; keep?: boolean };
+
+export type ClipPlanInput = {
+  game: Take;
+  end?: Take;
+  /** Target length in seconds; the story shots get what the opening, the explodes and the end leave. */
+  seconds: number;
+  /** Seconds of each place's explode take, for the places that have one. */
+  explodes?: Record<string, number>;
+  firstPlace: string;
+  lastPlace: string;
+  caption: string;
+};
+
+/**
+ * The shot list: the first click opens the clip 0.2 s in and the hero walks for two seconds, then the first place
+ * comes apart into its layers; a shot per story click (0.5 s before, 2-3 s after, or round its effect when the hero
+ * had far to walk), at least one per place, each place's first shot after its own explode, trimmed to the target
+ * length; the last place's explode, the last place, and its explode again under the end card.
+ */
+export function pickShots(input: ClipPlanInput): Shot[] {
+  const { game } = input;
+  const clicks = game.events.filter((e): e is Extract<TakeEvent, { kind: 'click' }> => e.kind === 'click');
+  if (!clicks.length) throw new Error('the game take has no clicks');
+  const first = clicks[0];
+  const from = round(Math.max(0, first.t - OPEN_LEAD));
+  const open: Shot = {
+    source: 'game', from, to: round(Math.min(game.duration, from + OPEN_SHOT)), speed: 1, place: placeAt(game, first.t),
+    cropX: cropFor(heroAt(game, from, from + OPEN_SHOT), first.x), why: 'opening click',
+  };
+  const exploded = new Set<string>();
+  const explode = (place: string, why: string): Shot[] => {
+    const seconds = input.explodes?.[place];
+    if (!seconds) return [];
+    exploded.add(place);
+    return [{ source: 'explode', from: 0, to: round(Math.min(seconds, EXPLODE)), speed: 1, place, cropX: cropFor(undefined), why }];
+  };
+  const skip = input.end ? input.lastPlace : undefined;
+  // What the explodes will take: one per place the story shots visit, one before the last place, one under the card.
+  const places = new Set([open.place, ...game.events.flatMap((e) => (e.kind === 'enter' && e.place !== skip ? [e.place] : []))]);
+  const explodeSeconds = (place: string) => Math.min(input.explodes?.[place] ?? 0, EXPLODE);
+  const tailPlaces = input.end && !places.has(input.lastPlace) ? [input.lastPlace, input.lastPlace] : [input.lastPlace];
+  const fixed = shotSeconds(open) + [...places, ...tailPlaces].reduce((n, p) => n + explodeSeconds(p), 0) + (input.end ? END_SHOT : 0) + END_HOLD;
+  const budget = Math.max(input.seconds - fixed, LEAST_PER_PLACE * Math.max(1, places.size));
+  // The last place is the end take's: the game take's few seconds there are left out.
+  const middle = middleShots(game, clicks.slice(1), open.to, darkSpans(game), budget, skip);
+  const shots: Shot[] = [open, ...explode(open.place, `${open.place} comes apart into its layers`)];
+  for (const s of middle) {
+    if (!exploded.has(s.place)) shots.push(...explode(s.place, `${s.place} comes apart into its layers`));
+    shots.push(s);
+  }
+  return [...shots, ...endShots(input, exploded, explode)];
+}
+
+function endShots(input: ClipPlanInput, exploded: Set<string>, explode: (place: string, why: string) => Shot[]): Shot[] {
+  const { end, lastPlace: place } = input;
+  const out: Shot[] = [];
+  if (end) {
+    if (!exploded.has(place)) out.push(...explode(place, `${place} comes apart into its layers`));
+    const click = end.events.find((e): e is Extract<TakeEvent, { kind: 'click' }> => e.kind === 'click');
+    const from = click ? Math.max(0, click.t - 0.3) : Math.min(1, end.duration / 4);
+    const to = Math.min(end.duration, from + END_SHOT);
+    out.push({ source: 'end', from: round(from), to: round(to), speed: 1, place, cropX: cropFor(heroAt(end, from, to), click?.x), why: 'the last place' });
+  }
+  const card = explode(place, `${place} comes apart under the end card`);
+  if (card.length) out.push({ ...card[0], caption: input.caption, hold: END_HOLD });
+  else if (out.length) Object.assign(out[out.length - 1], { caption: input.caption, hold: END_HOLD });
+  return out;
+}
+
+function middleShots(game: Take, clicks: Array<Extract<TakeEvent, { kind: 'click' }>>, after: number, dark: Array<[number, number]>, budget: number, skip?: string): Shot[] {
+  const cands: Candidate[] = [];
+  const nextClick = (t: number) => clicks.find((c) => c.t > t + 0.01)?.t ?? game.duration;
+  for (const c of clicks) {
+    const a = c.action;
+    // Answers are UI text, not the place: never a shot of their own.
+    if (!a || a.kind === 'choose' || a.kind === 'leave') continue;
+    const target = targetOf(a);
+    const use = game.events.find((e) => e.kind === 'use' && e.t >= c.t && e.t <= nextClick(c.t) && e.thing === target);
+    let from: number, to: number, at: number;
+    if (!use || use.t - c.t <= AFTER) {
+      from = c.t - BEFORE;
+      to = Math.min(c.t + 3.5, Math.max(c.t + AFTER, use ? use.t + 1.2 : 0));
+      at = c.t;
+    } else {
+      // A long walk: the shot is the hero arriving and using it, not the cursor.
+      from = use.t - 1.8;
+      to = use.t + 1.4;
+      at = use.t;
+    }
+    const priority = a.kind === 'go' ? 3 : a.kind === 'do' && a.with ? 3 : a.kind === 'do' ? 2 : 1;
+    cands.push({ from, to, at, click: from <= c.t ? c.x : undefined, priority, place: c.place, why: `${a.kind} ${target}` });
+  }
+  // One shot per place change: the hero just arrived, unless a click shot already opens soon after.
+  for (const e of game.events) {
+    if (e.kind !== 'enter') continue;
+    const from = e.t + FADE_IN;
+    if (cands.some((c) => c.from >= e.t && c.from - from < 2)) continue;
+    cands.push({ from, to: from + 2.4, at: from, priority: 2.5, place: e.place, why: `arrive in ${e.place}` });
+  }
+  // Out of the dark and fades, after the opening shot, inside the take.
+  const clean: Candidate[] = [];
+  for (const c of cands) {
+    let { from, to } = c;
+    from = Math.max(from, after);
+    to = Math.min(to, game.duration);
+    for (const [d0, d1] of dark) {
+      if (to <= d0 || from >= d1) continue;
+      if (c.at < d0) to = Math.min(to, d0);
+      else from = Math.max(from, d1);
+    }
+    const place = placeAt(game, (from + to) / 2);
+    if (to - from >= SHORTEST && place !== skip) clean.push({ ...c, from, to, place });
+  }
+  clean.sort((a, b) => a.from - b.from);
+  // No overlaps: a later shot starts where the earlier one ends; a stub left over goes.
+  const shots: Candidate[] = [];
+  for (const c of clean) {
+    const prev = shots.at(-1);
+    if (prev && c.from < prev.to) {
+      if (c.priority > prev.priority && c.from - prev.from >= SHORTEST) prev.to = c.from;
+      else c.from = prev.to;
+    }
+    if (c.to - c.from >= SHORTEST) shots.push(c);
+  }
+  // Each place keeps its best shot while others are dropped.
+  for (const place of new Set(shots.map((s) => s.place))) {
+    const best = shots.filter((s) => s.place === place).sort((a, b) => b.priority - a.priority || a.from - b.from)[0];
+    best.keep = true;
+  }
+  const total = () => shots.reduce((n, s) => n + s.to - s.from, 0);
+  // Over the budget: tails come in to 2.5 s, then the least telling shots go (the latest first among equals).
+  for (const s of shots) if (total() > budget && s.to - s.from > 2.5) s.to = Math.max(s.from + 2.5, s.to - (total() - budget));
+  while (total() > budget && shots.length) {
+    const drop = [...shots].sort((a, b) => Number(!!a.keep) - Number(!!b.keep) || a.priority - b.priority || b.from - a.from)[0];
+    if (drop.keep && total() - (drop.to - drop.from) < budget * 0.6) {
+      // Only kept shots are left: shorten them all instead of losing a place.
+      const k = budget / total();
+      for (const s of shots) s.to = s.from + Math.max(SHORTEST, (s.to - s.from) * k);
+      break;
+    }
+    shots.splice(shots.indexOf(drop), 1);
+  }
+  // Under the budget: tails grow up to LONGEST, never into the dark or the next shot.
+  for (const [i, s] of shots.entries()) {
+    const room = budget - total();
+    if (room <= 0.05) break;
+    const limit = Math.min(shots[i + 1]?.from ?? game.duration, ...dark.filter(([d0]) => d0 >= s.to).map(([d0]) => d0), s.from + LONGEST);
+    s.to = Math.max(s.to, Math.min(limit, s.to + room));
+  }
+  return shots.map((s) => ({ source: 'game', from: round(s.from), to: round(s.to), speed: 1, place: s.place, cropX: cropFor(heroAt(game, s.from, s.to), s.click), why: s.why }));
+}
+
+/** One sound under the clip: a file at a clip time, normalised to `lufs`, then scaled by `gain`. */
+export type Cue = { kind: 'bed' | 'loop' | 'voice' | 'music'; file: string; at: number; seconds: number; lufs: number; gain: number; loop: boolean; fadeIn: number; fadeOut: number };
+
+/** Gain of a place's loop by its planned level (1 quiet, 3 loud). */
+const LOOP_GAIN = [0, 0.35, 0.5, 0.9];
+export const LEVELS = { bed: -26, voice: -17, music: -21, final: -14 };
+
+/**
+ * The soundtrack from the world's own files: each place's bed and its always-on loops under that place's shots,
+ * the voiced lines where they were said in a shot that plays at normal speed (never two at once), and the music
+ * under all of it, fading out over the end card.
+ */
+export function audioPlan(shots: Shot[], takes: Partial<Record<ShotSource, Take>>, sound: SoundData | undefined, music: string | undefined): Cue[] {
+  const starts = shotStarts(shots);
+  const total = shots.reduce((n, s) => n + shotSeconds(s), 0);
+  const cues: Cue[] = [];
+  // Consecutive shots in one place share one bed, so it does not restart at each cut.
+  const spans: Array<{ place: string; from: number; to: number }> = [];
+  shots.forEach((s, i) => {
+    const prev = spans.at(-1);
+    const to = starts[i] + shotSeconds(s);
+    if (prev && prev.place === s.place) prev.to = to;
+    else spans.push({ place: s.place, from: starts[i], to });
+  });
+  for (const span of spans) {
+    const p = sound?.places[span.place];
+    if (!p) continue;
+    const seconds = round(span.to - span.from);
+    const fade = (n: number) => Math.min(n, seconds / 4);
+    const bed = p.bed && sound.files[p.bed];
+    if (bed) cues.push({ kind: 'bed', file: bed.file, at: round(span.from), seconds, lufs: LEVELS.bed, gain: 0.9, loop: true, fadeIn: fade(0.25), fadeOut: fade(0.3) });
+    for (const l of p.loops) {
+      const f = sound.files[l.id];
+      if (!f || l.when.length) continue;
+      cues.push({ kind: 'loop', file: f.file, at: round(span.from), seconds, lufs: LEVELS.bed, gain: LOOP_GAIN[l.level] ?? 0.5, loop: true, fadeIn: fade(0.25), fadeOut: fade(0.3) });
+    }
+  }
+  let free = 0;
+  shots.forEach((s, i) => {
+    const take = (s.source === 'game' || s.source === 'end') && s.speed === 1 ? takes[s.source] : undefined;
+    for (const e of take?.events ?? []) {
+      if (e.kind !== 'say' || e.t < s.from || e.t >= s.to) continue;
+      const line = spokenLine(sound, e.text, e.who);
+      if (!line) continue;
+      const at = starts[i] + (e.t - s.from);
+      if (at < free - 0.05 || total - at < 0.6) continue;
+      const seconds = Math.min(line.seconds, total - at);
+      cues.push({ kind: 'voice', file: line.file, at: round(at), seconds: round(seconds), lufs: LEVELS.voice, gain: 1, loop: false, fadeIn: 0.02, fadeOut: seconds < line.seconds ? 0.3 : 0.05 });
+      free = at + line.seconds;
+    }
+  });
+  if (music) cues.push({ kind: 'music', file: music, at: 0, seconds: round(total), lufs: LEVELS.music, gain: 1, loop: true, fadeIn: 0.2, fadeOut: Math.min(2.2, total / 4) });
+  return cues;
+}
+
+/**
+ * The world's sound for a clip, from what was made (sound/sounds.json) rather than what the page would play: the page
+ * leaves out a bed whose prompt the plan has since changed, but the file made is still that place's sound. Each
+ * place gets its `<place>-bed` and its `<place>-<thing>-loop` files (a loop belongs to the longest place key it starts
+ * with); a loop keeps the conditions the page knows for it, else plays always. Spoken lines come from `page`.
+ */
+export function clipSound(page: SoundData | undefined, made: Record<string, { file: string; kind: string; level: number }>, places: string[]): SoundData {
+  const files: SoundData['files'] = {};
+  for (const [id, s] of Object.entries(made)) {
+    if (s.kind === 'music' || s.kind === 'bed' || s.kind === 'loop' || s.kind === 'step' || s.kind === 'shot') files[id] = { file: s.file, kind: s.kind, level: s.level };
+  }
+  const owner = (id: string) => [...places].sort((a, b) => b.length - a.length).find((p) => id.startsWith(`${p}-`));
+  const out: SoundData = { ...(page ?? { shots: {} }), files, music: files.music ? 'music' : page?.music, places: {} };
+  for (const place of places) {
+    const bed = files[`${place}-bed`]?.kind === 'bed' ? `${place}-bed` : page?.places[place]?.bed;
+    const loops = Object.keys(files).filter((id) => files[id].kind === 'loop' && id.endsWith('-loop') && owner(id) === place).map((id) => {
+      const thing = id.slice(place.length + 1, -'-loop'.length);
+      const known = page?.places[place]?.loops.find((l) => l.id === id);
+      return { id, thing, level: files[id].level, when: known?.when ?? [] };
+    });
+    out.places[place] = { bed, loops, zones: page?.places[place]?.zones ?? [] };
+  }
+  return out;
+}

package/app/src/film/reel.ts

@@ -0,0 +1,126 @@
+// The build-up reel: one still per step of a place's build, from the files the steps left behind.
+// The demo film plays these on a beat, so each still says which step it is and which model made it.
+import fs from 'node:fs';
+import sharp from 'sharp';
+import { FRAME } from '../scene/layout.ts';
+import type { Critique } from '../scene/critic.ts';
+import type { World } from '../world/schema.ts';
+import type { WorldFiles } from '../world/files.ts';
+
+export const SCREEN = { width: 1920, height: 1080 } as const;
+// The painting sits centred, scaled to leave room for the caption under it.
+const SCALE = 1000 / FRAME.height;
+const BOX = { width: Math.round(FRAME.width * SCALE), height: 1000 };
+const LEFT = Math.round((SCREEN.width - BOX.width) / 2);
+const TOP = 16;
+
+export type Beat = {
+  key: string;
+  title: string;
+  /** Who made what this beat shows: a Meta model, or code. */
+  by: string;
+  still: Buffer;
+  /** How this beat comes in over the one before (an ffmpeg xfade transition). */
+  enter: 'fade' | 'wipeleft' | 'circleopen';
+};
+
+const esc = (s: string) => s.replace(/&/g, '&amp;').replace(/</g, '&lt;').replace(/>/g, '&gt;');
+
+/** The caption under the picture: step title and a badge naming the model. */
+function captionSvg(title: string, by: string): string {
+  const badge = Math.round(by.length * 14 + 44);
+  return `<rect x="${LEFT}" y="${SCREEN.height - 62}" width="${badge}" height="44" rx="22" fill="#2f5bea"/>
+<text x="${LEFT + 18}" y="${SCREEN.height - 32}" font-family="DejaVu Sans, sans-serif" font-size="22" font-weight="bold" fill="#fff">${esc(by)}</text>
+<text x="${LEFT + badge + 20}" y="${SCREEN.height - 31}" font-family="DejaVu Sans, sans-serif" font-size="26" fill="#e8e6ee">${esc(title)}</text>`;
+}
+
+/** A painting-sized picture placed on the black screen, with optional drawing over it (in painting pixels). */
+async function onScreen(picture: Buffer | null, over: string, title: string, by: string, clean: boolean): Promise<Buffer> {
+  const layers: sharp.OverlayOptions[] = [];
+  if (picture) {
+    const fitted = await sharp(picture).resize(BOX.width, BOX.height, { fit: 'fill' }).png().toBuffer();
+    layers.push({ input: fitted, left: LEFT, top: TOP });
+  }
+  const scaled = over && `<g transform="translate(${LEFT} ${TOP}) scale(${SCALE})">${over}</g>`;
+  const svg = `<svg xmlns="http://www.w3.org/2000/svg" width="${SCREEN.width}" height="${SCREEN.height}">${scaled}${clean ? '' : captionSvg(title, by)}</svg>`;
+  layers.push({ input: Buffer.from(svg), left: 0, top: 0 });
+  return sharp({ create: { width: SCREEN.width, height: SCREEN.height, channels: 3, background: '#000' } })
+    .composite(layers).png().toBuffer();
+}
+
+/** The world as Spark wrote it: places as cards, exits as lines, in painting pixels. */
+export function worldSvg(world: World, focus: string): string {
+  const n = world.places.length;
+  const w = 340, h = 140;
+  // Up to three places stand in a row; more go round an ellipse.
+  const pos = new Map(world.places.map((p, i) => {
+    if (n <= 3) return [p.key, { x: FRAME.width / 2 + (i - (n - 1) / 2) * (w + 80) - w / 2, y: 470 - h / 2 }];
+    const a = (i / n) * Math.PI * 2 - Math.PI / 2;
+    return [p.key, { x: FRAME.width / 2 + Math.cos(a) * 360 - w / 2, y: 500 + Math.sin(a) * 230 - h / 2 }];
+  }));
+  const lines = world.places.flatMap((p) => p.exits.map((e) => {
+    const a = pos.get(p.key)!, b = pos.get(e.to);
+    return b ? `<line x1="${a.x + w / 2}" y1="${a.y + h / 2}" x2="${b.x + w / 2}" y2="${b.y + h / 2}" stroke="#7d86a8" stroke-width="4" stroke-dasharray="10 8"/>` : '';
+  }));
+  const cards = world.places.map((p) => {
+    const { x, y } = pos.get(p.key)!;
+    const hot = p.key === focus;
+    return `<rect x="${x}" y="${y}" width="${w}" height="${h}" rx="14" fill="${hot ? '#2f5bea' : '#1b1d26'}" stroke="${hot ? '#9fb4ff' : '#3a3f55'}" stroke-width="3"/>
+<text x="${x + 22}" y="${y + 56}" font-family="DejaVu Sans, sans-serif" font-size="${Math.min(34, Math.floor((w - 44) / (p.key.length * 0.62)))}" font-weight="bold" fill="#fff">${esc(p.key)}</text>
+<text x="${x + 22}" y="${y + 104}" font-family="DejaVu Sans, sans-serif" font-size="24" fill="#c9cde0">${p.things.length} things · ${p.exits.length} exit${p.exits.length === 1 ? '' : 's'}</text>`;
+  });
+  const pitch = world.pitch.length > 110 ? `${world.pitch.slice(0, 107)}…` : world.pitch;
+  return `<text x="60" y="90" font-family="DejaVu Serif, serif" font-size="52" font-weight="bold" fill="#fff">${esc(world.title)}</text>
+<text x="60" y="140" font-family="DejaVu Sans, sans-serif" font-size="22" fill="#a9adc0">${esc(pitch)}</text>
+${lines.join('')}${cards.join('')}`;
+}
+
+/** Numbered frames round the regions the critic asked to repaint, and the list of fixes in a panel. */
+export function critiqueSvg(found: Critique, colour = '#ff4a3d', list = true): string {
+  const frames = found.issues.map((i, n) => {
+    const [x0, y0, x1, y1] = i.region;
+    return `<rect x="${x0}" y="${y0}" width="${x1 - x0}" height="${y1 - y0}" fill="none" stroke="${colour}" stroke-width="5" rx="6"/>
+<circle cx="${x0}" cy="${y0}" r="20" fill="${colour}"/>
+<text x="${x0}" y="${y0 + 8}" text-anchor="middle" font-family="DejaVu Sans, sans-serif" font-size="22" font-weight="bold" fill="#fff">${n + 1}</text>`;
+  });
+  if (!list) return frames.join('');
+  const rows = found.issues.map((i, n) => {
+    const fix = i.fix.length > 78 ? `${i.fix.slice(0, 75)}…` : i.fix;
+    return `<text x="40" y="${FRAME.height - 40 - (found.issues.length - 1 - n) * 34}" font-family="DejaVu Sans, sans-serif" font-size="22" fill="#fff">${n + 1}. ${esc(i.problem)}: ${esc(fix)}</text>`;
+  });
+  const top = FRAME.height - 70 - found.issues.length * 34;
+  return `${frames.join('')}<rect x="20" y="${top}" width="${FRAME.width - 40}" height="${FRAME.height - 20 - top}" rx="12" fill="#000" fill-opacity="0.66"/>${rows.join('')}`;
+}
+
+/** Every beat the files of one place allow, in build order; steps not run yet are left out. */
+export async function placeBeats(files: WorldFiles, placeKey: string, clean = false): Promise<Beat[]> {
+  const world = files.world();
+  const place = world.places.find((p) => p.key === placeKey);
+  if (!place) throw new Error(`no place "${placeKey}" in ${files.slug}; places: ${world.places.map((p) => p.key).join(', ')}`);
+  const at = (...parts: string[]) => files.place(place.key, ...parts);
+  const has = (...parts: string[]) => fs.existsSync(at(...parts));
+  const read = (...parts: string[]) => fs.readFileSync(at(...parts));
+  const beats: Beat[] = [];
+  const add = async (key: string, title: string, by: string, picture: Buffer | null, over = '', enter: Beat['enter'] = 'fade') =>
+    beats.push({ key, title, by, enter, still: await onScreen(picture, over, title, by, clean) });
+
+  await add('world', `One sentence becomes a world: ${world.places.length} places, a hero, a story`, 'Muse Spark', null, worldSvg(world, place.key));
+  if (has('photo.png')) await add('photo', 'A real photo is the first place', 'Camera', read('photo.png'));
+  if (has('blockin.png')) await add('layout', 'Spark plans every thing, light and floor', 'Muse Spark', read(has('plan.png') ? 'plan.png' : 'blockin.png'));
+  if (has('painting-first.png')) await add('paint', 'Muse paints over the block-in', 'Muse Image', read('painting-first.png'), '', 'wipeleft');
+  const critique = has('critique.json') ? (JSON.parse(read('critique.json').toString()) as Critique) : null;
+  if (critique?.issues.length && has('painting-first.png')) {
+    await add('critic', `Spark finds ${critique.issues.length} thing${critique.issues.length === 1 ? '' : 's'} to fix`, 'Muse Spark', read('painting-first.png'), critiqueSvg(critique));
+    if (has('painting.png')) await add('repaint', 'Muse repaints only those regions', 'Muse Image', read('painting.png'), critiqueSvg(critique, '#3ddc84', false));
+  } else if (has('painting.png') && !has('painting-first.png')) await add('paint', 'Muse paints over the block-in', 'Muse Image', read('painting.png'), '', 'wipeleft');
+  if (has('masks', 'overlay.png')) await add('masks', 'SAM 3.1 cuts out every thing and the floor', 'SAM 3.1', read('masks', 'overlay.png'), '', 'circleopen');
+  if (has('walk.png')) await add('walk', 'Floor, scale and occlusion measured from the masks', 'SAM 3.1 + code', read('walk.png'));
+  if (has('verify.json') && has('painting.png')) {
+    const v = JSON.parse(read('verify.json').toString()) as { ok: boolean; errors: unknown[]; flags: unknown[] };
+    const mark = v.ok ? '#3ddc84' : '#ff4a3d';
+    const over = `<rect x="0" y="0" width="${FRAME.width}" height="${FRAME.height}" fill="none" stroke="${mark}" stroke-width="12"/>
+<text x="${FRAME.width - 40}" y="80" text-anchor="end" font-family="DejaVu Sans, sans-serif" font-size="54" font-weight="bold" fill="${mark}" stroke="#000" stroke-width="5" paint-order="stroke">${v.ok ? 'PLAYABLE' : `${v.errors.length} ERRORS`}</text>`;
+    await add('verify', v.ok ? 'Every check passes: the scene is playable' : 'Verify found problems to fix', 'code', read('painting.png'), over);
+  }
+  return beats;
+}

package/app/src/film/screencast.ts

@@ -0,0 +1,123 @@
+// Smooth real-time recording of the play page: Chrome itself (new headless, on the GPU, ~60 fps; Playwright's bundled
+// headless shell renders on SwiftShader and looks choppy), every painted frame from a CDP screencast, a drawn cursor
+// that glides like a hand, and the frames laid out to a constant 60 fps by their timestamps.
+import { execFileSync } from 'node:child_process';
+import fs from 'node:fs';
+import path from 'node:path';
+import type { Browser, CDPSession, Page } from 'playwright';
+import { chromium } from '../commands/explore.ts';
+
+/** A white arrow drawn in the page, following the mouse; it dips on press. */
+const CURSOR = `(() => {
+  const c = document.createElement('div');
+  c.id = '__cursor';
+  c.innerHTML = '<svg width="34" height="40" viewBox="0 0 17 20"><path d="M1 1 L1 16 L5 12.5 L8 19 L10.5 18 L7.6 11.6 L13 11.6 Z" fill="white" stroke="black" stroke-width="1.2" stroke-linejoin="round"/></svg>';
+  Object.assign(c.style, { position: 'fixed', left: '0', top: '0', zIndex: 2147483647, pointerEvents: 'none', transform: 'translate(-100px,-100px)', filter: 'drop-shadow(0 2px 3px rgba(0,0,0,.35))', transition: 'scale .08s' });
+  const put = () => document.documentElement.appendChild(c);
+  document.readyState === 'loading' ? addEventListener('DOMContentLoaded', put) : put();
+  addEventListener('mousemove', (e) => { c.style.transform = 'translate(' + (e.clientX - 3) + 'px,' + (e.clientY - 2) + 'px)'; }, true);
+  addEventListener('mousedown', () => { c.style.scale = '0.85'; }, true);
+  addEventListener('mouseup', () => { c.style.scale = '1'; }, true);
+})()`;
+
+const ease = (u: number) => (u < 0.5 ? 4 * u * u * u : 1 - Math.pow(-2 * u + 2, 3) / 2);
+
+export type Recorder = {
+  page: Page;
+  /** Wall-clock seconds (Date.now() / 1000) of each click. */
+  clicks: number[];
+  start(): Promise<void>;
+  glide(x: number, y: number, ms?: number): Promise<void>;
+  click(x: number, y: number, opts?: { ms?: number; dwell?: number; button?: 'left' | 'right' }): Promise<number>;
+  /** The cursor dips as in a click, without a click reaching the page. */
+  tap(x: number, y: number, opts?: { ms?: number; dwell?: number }): Promise<number>;
+  /** Writes `mp4` at a constant `fps`; returns the wall-clock second of its first frame and its length. */
+  stop(mp4: string, fps?: number): Promise<{ first: number; seconds: number; frames: number }>;
+  close(): Promise<void>;
+};
+
+/** Opens a page of width x height (page pixels) recording into `work` (frames are kept there until close). */
+export async function openRecorder(width: number, height: number, work: string, log: (s: string) => void = console.log): Promise<Recorder> {
+  fs.rmSync(work, { recursive: true, force: true });
+  fs.mkdirSync(path.join(work, 'frames'), { recursive: true });
+  const pw = (await chromium()) as unknown as typeof import('playwright').chromium;
+  const args = ['--autoplay-policy=no-user-gesture-required', '--hide-scrollbars'];
+  let browser: Browser;
+  try {
+    browser = await pw.launch({ channel: 'chrome', args });
+  } catch (error) {
+    log(`Chrome is not installed (${(error as Error).message.split('\n')[0]}); recording with Playwright's own browser, which may be choppy`);
+    browser = await pw.launch({ args });
+  }
+  const ctx = await browser.newContext({ viewport: { width, height }, deviceScaleFactor: 1 });
+  await ctx.addInitScript(CURSOR);
+  const page = await ctx.newPage();
+  const cdp: CDPSession = await ctx.newCDPSession(page);
+  const frames: Array<{ file: string; t: number }> = [];
+  let recording = false;
+  cdp.on('Page.screencastFrame', ({ data, metadata, sessionId }) => {
+    cdp.send('Page.screencastFrameAck', { sessionId }).catch(() => {});
+    if (!recording || metadata.timestamp === undefined) return;
+    const file = path.join(work, 'frames', `${String(frames.length).padStart(6, '0')}.jpg`);
+    fs.writeFileSync(file, Buffer.from(data, 'base64'));
+    frames.push({ file, t: metadata.timestamp });
+  });
+  let mouse = { x: width / 2, y: height / 2 };
+  const glide = async (x: number, y: number, ms = 550) => {
+    const from = { ...mouse }, n = Math.max(8, Math.round(ms / 16));
+    for (let i = 1; i <= n; i++) {
+      const u = ease(i / n);
+      await page.mouse.move(from.x + (x - from.x) * u, from.y + (y - from.y) * u);
+      await page.waitForTimeout(ms / n);
+    }
+    mouse = { x, y };
+  };
+  const rec: Recorder = {
+    page,
+    clicks: [],
+    async start() {
+      await cdp.send('Page.startScreencast', { format: 'jpeg', quality: 90, maxWidth: width, maxHeight: height, everyNthFrame: 1 });
+      recording = true;
+    },
+    glide,
+    async click(x, y, opts = {}) {
+      await glide(x, y, opts.ms);
+      await page.waitForTimeout(opts.dwell ?? 300);
+      const at = Date.now() / 1000;
+      rec.clicks.push(at);
+      await page.mouse.down({ button: opts.button ?? 'left' });
+      await page.waitForTimeout(70);
+      await page.mouse.up({ button: opts.button ?? 'left' });
+      return at;
+    },
+    async tap(x, y, opts = {}) {
+      await glide(x, y, opts.ms);
+      await page.waitForTimeout(opts.dwell ?? 300);
+      const at = Date.now() / 1000;
+      rec.clicks.push(at);
+      await page.evaluate(`document.getElementById('__cursor') && (document.getElementById('__cursor').style.scale = '0.85')`);
+      await page.waitForTimeout(90);
+      await page.evaluate(`document.getElementById('__cursor') && (document.getElementById('__cursor').style.scale = '1')`);
+      return at;
+    },
+    async stop(mp4, fps = 60) {
+      recording = false;
+      await cdp.send('Page.stopScreencast').catch(() => {});
+      const end = Date.now() / 1000;
+      if (!frames.length) throw new Error('the screencast recorded no frames');
+      // Each frame holds until the next one was painted (the screencast may deliver them slightly out of order).
+      frames.sort((a, b) => a.t - b.t);
+      const lines = frames.map((f, i) => `file '${f.file}'\nduration ${Math.max(0.001, (frames[i + 1]?.t ?? end) - f.t).toFixed(4)}\n`);
+      const list = path.join(work, 'frames.txt');
+      fs.writeFileSync(list, lines.join('') + `file '${frames.at(-1)!.file}'\n`);
+      execFileSync('ffmpeg', ['-v', 'error', '-y', '-f', 'concat', '-safe', '0', '-i', list, '-vf', `scale=${width}:${height}:flags=lanczos,fps=${fps},format=yuv420p`,
+        '-c:v', 'libx264', '-crf', '16', '-preset', 'medium', mp4]);
+      return { first: frames[0].t, seconds: end - frames[0].t, frames: frames.length };
+    },
+    async close() {
+      await browser.close().catch(() => {});
+      fs.rmSync(path.join(work, 'frames'), { recursive: true, force: true });
+    },
+  };
+  return rec;
+}