@vfrey/vfrey 0.0.1 → 1.0.0-beta.0

package/src/scene/render.ts

@@ -0,0 +1,292 @@
+import { cp, mkdir, readdir, readFile, rename, rm, stat, writeFile } from "node:fs/promises";
+import { createRequire } from "node:module";
+import os from "node:os";
+import path from "node:path";
+import { fileURLToPath } from "node:url";
+import type { WebpackOverrideFn } from "@remotion/bundler";
+import { commandFor, FRAMEY_ENTRY, hyperframesCli } from "../product/probes.js";
+import { ensureChrome, run } from "../render/html-engine.js";
+import { probeVideo, type RenderedVideo } from "../render/probe.js";
+import { withJsExtensions } from "../render/webpack.js";
+import { sceneTimingFor } from "./audio.js";
+import { burnLabel, burnLabelStill } from "./label.js";
+import { startNetGuard } from "./net-guard.js";
+import { SceneError } from "./scene.js";
+import { SUPPLY_DIR, writeSupply } from "./supply.js";
+import type { BeatInput, SceneTiming } from "./timing.js";
+
+type Env = Record<string, string | undefined>;
+
+export type SceneEngine = "vfrey-framey" | "hyperframes" | "remotion";
+
+export const SCENE_FRAME = { width: 1080, height: 1920, fps: 30 } as const;
+
+export interface SceneRenderInput {
+  engine: SceneEngine;
+  /** The scene folder (vfrey's stored copy of it). It is never written to. */
+  sceneDir: string;
+  title: string;
+  /** The spoken lines, in order. */
+  lines: string[];
+  beats: BeatInput[];
+  voiceoverPath: string | null;
+  /** The mp4 to write, or the image when `frameAt` is set. */
+  outputPath: string;
+  /** A folder this render owns. It is removed when the render ends, good or bad. */
+  workDir: string;
+  /** Render one frame at this time (seconds) instead of the whole video. */
+  frameAt?: number;
+  env?: Env;
+  cwd?: string;
+  /** Called for each thing worth telling the person: a blocked network request, a rewritten script tag. */
+  warn?: (message: string) => void;
+}
+
+export interface SceneRenderResult {
+  timing: SceneTiming;
+  /** The real length of the written video, from ffprobe. Null for a single frame. */
+  durationSec: number | null;
+  /** What ffprobe says the file is (the same probe the other engines record). */
+  rendered: RenderedVideo | null;
+}
+
+export const GSAP_CDN = /<script\b[^>]*\bsrc\s*=\s*["']https?:\/\/(?:cdn\.jsdelivr\.net\/npm\/gsap@[^"']*|cdnjs\.cloudflare\.com\/ajax\/libs\/gsap\/[^"']*|unpkg\.com\/gsap[^"']*|code\.jquery\.com\/[^"']*gsap[^"']*)["'][^>]*>\s*<\/script>/gi;
+
+const num = (value: number) => String(Math.round(value * 1000) / 1000);
+
+/** How much of the scene after a `--frame` is rendered with it, so a transition under way at that frame is complete in the clip. */
+export const TRANSITION_TAIL_SECONDS = 4;
+
+/** True when the scene's HTML loads the shader-transitions runtime or calls `vfrey.transition`. */
+export function usesTransitions(html: string): boolean {
+  return /\bvfrey\.transition\s*\(|vfrey\/transitions\.js/.test(html);
+}
+
+/**
+ * Framey decides how to capture a render (layered, with the shaders composited) by reading the page's inline scripts for
+ * `HyperShader.init(` or `__hf.transitions =` before it loads anything. `vfrey.transition` makes that call from
+ * vfrey.js, which the check cannot see, so a scene that uses it gets this one inline line. It only makes sure the field
+ * exists; the transitions runtime fills it in.
+ */
+export const TRANSITIONS_MARKER = "<script>if (window.__hf) window.__hf.transitions = window.__hf.transitions || [];</script>";
+
+/**
+ * The composition HTML as the renderer needs it: the root is 1080 by 1920 and lasts exactly `durationSec`, the voice
+ * is on the timeline, and a GSAP script tag that points at a CDN points at vfrey's local copy instead.
+ */
+export function patchComposition(html: string, timing: Pick<SceneTiming, "durationSec">, hasVoice: boolean): { html: string; notes: string[] } {
+  const notes: string[] = [];
+  const root = /<([a-zA-Z][\w-]*)((?:\s+(?:[^>"']|"[^"]*"|'[^']*')*)?\bdata-composition-id\s*=(?:\s+(?:[^>"']|"[^"]*"|'[^']*')*|[^>])*)>/.exec(html);
+  if (!root) {
+    throw new SceneError(
+      "index.html has no element with data-composition-id, so there is nothing to render.",
+      'wrap the scene in <div id="root" data-composition-id="main">...</div>, as `npx vfrey scene init DIR` shows.',
+      "scene_no_root",
+    );
+  }
+  let attrs = root[2]!;
+  const set = (name: string, value: string) => {
+    const pattern = new RegExp(`\\s${name}\\s*=\\s*("[^"]*"|'[^']*'|[^\\s>]+)`, "i");
+    attrs = pattern.test(attrs) ? attrs.replace(pattern, ` ${name}="${value}"`) : `${attrs} ${name}="${value}"`;
+  };
+  set("data-start", "0");
+  set("data-duration", num(timing.durationSec));
+  set("data-width", String(SCENE_FRAME.width));
+  set("data-height", String(SCENE_FRAME.height));
+  const audio = hasVoice
+    ? `\n<audio id="vfrey-voice" src="${SUPPLY_DIR}/voice.wav" data-start="0" data-duration="${num(timing.durationSec)}" data-track-index="0" data-volume="1"></audio>`
+    : "";
+  let out = `${html.slice(0, root.index)}<${root[1]}${attrs}>${audio}${html.slice(root.index + root[0].length)}`;
+  if (usesTransitions(out) && !out.includes(TRANSITIONS_MARKER)) {
+    out = /<\/head>/i.test(out) ? out.replace(/<\/head>/i, `${TRANSITIONS_MARKER}\n</head>`) : `${TRANSITIONS_MARKER}\n${out}`;
+  }
+  out = out.replace(GSAP_CDN, () => {
+    notes.push(`The GSAP script tag that loads from a CDN now loads vfrey's local copy (${SUPPLY_DIR}/gsap.min.js).`);
+    return `<script src="${SUPPLY_DIR}/gsap.min.js"></script>`;
+  });
+  return { html: out, notes };
+}
+
+function blockedWarnings(hosts: string[]): string[] {
+  return hosts.map((host) => `The scene tried to reach ${host} over the network. The network is blocked while a scene renders, so the request failed.`);
+}
+
+async function chromeFor(env: Env, cwd: string): Promise<string> {
+  return ensureChrome(env, cwd);
+}
+
+/** Renders a scene (or one frame of it) with the chosen engine, then puts the AI label on it. */
+export async function renderScene(input: SceneRenderInput): Promise<SceneRenderResult> {
+  const env = input.env ?? process.env;
+  const cwd = input.cwd ?? process.cwd();
+  await rm(input.workDir, { recursive: true, force: true });
+  await mkdir(input.workDir, { recursive: true });
+  const guardDir = `${input.workDir}.guard`;
+  let guard: Awaited<ReturnType<typeof startNetGuard>> | null = null;
+  try {
+    const timing = await sceneTimingFor({ title: input.title, lines: input.lines, beats: input.beats }, input.voiceoverPath, env);
+    const chrome = await chromeFor(env, cwd);
+    guard = await startNetGuard(chrome, guardDir);
+    await mkdir(path.dirname(input.outputPath), { recursive: true });
+    await rm(input.outputPath, { force: true });
+    if (input.engine === "remotion") await renderTsx(input, timing, guard.chrome, env, cwd);
+    else await renderHtml(input, timing, guard.chrome, env, cwd);
+    const info = await stat(input.outputPath).catch(() => null);
+    if (!info || info.size <= 0) throw new Error("the renderer wrote no output");
+    for (const message of blockedWarnings(guard.blocked())) input.warn?.(message);
+    const labelDir = path.join(input.workDir, "label");
+    if (input.frameAt === undefined) await burnLabel(input.outputPath, labelDir, env);
+    else await burnLabelStill(input.outputPath, labelDir, env);
+    const rendered = input.frameAt === undefined ? await probeVideo(input.outputPath, env) : null;
+    return { timing, rendered, durationSec: rendered?.durationSec ?? null };
+  } finally {
+    await guard?.close().catch(() => undefined);
+    await rm(guardDir, { recursive: true, force: true });
+    await rm(input.workDir, { recursive: true, force: true });
+  }
+}
+
+async function renderHtml(input: SceneRenderInput, timing: SceneTiming, chrome: string, env: Env, cwd: string): Promise<void> {
+  const project = path.join(input.workDir, "project");
+  await cp(input.sceneDir, project, { recursive: true });
+  await writeSupply({ dir: project, timing, voiceFile: input.voiceoverPath, kind: "html", env });
+  const indexFile = path.join(project, "index.html");
+  const originalHtml = await readFile(indexFile, "utf8");
+  const patched = patchComposition(originalHtml, timing, input.voiceoverPath !== null);
+  for (const note of patched.notes) input.warn?.(note);
+  await writeFile(indexFile, patched.html);
+  let command: string;
+  let args: string[];
+  if (input.engine === "vfrey-framey") {
+    command = process.execPath;
+    args = [FRAMEY_ENTRY];
+  } else {
+    const cli = hyperframesCli(env, cwd);
+    if (!cli) throw new Error("the HyperFrames CLI was not found");
+    ({ command, args } = commandFor(cli.bin));
+  }
+  const runEnv = {
+    ...env,
+    PRODUCER_HEADLESS_SHELL_PATH: chrome,
+    HYPERFRAMES_BROWSER_PATH: chrome,
+    HYPERFRAMES_NO_TELEMETRY: "1",
+    HYPERFRAMES_NO_UPDATE_CHECK: "1",
+    DO_NOT_TRACK: "1",
+  } as NodeJS.ProcessEnv;
+  if (input.frameAt !== undefined && usesTransitions(patched.html)) {
+    await renderTransitionFrame(input, project, timing, originalHtml, command, args, runEnv);
+    return;
+  }
+  if (input.frameAt !== undefined) {
+    const shots = path.join(input.workDir, "shots");
+    await run(command, [...args, "snapshot", project, "--at", String(input.frameAt), "--no-end", "--describe", "false", "-o", shots], runEnv, project);
+    const png = (await readdir(shots)).find((name) => name.toLowerCase().endsWith(".png"));
+    if (!png) throw new Error("the renderer wrote no frame");
+    await rename(path.join(shots, png), input.outputPath);
+    return;
+  }
+  await run(command, [...args, "render", project, "-o", input.outputPath, "--quality", "standard", "--fps", String(SCENE_FRAME.fps)], runEnv, project);
+}
+
+/**
+ * One frame of a scene with shader transitions. A snapshot would show the shots stacked, because the shaders are
+ * composited by the render pipeline, not by the page. So the scene is rendered, cut off a few seconds after the frame
+ * asked for, and that frame is taken from the clip. The tail matters: the pipeline composites a transition only when the
+ * clip lasts past its end (cut at the frame itself, a frame in the middle of a transition shows the target shot alone).
+ */
+async function renderTransitionFrame(input: SceneRenderInput, project: string, timing: SceneTiming, html: string, command: string, args: string[], runEnv: NodeJS.ProcessEnv): Promise<void> {
+  const fps = SCENE_FRAME.fps;
+  const total = Math.max(1, Math.round(timing.durationSec * fps));
+  const index = Math.min(total - 1, Math.max(0, Math.round(input.frameAt! * fps)));
+  const clip = patchComposition(html, { durationSec: Math.min(timing.durationSec, (index + 1) / fps + TRANSITION_TAIL_SECONDS) }, false);
+  await writeFile(path.join(project, "index.html"), clip.html);
+  const video = path.join(input.workDir, "frame-clip.mp4");
+  await run(command, [...args, "render", project, "-o", video, "--quality", "standard", "--fps", String(fps)], runEnv, project);
+  await run("ffmpeg", ["-y", "-v", "error", "-i", video, "-vf", `select=eq(n\\,${index})`, "-frames:v", "1", input.outputPath], runEnv, project);
+}
+
+/** Where webpack finds `react`, `remotion` and the rest for a scene that lives outside the vfrey install. */
+function resolveRoots(): { modules: string[]; alias: Record<string, string> } {
+  const require = createRequire(import.meta.url);
+  const alias: Record<string, string> = {};
+  for (const name of ["react", "react-dom", "remotion"]) alias[name] = path.dirname(require.resolve(`${name}/package.json`));
+  return { modules: require.resolve.paths("react") ?? [], alias };
+}
+
+export const SCENE_RUNTIME_ENTRY = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "runtime/react.tsx");
+
+export function sceneWebpack(): WebpackOverrideFn {
+  const roots = resolveRoots();
+  return (current) => {
+    const base = withJsExtensions(current) as typeof current;
+    return {
+      ...base,
+      resolve: {
+        ...base.resolve,
+        modules: [...(base.resolve?.modules ?? ["node_modules"]), ...roots.modules],
+        alias: { ...(Array.isArray(base.resolve?.alias) ? {} : base.resolve?.alias), ...roots.alias, "@vfrey/scene": SCENE_RUNTIME_ENTRY },
+      },
+    };
+  };
+}
+
+/** The Remotion entry vfrey generates around the scene: the provider, the voice, and one composition. */
+export function remotionEntry(timing: SceneTiming, hasVoice: boolean): string {
+  return `import React from "react";
+import { AbsoluteFill, Audio, Composition, registerRoot, staticFile } from "remotion";
+import { VfreyProvider } from "@vfrey/scene";
+import * as sceneModule from "./scene/index";
+import timing from "./scene/${SUPPLY_DIR}/timing.json";
+
+const Scene: React.ComponentType = (sceneModule as any).default ?? (sceneModule as any).Scene;
+if (!Scene) throw new Error("index.tsx must export a React component: export default function Scene() {...}");
+
+function Wrapped() {
+  return (
+    <VfreyProvider timing={timing as any}>
+      ${hasVoice ? `<Audio src={staticFile("${SUPPLY_DIR}/voice.wav")} />` : ""}
+      <AbsoluteFill>
+        <Scene />
+      </AbsoluteFill>
+    </VfreyProvider>
+  );
+}
+
+registerRoot(() => (
+  <Composition id="Scene" component={Wrapped} durationInFrames={${Math.max(1, Math.round(timing.durationSec * SCENE_FRAME.fps))}} fps={${SCENE_FRAME.fps}} width={${SCENE_FRAME.width}} height={${SCENE_FRAME.height}} />
+));
+`;
+}
+
+async function renderTsx(input: SceneRenderInput, timing: SceneTiming, chrome: string, env: Env, _cwd: string): Promise<void> {
+  const sceneCopy = path.join(input.workDir, "scene");
+  await cp(input.sceneDir, sceneCopy, { recursive: true });
+  await writeSupply({ dir: sceneCopy, timing, voiceFile: input.voiceoverPath, kind: "tsx", env });
+  const entry = path.join(input.workDir, "entry.tsx");
+  await writeFile(entry, remotionEntry(timing, input.voiceoverPath !== null));
+  const { bundle } = await import("@remotion/bundler");
+  const { renderMedia, renderStill, selectComposition } = await import("@remotion/renderer");
+  const serveUrl = await bundle({
+    entryPoint: entry,
+    publicDir: sceneCopy,
+    outDir: path.join(input.workDir, "bundle"),
+    enableCaching: false,
+    webpackOverride: sceneWebpack(),
+  });
+  const port = Number(env.VFREY_REMOTION_PORT) || undefined;
+  const common = { serveUrl, browserExecutable: chrome, ...(port ? { port } : {}) };
+  const composition = await selectComposition({ ...common, id: "Scene", inputProps: {} });
+  if (input.frameAt !== undefined) {
+    const frame = Math.min(composition.durationInFrames - 1, Math.max(0, Math.round(input.frameAt * SCENE_FRAME.fps)));
+    await renderStill({ ...common, composition, frame, output: input.outputPath, imageFormat: "png", inputProps: {} });
+    return;
+  }
+  await renderMedia({
+    ...common,
+    composition,
+    codec: "h264",
+    outputLocation: input.outputPath,
+    inputProps: {},
+    concurrency: Math.max(1, Math.min(4, os.availableParallelism())),
+  });
+}

package/src/scene/runtime/browser.ts

@@ -0,0 +1,199 @@
+/**
+ * `vfrey/vfrey.js`: the helper an HTML scene includes with one script tag. It is bundled to a classic script at
+ * render time, with the timing file embedded, so `vfrey.timing` is there the moment the script has run.
+ */
+import { plan, type TransitionSpec } from "./transitions.js";
+import { captionCss, captionGroupHtml, captionGroups, lineView, wordAt, type CaptionOptions, type SceneTiming, type TimingWord } from "./core.js";
+
+interface Timeline {
+  set(target: unknown, vars: Record<string, unknown>, position?: number): Timeline;
+  [key: string]: unknown;
+}
+interface Gsap {
+  timeline(vars?: Record<string, unknown>): Timeline;
+}
+
+type Cue<T> = (tl: any, t: number, subject: T) => void;
+
+const g = globalThis as unknown as {
+  gsap?: Gsap;
+  document: Document;
+  __timelines?: Record<string, Timeline>;
+  __vfreyTiming?: SceneTiming;
+  vfrey?: unknown;
+  HyperShader?: { init(config: Record<string, unknown>): Timeline };
+};
+
+const timing = g.__vfreyTiming as SceneTiming;
+if (!timing) throw new Error("vfrey: the timing data is missing. Is vfrey/vfrey.js the file vfrey wrote at render time?");
+
+let current: Timeline | null = null;
+let captionBlocks = 0;
+
+interface TransitionCall extends TransitionSpec {
+  from: HTMLElement;
+  to: HTMLElement;
+  names: [string, string];
+  accent?: string;
+  background?: string;
+}
+const transitionCalls: TransitionCall[] = [];
+let transitionsScheduled = false;
+let shotIds = 0;
+
+/** Wires every `vfrey.transition` call into Framey's shader runtime, once, when the page's scripts have all run. */
+function applyTransitions(): void {
+  const shaders = g.HyperShader;
+  if (!shaders) throw new Error('vfrey.transition: the transitions runtime is not loaded. Add <script src="vfrey/transitions.js"></script> before <script src="vfrey/vfrey.js"></script>.');
+  const names = new Map<HTMLElement, string>();
+  for (const call of transitionCalls) {
+    names.set(call.from, call.names[0]);
+    names.set(call.to, call.names[1]);
+  }
+  const chain = plan(transitionCalls, (el: HTMLElement) => names.get(el) ?? "a shot");
+  if (!g.document.getElementById("vfrey-shots-css")) {
+    const css = g.document.createElement("style");
+    css.id = "vfrey-shots-css";
+    // Zero specificity: a scene's own CSS for its shots wins. Shots stack on top of each other in the frame.
+    css.textContent = ":where(.scene){position:absolute;left:0;top:0;width:100%;height:100%;}";
+    g.document.head.appendChild(css);
+  }
+  for (const shot of chain.scenes) {
+    if (!shot.id) shot.id = `vf-shot-${(shotIds += 1)}`;
+    shot.classList.add("scene");
+  }
+  const options = transitionCalls.find((c) => c.accent || c.background);
+  shaders.init({
+    bgColor: options?.background ?? "#000000",
+    ...(options?.accent ? { accentColor: options.accent } : {}),
+    scenes: chain.scenes.map((shot) => shot.id),
+    transitions: chain.transitions,
+    timeline: pick(),
+  });
+}
+
+function rootId(): string {
+  return g.document.querySelector("[data-composition-id]")?.getAttribute("data-composition-id") ?? "main";
+}
+
+/** Makes the paused GSAP timeline the scene animates on, and registers it where the renderer looks for it. */
+function makeTimeline(): Timeline {
+  if (!g.gsap) throw new Error('vfrey: GSAP is not loaded. Add <script src="vfrey/gsap.min.js"></script> before <script src="vfrey/vfrey.js"></script>.');
+  const tl = g.gsap.timeline({ paused: true });
+  g.__timelines = g.__timelines || {};
+  g.__timelines[rootId()] = tl;
+  current = tl;
+  return tl;
+}
+
+function pick(tl?: Timeline): Timeline {
+  return tl ?? current ?? makeTimeline();
+}
+
+function element(target: string | Element): HTMLElement {
+  const found = typeof target === "string" ? g.document.querySelector(target) : target;
+  if (!found) throw new Error(`vfrey.captions: nothing matches ${String(target)}.`);
+  return found as HTMLElement;
+}
+
+const vfrey = {
+  /** Words, lines and beats with start and end times in seconds, and `durationSec`. The same data as vfrey/timing.json. */
+  timing,
+  /** Creates the scene's paused GSAP timeline and registers it. Call it once; `at`, `onWord` and `captions` add to it. */
+  timeline: makeTimeline,
+  /** Uses a timeline you made yourself as the one `at`, `onWord` and `captions` add to. */
+  use(tl: Timeline): Timeline {
+    current = tl;
+    return tl;
+  },
+  /** `vfrey.at(2.5, (tl, t) => tl.from("#x", { opacity: 0, duration: 0.3 }, t))`. Runs `fn` while the timeline is built, so a frame is the same on every render. */
+  at(t: number, fn: Cue<number>, tl?: Timeline): void {
+    if (!Number.isFinite(t) || t < 0) throw new Error(`vfrey.at: ${String(t)} is not a time in seconds.`);
+    fn(pick(tl), t, t);
+  },
+  /**
+   * Times a block from `npx vfrey scene add`: the block mounted on `target` (a selector or an element) starts at `start`
+   * seconds, and lasts `duration` seconds when given. `vfrey.onLine(1, (tl, t) => vfrey.block("#block-lt-clean-bar", t))`.
+   */
+  block(target: string | Element, start: number, duration?: number): void {
+    if (!Number.isFinite(start) || start < 0) throw new Error(`vfrey.block: ${String(start)} is not a time in seconds.`);
+    const found = typeof target === "string" ? g.document.querySelector(target) : target;
+    if (!found) throw new Error(`vfrey.block: nothing matches ${String(target)}. Paste the snippet \`npx vfrey scene add\` printed into your root element first.`);
+    found.setAttribute("data-start", String(Math.round(start * 1000) / 1000));
+    if (duration !== undefined) found.setAttribute("data-duration", String(duration));
+  },
+  /**
+   * A shader transition between two shots (elements that fill the frame, stacked on top of each other): at `at`
+   * seconds, for `duration` (default 0.7), `from` becomes `to`. `type` is one of `npx vfrey scene transitions` (default
+   * cross-warp-morph; `crossfade` needs no GPU). Calls chain: each `from` is the previous call's `to`, and they may be
+   * made in any order. Seekable and deterministic: the shots' opacity is on the scene's GSAP timeline, and the renderer
+   * composites the shader per frame. Needs `<script src="vfrey/transitions.js">` first. Options: ease, accent, background (hex).
+   * `vfrey.transition("#intro", "#demo", { type: "glitch", at: 3.2, duration: 0.6 })`.
+   */
+  transition(from: string | Element, to: string | Element, options: Omit<TransitionSpec, "from" | "to"> & { accent?: string; background?: string }): void {
+    const find = (target: string | Element, role: string): HTMLElement => {
+      const found = typeof target === "string" ? g.document.querySelector(target) : target;
+      if (!found) throw new Error(`vfrey.transition: nothing matches ${String(target)} (the ${role} shot).`);
+      return found as HTMLElement;
+    };
+    if (!options || typeof options !== "object") throw new Error('vfrey.transition: pass { type, at, duration }, for example { type: "glitch", at: 3.2 }.');
+    const label = (t: string | Element, el: HTMLElement) => (typeof t === "string" ? t : el.id ? `#${el.id}` : el.tagName.toLowerCase());
+    const a = find(from, "from");
+    const b = find(to, "to");
+    transitionCalls.push({ ...options, from: a, to: b, names: [label(from, a), label(to, b)] });
+    if (!transitionsScheduled) {
+      transitionsScheduled = true;
+      if (g.document.readyState === "loading") g.document.addEventListener("DOMContentLoaded", applyTransitions);
+      else Promise.resolve().then(applyTransitions);
+    }
+  },
+  /** Cues `fn` at the moment word `i` starts being spoken. */
+  onWord(i: number, fn: Cue<TimingWord>, tl?: Timeline): void {
+    const word = wordAt(timing, i);
+    fn(pick(tl), word.start, word);
+  },
+  /** Cues `fn` at the moment line `i` starts being spoken. */
+  onLine(i: number, fn: Cue<ReturnType<typeof lineView>>, tl?: Timeline): void {
+    const line = lineView(timing, i);
+    fn(pick(tl), line.start, line);
+  },
+  /** One spoken line: `{ i, text, start, end, words }`. */
+  line: (i: number) => lineView(timing, i),
+  word: (i: number) => wordAt(timing, i),
+  beat(i: number) {
+    const beat = timing.beats[i];
+    if (!beat) throw new Error(`vfrey: there is no beat ${i}; the script has ${timing.beats.length} beats.`);
+    return beat;
+  },
+  /**
+   * Word-highlighted captions in `el` (a selector or an element): groups of a few words, the spoken word marked.
+   * Position the element anywhere. Options: style box|color|underline|pop|plain, size, color, highlight,
+   * highlightText, font, maxWords, maxChars, align, stroke. Returns the groups it made.
+   */
+  captions(target: string | Element, options: CaptionOptions & { timeline?: Timeline } = {}) {
+    const root = element(target);
+    const tl = pick(options.timeline);
+    captionBlocks += 1;
+    const id = `vf${captionBlocks}`;
+    const scope = `.vf-cap-${captionBlocks}`;
+    const css = g.document.createElement("style");
+    css.textContent = captionCss(options, scope);
+    g.document.head.appendChild(css);
+    root.classList.add("vf-captions", `vf-cap-${captionBlocks}`);
+    const groups = captionGroups(timing, options);
+    root.innerHTML = groups.map((group, i) => captionGroupHtml(group, i, id)).join("");
+    groups.forEach((group, i) => {
+      const box = g.document.getElementById(`${id}-g-${i}`)!;
+      tl.set(box, { className: "vf-grp on" }, group.start);
+      group.words.forEach((word, k) => {
+        const span = g.document.getElementById(`${id}-w-${i}-${k}`)!;
+        tl.set(span, { className: "vf-w on" }, word.start);
+        tl.set(span, { className: "vf-w" }, word.until);
+      });
+      tl.set(box, { className: "vf-grp" }, group.end);
+    });
+    return groups;
+  },
+};
+
+g.vfrey = vfrey;

package/src/scene/runtime/core.ts

@@ -0,0 +1,181 @@
+/**
+ * The scene helpers both runtimes share: word groups for captions, the caption stylesheet, and timing lookups.
+ * No DOM and no Node here, so the same code runs in the browser helper (`vfrey.js`) and in React (`@vfrey/scene`).
+ */
+import type { SceneTiming, TimingLine, TimingWord } from "../timing.js";
+
+export type { SceneTiming, TimingLine, TimingWord };
+
+/** Longest caption group, in characters, that fits two lines at the default size. */
+export const GROUP_MAX_CHARS = 30;
+
+/**
+ * Groups of about three to six words, ending at punctuation when that is possible. A small dynamic program: each
+ * cut costs more the further its size is from four, and a cut after a word with punctuation is free.
+ */
+export function groupSizes(words: string[], maxWords = 6, maxChars = GROUP_MAX_CHARS): number[] {
+  const n = words.length;
+  if (n === 0) return [];
+  const best: number[] = new Array(n + 1).fill(Infinity);
+  const cut: number[] = new Array(n + 1).fill(0);
+  best[0] = 0;
+  for (let end = 1; end <= n; end += 1) {
+    for (let size = 1; size <= maxWords && size <= end; size += 1) {
+      const slice = words.slice(end - size, end);
+      const chars = slice.join(" ").length;
+      if (chars > maxChars && size > 1) continue;
+      let c = best[end - size]! + Math.abs(size - 4) * 0.6 + (size < 3 ? 2.5 : 0);
+      if (end < n && !/[,.;:!?—]$/.test(words[end - 1]!)) c += 0.7;
+      if (c < best[end]!) {
+        best[end] = c;
+        cut[end] = size;
+      }
+    }
+  }
+  const sizes: number[] = [];
+  for (let end = n; end > 0; end -= cut[end]!) sizes.unshift(cut[end]!);
+  return sizes;
+}
+
+export const CAPTION_STYLES = ["box", "color", "underline", "pop", "plain"] as const;
+export type CaptionStyle = (typeof CAPTION_STYLES)[number];
+
+export interface CaptionOptions {
+  /** `box` (default): the spoken word sits in a filled box. `color`: it changes color. `underline`: a bar under it. `pop`: it grows and changes color. `plain`: no highlight. */
+  style?: CaptionStyle;
+  /** Font size in px. Default 68. */
+  size?: number;
+  /** Text color. Default #000000. */
+  color?: string;
+  /** Box, color or underline color of the spoken word. Default #2196F3. */
+  highlight?: string;
+  /** Text color of the spoken word inside a box. Default #FFFFFF. */
+  highlightText?: string;
+  /** CSS font-family. Default "Titan One", which `vfrey/fonts/fonts.css` provides. */
+  font?: string;
+  /** Most words in one group. Default 6. */
+  maxWords?: number;
+  /** Most characters in one group. Default 30. */
+  maxChars?: number;
+  /** `center` (default), `left` or `right`. */
+  align?: "center" | "left" | "right";
+  /** Add a thick outline of this color around the text, for captions over busy pictures. */
+  stroke?: string;
+}
+
+export interface CaptionWord {
+  /** Index in `timing.words`. */
+  index: number;
+  text: string;
+  start: number;
+  /** Until the next word starts (or the group ends). */
+  until: number;
+}
+
+export interface CaptionGroup {
+  line: number;
+  words: CaptionWord[];
+  /** The group shows from `start` to `end`. */
+  start: number;
+  end: number;
+}
+
+/** The caption groups for a timing file: never across a line, each shown until the next begins or half a second after its last word. */
+export function captionGroups(timing: Pick<SceneTiming, "words" | "lines">, options: CaptionOptions = {}): CaptionGroup[] {
+  const groups: CaptionGroup[] = [];
+  for (const line of timing.lines) {
+    const words = timing.words.slice(line.firstWord, line.lastWord + 1);
+    const sizes = groupSizes(words.map((w) => w.text), options.maxWords ?? 6, options.maxChars ?? GROUP_MAX_CHARS);
+    let at = 0;
+    for (const size of sizes) {
+      const slice = words.slice(at, at + size);
+      at += size;
+      groups.push({
+        line: line.i,
+        words: slice.map((w) => ({ index: w.i, text: w.text, start: w.start, until: w.end })),
+        start: slice[0]!.start,
+        end: 0,
+      });
+    }
+  }
+  groups.forEach((g, i) => {
+    const next = groups[i + 1];
+    const lastEnd = g.words[g.words.length - 1]!.until;
+    g.end = Math.min(next ? next.start : Infinity, lastEnd + 0.5);
+    g.words.forEach((w, k) => {
+      const after = g.words[k + 1];
+      w.until = after ? after.start : g.end;
+    });
+  });
+  return groups;
+}
+
+/** The group on screen at time t and the spoken word inside it, or -1 for none. */
+export function captionStateAt(groups: CaptionGroup[], t: number): { group: number; word: number } {
+  for (let i = 0; i < groups.length; i += 1) {
+    const g = groups[i]!;
+    if (t >= g.start && t < g.end) {
+      let word = -1;
+      for (let k = 0; k < g.words.length; k += 1) if (g.words[k]!.start <= t) word = k;
+      return { group: i, word };
+    }
+  }
+  return { group: -1, word: -1 };
+}
+
+/** The stylesheet for `.vf-captions`. */
+export function captionCss(options: CaptionOptions = {}, scope = ".vf-captions"): string {
+  const style = options.style ?? "box";
+  const size = options.size ?? 68;
+  const color = options.color ?? "#000000";
+  const hi = options.highlight ?? "#2196F3";
+  const hiText = options.highlightText ?? "#FFFFFF";
+  const font = options.font ?? '"Titan One", "Arial Black", sans-serif';
+  const justify = options.align === "left" ? "flex-start" : options.align === "right" ? "flex-end" : "center";
+  const outline = options.stroke ? `-webkit-text-stroke: ${Math.max(4, Math.round(size / 9))}px ${options.stroke}; paint-order: stroke fill;` : "";
+  const on =
+    style === "box"
+      ? `background: ${hi}; color: ${hiText};`
+      : style === "color"
+        ? `color: ${hi};`
+        : style === "underline"
+          ? `box-shadow: inset 0 -0.14em 0 ${hi};`
+          : style === "pop"
+            ? `color: ${hi}; transform: scale(1.14);`
+            : "";
+  return `
+${scope} { display: flex; flex-direction: column; justify-content: flex-end; align-items: ${justify}; font-family: ${font}; font-size: ${size}px; line-height: 1.12; color: ${color}; ${outline} }
+${scope} * { box-sizing: border-box; }
+${scope} .vf-grp { display: none; flex-wrap: wrap; justify-content: ${justify}; align-items: center; gap: 8px 10px; }
+${scope} .vf-grp.on { display: flex; }
+${scope} .vf-w { display: inline-block; padding: 4px 16px 8px; border-radius: 14px; background: transparent; ${style === "pop" ? "transform-origin: center bottom;" : ""} }
+${scope} .vf-w.on { ${on} }`;
+}
+
+export function escapeHtml(text: string): string {
+  return text.replace(/&/g, "&amp;").replace(/</g, "&lt;").replace(/>/g, "&gt;").replace(/"/g, "&quot;");
+}
+
+/** One caption group as markup. `ids` gives every group and word an id the timeline cues point at. */
+export function captionGroupHtml(group: CaptionGroup, index: number, prefix: string): string {
+  const spans = group.words.map((w, k) => `<span id="${prefix}-w-${index}-${k}" class="vf-w">${escapeHtml(w.text)}</span>`).join("");
+  return `<div id="${prefix}-g-${index}" class="vf-grp">${spans}</div>`;
+}
+
+export function lineAt(timing: Pick<SceneTiming, "lines">, i: number): TimingLine {
+  const line = timing.lines[i];
+  if (!line) throw new Error(`vfrey: there is no line ${i}; the script has ${timing.lines.length} lines (0 to ${timing.lines.length - 1}).`);
+  return line;
+}
+
+export function wordAt(timing: Pick<SceneTiming, "words">, i: number): TimingWord {
+  const word = timing.words[i];
+  if (!word) throw new Error(`vfrey: there is no word ${i}; the script has ${timing.words.length} words (0 to ${timing.words.length - 1}).`);
+  return word;
+}
+
+/** A line with its words, the shape `vfrey.line(i)` returns. */
+export function lineView(timing: Pick<SceneTiming, "lines" | "words">, i: number): TimingLine & { words: TimingWord[] } {
+  const line = lineAt(timing, i);
+  return { ...line, words: timing.words.slice(line.firstWord, line.lastWord + 1) };
+}