@foldspace_npm/harness 0.1.31 → 0.1.33

package/src/video/compose.mjs

@@ -0,0 +1,476 @@
+// Making the MP4 from a recording, in a hidden Chrome.
+//
+// No ffmpeg and nothing to install: a separate headless instance of the same
+// Chrome `inject` uses loads the composer page from a short-lived server on
+// 127.0.0.1, draws the video on a canvas, encodes H.264 with WebCodecs and
+// posts the MP4 back. The headless Chrome gets a throwaway profile, and
+// --use-mock-keychain --password-store=basic: without them macOS asks for the
+// keychain password the moment Chrome starts.
+
+import { spawn } from "node:child_process";
+import crypto from "node:crypto";
+import fs from "node:fs";
+import http from "node:http";
+import os from "node:os";
+import path from "node:path";
+import { fileURLToPath } from "node:url";
+
+import { findChrome } from "../chrome-binary.mjs";
+import { scenePlace } from "./scenes.mjs";
+import { TIMING, fitToLength, layoutClips, sceneTimeMap } from "./timeline.mjs";
+
+const videoRoot = path.dirname(fileURLToPath(import.meta.url));
+
+export const PRESETS = Object.freeze({
+  landscape: { width: 1920, height: 1080, bitrate: 8_000_000 },
+  square: { width: 1080, height: 1080, bitrate: 6_000_000 },
+});
+
+export class VideoError extends Error {
+  constructor(code, message) {
+    super(message);
+    this.code = code;
+  }
+}
+
+// ------------------------------------------------------------ recording on disk
+
+const RECORD_AGAIN = "Record again: `npx foldspace record`.";
+
+/**
+ * A recording folder: record.json plus scene-<n>/{events.json, frames.json,
+ * frames/}. Only a finished one makes a video: record.json says `complete`
+ * once every scene's answer settled. A scene that failed or was cut off would
+ * put a caption over an unanswered "Thinking...".
+ */
+export function loadRecording(recordDir) {
+  const manifestPath = path.join(recordDir, "record.json");
+  let manifest;
+  try {
+    manifest = JSON.parse(fs.readFileSync(manifestPath, "utf8"));
+  } catch {
+    throw new VideoError("no_recording", `No recording in ${recordDir} (record.json is missing).`);
+  }
+  if (manifest.complete !== true) {
+    const stopped = (manifest.scenes || []).find((scene) => scene.ok !== true);
+    throw new VideoError(
+      "recording_incomplete",
+      stopped
+        ? `Scene ${stopped.n} of this recording never finished${stopped.reason ? ` (${stopped.reason})` : ""}, so it makes no video. ${RECORD_AGAIN}`
+        : `This recording stopped before it finished, so it makes no video. ${RECORD_AGAIN}`,
+    );
+  }
+  const scenes = (manifest.scenes || []).map((scene) => {
+    const dir = path.join(recordDir, scene.dir);
+    try {
+      const events = JSON.parse(fs.readFileSync(path.join(dir, "events.json"), "utf8"));
+      const frames = JSON.parse(fs.readFileSync(path.join(dir, "frames.json"), "utf8"));
+      return { ...scene, events, frames };
+    } catch {
+      throw new VideoError("recording_incomplete", `Scene ${scene.n} of this recording is missing its files. ${RECORD_AGAIN}`);
+    }
+  });
+  return { dir: recordDir, manifest, scenes };
+}
+
+/** A scene's events and picture times in seconds, for the timeline. */
+export function sceneTimes(scene) {
+  const seconds = (ms) => (ms == null ? null : ms / 1000);
+  const { events } = scene;
+  return {
+    events: {
+      typingStart: seconds(events.typingStart),
+      sent: seconds(events.sent),
+      firstReply: seconds(events.firstReply),
+      settled: seconds(events.settledAt),
+    },
+    frameTimes: scene.frames.map((frame) => frame.t / 1000),
+  };
+}
+
+const CAPTIONS_ONLY =
+  "Only the product name, the capabilities, the lines and the end card can change without recording again.";
+
+/** The recording must be of the questions in the scenes file now; the rest of each caption may change. */
+export function checkRecordingMatches(recording, config) {
+  const recorded = recording.scenes.map((scene) => scene.question);
+  const wanted = config.scenes.map((scene) => scene.question);
+  if (recorded.length !== wanted.length) {
+    return `The recording has ${recorded.length} scene(s) and the scenes file ${wanted.length}. ${CAPTIONS_ONLY}`;
+  }
+  const changed = wanted.findIndex((question, index) => question !== recorded[index]);
+  if (changed >= 0) {
+    return `Scene ${changed + 1}'s question is not the one recorded. ${CAPTIONS_ONLY}`;
+  }
+  return null;
+}
+
+// ------------------------------------------------------------ plan
+
+/**
+ * Everything the composer page needs, as JSON: the frame size, the clips in
+ * order with their time maps and captions, and where the poster is taken.
+ * The opening card meets the product's agent; each caption shows the scene's
+ * capabilities, where the video is ("2 of 4"), the question and its line.
+ * The colours are Foldspace's own (brand.mjs), never the app's.
+ */
+export function buildPlan({ recording, config, preset = "landscape", endCard = true, timing = TIMING }) {
+  const size = PRESETS[preset];
+  if (!size) throw new VideoError("bad_preset", `Unknown preset '${preset}': landscape or square.`);
+  const timed = recording.scenes.map(sceneTimes);
+  const fit = fitToLength(timed, { titleCard: true, endCard, timing });
+  const product = config.product;
+  const total = config.scenes.length;
+  const clips = [{ kind: "opening", product, fadeInS: 0.5, durationS: timing.openingS }];
+  recording.scenes.forEach((scene, index) => {
+    const map = sceneTimeMap(timed[index].events, timed[index].frameTimes, { k: fit.k, timing });
+    const { question, capabilities, line } = config.scenes[index];
+    clips.push({
+      kind: "scene",
+      n: index + 1,
+      dir: scene.dir,
+      frames: scene.frames.map((frame) => frame.file),
+      frameTimes: timed[index].frameTimes,
+      segments: map.segments,
+      caption: map.caption,
+      captionText: { capabilities, place: scenePlace(index + 1, total), question, line },
+      viewport: scene.events.viewport || null,
+      panel: (scene.events.panel || []).map(({ x, y, width, height }) => ({ x, y, width, height })),
+      durationS: map.durationS,
+    });
+  });
+  if (endCard) {
+    clips.push({
+      kind: "end",
+      eyebrow: `${product}  ×  Foldspace`,
+      title: "An agent that does the work.",
+      sub: "Built with Foldspace.   foldspace.ai",
+      mark: true,
+      durationS: timing.endS,
+    });
+  }
+  const layout = layoutClips(
+    clips.map((clip) => clip.durationS),
+    timing.dissolveS,
+  );
+  // The poster: the first scene's answer with its caption fully in.
+  const first = clips.findIndex((clip) => clip.kind === "scene");
+  const posterS = first >= 0 ? layout.starts[first] + (clips[first].caption.inEnd + clips[first].caption.outStart) / 2 : 1;
+  return {
+    ...size,
+    preset,
+    product,
+    clips,
+    layout,
+    posterS,
+    fit,
+  };
+}
+
+// ------------------------------------------------------------ hidden Chrome
+
+const HEADLESS_ARGS = [
+  "--headless=new",
+  "--use-mock-keychain",
+  "--password-store=basic",
+  "--no-first-run",
+  "--no-default-browser-check",
+  "--disable-extensions",
+  "--disable-sync",
+  "--disable-default-apps",
+  "--disable-background-networking",
+  "--disable-component-update",
+  "--disable-background-timer-throttling",
+  "--disable-renderer-backgrounding",
+  "--mute-audio",
+  "--hide-scrollbars",
+];
+
+// Every hidden Chrome this process has started and not yet closed. A run that
+// is stopped (Ctrl-C, a timeout) closes them through closeAllHeadless(); one
+// that exits any other way kills them on the way out. Left behind, each would
+// keep running with a profile in the temp folder.
+const launched = new Set();
+let exitHookSet = false;
+
+function killLeftovers() {
+  for (const chrome of launched) chrome.killNow();
+}
+
+/** Close every hidden Chrome this process started (for an interrupted run). */
+export async function closeAllHeadless() {
+  await Promise.allSettled([...launched].map((chrome) => chrome.close()));
+}
+
+/**
+ * Start a hidden Chrome on `url` with a throwaway profile. `extraArgs` lets a
+ * test add a debugging port. Returns { child, profileDir, close() }.
+ */
+export function launchHeadless(chromePath, url, { extraArgs = [] } = {}) {
+  const profileDir = fs.mkdtempSync(path.join(os.tmpdir(), "foldspace-video-chrome-"));
+  const child = spawn(chromePath, [...HEADLESS_ARGS, `--user-data-dir=${profileDir}`, ...extraArgs, url], {
+    stdio: "ignore",
+  });
+  let exited = false;
+  child.on("exit", () => {
+    exited = true;
+  });
+  child.on("error", () => {
+    exited = true;
+  });
+  const chrome = {
+    child,
+    profileDir,
+    get exited() {
+      return exited;
+    },
+    // Synchronous, for process exit: no time to wait for a clean close.
+    killNow() {
+      launched.delete(chrome);
+      if (!exited) {
+        try {
+          child.kill("SIGKILL");
+        } catch {
+          // already gone
+        }
+      }
+      try {
+        fs.rmSync(profileDir, { recursive: true, force: true });
+      } catch {
+        // Chrome may still hold it for a moment; the temp folder is the OS's to clear
+      }
+    },
+    async close() {
+      launched.delete(chrome);
+      if (!exited) {
+        child.kill("SIGTERM");
+        for (let tick = 0; tick < 30 && !exited; tick++) await new Promise((resolve) => setTimeout(resolve, 100));
+        if (!exited) child.kill("SIGKILL");
+      }
+      // Chrome can still be letting go of its profile for a moment.
+      for (let attempt = 0; attempt < 5; attempt++) {
+        try {
+          fs.rmSync(profileDir, { recursive: true, force: true });
+          break;
+        } catch {
+          await new Promise((resolve) => setTimeout(resolve, 200));
+        }
+      }
+    },
+  };
+  launched.add(chrome);
+  if (!exitHookSet) {
+    exitHookSet = true;
+    process.on("exit", killLeftovers);
+  }
+  return chrome;
+}
+
+const TYPES = {
+  ".html": "text/html; charset=utf-8",
+  ".mjs": "text/javascript; charset=utf-8",
+  ".js": "text/javascript; charset=utf-8",
+  ".json": "application/json",
+  ".woff2": "font/woff2",
+  ".svg": "image/svg+xml",
+  ".jpg": "image/jpeg",
+};
+
+// What the composer page may read from this package.
+const STATIC_ROOTS = ["composer", "vendor", "fonts", "assets"];
+const STATIC_FILES = ["timeline.mjs", "brand.mjs"];
+
+function inside(root, relative) {
+  const full = path.resolve(root, relative);
+  return full === root || full.startsWith(`${root}${path.sep}`) ? full : null;
+}
+
+/**
+ * The page's server: 127.0.0.1 only, every path under a random token, so no
+ * other page or process on the machine can guess its way to the recorded
+ * pictures (they show the customer's real data). Resolves `done` when the
+ * page has posted the video, or fails with what the page reported.
+ */
+function startServer({ plan, recordDir: givenDir, onProgress }) {
+  const recordDir = path.resolve(givenDir);
+  const token = crypto.randomBytes(16).toString("hex");
+  const posted = {};
+  let settle;
+  const finished = new Promise((resolve, reject) => {
+    settle = { resolve, reject };
+  });
+  const server = http.createServer((request, response) => {
+    const url = new URL(request.url, "http://127.0.0.1");
+    const prefix = `/${token}/`;
+    if (!url.pathname.startsWith(prefix)) {
+      response.writeHead(404).end();
+      return;
+    }
+    const rest = decodeURIComponent(url.pathname.slice(prefix.length));
+    if (request.method === "POST") {
+      const chunks = [];
+      request.on("data", (chunk) => chunks.push(chunk));
+      request.on("end", () => {
+        const body = Buffer.concat(chunks);
+        response.writeHead(204).end();
+        if (rest === "progress") {
+          try {
+            onProgress?.(JSON.parse(body.toString("utf8")));
+          } catch {
+            // progress is only for the log
+          }
+        } else if (rest === "fail") {
+          let report = {};
+          try {
+            report = JSON.parse(body.toString("utf8"));
+          } catch {
+            // keep the generic message
+          }
+          settle.reject(new VideoError(report.code || "page_error", report.message || "The composer page failed."));
+        } else if (rest === "done") {
+          try {
+            settle.resolve({ ...posted, done: JSON.parse(body.toString("utf8")) });
+          } catch {
+            settle.resolve({ ...posted, done: {} });
+          }
+        } else {
+          posted[rest] = body;
+        }
+      });
+      return;
+    }
+    let file = null;
+    if (rest === "plan.json") {
+      response.writeHead(200, { "content-type": TYPES[".json"], "cache-control": "no-store" });
+      response.end(JSON.stringify(plan));
+      return;
+    }
+    if (rest.startsWith("rec/")) file = inside(recordDir, rest.slice(4));
+    else if (STATIC_FILES.includes(rest) || STATIC_ROOTS.includes(rest.split("/")[0])) file = inside(videoRoot, rest);
+    if (!file || !fs.existsSync(file) || !fs.statSync(file).isFile()) {
+      response.writeHead(404).end();
+      return;
+    }
+    response.writeHead(200, {
+      "content-type": TYPES[path.extname(file)] || "application/octet-stream",
+      "cache-control": "no-store",
+    });
+    fs.createReadStream(file).pipe(response);
+  });
+  return new Promise((resolve) => {
+    server.listen(0, "127.0.0.1", () => {
+      const { port } = server.address();
+      resolve({
+        url: `http://127.0.0.1:${port}/${token}/composer/index.html`,
+        finished,
+        close: () => new Promise((done) => server.close(() => done())),
+        fail: (error) => settle.reject(error),
+      });
+    });
+  });
+}
+
+/** Run the composer page in a hidden Chrome; returns { mp4, poster, done }. */
+export async function renderInChrome({ plan, recordDir, chromePath, onProgress, timeoutMs = 15 * 60 * 1000 }) {
+  const server = await startServer({ plan, recordDir, onProgress });
+  const chrome = launchHeadless(chromePath, server.url);
+  chrome.child.on("exit", (code, signal) =>
+    server.fail(
+      new VideoError(
+        "chrome_exited",
+        `The hidden Chrome closed before the video was made (${signal ? `stopped by ${signal}` : `exit code ${code}`}).`,
+      ),
+    ),
+  );
+  chrome.child.on("error", (error) =>
+    server.fail(new VideoError("chrome_failed", `Could not start Chrome at ${chromePath}: ${error.message}`)),
+  );
+  const timer = setTimeout(
+    () => server.fail(new VideoError("timeout", `The video was not made within ${Math.round(timeoutMs / 60000)} minutes.`)),
+    timeoutMs,
+  );
+  try {
+    const result = await server.finished;
+    if (!result.out) throw new VideoError("page_error", "The composer page finished without a video.");
+    return { mp4: result.out, poster: result.poster || null, done: result.done };
+  } finally {
+    clearTimeout(timer);
+    await chrome.close();
+    await server.close();
+  }
+}
+
+const PROBE_PAGE = `<!doctype html><meta charset="utf-8"><script type="module">
+const configs = ["avc1.640028", "avc1.4d0028", "avc1.42e028"].map((codec) => ({
+  codec, width: 1920, height: 1080, bitrate: 8000000, framerate: 30, avc: { format: "avc" } }));
+let codec = null;
+if (typeof VideoEncoder === "function") {
+  for (const config of configs) {
+    try { if ((await VideoEncoder.isConfigSupported(config)).supported) { codec = config.codec; break; } } catch {}
+  }
+}
+await fetch("result", { method: "POST", body: JSON.stringify({ codec }) });
+</script>`;
+
+/**
+ * Whether this Chrome can encode H.264, asked in a hidden Chrome the same way
+ * the composer starts one: { ok, codec } or { ok: false, code, message }.
+ * Run before recording, so a Chrome that cannot make the video says so before
+ * minutes of recording rather than after.
+ */
+export async function probeEncoder({ chromePath = findChrome(), timeoutMs = 60_000 } = {}) {
+  if (!chromePath) return { ok: false, code: "no_chrome", message: needsChrome(null) };
+  const token = crypto.randomBytes(16).toString("hex");
+  let answer;
+  const answered = new Promise((resolve) => {
+    answer = resolve;
+  });
+  const server = http.createServer((request, response) => {
+    if (request.url === `/${token}/` && request.method === "GET") {
+      response.writeHead(200, { "content-type": TYPES[".html"] }).end(PROBE_PAGE);
+      return;
+    }
+    if (request.url === `/${token}/result` && request.method === "POST") {
+      const chunks = [];
+      request.on("data", (chunk) => chunks.push(chunk));
+      request.on("end", () => {
+        response.writeHead(204).end();
+        try {
+          answer(JSON.parse(Buffer.concat(chunks).toString("utf8")));
+        } catch {
+          answer({ codec: null });
+        }
+      });
+      return;
+    }
+    response.writeHead(404).end();
+  });
+  await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve));
+  const chrome = launchHeadless(chromePath, `http://127.0.0.1:${server.address().port}/${token}/`);
+  chrome.child.on("error", () => answer({ failed: true }));
+  chrome.child.on("exit", () => answer({ failed: true }));
+  const timer = setTimeout(() => answer({ failed: true, timeout: true }), timeoutMs);
+  try {
+    const result = await answered;
+    if (result.failed) {
+      return {
+        ok: false,
+        code: "chrome_failed",
+        message: `Chrome at ${chromePath} did not start in the background${result.timeout ? " in time" : ""}, so the video cannot be made.`,
+      };
+    }
+    if (!result.codec) return { ok: false, code: "no_h264", message: needsChrome(chromePath) };
+    return { ok: true, codec: result.codec, chromePath };
+  } finally {
+    clearTimeout(timer);
+    await chrome.close();
+    await new Promise((resolve) => server.close(() => resolve()));
+  }
+}
+
+/** The message for a Chrome that cannot make the video. */
+export function needsChrome(chromePath) {
+  return chromePath
+    ? `The video needs Google Chrome: this Chrome (${chromePath}) cannot encode H.264 video, as some Chromium builds cannot. Install Google Chrome, or set CHROME_PATH to it.`
+    : "The video needs Google Chrome, and none was found. Install Google Chrome, or set CHROME_PATH to it.";
+}