overlay-factory-worker 0.1.0

package/scripts/place-overlay.ts

@@ -0,0 +1,359 @@
+/**
+ * Pick where the code card and the difficulty meter should sit.
+ *
+ * Samples frames from the b-roll, scores regions by how busy they are (edge
+ * energy) plus a penalty for skin tones, and moves the overlays to the calmest
+ * places that don't cover Erin or Instagram's own UI.
+ *
+ * The card is the important one: it's large and centred, so on footage where
+ * she's centre-frame it lands squarely on her face and crops her to a
+ * forehead. It can move down, and it can shrink — a smaller card that shows
+ * her face beats a bigger one that doesn't.
+ *
+ * A caveat worth keeping in mind when reading the scores: the skin test is
+ * per-pixel COLOUR classification, not face detection. It fires happily on
+ * wood, terracotta, and warm-lit walls, and it can't tell a face from a hand.
+ * It's a good "avoid this region" prior and a poor face locator.
+ *
+ * Usage: npx tsx scripts/place-overlay.ts episodes/what-prints-001.json
+ */
+import { execFileSync } from "node:child_process";
+import { readFileSync, writeFileSync, mkdtempSync, rmSync } from "node:fs";
+import { join } from "node:path";
+import { tmpdir } from "node:os";
+import sharp from "sharp";
+import { SAFE_AREA } from "../src/overlay/safeArea";
+import {
+  CARD_MIN_WIDTH_PCT,
+  CARD_WIDTH_PCT,
+  cardBox,
+  cardCenterRange,
+} from "../src/overlay/cardLayout";
+
+const W = 1080;
+const H = 1920;
+const ANALYSIS_W = 270; // quarter scale — plenty for a busyness map
+const SCALE = W / ANALYSIS_W;
+
+// Measured off a render, not guessed: the pill draws 400x146.
+const METER = { width: 400, height: 150 };
+const MARGIN = 70;
+
+/**
+ * The meter's home. A series is recognisable partly because things sit in the
+ * same place every time, so the badge stays here unless the footage underneath
+ * is genuinely too busy to read against — content-aware placement is the
+ * exception, not the rule.
+ */
+const HOME = { x: MARGIN, y: 1330 };
+
+/**
+ * How much busier than the calmest available spot HOME is allowed to be before
+ * we give up on consistency. Relative, not absolute: footage with Erin in frame
+ * scores high everywhere (the skin penalty dominates), so a fixed threshold
+ * would move the badge on every episode and we would never keep a house
+ * position at all.
+ */
+const HOME_TOLERANCE = 1.25;
+
+// Instagram's own chrome — see src/overlay/safeArea.ts.
+const IG_BOTTOM_RESERVED = SAFE_AREA.bottom;
+const IG_RIGHT_RESERVED = SAFE_AREA.right;
+
+type Episode = {
+  broll: string;
+  card: string;
+  durationSec: number;
+  revealAtSec: number;
+  trimBeforeSec: number;
+  meterX?: number;
+  meterY?: number;
+  cardY?: number;
+  cardWidthPct?: number;
+};
+
+/**
+ * How much busier the full-size, centred card may be than the best alternative
+ * before it gets moved or shrunk.
+ *
+ * Same reasoning as HOME_TOLERANCE: the card should sit in its usual place on
+ * most episodes, because a series is recognisable partly by things being where
+ * they always are. Moving it is for the episodes where it genuinely lands on
+ * her face.
+ */
+const CARD_TOLERANCE = 1.2;
+
+const frameAt = (video: string, seconds: number, out: string) => {
+  execFileSync("ffmpeg", [
+    "-y", "-loglevel", "error",
+    "-ss", String(seconds),
+    "-i", video,
+    "-frames:v", "1",
+    "-vf", `scale=${ANALYSIS_W}:-1`,
+    out,
+  ]);
+};
+
+/** Per-pixel "how much is going on here", 0..1, plus a skin-tone flag. */
+const analyzeFrame = async (file: string) => {
+  const { data, info } = await sharp(file)
+    .resize(ANALYSIS_W)
+    .raw()
+    .toBuffer({ resolveWithObject: true });
+  const { width, height, channels } = info;
+
+  const energy = new Float64Array(width * height);
+  const at = (x: number, y: number, c: number) =>
+    data[(y * width + x) * channels + c];
+
+  for (let y = 1; y < height - 1; y++) {
+    for (let x = 1; x < width - 1; x++) {
+      const lum = (xx: number, yy: number) =>
+        0.299 * at(xx, yy, 0) + 0.587 * at(xx, yy, 1) + 0.114 * at(xx, yy, 2);
+      const gx = lum(x + 1, y) - lum(x - 1, y);
+      const gy = lum(x, y + 1) - lum(x, y - 1);
+      let e = Math.min(1, Math.hypot(gx, gy) / 90);
+
+      // Skin tones: strongly penalized so the badge never lands on her hands.
+      const r = at(x, y, 0);
+      const g = at(x, y, 1);
+      const b = at(x, y, 2);
+      const isSkin =
+        r > 95 && g > 40 && b > 20 && r > g && r > b && r - Math.min(g, b) > 15;
+      if (isSkin) e += 1.5;
+
+      energy[y * width + x] = e;
+    }
+  }
+  return { energy, width, height };
+};
+
+/** Summed-area table so rectangle scores are O(1). */
+const integral = (src: Float64Array, width: number, height: number) => {
+  const sum = new Float64Array((width + 1) * (height + 1));
+  for (let y = 0; y < height; y++) {
+    let rowSum = 0;
+    for (let x = 0; x < width; x++) {
+      rowSum += src[y * width + x];
+      sum[(y + 1) * (width + 1) + (x + 1)] =
+        sum[y * (width + 1) + (x + 1)] + rowSum;
+    }
+  }
+  return sum;
+};
+
+const rectMean = (
+  sum: Float64Array,
+  width: number,
+  x0: number, y0: number, x1: number, y1: number,
+) => {
+  const s =
+    sum[y1 * (width + 1) + x1] -
+    sum[y0 * (width + 1) + x1] -
+    sum[y1 * (width + 1) + x0] +
+    sum[y0 * (width + 1) + x0];
+  const area = Math.max(1, (x1 - x0) * (y1 - y0));
+  return s / area;
+};
+
+const main = async () => {
+  const episodePath = process.argv[2];
+  if (!episodePath) {
+    console.error("usage: place-overlay.ts <episode.json>");
+    process.exit(1);
+  }
+  const episode = JSON.parse(readFileSync(episodePath, "utf8")) as Episode;
+  const video = join(import.meta.dirname, "../public", episode.broll);
+
+  const cardMeta = await sharp(
+    join(import.meta.dirname, "../public", episode.card),
+  ).metadata();
+  const aspect = cardMeta.height! / cardMeta.width!;
+
+  // Sample across the CARD's time on screen, not the meter's. The card is up
+  // from the first frame and the meter only appears part-way in, so the meter's
+  // window misses exactly the stretch where the card can land on her.
+  const dir = mkdtempSync(join(tmpdir(), "reel-place-"));
+  const cardStart = episode.trimBeforeSec;
+  const end = episode.trimBeforeSec + episode.durationSec;
+  const cardTimes = [0, 0.25, 0.5, 0.75, 0.95].map(
+    (f) => cardStart + (end - cardStart) * f,
+  );
+
+  const cardMaps = [];
+  for (const [i, t] of cardTimes.entries()) {
+    const f = join(dir, `c${i}.png`);
+    frameAt(video, t, f);
+    cardMaps.push(await analyzeFrame(f));
+  }
+  const cw = cardMaps[0].width;
+  const ch = cardMaps[0].height;
+  const cardSums = cardMaps.map((m) => integral(m.energy, cw, ch));
+
+  /** Worst-frame busyness under a box, in frame coordinates. */
+  const boxScore = (bx: number, by: number, bw: number, bh: number) =>
+    Math.max(
+      ...cardSums.map((sum) =>
+        rectMean(
+          sum,
+          cw,
+          Math.max(0, Math.floor(bx / SCALE)),
+          Math.max(0, Math.floor(by / SCALE)),
+          Math.min(cw, Math.ceil((bx + bw) / SCALE)),
+          Math.min(ch, Math.ceil((by + bh) / SCALE)),
+        ),
+      ),
+    );
+
+  // The card's usual place: full width, centred on the safe band. Everything
+  // else is measured against this, because staying put is the default.
+  const cardHome = cardBox(aspect);
+  const homeCardScore = boxScore(
+    cardHome.x,
+    cardHome.y,
+    cardHome.width,
+    cardHome.height,
+  );
+
+  // Candidates: a few widths, each swept down its legal vertical range. Larger
+  // is preferred, so the sort breaks ties toward keeping the card readable.
+  type CardCand = { widthPct: number; centerY: number; score: number };
+  const cardCands: CardCand[] = [];
+  for (const widthPct of [CARD_WIDTH_PCT, 0.78, 0.7, CARD_MIN_WIDTH_PCT]) {
+    const box = cardBox(aspect, widthPct);
+    // Leave the meter somewhere to live: it must sit below the card.
+    const [minC, maxC] = cardCenterRange(box.height, METER.height + 60);
+    for (let cy = minC; cy <= maxC; cy += 40) {
+      const y = cy - box.height / 2;
+      cardCands.push({
+        widthPct,
+        centerY: cy,
+        // Shrinking is a real cost — a smaller card is harder to read — so it
+        // has to buy a real improvement in busyness to win.
+        score:
+          boxScore(box.x, y, box.width, box.height) *
+          (1 + (CARD_WIDTH_PCT - widthPct) * 0.5),
+      });
+    }
+  }
+  cardCands.sort((a, b) => a.score - b.score);
+  const bestCard = cardCands[0];
+
+  const moveCard = bestCard && homeCardScore > bestCard.score * CARD_TOLERANCE;
+  if (moveCard) {
+    episode.cardY = Math.round(bestCard.centerY);
+    episode.cardWidthPct = Number(bestCard.widthPct.toFixed(3));
+    console.log(
+      `${episodePath}: card busy at home (${homeCardScore.toFixed(3)}) — ` +
+        `moved to y=${episode.cardY} at ${Math.round(bestCard.widthPct * 100)}% width ` +
+        `(${bestCard.score.toFixed(3)})`,
+    );
+  } else {
+    delete episode.cardY;
+    delete episode.cardWidthPct;
+    console.log(
+      `${episodePath}: card stays home (busyness ${homeCardScore.toFixed(3)})`,
+    );
+  }
+
+  // Whatever the card ended up as, the meter goes below THAT.
+  const placedCard = cardBox(
+    aspect,
+    episode.cardWidthPct ?? CARD_WIDTH_PCT,
+    episode.cardY,
+  );
+  const cardBottom = placedCard.bottom;
+
+  // Sample the window where the meter is on screen.
+  const start = episode.trimBeforeSec + episode.revealAtSec;
+  const times = [start, (start + end) / 2, Math.max(start, end - 0.2)];
+
+  const maps = [];
+  for (const [i, t] of times.entries()) {
+    const f = join(dir, `f${i}.png`);
+    frameAt(video, t, f);
+    maps.push(await analyzeFrame(f));
+  }
+
+  const width = maps[0].width;
+  const height = maps[0].height;
+  const sums = maps.map((m) => integral(m.energy, width, height));
+
+  // Candidate slots: the band under the card, clear of Instagram's UI.
+  const mw = METER.width / SCALE;
+  const mh = METER.height / SCALE;
+  const candidates: { x: number; y: number; score: number; raw: number }[] = [];
+
+  // Always below the code card — the meter reads as the answer to the snippet,
+  // so it should never sit above it.
+  const yBands: [number, number][] = [
+    [cardBottom + 40, H - IG_BOTTOM_RESERVED - METER.height],
+  ];
+
+  for (const [yStart, yEnd] of yBands) {
+    if (yEnd <= yStart) continue;
+    for (let py = Math.round(yStart); py <= yEnd; py += 30) {
+      for (
+        let px = MARGIN;
+        px <= W - IG_RIGHT_RESERVED - METER.width;
+        px += 40
+      ) {
+        const x0 = Math.floor(px / SCALE);
+        const y0 = Math.floor(py / SCALE);
+        const x1 = Math.min(width, Math.ceil(x0 + mw));
+        const y1 = Math.min(height, Math.ceil(y0 + mh));
+
+        // Worst frame decides — the spot must stay clear the whole time.
+        let score = 0;
+        for (const sum of sums) {
+          score = Math.max(score, rectMean(sum, width, x0, y0, x1, y1));
+        }
+        // Mild preference for the lower-left, where the badge has always sat.
+        const pull =
+          0.04 * (px / W) + 0.04 * (1 - py / H);
+        candidates.push({ x: px, y: py, score: score + pull, raw: score });
+      }
+    }
+  }
+
+  candidates.sort((a, b) => a.score - b.score);
+  const best = candidates[0];
+
+  // Score the home slot on the same frames, nudged down if a tall card reaches it.
+  const home = { x: HOME.x, y: Math.max(HOME.y, Math.round(cardBottom + 40)) };
+  const hx = Math.floor(home.x / SCALE);
+  const hy = Math.floor(home.y / SCALE);
+  let homeScore = 0;
+  for (const sum of sums)
+    homeScore = Math.max(
+      homeScore,
+      rectMean(
+        sum,
+        width,
+        hx,
+        hy,
+        Math.min(width, Math.ceil(hx + mw)),
+        Math.min(height, Math.ceil(hy + mh)),
+      ),
+    );
+
+  rmSync(dir, { recursive: true, force: true });
+
+  // Compare like with like: best.score carries the lower-left pull bias, best.raw doesn't.
+  // No candidate at all means the card left no legal band below it — keep the
+  // house position rather than failing the whole episode over a badge.
+  const useHome = !best || homeScore <= best.raw * HOME_TOLERANCE;
+  const chosen = useHome ? home : best;
+  episode.meterX = Math.round(chosen.x);
+  episode.meterY = Math.round(chosen.y);
+  writeFileSync(episodePath, JSON.stringify(episode, null, 2) + "\n");
+
+  console.log(
+    useHome
+      ? `${episodePath}: meter at home ${episode.meterX},${episode.meterY} (busyness ${homeScore.toFixed(3)})`
+      : `${episodePath}: home too busy (${homeScore.toFixed(3)} vs ${best.raw.toFixed(3)} available) — ` +
+        `moved to ${episode.meterX},${episode.meterY}`,
+  );
+};
+
+main();

package/scripts/prep-card.ts

@@ -0,0 +1,73 @@
+/**
+ * Prep code-card screenshots for overlay use:
+ * crop away the (near-)black background around the card, then
+ * mask to rounded corners so the output PNG has transparency.
+ *
+ * Usage: npx tsx scripts/prep-card.ts [file.jpg ...]   (default: all of cards-raw/)
+ */
+import sharp from "sharp";
+import { readdirSync, mkdirSync } from "node:fs";
+import { join, parse } from "node:path";
+
+const RAW = join(import.meta.dirname, "../public/assets/cards-raw");
+const OUT = join(import.meta.dirname, "../public/assets/cards");
+mkdirSync(OUT, { recursive: true });
+
+// The card body is ~#1c1c1e on a #000 background. Scan rows/columns of raw
+// pixels and find the bounding box of everything brighter than the background.
+const findCardBounds = async (file: string) => {
+  const { data, info } = await sharp(file)
+    .greyscale()
+    .raw()
+    .toBuffer({ resolveWithObject: true });
+  const { width, height } = info;
+  const THRESH = 12; // > pure-black background + JPEG noise
+  let top = height, bottom = 0, left = width, right = 0;
+  for (let y = 0; y < height; y++) {
+    for (let x = 0; x < width; x++) {
+      if (data[y * width + x] > THRESH) {
+        if (y < top) top = y;
+        if (y > bottom) bottom = y;
+        if (x < left) left = x;
+        if (x > right) right = x;
+      }
+    }
+  }
+  if (top >= bottom || left >= right) throw new Error(`no card found in ${file}`);
+  return { left, top, width: right - left + 1, height: bottom - top + 1 };
+};
+
+const roundedMask = (w: number, h: number, r: number) =>
+  Buffer.from(
+    `<svg width="${w}" height="${h}"><rect width="${w}" height="${h}" rx="${r}" ry="${r}" fill="#fff"/></svg>`,
+  );
+
+const prep = async (file: string) => {
+  const bounds = await findCardBounds(file);
+  const radius = Math.round(bounds.width * 0.045); // matches the cards' corner radius
+  const out = join(OUT, parse(file).name + ".png");
+  await sharp(file)
+    .extract(bounds)
+    .composite([{ input: roundedMask(bounds.width, bounds.height, radius), blend: "dest-in" }])
+    .png()
+    .toFile(out);
+  console.log(`${parse(file).base} -> ${parse(out).base} (${bounds.width}x${bounds.height})`);
+};
+
+const args = process.argv.slice(2);
+const files = args.length
+  ? args
+  : readdirSync(RAW)
+      .filter((f) => /\.jpe?g$/i.test(f))
+      .map((f) => join(RAW, f));
+
+const main = async () => {
+  for (const f of files) {
+    try {
+      await prep(f);
+    } catch (e) {
+      console.error(`FAILED ${f}: ${(e as Error).message}`);
+    }
+  }
+};
+main();

package/scripts/render-overlay.ts

@@ -0,0 +1,150 @@
+/**
+ * Burn overlay layers onto a clip without re-rendering the clip.
+ *
+ * Usage:
+ *   npx tsx scripts/render-overlay.ts <input video> <overlay json> <output mp4>
+ *
+ * Two passes:
+ *   1. Remotion renders the `Overlay` composition alpha-only to ProRes 4444,
+ *      at `--scale` so a 1080x1920 composition rasterizes natively at the
+ *      output resolution. Type and the dot matrix stay crisp at 4K without any
+ *      component knowing 4K exists.
+ *   2. ffmpeg scales/crops the source to the target frame and composites the
+ *      overlay over it, hardware-encoded via videotoolbox.
+ *
+ * The source video never enters Chrome. That's the point: pushing 4K frames
+ * through a headless browser is slow and re-encodes footage that was already
+ * fine, and it's what makes the round trip feel instant instead of awful.
+ */
+import { execFileSync } from "node:child_process";
+import { existsSync, mkdtempSync, readFileSync, rmSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join, resolve } from "node:path";
+import { overlaySchema, type OverlayProps } from "../src/overlay/types";
+
+const FRAME = { width: 1080, height: 1920 };
+
+const run = (cmd: string, args: string[]) =>
+  execFileSync(cmd, args, { stdio: "inherit", cwd: resolve(__dirname, "..") });
+
+const probe = (file: string) => {
+  const out = execFileSync(
+    "ffprobe",
+    [
+      "-v", "error",
+      "-select_streams", "v:0",
+      "-show_entries", "stream=width,height,duration",
+      "-show_entries", "format=duration",
+      "-of", "json",
+      file,
+    ],
+    { encoding: "utf8" },
+  );
+  const json = JSON.parse(out);
+  const stream = json.streams?.[0] ?? {};
+  const duration = Number(stream.duration ?? json.format?.duration ?? 0);
+  return {
+    width: Number(stream.width),
+    height: Number(stream.height),
+    duration,
+  };
+};
+
+/**
+ * Burn `props` onto `input`, writing `output`. Exported so the Overlay Factory
+ * worker can call it directly instead of shelling back out to this file.
+ */
+export const renderOverlay = (
+  input: string,
+  props: OverlayProps,
+  output: string,
+) => {
+  if (!existsSync(input)) throw new Error(`No such video: ${input}`);
+
+  const source = probe(input);
+  if (!source.width || !source.height) {
+    throw new Error(`Could not read video dimensions from ${input}`);
+  }
+
+  // Output keeps the source's vertical resolution, so 4K in means 4K out. The
+  // overlay is scaled to match rather than the footage being scaled to the
+  // overlay — the whole reason the footage stays untouched.
+  const targetHeight = source.height;
+  const targetWidth = Math.round((targetHeight * FRAME.width) / FRAME.height);
+  const scale = targetHeight / FRAME.height;
+
+  console.log(
+    `source ${source.width}x${source.height} ${source.duration.toFixed(1)}s ` +
+      `→ ${targetWidth}x${targetHeight} (overlay scale ${scale.toFixed(2)}x)`,
+  );
+
+  const tmp = mkdtempSync(join(tmpdir(), "overlay-"));
+  const alphaFile = join(tmp, "overlay.mov");
+
+  try {
+    // Pass 1 — overlay only, alpha preserved.
+    run("npx", [
+      "remotion", "render", "Overlay",
+      alphaFile,
+      "--codec", "prores",
+      "--prores-profile", "4444",
+      // ProRes 4444 is the only route to a real alpha channel out of Remotion;
+      // "transparent" without it silently composites onto black. The PNG frame
+      // format isn't optional either — JPEG frames have no alpha to carry, and
+      // Remotion rejects the combination rather than quietly flattening it.
+      "--pixel-format", "yuva444p10le",
+      "--image-format", "png",
+      "--scale", String(scale),
+      "--props", JSON.stringify(props),
+    ]);
+
+    // Pass 2 — composite. force_original_aspect_ratio=increase + crop is
+    // objectFit:cover, so footage that isn't already 9:16 fills the frame
+    // instead of letterboxing.
+    run("ffmpeg", [
+      "-y",
+      "-i", input,
+      "-i", alphaFile,
+      "-filter_complex",
+      `[0:v]scale=${targetWidth}:${targetHeight}:force_original_aspect_ratio=increase,` +
+        `crop=${targetWidth}:${targetHeight},setsar=1[bg];` +
+        `[1:v]scale=${targetWidth}:${targetHeight}[ov];` +
+        // format=yuv420p is not cosmetic: the overlay filter hands back BGRA,
+        // and an HEVC file in BGRA imports into iOS Photos as a file that
+        // won't play. Everything downstream of here expects 4:2:0.
+        `[bg][ov]overlay=0:0:format=auto,format=yuv420p[out]`,
+      "-map", "[out]",
+      // Keep the original audio when there is one; -shortest stops at the
+      // overlay's length so a long clip trims to the overlay, not the reverse.
+      "-map", "0:a?",
+      "-c:v", "hevc_videotoolbox",
+      "-b:v", targetHeight >= 2000 ? "40M" : "12M",
+      "-tag:v", "hvc1",
+      "-c:a", "aac",
+      "-b:a", "192k",
+      "-shortest",
+      output,
+    ]);
+
+    console.log(`\n✓ ${output}`);
+  } finally {
+    rmSync(tmp, { recursive: true, force: true });
+  }
+};
+
+// CLI entry — skipped when the worker imports renderOverlay.
+if (require.main === module) {
+  const [input, overlayJson, output] = process.argv.slice(2);
+  if (!input || !overlayJson || !output) {
+    console.error(
+      "usage: render-overlay.ts <input video> <overlay json> <output mp4>",
+    );
+    process.exit(1);
+  }
+  if (!existsSync(overlayJson)) throw new Error(`No such JSON: ${overlayJson}`);
+  renderOverlay(
+    input,
+    overlaySchema.parse(JSON.parse(readFileSync(overlayJson, "utf8"))),
+    output,
+  );
+}