@ossclip/core 0.1.0

package/src/normalize.ts

@@ -0,0 +1,397 @@
+import { run } from "./exec";
+import type { ContentRectSegment } from "./content-rect";
+import type { WindowFace } from "./face";
+
+/**
+ * Framing normalization for a mixed-framing source (option (a), chosen with
+ * the author 2026-07-28).
+ *
+ * A source that alternates a letterboxed landscape strip with full-bleed
+ * portrait has no single framing — and rendering each segment's own framing
+ * cover-filled produced a ~3× apparent zoom jump at every boundary, nine
+ * times a minute on the motivating clip ("weird zoom-outs and zoom-ins").
+ * Smoothing the boundaries cannot fix that; the output would still alternate
+ * between two shots.
+ *
+ * The fix is editorial, applied at bake time: pick ONE field of view — the
+ * tightest the source ever shows, i.e. the strip, since the strip's pixels
+ * are all those stretches have — and crop every other segment down to a
+ * window of that same shape, placed on the measured face. The result is a
+ * single, uniform landscape source with constant apparent framing, and the
+ * ENTIRE downstream pipeline (face bias, source-text, cover, layouts, zoom)
+ * runs its ordinary uniform-source path on it. Tight but stable, by choice.
+ *
+ * When even the strip cannot cover the output frame without excessive
+ * upscaling, normalization refuses (`ok: false`) and the caller falls back to
+ * render-time FIT — the strip shown at its natural size rather than
+ * fake-zoomed (option (b)).
+ */
+
+/** One baked stretch: this window of the source, scaled to the canvas. */
+export interface NormalizeSegment {
+  startSec: number;
+  endSec: number;
+  /** Crop window in SOURCE pixels — always the canvas's aspect. */
+  window: { x: number; y: number; w: number; h: number };
+}
+
+export interface NormalizePlan {
+  /** The common frame every segment is cropped+scaled to. */
+  canvas: { width: number; height: number };
+  segments: NormalizeSegment[];
+  /**
+   * Per segment, the face's height as a fraction of the CANVAS after baking —
+   * what the framing actually achieved, as opposed to what it aimed for. This
+   * is the input to `assessCueFraming`, and eventually to telling the producer
+   * which windows can host which layouts.
+   */
+  faceFracOfCanvas: number[];
+  /**
+   * The upscale a full-bleed cover of the OUTPUT implies. The quality gate:
+   * past `MAX_NORMALIZE_UPSCALE` the picture would be visibly soft, and a
+   * soft fake is worse than an honest fit.
+   */
+  coverUpscale: number;
+  ok: boolean;
+}
+
+/**
+ * Ceiling on how far the canvas may be upscaled when a full-bleed layout
+ * covers the output with it. The motivating clip sits at 1920/808 ≈ 2.38 —
+ * soft but within reel norms; a strip much shorter than that is not worth
+ * faking a full-frame shot from.
+ */
+export const MAX_NORMALIZE_UPSCALE = 2.6;
+
+/**
+ * A head is about 1.55x the detector's face box tall — the box bounds eyes,
+ * nose and mouth, and `stage.ts` models the crown at 0.35x above it and the
+ * chin at 0.2x below (FINDINGS §19).
+ */
+const HEAD_PER_FACE = 1.55;
+const HEAD_ABOVE = 0.85;
+const HEAD_BELOW = 0.7;
+
+/**
+ * The tightest the framing may ever get, as face-box height over frame height.
+ *
+ * Derived, not tuned: the head must survive the idle zoom with margin left, so
+ * `HEAD_PER_FACE x F x ZOOM_MAX_SCALE <= 1 - 2 x margin`. At a 6% margin top
+ * and bottom that is `1.55 x F x 1.05 <= 0.88`, i.e. F <= 0.54.
+ *
+ * This is a CEILING on cropping in, never a target to reach. A segment whose
+ * face is already larger than this is left at its own framing rather than
+ * cropped further — there is no version of "consistent framing" worth cutting
+ * someone's forehead off for, which is exactly what the previous round did.
+ */
+export const MAX_FACE_FRACTION = 0.54;
+
+const even = (v: number): number => 2 * Math.floor(v / 2);
+const clamp = (v: number, lo: number, hi: number): number => Math.min(Math.max(v, lo), hi);
+
+const median = (xs: number[]): number => {
+  const s = [...xs].sort((a, b) => a - b);
+  const m = Math.floor(s.length / 2);
+  return s.length % 2 ? s[m]! : (s[m - 1]! + s[m]!) / 2;
+};
+
+/**
+ * Decide the canvas and each segment's crop window.
+ *
+ * `faces` is parallel to `timeline` — each face in ITS segment rect's own
+ * fractions (`measureFaceInWindows`).
+ *
+ * The window is sized by the FACE, not by a constant rect. Equalizing the
+ * canvas alone is not enough, and the author's clip proved it: its two
+ * framings are the same camera shot presented differently — the letterboxed
+ * strip is the whole landscape frame, the full-bleed stretches are a zoomed
+ * crop of it. So the face measures 0.28-0.44 of the frame in one state and
+ * 0.48-0.57 in the other, and cropping both to a constant-height window put
+ * the face at 108% of output height in the full-bleed stretches (head taller
+ * than the frame) against 57% in the strips. Same subject, wildly different
+ * size, at every boundary.
+ *
+ * Sizing each window as `faceHeight / targetFraction` makes the SUBJECT the
+ * constant instead, which is what "one consistent framing" has to mean. The
+ * target is the MEDIAN measured fraction: a segment whose face is smaller
+ * than the target crops in to match, and one whose face is larger can only
+ * zoom out as far as its own rect — clamping there rather than inventing
+ * pixels. The median (not the max) is what keeps that clamping rare and the
+ * upscale inside the quality gate.
+ */
+export function planNormalization(
+  timeline: readonly ContentRectSegment[],
+  faces: ReadonlyArray<WindowFace | null>,
+  output: { width: number; height: number },
+): NormalizePlan {
+  const boxed = timeline.filter((s) => !s.rect.full);
+  // Callers only reach here for a mixed source, but refuse rather than crash.
+  if (boxed.length === 0 || timeline.length < 2) {
+    return {
+      canvas: { width: 0, height: 0 },
+      segments: [],
+      faceFracOfCanvas: [],
+      coverUpscale: Infinity,
+      ok: false,
+    };
+  }
+
+  /**
+   * The LARGEST face fraction in each segment, not the median. A window sized
+   * on the median is correct only at the median moment: the author's clip
+   * moves 29%-48% inside one 12s stretch, and sizing on 34% put the head past
+   * the frame edge whenever they leaned in — which is precisely the frame they
+   * flagged. Sizing on the maximum makes the tightest moment the safe one and
+   * every other moment merely roomier.
+   */
+  const measured = timeline.map((_, i) => faces[i]?.sizeFracMax ?? faces[i]?.sizeFrac ?? null);
+  const known = measured.filter((v): v is number => v !== null);
+
+  // ---- Window heights ------------------------------------------------------
+  // Without a single measurement there is no subject to hold constant, so the
+  // rect-shaped fallback stands: the tightest field of view, uniformly.
+  const target = known.length > 0 ? Math.min(median(known), MAX_FACE_FRACTION) : null;
+  const rectShapedHeights = (): number[] => {
+    const canvasRect = boxed.reduce((a, b) => (b.rect.h < a.rect.h ? b : a)).rect;
+    const a = canvasRect.w / canvasRect.h;
+    return timeline.map((s) => even(Math.min(s.rect.w, s.rect.h * a) / a));
+  };
+  const windowHeights =
+    target === null
+      ? rectShapedHeights()
+      : timeline.map((s, i) => {
+          // An unmeasured segment inherits the median fraction of the segments
+          // framed like it (same rect height), so it is sized in ITS OWN class
+          // rather than averaged across two different shots.
+          const sameClass = timeline.flatMap((o, j) =>
+            measured[j] !== null && Math.abs(o.rect.h - s.rect.h) <= 2 ? [measured[j]!] : [],
+          );
+          const frac = measured[i] ?? (sameClass.length > 0 ? median(sameClass) : median(known));
+          return even(clamp((frac * s.rect.h) / target, 16, s.rect.h));
+        });
+
+  // ---- Canvas --------------------------------------------------------------
+  // The widest aspect every window can actually hold. Wider than the output's
+  // own aspect leaves the stage some horizontal freedom for the face bias;
+  // narrower simply means the output crops height, which cover already does.
+  const aspect = timeline.reduce(
+    (a, s, i) => Math.min(a, s.rect.w / windowHeights[i]!),
+    Number.POSITIVE_INFINITY,
+  );
+  // The smallest window, so baking never upscales — the tightest segment sets
+  // the resolution and every other one is downscaled into it.
+  const canvasHeight = even(Math.min(...windowHeights));
+  const canvas = { width: even(canvasHeight * aspect), height: canvasHeight };
+
+  // ---- Face placement inside the window ------------------------------------
+  // Taken from the segments whose window IS their rect: their framing is the
+  // author's own and survives untouched, so it is the one to reproduce.
+  let wx = 0;
+  let wy = 0;
+  let weight = 0;
+  timeline.forEach((seg, i) => {
+    const f = faces[i];
+    if (!f || windowHeights[i]! < seg.rect.h - 2) return;
+    const dur = Math.max(1e-6, seg.endSec - seg.startSec);
+    wx += f.centerXFrac * dur;
+    wy += f.centerYFrac * dur;
+    weight += dur;
+  });
+  const targetX = weight > 0 ? wx / weight : 0.5;
+  const targetY = weight > 0 ? wy / weight : 0.45;
+
+  const segments: NormalizeSegment[] = timeline.map((seg, i) => {
+    const r = seg.rect;
+    const wH = windowHeights[i]!;
+    const wW = even(Math.min(r.w, wH * aspect));
+    const f = faces[i];
+    // Face position in source px; the rect centre when nothing was measured.
+    const faceX = r.x + (f ? f.centerXFrac : 0.5) * r.w;
+    const faceY = r.y + (f ? f.centerYFrac : 0.5) * r.h;
+    const x = even(clamp(faceX - targetX * wW, r.x, r.x + r.w - wW));
+
+    let y = clamp(faceY - targetY * wH, r.y, r.y + r.h - wH);
+    // Then slide — never resize — so the whole HEAD is inside the window at
+    // the segment's largest face, since the aesthetic anchor above is about
+    // where the face sits, and this is about not amputating it. Bounded by
+    // the rect: if the head genuinely runs past the source's own edge there
+    // is nothing to slide toward, and the clamp leaves it where it was.
+    if (f) {
+      const maxFace = (f.sizeFracMax ?? f.sizeFrac) * r.h;
+      const headTop = faceY - HEAD_ABOVE * maxFace;
+      const headBottom = faceY + HEAD_BELOW * maxFace;
+      if (headBottom - headTop <= wH) {
+        y = clamp(y, headBottom - wH, headTop);
+      } else {
+        // Head taller than the window: centre it, so what is lost is shared
+        // between crown and chin instead of taking the whole bite off one end.
+        y = (headTop + headBottom) / 2 - wH / 2;
+      }
+      y = clamp(y, r.y, r.y + r.h - wH);
+    }
+    return {
+      startSec: seg.startSec,
+      endSec: seg.endSec,
+      window: { x, y: even(y), w: wW, h: wH },
+    };
+  });
+
+  // Cover the output with the canvas: for a canvas wider than the output's
+  // aspect the height binds, otherwise the width does.
+  const coverUpscale =
+    canvas.width / canvas.height > output.width / output.height
+      ? output.height / canvas.height
+      : output.width / canvas.width;
+
+  // What each segment actually achieved once its window is scaled to the
+  // canvas — measured from the plan, never assumed to equal the target: a
+  // segment clamped at its own rect lands wherever its rect put it.
+  const faceFracOfCanvas = timeline.map((seg, i) => {
+    const frac = measured[i];
+    if (frac === null || frac === undefined) return target ?? 0;
+    return (frac * seg.rect.h) / segments[i]!.window.h;
+  });
+
+  return {
+    canvas,
+    segments,
+    faceFracOfCanvas,
+    coverUpscale,
+    ok: coverUpscale <= MAX_NORMALIZE_UPSCALE,
+  };
+}
+
+/** A cue's video slot, in output pixels — `layoutSlots(cue.layout).video.rect`. */
+export interface CueSlot {
+  id: string;
+  layout: string;
+  /** SOURCE seconds, so this intersects the content timeline directly. */
+  startSec: number;
+  endSec: number;
+  slot: { width: number; height: number };
+}
+
+export interface FramingIssue {
+  cueId: string;
+  layout: string;
+  /** Face height over the SLOT's height, after cover crops the canvas to it. */
+  faceFracOfSlot: number;
+  /** The whole head, under the idle zoom. Above 1 it does not fit. */
+  headFracOfSlot: number;
+}
+
+/**
+ * How each scene's slot actually frames the speaker (plan step D).
+ *
+ * A slot WIDER than the canvas gets cover-cropped vertically, so it shows only
+ * `canvasAspect / slotAspect` of the canvas height — and the face grows by the
+ * inverse of that. `video-top` is a 1080x806 band against a portrait canvas:
+ * it shows ~42% of the canvas height, so a face occupying 44% of the canvas
+ * occupies 105% of the band. That is the crown trimming, and it is a property
+ * of the LAYOUT, not of the source or of any global constant.
+ *
+ * This reports it per cue rather than fixing it, deliberately. It is not
+ * fixable by cropping: the pixels a wide band wants do not exist in a portrait
+ * close-up. It is fixable by not putting a wide band on that moment — which is
+ * a producer decision, and this is the evidence it needs.
+ */
+/**
+ * Face height over a SLOT's height, once cover crops the canvas into it.
+ *
+ * A slot WIDER than the canvas is cover-cropped vertically and shows only
+ * `canvasAspect / slotAspect` of the canvas height, so the face grows by the
+ * inverse; a slot no wider than the canvas crops width, and the face's height
+ * fraction is unchanged. THE arithmetic behind every framing judgement —
+ * per-cue assessment, the producer's brief, and the layout repair pass all
+ * call this one function, so they cannot disagree.
+ */
+export function faceFracInSlot(
+  faceFracOfCanvas: number,
+  canvasAspect: number,
+  slotAspect: number,
+): number {
+  if (canvasAspect <= 0 || slotAspect <= 0) return faceFracOfCanvas;
+  return faceFracOfCanvas / Math.min(1, canvasAspect / slotAspect);
+}
+
+/** The whole head, under the idle zoom — above 1 the crop trims it. */
+export function headFracInSlot(
+  faceFracOfCanvas: number,
+  canvasAspect: number,
+  slotAspect: number,
+  zoom: number,
+): number {
+  return HEAD_PER_FACE * faceFracInSlot(faceFracOfCanvas, canvasAspect, slotAspect) * zoom;
+}
+
+export function assessCueFraming(
+  cues: readonly CueSlot[],
+  segments: ReadonlyArray<{ startSec: number; endSec: number }>,
+  faceFracOfCanvas: readonly number[],
+  canvas: { width: number; height: number },
+  zoom: number,
+): FramingIssue[] {
+  if (canvas.width <= 0 || canvas.height <= 0) return [];
+  const canvasAspect = canvas.width / canvas.height;
+  const out: FramingIssue[] = [];
+  for (const cue of cues) {
+    // The worst framing the cue is on screen for — a cue spanning a boundary
+    // is judged by its tightest moment, not by an average of the two.
+    let frac = 0;
+    segments.forEach((seg, i) => {
+      if (seg.startSec < cue.endSec && seg.endSec > cue.startSec) {
+        frac = Math.max(frac, faceFracOfCanvas[i] ?? 0);
+      }
+    });
+    if (frac <= 0 || cue.slot.width <= 0 || cue.slot.height <= 0) continue;
+    const slotAspect = cue.slot.width / cue.slot.height;
+    out.push({
+      cueId: cue.id,
+      layout: cue.layout,
+      faceFracOfSlot: faceFracInSlot(frac, canvasAspect, slotAspect),
+      headFracOfSlot: headFracInSlot(frac, canvasAspect, slotAspect, zoom),
+    });
+  }
+  return out;
+}
+
+/**
+ * The ffmpeg filter graph baking the plan: each segment trimmed, cropped to
+ * its window, scaled to the canvas, and the pieces concatenated back into one
+ * continuous stream. The segment boundaries partition the source exactly, so
+ * the output timeline equals the input's and the untouched audio stays in
+ * sync.
+ */
+export function normalizationFilterGraph(plan: NormalizePlan): string {
+  const parts = plan.segments.map((s, i) => {
+    const w = s.window;
+    return (
+      `[0:v]trim=start=${s.startSec.toFixed(3)}:end=${s.endSec.toFixed(3)},` +
+      `setpts=PTS-STARTPTS,crop=${w.w}:${w.h}:${w.x}:${w.y},` +
+      `scale=${plan.canvas.width}:${plan.canvas.height}[v${i}]`
+    );
+  });
+  const inputs = plan.segments.map((_, i) => `[v${i}]`).join("");
+  return `${parts.join(";")};${inputs}concat=n=${plan.segments.length}:v=1:a=0[v]`;
+}
+
+/**
+ * Bake the normalized source. Encoded with the mezzanine's own settings
+ * (dense keyframes) because it REPLACES the mezzanine — normalizing and then
+ * re-encoding for seekability would be two generations of loss for nothing.
+ */
+export async function bakeNormalizedSource(
+  tools: { ffmpegPath: string },
+  input: string,
+  plan: NormalizePlan,
+  outPath: string,
+): Promise<void> {
+  await run(tools.ffmpegPath, [
+    "-y", "-i", input,
+    "-filter_complex", normalizationFilterGraph(plan),
+    "-map", "[v]", "-map", "0:a?",
+    "-c:v", "libx264", "-preset", "veryfast", "-crf", "18", "-g", "30", "-pix_fmt", "yuv420p",
+    "-c:a", "aac", "-b:a", "192k",
+    outPath,
+  ]);
+}