@ossclip/core 0.1.0

package/src/framing.ts

@@ -0,0 +1,277 @@
+import type { Transcript } from "./schema";
+import { SCENE_REGISTRY } from "./scene-registry";
+import type { Layout } from "./scene-schema";
+import type { Moment } from "./producer/beats";
+import { headFracInSlot } from "./normalize";
+
+/**
+ * Producer framing awareness (PLAN 2026-07-28, Tasks A and B).
+ *
+ * The beat-sheet prompt used to contain zero geometry — intent, duration,
+ * component menu, transcript. So the producer happily put `video-top` (a wide
+ * band) on moments where the speaker fills 44% of a portrait canvas, and the
+ * band cover-cropped the head to 206% of its height. Three rounds of tuning a
+ * global crop constant could not fix that, because it is not a cropping
+ * problem: the pixels a wide band wants do not exist in a portrait close-up.
+ * The fix is editorial — don't put a wide band on a close-up moment — which
+ * makes it the PRODUCER's decision, and this module is how the producer gets
+ * the evidence (Task A) and how its choice is checked (Task B).
+ *
+ * Two design rules, from the plan:
+ *   - The model never does pixel math. The brief is qualitative — CLOSE /
+ *     MEDIUM / WIDE plus which layouts that rules out. The arithmetic stays
+ *     in `headFracInSlot`, where it is tested.
+ *   - The repair pass ships regardless of how well the prompt behaves. A
+ *     prompt is a request; `repairMomentLayouts` is the constraint (the §35
+ *     lesson).
+ */
+
+/** One framing window in SOURCE time, from the normalization plan. */
+export interface FramingWindow {
+  startSec: number;
+  endSec: number;
+  /** Face height as a fraction of the (normalized) canvas during this window. */
+  faceFracOfCanvas: number;
+}
+
+/**
+ * A layout's video-slot shape. Injected by the CLI from `layoutSlots` —
+ * core stays React- and scenes-free, and the feasibility question needs only
+ * the slot's ASPECT and whether the video is the subject there at all.
+ */
+export interface LayoutFraming {
+  layout: Layout;
+  /** Slot width over height, in output pixels. */
+  slotAspect: number;
+  /**
+   * False for slots where the head-fits rule does not apply: a pip bubble is
+   * MEANT to be a tight head shot, and a `graphic-only` slot never draws.
+   */
+  primary: boolean;
+}
+
+/** Everything a framing judgement needs, assembled once by the CLI. */
+export interface FramingContext {
+  windows: FramingWindow[];
+  /** The normalized canvas's width over height. */
+  canvasAspect: number;
+  layouts: LayoutFraming[];
+  /** The idle zoom's peak — the head must fit at the tightest moment. */
+  zoom: number;
+}
+
+/** The moment's span in SOURCE seconds, off the transcript's own stamps. */
+export function momentSourceWindow(
+  transcript: Transcript,
+  startWord: number,
+  endWord: number,
+): { startSec: number; endSec: number } | null {
+  const first = transcript.words[startWord];
+  const last = transcript.words[endWord];
+  if (!first || !last) return null;
+  return { startSec: first.start, endSec: last.end };
+}
+
+/** The tightest framing inside [startSec, endSec] — a span is judged by its
+ * worst moment, not its average. */
+export function worstFaceFrac(
+  windows: readonly FramingWindow[],
+  startSec: number,
+  endSec: number,
+): number {
+  let frac = 0;
+  for (const w of windows) {
+    if (w.startSec < endSec && w.endSec > startSec) frac = Math.max(frac, w.faceFracOfCanvas);
+  }
+  return frac;
+}
+
+/** Can this layout's slot hold the whole head at this framing? Non-primary
+ * slots are always feasible — the rule does not apply to them. */
+export function layoutFeasible(ctx: FramingContext, layout: Layout, faceFrac: number): boolean {
+  const entry = ctx.layouts.find((l) => l.layout === layout);
+  if (!entry || !entry.primary || faceFrac <= 0) return true;
+  return headFracInSlot(faceFrac, ctx.canvasAspect, entry.slotAspect, ctx.zoom) <= 1;
+}
+
+/** The primary layouts this framing rules out, worst offender first. */
+export function infeasibleLayouts(ctx: FramingContext, faceFrac: number): Layout[] {
+  return ctx.layouts
+    .filter((l) => !layoutFeasible(ctx, l.layout, faceFrac))
+    .sort(
+      (a, b) =>
+        headFracInSlot(faceFrac, ctx.canvasAspect, b.slotAspect, ctx.zoom) -
+        headFracInSlot(faceFrac, ctx.canvasAspect, a.slotAspect, ctx.zoom),
+    )
+    .map((l) => l.layout);
+}
+
+/** Qualitative label for the brief. CLOSE is defined by CONSEQUENCE — some
+ * layout is unavailable — not by an arbitrary fraction threshold. */
+function shotLabel(ctx: FramingContext, faceFrac: number): "CLOSE" | "MEDIUM" | "WIDE" {
+  if (infeasibleLayouts(ctx, faceFrac).length > 0) return "CLOSE";
+  return faceFrac >= 0.3 ? "MEDIUM" : "WIDE";
+}
+
+/**
+ * The framing brief for the beat-sheet prompt (Task A).
+ *
+ * One line per framing stretch, in WORD INDICES — the producer reasons in
+ * word spans and never sees a second. Adjacent windows whose constraint is
+ * identical merge, so a ten-segment source with two real framings reads as a
+ * handful of lines, not a table. Windows with no face measurement carry no
+ * signal and produce no line; a source with no measured window at all returns
+ * "" and the prompt is unchanged.
+ */
+export function buildFramingBrief(ctx: FramingContext, transcript: Transcript): string {
+  if (transcript.words.length === 0) return "";
+
+  // Word range per window: a word belongs to the window containing its middle.
+  const mid = (i: number): number => {
+    const w = transcript.words[i]!;
+    return (w.start + w.end) / 2;
+  };
+  type Line = { fromWord: number; toWord: number; faceFrac: number; avoid: Layout[] };
+  const lines: Line[] = [];
+  for (const win of ctx.windows) {
+    if (win.faceFracOfCanvas <= 0) continue;
+    let from = -1;
+    let to = -1;
+    for (let i = 0; i < transcript.words.length; i++) {
+      if (mid(i) >= win.startSec && mid(i) < win.endSec) {
+        if (from === -1) from = i;
+        to = i;
+      }
+    }
+    if (from === -1) continue;
+    const avoid = infeasibleLayouts(ctx, win.faceFracOfCanvas);
+    const prev = lines[lines.length - 1];
+    if (prev && prev.toWord === from - 1 && sameAvoid(prev.avoid, avoid)) {
+      // Same constraint, contiguous words: one line. The face fraction kept is
+      // the WORST, consistent with how spans are judged everywhere else.
+      prev.toWord = to;
+      prev.faceFrac = Math.max(prev.faceFrac, win.faceFracOfCanvas);
+    } else {
+      lines.push({ fromWord: from, toWord: to, faceFrac: win.faceFracOfCanvas, avoid });
+    }
+  }
+  if (lines.length === 0) return "";
+
+  const body = lines
+    .map((l) => {
+      const label = shotLabel(ctx, l.faceFrac);
+      const detail = `the face fills ~${Math.round(l.faceFrac * 100)}% of the frame height`;
+      return l.avoid.length > 0
+        ? `- words ${l.fromWord}-${l.toWord}: ${label} shot (${detail}) — layouts ` +
+            `${l.avoid.join(", ")} would crop the head here and are UNAVAILABLE`
+        : `- words ${l.fromWord}-${l.toWord}: ${label} shot (${detail}) — any layout works`;
+    })
+    .join("\n");
+  return (
+    "Camera framing by word range (measured from the footage — hard constraints, not suggestions):\n" +
+    body
+  );
+}
+
+function sameAvoid(a: readonly Layout[], b: readonly Layout[]): boolean {
+  return a.length === b.length && a.every((l, i) => l === b[i]);
+}
+
+export interface LayoutRepair {
+  /** Index into the moments array. */
+  moment: number;
+  issue: string;
+}
+
+/**
+ * The Task B safety net: rewrite any moment whose layout cannot physically
+ * hold the head at that moment's framing.
+ *
+ * Runs AFTER `normalizeBeatSheet` (word spans are already valid) and before
+ * scene generation. The producer's explicit choice is kept when feasible;
+ * an infeasible one — or an infeasible registry default when the producer
+ * said nothing — is swapped for the first feasible layout among the
+ * component's default and alternates. When NOTHING is feasible the least-bad
+ * candidate is taken and the issue says so, because a scene that trims a
+ * little is still better than silently rendering the worst option.
+ */
+export function repairMomentLayouts(
+  moments: readonly Moment[],
+  transcript: Transcript,
+  ctx: FramingContext,
+): { moments: Moment[]; issues: LayoutRepair[] } {
+  const issues: LayoutRepair[] = [];
+  const out = moments.map((m, idx) => {
+    if (m.sceneKind === "none") return m;
+    const meta = SCENE_REGISTRY[m.sceneKind];
+    const window = momentSourceWindow(transcript, m.startWord, m.endWord);
+    if (!meta || !window) return m;
+    const faceFrac = worstFaceFrac(ctx.windows, window.startSec, window.endSec);
+    if (faceFrac <= 0) return m;
+
+    const requested = m.layout ?? meta.defaultLayout;
+    if (layoutFeasible(ctx, requested, faceFrac)) return m;
+
+    const candidates = [...new Set<Layout>([meta.defaultLayout, ...meta.altLayouts])];
+    const feasible = candidates.filter((l) => layoutFeasible(ctx, l, faceFrac));
+    if (feasible.length > 0) {
+      issues.push({
+        moment: idx,
+        issue: `layout ${requested} would crop the head on this close shot; using ${feasible[0]}`,
+      });
+      return { ...m, layout: feasible[0]! };
+    }
+    // Nothing fits: take the candidate that trims least, and say so.
+    const leastBad = candidates.reduce((a, b) => {
+      const fr = (l: Layout): number => {
+        const entry = ctx.layouts.find((e) => e.layout === l);
+        return entry && entry.primary
+          ? headFracInSlot(faceFrac, ctx.canvasAspect, entry.slotAspect, ctx.zoom)
+          : 0;
+      };
+      return fr(b) < fr(a) ? b : a;
+    });
+    issues.push({
+      moment: idx,
+      issue:
+        `no ${m.sceneKind} layout fully fits the head on this close shot; ` +
+        `${leastBad} trims least`,
+    });
+    return { ...m, layout: leastBad };
+  });
+  return { moments: out, issues };
+}
+
+/**
+ * Layouts that make sense in a LANDSCAPE frame (R15).
+ *
+ * `video-top`, `pip-bubble` and `graphic-only` are vertical-format ideas: they
+ * exist to split a tall frame between a face and a card, or to demote the
+ * speaker to an inset because a 9:16 frame cannot show both at a readable
+ * size. A 16:9 export has room the vertical frame never had, and §54 gives it
+ * layouts that USE that room instead of blurring it away: a broadcast
+ * `lower-third` (picture whole, card in the bottom band) and `split-left`/
+ * `split-right` (speaker fills one half, graphic the other — two layouts
+ * because the side matters: eyeline and the room's contents differ left vs
+ * right, and the producer or the editor should be able to pick).
+ */
+export const LANDSCAPE_LAYOUTS: readonly Layout[] = [
+  "full-bleed",
+  "blurred-behind",
+  "lower-third",
+  "split-left",
+  "split-right",
+];
+
+/**
+ * Nearest landscape-appropriate layout. Identity for the five that qualify;
+ * the vertical splits map to their landscape analog — `video-top` and
+ * `pip-bubble` both mean "face and graphic share the frame", which in 16:9 is
+ * a side-by-side split, not a blur — and `graphic-only` (the graphic IS the
+ * subject) keeps `blurred-behind`, the layout that lets a card own the frame
+ * without discarding the picture.
+ */
+export function landscapeLayout(layout: Layout): Layout {
+  if (LANDSCAPE_LAYOUTS.includes(layout)) return layout;
+  return layout === "graphic-only" ? "blurred-behind" : "split-left";
+}

package/src/grounding.ts

@@ -0,0 +1,130 @@
+import type { Scene } from "./scene-schema";
+import type { Transcript } from "./schema";
+import { soundsSimilar } from "./phonetics";
+
+/**
+ * Copy-grounding post-check (FINDINGS §14a): label-ish props must reuse
+ * nouns the take actually contains. The producer prompt demands grounding in
+ * the moment's slice; this check is deliberately looser — it flags a token
+ * only when it appears NOWHERE in the transcript — so what it does flag is a
+ * high-confidence hallucination ("REVENUE" on a code-churn stat), visible in
+ * the report without watching the video.
+ *
+ * Run this against the REPAIRED transcript, never the raw one. §17 was this
+ * check fighting the mishearing mitigation: the producer correctly wrote
+ * "code churn" for a take transcribed as "coach and", and the check reported
+ * the repair as an invention. Repairing once, up front, removes the conflict
+ * at its source — which is why no phonetic tolerance lives here. Tolerance
+ * was tried and rejected: matching copy against the take by sound absolves
+ * real hallucinations too ("CODECHUN REVENUE" reads as a repair of "code
+ * churn"), and a second mitigation pulling against the first is exactly what
+ * produced §17.
+ */
+
+export interface GroundingIssue {
+  sceneId: string;
+  component: string;
+  field: string;
+  token: string;
+}
+
+/**
+ * Fields that carry factual labels, per component. Conversational/stylized
+ * copy (ChatMock messages, TerminalMock lines) is invented by design and is
+ * not checked.
+ */
+const CHECKED_FIELDS: Record<string, string[]> = {
+  TitleCard: ["eyebrow", "title", "sub"],
+  StatCard: ["label", "caption"],
+  RuleCard: ["kicker", "text", "struck"],
+  StrikethroughReveal: ["lines"],
+  FlowDiagram: ["nodes"],
+  ScreenshotFrame: ["label"],
+};
+
+/**
+ * Function words carry no factual claim, so flagging them is pure noise —
+ * this check is only worth anything if it is precise (FINDINGS §30, where
+ * `caption "but"` was reported as ungrounded copy). Kept to grammar and
+ * generic connectives: domain nouns stay checkable, because inventing one is
+ * exactly the failure this exists to catch.
+ */
+const STOPWORDS = new Set([
+  "the", "a", "an", "of", "to", "in", "on", "at", "for", "and", "or", "vs",
+  "no", "not", "non", "per", "than", "then", "with", "without", "into",
+  "over", "under", "more", "less", "most", "least", "your", "our", "my",
+  "his", "her", "their", "its", "it", "is", "are", "was", "be", "this",
+  "that", "these", "those", "you", "we", "they", "one", "two", "all",
+  "every", "each", "when", "how", "why", "what", "now", "today", "rule",
+  "step", "do", "dont", "done",
+  // §30: conjunctions, auxiliaries, prepositions and degree words.
+  "but", "so", "yet", "if", "else", "because", "while", "since", "until",
+  "from", "about", "after", "before", "during", "between", "through",
+  "up", "down", "off", "out", "away", "back", "here", "there", "again",
+  "just", "very", "really", "quite", "even", "still", "only", "also", "too",
+  "am", "were", "been", "being", "has", "have", "had", "can", "cant",
+  "will", "wont", "would", "should", "could", "may", "might", "must",
+  "get", "gets", "got", "let", "lets", "make", "makes", "made",
+  "some", "any", "many", "much", "few", "both", "own", "same", "other",
+  "another", "such", "which", "who", "whom", "whose", "where",
+  "always", "never", "often", "sometimes", "usually", "already", "soon",
+  "actually", "basically", "literally", "probably", "maybe", "perhaps",
+]);
+
+function tokenize(s: string): string[] {
+  return s
+    .toLowerCase()
+    .split(/[^a-z0-9]+/)
+    .filter(Boolean);
+}
+
+/** Numbers, units and short glue don't need transcript support. */
+function needsSupport(token: string): boolean {
+  if (token.length < 3) return false;
+  if (/\d/.test(token)) return false;
+  return !STOPWORDS.has(token);
+}
+
+function stringsOf(value: unknown): string[] {
+  if (typeof value === "string") return [value];
+  if (Array.isArray(value)) return value.flatMap(stringsOf);
+  if (value && typeof value === "object") return Object.values(value).flatMap(stringsOf);
+  return [];
+}
+
+export function checkGrounding(
+  scenes: readonly Scene[],
+  transcript: Transcript,
+  /**
+   * Who the speaker is (`--speaker`). Their own name and brand are legitimate
+   * on screen even when the recognizer mangled every utterance of them, so the
+   * hint counts as spoken vocabulary — otherwise the check fights the repair
+   * pass, which is the §17 mistake in a new place.
+   */
+  speaker?: string,
+): GroundingIssue[] {
+  const spoken = new Set([
+    ...transcript.words.flatMap((w) => tokenize(w.text)),
+    ...(speaker ? tokenize(speaker) : []),
+  ]);
+  const supported = (token: string): boolean =>
+    spoken.has(token) ||
+    spoken.has(`${token}s`) ||
+    (token.endsWith("s") && spoken.has(token.slice(0, -1)));
+
+  const issues: GroundingIssue[] = [];
+  for (const scene of scenes) {
+    const fields = CHECKED_FIELDS[scene.component] ?? [];
+    const merged = { ...scene.props, ...scene.overrides };
+    for (const field of fields) {
+      for (const text of stringsOf(merged[field])) {
+        for (const token of tokenize(text)) {
+          if (needsSupport(token) && !supported(token)) {
+            issues.push({ sceneId: scene.id, component: scene.component, field, token });
+          }
+        }
+      }
+    }
+  }
+  return issues;
+}

package/src/index.ts

@@ -0,0 +1,27 @@
+export * from "./schema";
+export * from "./scene-schema";
+export * from "./overrides";
+export * from "./scene-registry";
+export * from "./assemble";
+export * from "./fill";
+export * from "./producer/index";
+export * from "./timemap";
+export * from "./ingest";
+export * from "./transcribe";
+export * from "./analyze";
+export * from "./cutlist";
+export * from "./clip";
+export * from "./captions";
+export * from "./zoom";
+export * from "./grounding";
+export * from "./cta";
+export * from "./content-rect";
+export * from "./content-rect-detect";
+export * from "./normalize";
+export * from "./framing";
+export * from "./face";
+export * from "./cover";
+export * from "./source-text";
+export * from "./report";
+export * from "./config";
+export { run } from "./exec";

package/src/ingest.ts

@@ -0,0 +1,83 @@
+import { run } from "./exec";
+import type { Probe } from "./schema";
+
+export interface IngestTools {
+  ffmpegPath: string;
+  ffprobePath: string;
+}
+
+export async function probe(tools: IngestTools, path: string): Promise<Probe> {
+  const { stdout } = await run(tools.ffprobePath, [
+    "-v", "error",
+    "-print_format", "json",
+    "-show_streams",
+    "-show_format",
+    path,
+  ]);
+  const info = JSON.parse(stdout) as {
+    streams?: Array<{
+      codec_type?: string;
+      width?: number;
+      height?: number;
+      avg_frame_rate?: string;
+      r_frame_rate?: string;
+    }>;
+    format?: { duration?: string };
+  };
+  const video = info.streams?.find((s) => s.codec_type === "video");
+  const audio = info.streams?.find((s) => s.codec_type === "audio");
+  if (!video) throw new Error(`no video stream in ${path}`);
+  const rate = video.avg_frame_rate && video.avg_frame_rate !== "0/0" ? video.avg_frame_rate : video.r_frame_rate;
+  const [num, den] = (rate ?? "30/1").split("/").map(Number);
+  const duration = Number(info.format?.duration);
+  if (!Number.isFinite(duration) || duration <= 0) throw new Error(`could not determine duration of ${path}`);
+  return {
+    duration,
+    width: video.width ?? 0,
+    height: video.height ?? 0,
+    fps: den ? (num ?? 30) / den : 30,
+    hasAudio: Boolean(audio),
+  };
+}
+
+/** Extract 16 kHz mono PCM audio for ASR + silence analysis. */
+export async function extractAudio(tools: IngestTools, src: string, outWav: string): Promise<void> {
+  await run(tools.ffmpegPath, [
+    "-y", "-i", src,
+    "-vn", "-ar", "16000", "-ac", "1", "-c:a", "pcm_s16le",
+    outWav,
+  ]);
+}
+
+/**
+ * Re-encode with dense keyframes so EDL playback (<OffthreadVideo> with many
+ * small trims) seeks fast. Optional — most sources play fine untouched —
+ * EXCEPT when the source is letterboxed: then this pass also trims the baked
+ * bars (`crop`), so everything downstream sees the picture, not picture+bars
+ * (PLAN Task 7), and the pass stops being optional.
+ */
+export async function makeMezzanine(
+  tools: IngestTools,
+  src: string,
+  out: string,
+  opts: { cropVf?: string } = {},
+): Promise<void> {
+  await run(tools.ffmpegPath, [
+    "-y", "-i", src,
+    ...(opts.cropVf ? ["-vf", opts.cropVf] : []),
+    "-c:v", "libx264", "-preset", "veryfast", "-crf", "18", "-g", "30", "-pix_fmt", "yuv420p",
+    "-c:a", "aac", "-b:a", "192k",
+    out,
+  ]);
+}
+
+/** EBU R128 loudness normalization post-pass; video stream is copied untouched. */
+export async function loudnorm(tools: IngestTools, src: string, out: string): Promise<void> {
+  await run(tools.ffmpegPath, [
+    "-y", "-i", src,
+    "-c:v", "copy",
+    "-af", "loudnorm=I=-16:TP=-1.5:LRA=11",
+    "-c:a", "aac", "-b:a", "192k",
+    out,
+  ]);
+}