cursedbelt-core 1.0.0

package/src/core/media/renderPlan.ts

@@ -0,0 +1,318 @@
+// Timeline → ffmpeg render plan. Pure compiler from a cwip/timeline document to
+// an ordered list of ffmpeg argv commands + auxiliary files — executor-agnostic:
+// the browser runs the SAME plan on ffmpeg.wasm (`react/media/ffmpeg-wasm`) and a
+// server can spawn native ffmpeg against it. Nothing here touches an FS.
+//
+// Pipeline shape (the two-pass normalize+concat that survives mixed sources):
+//   1. per video clip  — extract the source window, apply speed (setpts/atempo),
+//      volume, and NORMALIZE to one spec (scale+pad+fps, resampled audio, a
+//      silent bed when the source has no audio) → chunk_i
+//   2. concat          — all chunks share codec/fps/resolution, so `-c copy`
+//   3. audio mix       — audio-track clips are extracted the same way and
+//      `adelay`ed to their timeline offsets, then `amix`ed under the video lane
+//
+// v1 model notes: ONE video lane (the first video track) — clips are laid
+// end-to-end in timeline order, so close gaps before exporting if you want cuts
+// where the gaps were. Audio clip offsets are their absolute `timelineStart`.
+
+import { clipDuration, roundSeconds, type TimelineClip, type TimelineTrack } from 'cwip/timeline';
+import { videoCodecPlan } from './ffmpeg';
+import type { VideoFormat, VideoQuality } from './types';
+
+/** An auxiliary text file the executor writes before running the commands. */
+export interface RenderPlanFile {
+  name: string;
+  text: string;
+}
+
+/** One ffmpeg invocation (argv without the binary). */
+export interface FfmpegCommand {
+  args: string[];
+  /** The file this command produces. */
+  outputName: string;
+  /** Output seconds this command encodes — weight for progress reporting
+   * (0 for stream-copy steps, which are ~instant). */
+  encodeSeconds: number;
+  label: string;
+}
+
+export interface TimelineRenderPlan {
+  files: RenderPlanFile[];
+  commands: FfmpegCommand[];
+  outputName: string;
+  /** Intermediates to delete after the export (NOT the inputs or the output). */
+  intermediateNames: string[];
+  totalEncodeSeconds: number;
+  /** Asset ids the executor must materialize under their mapped file names. */
+  requiredAssetIds: string[];
+}
+
+export interface NormalizeSpec {
+  width: number;
+  height: number;
+  fps: number;
+  audioRate: number;
+}
+
+/** 720p30 keeps in-browser (wasm) export times tolerable; hosts can raise it. */
+export const DEFAULT_NORMALIZE: NormalizeSpec = {
+  width: 1280,
+  height: 720,
+  fps: 30,
+  audioRate: 44100,
+};
+
+export interface TimelineRenderOptions {
+  /** assetId → file name in the executor's filesystem. */
+  assetFileNames: Record<string, string>;
+  /** assetId → whether the file HAS an audio stream (default true). Sources
+   * without one get a silent bed so every chunk concats consistently. */
+  assetHasAudio?: Record<string, boolean>;
+  outputName?: string;
+  format?: VideoFormat;
+  quality?: VideoQuality;
+  normalize?: NormalizeSpec;
+  /** Prefix for every intermediate file (default "csrp_"). */
+  intermediatePrefix?: string;
+}
+
+/**
+ * ffmpeg's `atempo` only accepts 0.5–2.0 — chain filters for anything beyond
+ * (4× → `atempo=2.0,atempo=2.0`).
+ */
+export const buildAtempoChain = (speed: number): string => {
+  const filters: string[] = [];
+  let remaining = speed;
+  while (remaining > 2) {
+    filters.push('atempo=2.0');
+    remaining /= 2;
+  }
+  while (remaining < 0.5) {
+    filters.push('atempo=0.5');
+    remaining /= 0.5;
+  }
+  filters.push(`atempo=${remaining.toFixed(4)}`);
+  return filters.join(',');
+};
+
+/** Video/audio speed filter fragments for a clip (null at speed 1). */
+export const speedFilters = (speed: number): { video: string; audio: string } | null =>
+  speed === 1
+    ? null
+    : { video: `setpts=${(1 / speed).toFixed(6)}*PTS`, audio: buildAtempoChain(speed) };
+
+const scalePad = (n: NormalizeSpec): string =>
+  `scale=${n.width}:${n.height}:force_original_aspect_ratio=decrease,` +
+  `pad=${n.width}:${n.height}:(ow-iw)/2:(oh-ih)/2,fps=${n.fps}`;
+
+/** The `-vf` chain for one clip: reset timestamps, speed, then normalize.
+ * `setpts=PTS-STARTPTS` rebases the extracted window to t=0 — without it a
+ * source whose first frame carries a non-zero PTS (common for screen/live
+ * recordings) makes the `fps` filter emit a BLACK frame to fill the gap before
+ * the real first frame, so the export opens on black instead of the picture. */
+const videoFilter = (clip: TimelineClip, normalize: NormalizeSpec): string => {
+  const speed = speedFilters(clip.speed);
+  return ['setpts=PTS-STARTPTS', speed?.video, scalePad(normalize)].filter(Boolean).join(',');
+};
+
+/** The audio chain for one clip: reset timestamps, speed, volume, resample.
+ * `asetpts=PTS-STARTPTS` mirrors the video rebase so audio stays aligned. */
+const audioFilter = (clip: TimelineClip, normalize: NormalizeSpec): string => {
+  const speed = speedFilters(clip.speed);
+  const volume = clip.volume !== 1 ? `volume=${clip.volume.toFixed(3)}` : null;
+  return ['asetpts=PTS-STARTPTS', speed?.audio, volume, `aresample=${normalize.audioRate}`]
+    .filter(Boolean)
+    .join(',');
+};
+
+const BASE_ARGS = ['-hide_banner', '-nostdin', '-y'];
+
+interface ChunkContext {
+  options: Required<Pick<TimelineRenderOptions, 'assetFileNames'>> & TimelineRenderOptions;
+  normalize: NormalizeSpec;
+  prefix: string;
+}
+
+const sourceFileOf = (ctx: ChunkContext, clip: TimelineClip): string => {
+  const file = ctx.options.assetFileNames[clip.sourceAssetId];
+  if (!file) {
+    throw new Error(
+      `[renderPlan] no file name mapped for asset "${clip.sourceAssetId}" (clip "${clip.name}") — add it to assetFileNames`,
+    );
+  }
+  return file;
+};
+
+const hasAudio = (ctx: ChunkContext, clip: TimelineClip): boolean =>
+  ctx.options.assetHasAudio?.[clip.sourceAssetId] ?? true;
+
+/** Extract + normalize one VIDEO clip into a self-contained chunk. */
+const videoChunkCommand = (ctx: ChunkContext, clip: TimelineClip, index: number): FfmpegCommand => {
+  const plan = videoCodecPlan(ctx.options.format, ctx.options.quality);
+  const outputName = `${ctx.prefix}v${index}.${ctx.options.format === 'webm' ? 'webm' : 'mp4'}`;
+  const silent = !hasAudio(ctx, clip);
+  const args = [
+    ...BASE_ARGS,
+    '-ss',
+    String(clip.trackIn),
+    '-to',
+    String(clip.trackOut),
+    '-i',
+    sourceFileOf(ctx, clip),
+    // A silent stereo bed when the source has no audio track, so EVERY chunk
+    // carries an audio stream and the concat/copy step stays consistent.
+    ...(silent
+      ? [
+          '-f',
+          'lavfi',
+          '-t',
+          String(clipDuration(clip)),
+          '-i',
+          `anullsrc=channel_layout=stereo:sample_rate=${ctx.normalize.audioRate}`,
+        ]
+      : []),
+    '-vf',
+    videoFilter(clip, ctx.normalize),
+    ...(silent ? ['-map', '0:v:0', '-map', '1:a:0'] : ['-af', audioFilter(clip, ctx.normalize)]),
+    '-c:v',
+    plan.videoCodec,
+    '-preset',
+    plan.preset,
+    '-crf',
+    String(plan.crf),
+    '-c:a',
+    plan.audioCodec,
+    ...plan.extraOutputArgs,
+    outputName,
+  ];
+  return {
+    args,
+    outputName,
+    encodeSeconds: clipDuration(clip),
+    label: `video clip ${index + 1} (${clip.name})`,
+  };
+};
+
+/** Extract one AUDIO-track clip as a lone audio chunk (m4a). */
+const audioChunkCommand = (ctx: ChunkContext, clip: TimelineClip, index: number): FfmpegCommand => {
+  const outputName = `${ctx.prefix}a${index}.m4a`;
+  const args = [
+    ...BASE_ARGS,
+    '-ss',
+    String(clip.trackIn),
+    '-to',
+    String(clip.trackOut),
+    '-i',
+    sourceFileOf(ctx, clip),
+    '-vn',
+    '-af',
+    audioFilter(clip, ctx.normalize),
+    '-c:a',
+    'aac',
+    outputName,
+  ];
+  return {
+    args,
+    outputName,
+    encodeSeconds: clipDuration(clip),
+    label: `audio clip ${index + 1} (${clip.name})`,
+  };
+};
+
+/**
+ * Compile the timeline into an ordered command list. Throws when the first
+ * video track has no clips (nothing to render) or an asset is unmapped.
+ */
+export const buildTimelineRenderPlan = (
+  tracks: TimelineTrack[],
+  options: TimelineRenderOptions,
+): TimelineRenderPlan => {
+  const normalize = options.normalize ?? DEFAULT_NORMALIZE;
+  const prefix = options.intermediatePrefix ?? 'csrp_';
+  const outputName = options.outputName ?? 'output.mp4';
+  const ctx: ChunkContext = { options: options as ChunkContext['options'], normalize, prefix };
+
+  const videoTrack = tracks.find((t) => t.type === 'video');
+  const videoClips = [...(videoTrack?.clips ?? [])].sort(
+    (a, b) => a.timelineStart - b.timelineStart,
+  );
+  if (videoClips.length === 0) {
+    throw new Error('[renderPlan] the timeline has no video clips — nothing to render');
+  }
+  const audioClips = tracks
+    .filter((t) => t.type === 'audio')
+    .flatMap((t) => t.clips)
+    .sort((a, b) => a.timelineStart - b.timelineStart);
+
+  const commands: FfmpegCommand[] = [];
+  const files: RenderPlanFile[] = [];
+  const intermediateNames: string[] = [];
+
+  // 1. video chunks
+  const videoChunks = videoClips.map((clip, i) => videoChunkCommand(ctx, clip, i));
+  commands.push(...videoChunks);
+  intermediateNames.push(...videoChunks.map((c) => c.outputName));
+
+  // 2. concat (stream copy — chunks are normalized to one spec)
+  const listName = `${prefix}concat.txt`;
+  files.push({ name: listName, text: videoChunks.map((c) => `file '${c.outputName}'`).join('\n') });
+  intermediateNames.push(listName);
+  const needsMix = audioClips.length > 0;
+  const videoLaneName = needsMix ? `${prefix}lane.mp4` : outputName;
+  commands.push({
+    args: [...BASE_ARGS, '-f', 'concat', '-safe', '0', '-i', listName, '-c', 'copy', videoLaneName],
+    outputName: videoLaneName,
+    encodeSeconds: 0,
+    label: 'concat video lane',
+  });
+
+  // 3. audio mix under the lane
+  if (needsMix) {
+    intermediateNames.push(videoLaneName);
+    const audioChunks = audioClips.map((clip, i) => audioChunkCommand(ctx, clip, i));
+    commands.push(...audioChunks);
+    intermediateNames.push(...audioChunks.map((c) => c.outputName));
+
+    const inputs = [videoLaneName, ...audioChunks.map((c) => c.outputName)];
+    const delays = audioClips.map((clip, i) => {
+      const ms = Math.max(0, Math.round(clip.timelineStart * 1000));
+      return `[${i + 1}:a]adelay=${ms}|${ms}[d${i}]`;
+    });
+    const mixInputs = ['[0:a]', ...audioClips.map((_, i) => `[d${i}]`)].join('');
+    const filterComplex = [
+      ...delays,
+      `${mixInputs}amix=inputs=${audioClips.length + 1}:duration=first:normalize=0[aout]`,
+    ].join(';');
+    commands.push({
+      args: [
+        ...BASE_ARGS,
+        ...inputs.flatMap((name) => ['-i', name]),
+        '-filter_complex',
+        filterComplex,
+        '-map',
+        '0:v',
+        '-map',
+        '[aout]',
+        '-c:v',
+        'copy',
+        '-c:a',
+        'aac',
+        outputName,
+      ],
+      outputName,
+      // re-encodes the full audio lane under a copied video stream — cheap but
+      // not instant; weight it by the lane length.
+      encodeSeconds: roundSeconds(videoClips.reduce((sum, c) => sum + clipDuration(c), 0)),
+      label: 'mix audio tracks',
+    });
+  }
+
+  return {
+    files,
+    commands,
+    outputName,
+    intermediateNames,
+    totalEncodeSeconds: roundSeconds(commands.reduce((sum, c) => sum + c.encodeSeconds, 0)),
+    requiredAssetIds: [...new Set([...videoClips, ...audioClips].map((c) => c.sourceAssetId))],
+  };
+};

package/src/core/media/segment.spec.ts

@@ -0,0 +1,159 @@
+import { describe, expect, test } from 'bun:test';
+import {
+  closeMask,
+  createRegionSegmenter,
+  dilateMask,
+  downscaleRgba,
+  erodeMask,
+  fitWorkingSize,
+  type SegmentSource,
+  sensitivityToTolerance,
+  upscaleMask,
+} from './segment';
+
+/** Build an RGBA source from a per-pixel color function. */
+const makeSource = (
+  width: number,
+  height: number,
+  colorAt: (x: number, y: number) => [number, number, number],
+): SegmentSource => {
+  const data = new Uint8ClampedArray(width * height * 4);
+  for (let y = 0; y < height; y++) {
+    for (let x = 0; x < width; x++) {
+      const [r, g, b] = colorAt(x, y);
+      const o = (y * width + x) << 2;
+      data[o] = r;
+      data[o + 1] = g;
+      data[o + 2] = b;
+      data[o + 3] = 255;
+    }
+  }
+  return { data, width, height };
+};
+
+const at = (m: Uint8ClampedArray, w: number, x: number, y: number) => m[y * w + x];
+
+describe('createRegionSegmenter — point (magic wand)', () => {
+  // A red square (the object) on a blue field — a clean object/background split.
+  const source = makeSource(40, 40, (x, y) =>
+    x >= 10 && x < 30 && y >= 10 && y < 30 ? [220, 30, 30] : [30, 30, 220],
+  );
+  const seg = createRegionSegmenter();
+
+  test('selects the clicked object and nothing of the background', () => {
+    const mask = seg.segment(source, { type: 'point', x: 20, y: 20 }) as Uint8ClampedArray;
+    expect(mask.length).toBe(40 * 40);
+    // inside the red square → selected
+    expect(at(mask, 40, 20, 20)).toBe(255);
+    expect(at(mask, 40, 12, 12)).toBe(255);
+    expect(at(mask, 40, 28, 28)).toBe(255);
+    // out in the blue field → not selected
+    expect(at(mask, 40, 2, 2)).toBe(0);
+    expect(at(mask, 40, 38, 38)).toBe(0);
+  });
+
+  test('a tight tolerance still selects a flat-color object; output is pure binary', () => {
+    const mask = seg.segment(
+      source,
+      { type: 'point', x: 20, y: 20 },
+      { tolerance: 8 },
+    ) as Uint8ClampedArray;
+    expect(at(mask, 40, 20, 20)).toBe(255);
+    for (const v of mask) expect(v === 0 || v === 255).toBe(true);
+  });
+
+  test('clicking the background selects the background region, not the object', () => {
+    const mask = seg.segment(source, { type: 'point', x: 2, y: 2 }) as Uint8ClampedArray;
+    expect(at(mask, 40, 2, 2)).toBe(255);
+    expect(at(mask, 40, 20, 20)).toBe(0); // the red object is excluded
+  });
+
+  test('an out-of-range click is clamped into bounds (no throw, plausible mask)', () => {
+    const mask = seg.segment(source, { type: 'point', x: 9999, y: -50 }) as Uint8ClampedArray;
+    expect(mask.length).toBe(40 * 40);
+    expect(at(mask, 40, 39, 0)).toBe(255); // clamped to the corner (blue field)
+  });
+});
+
+describe('createRegionSegmenter — box', () => {
+  // A green disc-ish object centered in the box, on a gray field.
+  const source = makeSource(60, 60, (x, y) => {
+    const dx = x - 30;
+    const dy = y - 30;
+    return dx * dx + dy * dy < 14 * 14 ? [40, 200, 60] : [180, 180, 180];
+  });
+  const seg = createRegionSegmenter();
+
+  test('selects the object inside the box and clips to the box bounds', () => {
+    const mask = seg.segment(source, {
+      type: 'box',
+      x0: 16,
+      y0: 16,
+      x1: 44,
+      y1: 44,
+    }) as Uint8ClampedArray;
+    expect(at(mask, 60, 30, 30)).toBe(255); // object center
+    // nothing selected outside the box (clipped)
+    expect(at(mask, 60, 2, 2)).toBe(0);
+    expect(at(mask, 60, 58, 58)).toBe(0);
+  });
+
+  test('normalizes a reversed/“drag-up-left” box', () => {
+    const a = seg.segment(source, {
+      type: 'box',
+      x0: 44,
+      y0: 44,
+      x1: 16,
+      y1: 16,
+    }) as Uint8ClampedArray;
+    expect(at(a, 60, 30, 30)).toBe(255);
+  });
+});
+
+describe('morphology', () => {
+  test('dilate grows, erode shrinks, close fills a 1px hole', () => {
+    const w = 5;
+    const h = 5;
+    const m = new Uint8ClampedArray(w * h);
+    // a filled 3×3 block with a single-pixel hole at its center
+    for (let y = 1; y <= 3; y++) for (let x = 1; x <= 3; x++) m[y * w + x] = 255;
+    m[2 * w + 2] = 0; // poke the hole
+
+    expect(at(closeMask(m, w, h, 1), w, 2, 2)).toBe(255); // hole filled
+    expect(at(dilateMask(m, w, h), w, 0, 2)).toBe(255); // grew left edge
+    expect(at(erodeMask(m, w, h), w, 1, 1)).toBe(0); // corner eroded away
+  });
+});
+
+describe('resolution helpers', () => {
+  test('fitWorkingSize caps to maxPixels and preserves aspect, never upsizing', () => {
+    const small = fitWorkingSize(800, 600, 2_000_000);
+    expect(small).toEqual({ width: 800, height: 600, scale: 1 });
+
+    const big = fitWorkingSize(4000, 3000, 1_000_000);
+    expect(big.width * big.height).toBeLessThanOrEqual(1_000_000);
+    expect(big.scale).toBeLessThan(1);
+    expect(Math.abs(big.width / big.height - 4000 / 3000)).toBeLessThan(0.02);
+  });
+
+  test('downscale then upscale round-trips a region back to natural dims', () => {
+    const src = makeSource(8, 8, (x) => (x < 4 ? [255, 0, 0] : [0, 0, 255]));
+    const down = downscaleRgba(src.data, 8, 8, 4, 4);
+    expect(down.length).toBe(4 * 4 * 4);
+    const small = new Uint8ClampedArray(4 * 4).fill(0);
+    small[0] = 255; // mark one working-res pixel
+    const up = upscaleMask(small, 4, 4, 8, 8);
+    expect(up.length).toBe(8 * 8);
+    expect(at(up, 8, 0, 0)).toBe(255);
+    expect(at(up, 8, 1, 1)).toBe(255); // the 1px block maps to a 2×2 natural block
+  });
+});
+
+describe('sensitivityToTolerance', () => {
+  test('is monotonic and bounded across the slider range', () => {
+    expect(sensitivityToTolerance(0)).toBeLessThan(sensitivityToTolerance(50));
+    expect(sensitivityToTolerance(50)).toBeLessThan(sensitivityToTolerance(100));
+    expect(sensitivityToTolerance(-10)).toBe(sensitivityToTolerance(0)); // clamped
+    expect(sensitivityToTolerance(999)).toBe(sensitivityToTolerance(100));
+  });
+});