@dailies-studio/shot-editor 0.1.0 → 0.2.0

package/src/panels/timeline/Filmstrip.tsx

@@ -0,0 +1,361 @@
+// A VIDEO CLIP'S OWN FRAMES, UNDER ITS BAR.
+//
+// The picture half of a video row. The sound half is the waveform `AudioLane`
+// already draws, and this file keeps that file's one non-negotiable rule:
+//
+//   A DECORATION NEVER FAILS AN EDIT.
+//
+// No sidecar, no ffmpeg on the host, a 2d context jsdom will not give, an
+// `Image` that does not exist: each ends as a bar with no thumbnails in it,
+// which still trims, slips, splits and plays. Every throwing surface in here is
+// wrapped, and `filmstrip.ts` takes the same posture one layer down.
+//
+// ── WHY THIS IS A CANVAS IN A DOM LANE ────────────────────────────────────
+//
+// `panels/timeline/` is DOM and stays DOM: the bars are divs, the handles are
+// spans, and the gestures are pointer events on real elements. The two
+// exceptions are both PICTURES rather than controls (the waveform and this),
+// and both are the same argument. Sixty thumbnails is sixty `<img>` elements
+// per row with sixty layout boxes and sixty repaints per scroll, against one
+// element and one `drawImage` loop. Nothing here is clickable, so nothing here
+// loses a hit target by not being an element.
+//
+// ── WHAT MAKES IT CHEAP ───────────────────────────────────────────────────
+//
+// Three things, and all three are load-bearing:
+//
+// 1. IT DOES NOT SUBSCRIBE TO THE FRAME. Not to `useShotEditorHot`, not to the
+//    playhead, not to anything on the shot store at all. A strip drawn thirty
+//    times a second would cost more than the rows it decorates. See
+//    `useLive.ts`'s header for the rule; this component is stricter than it,
+//    because it has no live state to carry.
+// 2. IT ONLY DRAWS WHAT IS ON SCREEN. `viewport.ts` answers which content
+//    pixels are visible, quantised to 64 px, and a bar further than 512 px
+//    outside that skips its draw AND its tile request. A fifty-second film at a
+//    deep zoom is six figures of bar pixels; this is what keeps the canvas a
+//    screen wide instead.
+// 3. IT REDRAWS ONLY ON A REAL INPUT. The viewport quantum, a tile arrival, the
+//    zoom, the clip's window, the clip's source placement, and its measured
+//    box. Nothing else can invalidate the pixels, because nothing else is in
+//    them. In particular the THEME is not an input: the strip paints tiles and
+//    reads no token, so the bar's own four fills (rest, hover, live, selected)
+//    show through the gaps as the bar's own CSS and an appearance swap has
+//    nothing here to restate. The day a token is read in this file it joins the
+//    dependency list.
+//
+// ── WHERE THE TILES GO IS NOT THIS FILE'S ARITHMETIC ──────────────────────
+//
+// `filmstripLayout` in model.ts owns it, and its header states the property
+// that makes a trim read correctly. The summary: a column is a SOURCE FRAME
+// placed at the scene frame that shows it, so trimming the head slides the
+// footage under the window instead of re-cutting the strip.
+import { pickTile, requestFilmstrip, subscribeFilmstrip } from "@dailies-studio/timeline";
+import type { FilmstripTile } from "@dailies-studio/timeline";
+import { useCallback, useEffect, useMemo, useState, useSyncExternalStore } from "react";
+import type { Rational } from "@dailies-studio/engine";
+import type { ShotAsset, VideoLayer3D } from "@dailies-studio/shot/doc";
+import { FILMSTRIP_TILE_PX, filmstripLayout } from "./model.js";
+import type { FilmstripPlan } from "./model.js";
+import {
+  DOCK_VIEWPORT_UNMEASURED,
+  dockScrollerOf,
+  readDockViewport,
+  subscribeDockViewport,
+  visibleSlice,
+} from "./viewport.js";
+import type { DockViewport } from "./viewport.js";
+
+/** The backing store is capped at 2x. Past that a strip costs four times the
+ *  memory and the fill rate of a 1x one to add detail to a 48 px thumbnail
+ *  nobody is reading detail off. The same ceiling the stage's raster rungs
+ *  keep, and for the same reason. */
+export const FILMSTRIP_MAX_DPR = 2;
+
+/** This row's scroller, and its quantised visible range.
+ *
+ *  The scroller is found by walking UP from the canvas rather than threaded
+ *  down as a prop: the rows between the dock and this component are owned by
+ *  other files, and a decoration may not make them carry a ref for it. */
+function useDockViewport(el: Element | null): DockViewport {
+  const scroller = useMemo(() => dockScrollerOf(el), [el]);
+  const subscribe = useCallback((cb: () => void) => subscribeDockViewport(scroller, cb), [scroller]);
+  const snapshot = useCallback(() => readDockViewport(scroller), [scroller]);
+  // The server snapshot is the unmeasured one: there is no layout to measure
+  // without a browser, and it is the value that admits the whole bar.
+  return useSyncExternalStore(subscribe, snapshot, () => DOCK_VIEWPORT_UNMEASURED);
+}
+
+/** The canvas's CSS box. `null` until the observer has delivered one, which is
+ *  what stops a first paint sizing a backing store from a zero-height box. */
+function useMeasured(el: Element | null): { width: number; height: number } | null {
+  const [box, setBox] = useState<{ width: number; height: number } | null>(null);
+  useEffect(() => {
+    if (el === null || typeof ResizeObserver !== "function") return;
+    const ro = new ResizeObserver((entries) => {
+      const rect = entries[entries.length - 1]?.contentRect;
+      if (rect === undefined) return;
+      // Identity-stable: a `setState` with an equal box on every observation
+      // would re-run the draw effect on every scroll of a row whose size never
+      // changed.
+      setBox((prev) =>
+        prev !== null && prev.width === rect.width && prev.height === rect.height
+          ? prev
+          : { width: rect.width, height: rect.height },
+      );
+    });
+    ro.observe(el);
+    return () => ro.disconnect();
+  }, [el]);
+  return box;
+}
+
+/** Bumps when tiles land for this asset. A counter and not the tiles
+ *  themselves: the sheet lives in the loader's cache, and the only thing this
+ *  component needs to know is that `pickTile` will answer differently now. */
+function useTileArrivals(assetPath: string | undefined): number {
+  const [epoch, setEpoch] = useState(0);
+  useEffect(() => {
+    if (assetPath === undefined) return;
+    return subscribeFilmstrip(assetPath, () => setEpoch((n) => n + 1));
+  }, [assetPath]);
+  return epoch;
+}
+
+/**
+ * One tile into one column, COVER-cropped.
+ *
+ * The sheet's tiles carry the recording's own aspect (the CLI scales to a fixed
+ * width and an even computed height), and a column is the row's height by the
+ * strip's fixed pitch, so the two rarely agree. Stretching to fit would lie
+ * about the framing of every shot in the film, which is the one thing an author
+ * reads a filmstrip for. A centre crop loses the edges of the frame and keeps
+ * the subject, which is the trade every NLE makes here.
+ */
+function drawTile(
+  ctx: CanvasRenderingContext2D,
+  tile: FilmstripTile,
+  x: number,
+  width: number,
+  height: number,
+): void {
+  if (!(tile.width > 0) || !(tile.height > 0) || !(width > 0) || !(height > 0)) return;
+  const want = width / height;
+  const have = tile.width / tile.height;
+  let sx = tile.sx;
+  let sy = 0;
+  let sw = tile.width;
+  let sh = tile.height;
+  if (have > want) {
+    sw = tile.height * want;
+    sx = tile.sx + (tile.width - sw) / 2;
+  } else {
+    sh = tile.width / want;
+    sy = (tile.height - sh) / 2;
+  }
+  ctx.drawImage(tile.image, sx, sy, sw, sh, x, 0, width, height);
+}
+
+/**
+ * Paint the plan.
+ *
+ * STALE TILES ARE DRAWN AS THEY ARE. Not dimmed, not hatched, not held back
+ * behind a spinner: a dimmed strip reads as a broken asset, and the pixels of
+ * the sheet the loader has moved past are approximately right, which is the
+ * whole standard a filmstrip is held to. The loader's subscription is what
+ * replaces them when the fresh sheet lands.
+ */
+function paint(
+  canvas: HTMLCanvasElement,
+  assetPath: string,
+  plan: FilmstripPlan,
+  fromPx: number,
+  height: number,
+  dpr: number,
+): void {
+  // THE TILES BEFORE THE CONTEXT. A row whose sheet has not arrived, or whose
+  // asset has no strip at all (every video on a host with no ffmpeg), then costs
+  // no 2d context, no clear and no transform. It is also the only reason this is
+  // safe to skip: `pickTile` clamps, so a loaded sheet always answers, and "no
+  // tiles" therefore means "never had any" rather than "had some a moment ago".
+  // There is nothing stale left on the canvas to erase.
+  const drawable: { tile: FilmstripTile; x: number }[] = [];
+  for (const column of plan.columns) {
+    const tile = pickTile(assetPath, column.sourceFrame, plan.intervalFrames);
+    if (tile !== null) drawable.push({ tile, x: column.x - fromPx });
+  }
+  if (drawable.length === 0) return;
+
+  const ctx = canvas.getContext("2d");
+  // A browser answers null for a context already taken as webgl, and jsdom
+  // answers UNDEFINED: its `getContext` is a not-implemented stub that returns
+  // nothing at all, against a type that promises `… | null`. Both are "no
+  // strip", so the check is loose on purpose: `=== null` passes an undefined
+  // straight into `setTransform` and turns a decoration into a thrown render.
+  if (ctx === null || ctx === undefined) return;
+  ctx.setTransform(1, 0, 0, 1, 0, 0);
+  ctx.clearRect(0, 0, canvas.width, canvas.height);
+  ctx.scale(dpr, dpr);
+  for (const { tile, x } of drawable) drawTile(ctx, tile, x, FILMSTRIP_TILE_PX, height);
+}
+
+export interface FilmstripProps {
+  /** The row's layer, for `video.asset`, `video.offsetFrames` and
+   *  `video.timeScale`. Its `lifetime` is NOT read here, see `lifetimeIn`. */
+  layer: VideoLayer3D;
+  /** The asset row `layer.video.asset` names, for its content-addressed
+   *  `path`. Absent (a reference the document no longer carries) draws
+   *  nothing and asks for nothing. */
+  asset: ShotAsset | undefined;
+  /** The document's rate, which is what turns a source frame into the time the
+   *  sheet is indexed by. */
+  fps: Rational;
+  /**
+   * The bar's left edge in SCENE-LOCAL frames: the same `row.lifespan.in` the
+   * caller computed the bar's `left` from.
+   *
+   * Passed rather than read off `layer.lifetime`, because an absent lifetime is
+   * materialised by `lifespanOf` against the container's duration and the strip
+   * must be laid out from the number the bar was DRAWN from. One arithmetic,
+   * the rule `useLive` states for the same reason.
+   */
+  lifetimeIn: number;
+  ppf: number;
+  /** The bar's left edge in LANE pixels, exactly as the caller placed it. */
+  barLeftPx: number;
+  /** The bar's width in pixels, exactly as the caller placed it. */
+  barWidthPx: number;
+  /** The dock's label column, which is the offset between a lane pixel and a
+   *  content pixel. The viewport is measured in content pixels. */
+  labelWidth: number;
+}
+
+/**
+ * Mount inside the clip's bar. The bar is already a containing block (it is
+ * absolutely positioned itself) and is `overflow: hidden`, so this positions
+ * itself absolutely against it and lets the bar's own rounded clip cut the
+ * columns at both ends.
+ *
+ * The canvas sits at the VISIBLE slice of the bar, not across the whole bar: at
+ * 60 px/frame a fifty-second film is past the width a browser will allocate for
+ * a canvas at all, and a canvas that fails to allocate draws nothing with no
+ * error anywhere.
+ *
+ * ── TWO THINGS THE MOUNT OWES IT ──────────────────────────────────────────
+ *
+ * 1. THE BAR'S OWN CONTENT HAS TO OUTRANK IT. This is a positioned box, and
+ *    CSS paints positioned descendants over in-flow content regardless of
+ *    document order, so `.lifespan-bar-name` needs a position of its own (it
+ *    is a plain span today) or the label disappears under the footage. The two
+ *    handles are already absolute and already later in the tree, so they are
+ *    fine as they are.
+ * 2. IT IS OFF BY THE BAR'S BORDER. An absolutely positioned child is placed
+ *    against the PADDING box, so `left: 0` here is the inner edge of the bar's
+ *    1px border rather than its outer edge. Left uncorrected on purpose: one
+ *    pixel of phase on a 48px thumbnail is not visible, and subtracting a
+ *    border width in here would put a second copy of a CSS number into the
+ *    geometry, where a stylesheet change could silently invalidate it.
+ */
+export function Filmstrip({
+  layer,
+  asset,
+  fps,
+  lifetimeIn,
+  ppf,
+  barLeftPx,
+  barWidthPx,
+  labelWidth,
+}: FilmstripProps) {
+  const [canvas, setCanvas] = useState<HTMLCanvasElement | null>(null);
+  const view = useDockViewport(canvas);
+  const measured = useMeasured(canvas);
+  const assetPath = asset?.path;
+  const epoch = useTileArrivals(assetPath);
+
+  const slice = visibleSlice(view, labelWidth + barLeftPx, barWidthPx);
+  const fromPx = slice?.fromPx ?? 0;
+  const toPx = slice?.toPx ?? 0;
+
+  const offsetFrames = layer.video.offsetFrames ?? 0;
+  const timeScale = layer.video.timeScale ?? 1;
+  const rate = fps.den > 0 ? fps.num / fps.den : 0;
+  const height = measured?.height ?? 0;
+
+  useEffect(() => {
+    if (canvas === null || assetPath === undefined) return;
+    // The skip is the whole point of the viewport store: a row off screen draws
+    // nothing AND asks for nothing, so a scroll past forty video rows does not
+    // queue forty derives on a server that has to run ffmpeg for each.
+    if (!(toPx > fromPx) || !(height > 0)) return;
+
+    const plan = filmstripLayout({
+      lifetimeIn,
+      offsetFrames,
+      timeScale,
+      ppf,
+      tileWidth: FILMSTRIP_TILE_PX,
+      fromPx,
+      toPx,
+    });
+    if (plan.columns.length === 0) return;
+
+    const first = plan.columns[0];
+    const last = plan.columns[plan.columns.length - 1];
+    if (first === undefined || last === undefined) return;
+
+    try {
+      requestFilmstrip({
+        assetPath,
+        fps: rate,
+        firstSourceFrame: first.sourceFrame,
+        lastSourceFrame: last.sourceFrame,
+        intervalFrames: plan.intervalFrames,
+        tileWidth: FILMSTRIP_TILE_PX,
+        tileHeight: height,
+      });
+
+      const width = toPx - fromPx;
+      const dpr = Math.min(FILMSTRIP_MAX_DPR, Math.max(1, globalThis.devicePixelRatio || 1));
+      const backingW = Math.max(1, Math.round(width * dpr));
+      const backingH = Math.max(1, Math.round(height * dpr));
+      // Written only when it MOVED: assigning `canvas.width` clears the whole
+      // surface even when the value is unchanged, so an unconditional write
+      // would blank the strip on every redraw that had nothing new to say.
+      if (canvas.width !== backingW) canvas.width = backingW;
+      if (canvas.height !== backingH) canvas.height = backingH;
+      paint(canvas, assetPath, plan, fromPx, height, dpr);
+    } catch {
+      // A decoration never fails an edit.
+    }
+    // `epoch` is in the list for its side effect alone: it is the loader saying
+    // `pickTile` answers differently now.
+  }, [canvas, assetPath, fromPx, toPx, height, lifetimeIn, offsetFrames, timeScale, ppf, rate, epoch]);
+
+  // The ONLY early return, and it deliberately does not test the slice.
+  //
+  // An off-screen bar collapses its canvas to zero width and keeps the element
+  // mounted, because unmounting it would unmount the node the viewport is
+  // measured through: `canvas` would go null, the scroller with it, the
+  // unmeasured viewport would admit the bar again, and the row would mount and
+  // unmount its canvas forever. The skip that matters is the draw and the fetch
+  // above, both of which are already off.
+  if (assetPath === undefined) return null;
+
+  return (
+    <canvas
+      ref={setCanvas}
+      className="filmstrip"
+      aria-hidden="true"
+      // Inline, because the geometry is computed: `left` and `width` change with
+      // every viewport quantum, and the rest is what makes a canvas a backdrop
+      // rather than a box in the flow.
+      style={{
+        position: "absolute",
+        top: 0,
+        bottom: 0,
+        left: fromPx,
+        width: Math.max(0, toPx - fromPx),
+        pointerEvents: "none",
+      }}
+    />
+  );
+}