@dailies-studio/shot-editor 0.1.0 → 0.2.0

package/src/state/stillThumbs.ts

@@ -0,0 +1,474 @@
+// The board's app-drawn stills: one picture per Shot and frame, and one per
+// storyboard sketch, drawn on demand and kept only as object URLs in this
+// page. Never stored as files.
+//
+// ── WHERE THEY ARE DRAWN ─────────────────────────────────────────────────
+//
+// On the tab's still stage (stillStage.ts `tabStillStage`), through that
+// stage's own bridge `captureFrame`, at `storyboardStillSize`. A Shot is drawn
+// from `sliceShotToScene` at the Shot-local frame (a slice's master frame is
+// its local one). A SKETCH is drawn from `sketchStageDoc`, the converter the
+// shot board's Play and the agent's `screenshot_shot { sketch }` run, at the
+// sketch's own frame, its code components bound by the chrome's rule
+// (`sketchBinding`): so a card shows the sketch as the stage draws it, the
+// same picture the agent sees. Never on the author's stage: loading a slice
+// there would swap what they look at. Never through
+// `captureShotFrameToStoryboard`, which takes the export lock, and never
+// through the capture HTTP route.
+//
+// ── ONE AT A TIME, AND ALWAYS LAST ───────────────────────────────────────
+//
+// One serial queue for both kinds. Each draw is a background capture, so a
+// relayed agent still (store mcpRelay.ts gives it 90 s) never waits behind
+// thumbnails: the thumbnail steps aside and is asked for again a moment
+// later. Nothing is drawn while the author plays, exports or looks at another
+// tab, and a draw in flight when they start is abandoned and queued again.
+// Each draw is bounded by `STILL_READY_MS`; one that runs past it is a failed
+// thumbnail.
+//
+// ── WHAT A PICTURE IS KEYED ON ───────────────────────────────────────────
+//
+// A Shot: the Shot, the frame, the size, and the Shot's drawn identity, a
+// counter per Shot bumped whenever `changedShots` (drawn.ts) says a write may
+// have changed its picture, or `shotsPlacing` says a code save reached it. A
+// planning-only write (the storyboard, a still's frame list) bumps nothing,
+// so the board keeps every picture it has.
+//
+// A sketch: the sketch itself, what it cites (its cards, their assets, a
+// cited definition with every definition, asset and material graph it places,
+// `defClosureOf`), the brand colour and font its text and accent read, the
+// part's length and the Project's size and fps, and the same code-save signal
+// for the modules its code components (`sketchCodeComponents`) and cited
+// definitions, nested ones included, place. No content hash: the inputs are
+// the key.
+//
+// A stale picture is still shown while its redraw waits, and draws wait for
+// edits to settle.
+//
+// ── EVERY FAILURE IS SILENT ──────────────────────────────────────────────
+//
+// As with posters (posterCapture.ts), a thumbnail is decoration: a failure is
+// a status the board may show, never a thrown error.
+import {
+  changedShots,
+  defClosureOf,
+  instanceModuleOf,
+  shotsPlacing,
+  sketchCardOf,
+  sketchCodeComponents,
+  sketchStageDoc,
+  sliceShotToScene,
+  storyboardStillSize,
+} from "@dailies-studio/shot/doc";
+import type {
+  DefClosure,
+  Doc3D,
+  ReferenceItem3D,
+  SketchBindingFor,
+  StoryboardPart3D,
+  StoryboardSketch3D,
+} from "@dailies-studio/shot/doc";
+import { STILL_READY_MS, stillYielded, tabStillStage } from "./stillStage.js";
+import type { StillStage } from "./stillStage.js";
+
+/** Exactly what this module reads from the store, structural as in
+ *  posterCapture.ts so a test double needs only these. */
+export interface StillThumbsStore {
+  getState(): { doc: Doc3D | null; playing: boolean; exportLock: unknown };
+  /** The COLD lane: the document, Play and the export lock are never a
+   *  transport tick. */
+  subscribeCold(listener: () => void): () => void;
+}
+
+export type StillThumb =
+  | { readonly status: "ready"; readonly url: string }
+  /** `url` is the older picture, shown while the redraw waits. */
+  | { readonly status: "pending"; readonly url?: string }
+  | { readonly status: "failed"; readonly message: string; readonly url?: string };
+
+export interface StillThumbsDeps {
+  store: StillThumbsStore;
+  /** Default: the tab's still stage. */
+  stage?: () => StillStage;
+  /** A sketch's code components, bound as a placed one would be (the
+   *  chrome's `contractBinding`), read when the sketch is drawn. Absent:
+   *  unbound. */
+  sketchBinding?: SketchBindingFor;
+  /** Loads the contracts of a sketch's code components before it is staged,
+   *  so `sketchBinding` has them. Absent: nothing to load. */
+  loadContracts?: (componentIds: readonly string[]) => Promise<unknown>;
+  setTimeout?: (fn: () => void, ms: number) => unknown;
+  clearTimeout?: (handle: unknown) => void;
+  hidden?: () => boolean;
+  onVisibilityChange?: (listener: () => void) => () => void;
+  createObjectURL?: (blob: Blob) => string;
+  revokeObjectURL?: (url: string) => void;
+  maxEntries?: number;
+}
+
+export interface StillThumbs {
+  /** The picture of `sceneId` at Shot-local `frame`, queueing a draw when it
+   *  has none or an older one. Null with no document or no such Shot. */
+  thumb(sceneId: string, frame: number): StillThumb | null;
+  /** The picture of one storyboard sketch as the stage draws it, queueing a
+   *  draw the same way. Null with no document or no such sketch. */
+  sketch(partId: string, sketchId: string): StillThumb | null;
+  /** Called after each change to what `thumb` answers. Returns the
+   *  unsubscribe. */
+  subscribe(listener: () => void): () => void;
+  /** After a code save of `files`: every Shot and sketch placing them is
+   *  redrawn. */
+  codeSaved(files: readonly string[]): void;
+  dispose(): void;
+}
+
+/** How long edits must stop before a redraw starts. */
+const SETTLE_MS = 500;
+/** How long a draw that stepped aside waits before it asks again. */
+const YIELD_MS = 1_000;
+// ponytail: a fixed cap; a board showing more pictures than this redraws the
+// ones it evicted. Raise it, or size it from the board, if that shows.
+const MAX_ENTRIES = 128;
+
+/** What one picture shows: a Shot at a frame, or a sketch. */
+type Target =
+  | { readonly kind: "shot"; readonly sceneId: string; readonly frame: number }
+  | { readonly kind: "sketch"; readonly partId: string; readonly sketchId: string };
+
+interface Entry {
+  readonly target: Target;
+  url?: string;
+  /** The identity `url` was drawn at, null for none. */
+  drawn: string | null;
+  failed?: { readonly identity: string; readonly message: string };
+}
+
+interface Running {
+  readonly key: string;
+  readonly controller: AbortController;
+  /** Why it was abandoned, when it was. */
+  stopped?: "paused" | "timeout";
+}
+
+export function createStillThumbs(deps: StillThumbsDeps): StillThumbs {
+  const stage = deps.stage ?? tabStillStage;
+  const later = deps.setTimeout ?? ((fn, ms) => globalThis.setTimeout(fn, ms));
+  const cancel = deps.clearTimeout ?? ((h) => globalThis.clearTimeout(h as never));
+  const hidden = deps.hidden ?? (() => typeof document !== "undefined" && document.visibilityState === "hidden");
+  const onVisibilityChange =
+    deps.onVisibilityChange ??
+    ((listener: () => void) => {
+      if (typeof document === "undefined") return () => {};
+      document.addEventListener("visibilitychange", listener);
+      return () => document.removeEventListener("visibilitychange", listener);
+    });
+  const createUrl = deps.createObjectURL ?? ((blob: Blob) => URL.createObjectURL(blob));
+  const revokeUrl = deps.revokeObjectURL ?? ((url: string) => URL.revokeObjectURL(url));
+  const maxEntries = deps.maxEntries ?? MAX_ENTRIES;
+
+  /** Insertion order is recency: `thumb` moves an entry to the end. */
+  const entries = new Map<string, Entry>();
+  /** Keys owed a draw, oldest first. */
+  const queue = new Set<string>();
+  /** Each Shot's drawn identity. */
+  const identities = new Map<string, number>();
+  /** The code saves that reached each module, for the sketches placing it;
+   *  `allCode` counts the saves of every module at once. */
+  const codeSaves = new Map<string, number>();
+  let allCode = 0;
+  const listeners = new Set<() => void>();
+  let doc = deps.store.getState().doc;
+  let running: Running | null = null;
+  let timer: unknown = null;
+
+  const sizeOf = (of: Doc3D): string => {
+    const size = storyboardStillSize(of.size);
+    return `${size.width}x${size.height}`;
+  };
+  const keyOf = (target: Target, of: Doc3D): string =>
+    target.kind === "shot" ? `${target.sceneId}:${target.frame}:${sizeOf(of)}` : `sketch:${target.sketchId}:${sizeOf(of)}`;
+
+  /** What a component card's definition draws: it, the definitions it places
+   *  (nested ones too), their assets and material graphs (`defClosureOf`).
+   *  Null for any other card. */
+  const closureOf = (of: Doc3D, card: ReferenceItem3D | null | undefined): DefClosure | null =>
+    card?.type === "component" && card.def !== undefined ? defClosureOf(of, card.def) : null;
+
+  /** The modules a sketch's picture places: its code components' and those
+   *  of the definitions it cites, nested ones included. */
+  function sketchModules(of: Doc3D, sketch: StoryboardSketch3D): string[] {
+    const modules = new Set(sketchCodeComponents(of, sketch).map(instanceModuleOf));
+    for (const item of sketch.items) {
+      if (item.kind !== "ref") continue;
+      const closure = closureOf(of, sketchCardOf(of, item));
+      for (const def of closure === null ? [] : [closure.def, ...Object.values(closure.nested)]) {
+        for (const layer of Object.values(def.layersById)) {
+          if (layer.kind === "instance") modules.add(instanceModuleOf(layer.instance.component));
+        }
+      }
+    }
+    return [...modules].sort();
+  }
+
+  function sketchOf(of: Doc3D, partId: string, sketchId: string): { part: StoryboardPart3D; sketch: StoryboardSketch3D } | null {
+    const part = of.storyboard?.parts.find((p) => p.id === partId);
+    const sketch = part?.sketches?.find((s) => s.id === sketchId);
+    return part === undefined || sketch === undefined ? null : { part, sketch };
+  }
+
+  /** What `target`'s picture is drawn from, as a string, or null when it is
+   *  gone. A Shot's is its counter; a sketch's is its inputs. */
+  function identityOf(target: Target, of: Doc3D): string | null {
+    if (target.kind === "shot") {
+      return of.scenesById[target.sceneId] === undefined ? null : String(identities.get(target.sceneId) ?? 0);
+    }
+    const found = sketchOf(of, target.partId, target.sketchId);
+    if (found === null) return null;
+    const { part, sketch } = found;
+    const cards = sketch.items.flatMap((item) => (item.kind === "ref" ? [sketchCardOf(of, item) ?? null] : []));
+    const assets = cards.map((card) => (card?.type === "asset" ? (of.assets[card.asset] ?? null) : null));
+    const defs = cards.map((card) => closureOf(of, card));
+    // The brand colour and font the ground, the accent and the text read.
+    const brand = (of.resources?.branding ?? []).filter((item) => item.type === "color" || item.type === "asset");
+    const modules = sketchModules(of, sketch);
+    const code = [modules.length === 0 ? 0 : allCode, ...modules.map((m) => codeSaves.get(m) ?? 0)];
+    return JSON.stringify([sketch, part.durationFrames, cards, assets, defs, brand, of.size, of.fps, code]);
+  }
+
+  const paused = (): boolean => {
+    const state = deps.store.getState();
+    return state.playing || state.exportLock !== null || hidden();
+  };
+  const notify = (): void => {
+    for (const listener of listeners) listener();
+  };
+
+  /** Draw next after `ms`. A later draw already armed is kept unless
+   *  `reset`, which edits use so a draw waits for them to settle. */
+  function arm(ms: number, reset = false): void {
+    if (timer !== null) {
+      if (!reset) return;
+      cancel(timer);
+    }
+    timer = later(() => {
+      timer = null;
+      pump();
+    }, ms);
+  }
+
+  function drop(key: string): void {
+    const entry = entries.get(key);
+    if (entry?.url !== undefined) revokeUrl(entry.url);
+    entries.delete(key);
+    queue.delete(key);
+  }
+
+  function clear(): void {
+    if (running !== null) running.controller.abort();
+    running = null;
+    for (const key of [...entries.keys()]) drop(key);
+    identities.clear();
+    codeSaves.clear();
+    allCode = 0;
+  }
+
+  function invalidate(sceneIds: readonly string[]): void {
+    if (sceneIds.length === 0) return;
+    for (const id of sceneIds) identities.set(id, (identities.get(id) ?? 0) + 1);
+    arm(SETTLE_MS, true);
+    notify();
+  }
+
+  /** After a document change: redraw the sketches whose inputs moved, once
+   *  the edits settle. */
+  function invalidateSketches(of: Doc3D): void {
+    const moved = [...entries.values()].some(
+      (entry) => entry.target.kind === "sketch" && entry.drawn !== null && identityOf(entry.target, of) !== entry.drawn,
+    );
+    if (!moved) return;
+    arm(SETTLE_MS, true);
+    notify();
+  }
+
+  function pump(): void {
+    if (running !== null || paused() || doc === null) return;
+    const still = stage();
+    for (const key of queue) {
+      queue.delete(key);
+      const entry = entries.get(key);
+      if (entry === undefined) continue;
+      const identity = identityOf(entry.target, doc);
+      if (identity === null) {
+        drop(key);
+        continue;
+      }
+      draw(still, doc, key, entry, identity);
+      return;
+    }
+  }
+
+  /** The document a picture is drawn from, its frame, and the code
+   *  components to load first. */
+  function staged(target: Target, from: Doc3D): { doc: Doc3D; frame: number; load: readonly string[] } | null {
+    if (target.kind === "shot") {
+      const slice = sliceShotToScene(from, target.sceneId);
+      return slice === null ? null : { doc: slice, frame: target.frame, load: [] };
+    }
+    const found = sketchOf(from, target.partId, target.sketchId);
+    if (found === null) return null;
+    const load = sketchCodeComponents(from, found.sketch);
+    const length = Math.max(1, found.part.durationFrames);
+    return { doc: from, frame: Math.min(found.sketch.frame ?? 0, length - 1), load };
+  }
+
+  function draw(still: StillStage, from: Doc3D, key: string, entry: Entry, drawing: string): void {
+    const plan = staged(entry.target, from);
+    if (plan === null) {
+      drop(key);
+      return;
+    }
+    const size = storyboardStillSize(from.size);
+    const mine: Running = { key, controller: new AbortController() };
+    running = mine;
+    const bound = later(() => {
+      mine.stopped = "timeout";
+      mine.controller.abort();
+    }, STILL_READY_MS);
+    const target = entry.target;
+    // Non-null: `staged` just found the sketch in this very document.
+    const start = (): Promise<Blob> =>
+      still.capture(
+        target.kind === "sketch" ? sketchStageDoc(from, target.partId, target.sketchId, deps.sketchBinding)! : plan.doc,
+        plan.frame,
+        size.width,
+        size.height,
+        { format: "webp", background: true, signal: mine.controller.signal },
+      );
+    // A sketch's contracts load first, so its instances bind as a placed one
+    // would; it is staged only then, against the bindings that loaded.
+    const work =
+      plan.load.length === 0 || deps.loadContracts === undefined
+        ? start()
+        : deps.loadContracts(plan.load).catch(() => {}).then(start);
+    void work
+      .then(
+        (blob) => {
+          if (running !== mine || entries.get(key) !== entry) return;
+          if (doc === null || identityOf(target, doc) !== drawing) {
+            // An edit landed while it drew: drawn again once they settle.
+            queue.add(key);
+            return;
+          }
+          if (entry.url !== undefined) revokeUrl(entry.url);
+          entry.url = createUrl(blob);
+          entry.drawn = drawing;
+          delete entry.failed;
+          notify();
+        },
+        (error: unknown) => {
+          if (running !== mine || entries.get(key) !== entry) return;
+          if (mine.stopped === "paused" || stillYielded(error)) {
+            queue.add(key);
+            if (mine.stopped !== "paused") arm(YIELD_MS);
+            return;
+          }
+          const message =
+            mine.stopped === "timeout"
+              ? `the still took longer than ${STILL_READY_MS / 1000} s`
+              : error instanceof Error
+                ? error.message
+                : String(error);
+          entry.failed = { identity: drawing, message };
+          notify();
+        },
+      )
+      .finally(() => {
+        cancel(bound);
+        if (running !== mine) return;
+        running = null;
+        arm(0);
+      });
+  }
+
+  function pauseCheck(): void {
+    if (!paused()) {
+      if (queue.size > 0 && running === null) arm(0);
+      return;
+    }
+    if (running !== null && running.stopped === undefined) {
+      running.stopped = "paused";
+      running.controller.abort();
+    }
+  }
+
+  /** `target`'s picture, queueing a draw when it has none or an older one. */
+  function picture(target: Target, of: Doc3D): StillThumb | null {
+    const now = identityOf(target, of);
+    if (now === null) return null;
+    const key = keyOf(target, of);
+    let entry = entries.get(key);
+    if (entry === undefined) {
+      entry = { target, drawn: null };
+    } else {
+      entries.delete(key);
+    }
+    entries.set(key, entry);
+    for (const old of entries.keys()) {
+      if (entries.size <= maxEntries) break;
+      if (old !== key && old !== running?.key) drop(old);
+    }
+    const url = entry.url === undefined ? {} : { url: entry.url };
+    if (entry.url !== undefined && entry.drawn === now) return { status: "ready", url: entry.url };
+    if (entry.failed?.identity === now) return { status: "failed", message: entry.failed.message, ...url };
+    if (running?.key !== key && !queue.has(key)) {
+      queue.add(key);
+      arm(0);
+    }
+    return { status: "pending", ...url };
+  }
+
+  const unsubscribe = deps.store.subscribeCold(() => {
+    const next = deps.store.getState().doc;
+    if (next !== doc) {
+      const before = doc;
+      doc = next;
+      if (next === null || before === null || before.id !== next.id) {
+        clear();
+        notify();
+      } else {
+        invalidate(changedShots(before, next));
+        invalidateSketches(next);
+      }
+    }
+    pauseCheck();
+  });
+  const unwatchVisibility = onVisibilityChange(pauseCheck);
+
+  return {
+    thumb(sceneId, frame) {
+      return doc === null ? null : picture({ kind: "shot", sceneId, frame }, doc);
+    },
+    sketch(partId, sketchId) {
+      return doc === null ? null : picture({ kind: "sketch", partId, sketchId }, doc);
+    },
+    subscribe(listener) {
+      listeners.add(listener);
+      return () => listeners.delete(listener);
+    },
+    codeSaved(files) {
+      if (doc === null) return;
+      if (files.length === 0) allCode += 1;
+      for (const file of files) codeSaves.set(file, (codeSaves.get(file) ?? 0) + 1);
+      invalidate(shotsPlacing(doc, files));
+      invalidateSketches(doc);
+    },
+    dispose() {
+      unsubscribe();
+      unwatchVisibility();
+      if (timer !== null) cancel(timer);
+      timer = null;
+      clear();
+      listeners.clear();
+    },
+  };
+}