@dailies-studio/shot-editor 0.1.0 → 0.2.0

package/src/stage/NodePreviews.tsx

@@ -0,0 +1,628 @@
+// One node's value, drawn as a thumbnail — the stage half of the material
+// workspace's node previews (bridgeTypes: `requestNodePreviews`).
+//
+// ── WHY IT IS STRUCTURALLY UNREACHABLE FROM AN EXPORT ────────────────────
+//
+// `StageApp` mounts this only while `exporting === null` and the material
+// workspace names a preview target, so in export framing it is not a component
+// that declines to run — it is a component that does not exist. The CLI render
+// page never mounts `StageApp` at all, so the whole file is unreachable there.
+// `bridgeClient.requestNodePreviews` refuses on the same two conditions, which
+// is a belt over that structure rather than the only guard.
+//
+// ── A QUAD, NOT THE SPECIMEN MESH ────────────────────────────────────────
+//
+// A thumbnail is a VALUE MAP, not shading. The quad shows the whole UV domain
+// identically for every mesh bound to the graph, which buys three things: the
+// picture answers "what is in this branch" rather than "what does it look like
+// on this model", the rig cache keys on the GRAPH alone, and a 400k-triangle
+// GLB is never drawn 64 times for a wall of 96px pictures. `createDesignPlaneGeometry`
+// is the same quad the plane primitive uses, so uv (0,0) is the top-left corner
+// in both.
+//
+// ── ONE ATLAS, ONE READBACK ──────────────────────────────────────────────
+//
+// Every cell is rendered into ONE render target under a scissor, and the whole
+// target comes back in a single `readRenderTargetPixelsAsync`. That call is a
+// GPU stall — the fence has to settle before the buffer is readable — so the
+// cost of a batch is one stall regardless of how many nodes it holds. Sixty-four
+// separate readbacks would be sixty-four stalls on the canvas the author is
+// still dragging in.
+//
+// ── CELL 0 IS AN ORIENTATION PROBE ───────────────────────────────────────
+//
+// The two backends hand back rows in opposite orders. WebGL reads through
+// `gl.readPixels`, whose origin is bottom-left, so row 0 is the BOTTOM of the
+// target; WebGPU's `copyTextureToBuffer` copies a top-left-origin texture, so
+// row 0 is the TOP. Nothing in the renderer normalises that, and a wrong guess
+// is not a crash — it is every thumbnail silently upside down, on whichever
+// backend nobody happened to test.
+//
+// So the flip is MEASURED, never assumed: cell 0 paints `uv().y` as grey, and
+// the readback is asked which way that ramp runs. Not future-proofing: the
+// preview stage takes the WebGPU-first default and lands on whichever backend
+// the host browser has (packages/shot/src/stageBackend.ts), so both row
+// orders are live today.
+//
+// ── NOT A `useFrame` ─────────────────────────────────────────────────────
+//
+// Work happens inside a request and nowhere else, so playback and scrubbing
+// cost this file exactly nothing. The renderer's viewport is saved and restored
+// around the batch, and the scissor test is turned back off, because a batch
+// that walked away with a 96px viewport would draw the next frame into one
+// corner of the canvas.
+import { Suspense, useEffect, useLayoutEffect, useMemo, useRef } from "react";
+import type { ReactElement } from "react";
+import { useLoader, useThree } from "@react-three/fiber";
+import { Color, FrontSide, Mesh, OrthographicCamera, RenderTarget, Scene, Vector4 } from "three";
+import type { Material } from "three";
+import { MeshBasicNodeMaterial } from "three/webgpu";
+import { uv, vec3, vec4 } from "three/tsl";
+import { GLTFLoader } from "three/addons/loaders/GLTFLoader.js";
+import {
+  compileMaterialGraphNodePreview,
+  createDesignPlaneGeometry,
+  materialGraphSourceOf,
+  primitiveMaterialGraphSource,
+  useShotTextures,
+} from "@dailies-studio/shot";
+import type {
+  CompiledMaterialGraphPreview,
+  MaterialGraphFrame,
+  MaterialGraphSourceChannels,
+} from "@dailies-studio/shot";
+import {
+  DEFAULT_PRIMITIVE_COLOR,
+  isGraphMaterial,
+  materialBindingForSlot,
+  materialGraphNodeSockets,
+  materialGraphSemanticSignature,
+} from "@dailies-studio/shot/doc";
+import type {
+  Doc3D,
+  Layer3D,
+  MaterialGraph3D,
+  MaterialGraphBinding,
+} from "@dailies-studio/shot/doc";
+import { useStageAssetResolver } from "@dailies-studio/runtime";
+import type {
+  MaterialPreviewTarget,
+  NodePreviewBatch,
+  NodePreviewRequest,
+  NodePreviewResult,
+} from "../bridgeTypes.js";
+import type { StageClient } from "./bridgeClient.js";
+
+/**
+ * The renderer surface this file uses, as a STRUCTURAL type.
+ *
+ * Named rather than imported so the list of calls a node preview makes is
+ * readable in one place, and so a test can hand in a plain object without
+ * standing up a GL context. `useThree` types `gl` as the WebGL renderer while
+ * the shot stage runs the node renderer, so a cast happens either way.
+ */
+interface PreviewRenderer {
+  render(scene: Scene, camera: OrthographicCamera): void;
+  setRenderTarget(target: RenderTarget | null): void;
+  setViewport(x: number, y: number, width: number, height: number): void;
+  setScissor(x: number, y: number, width: number, height: number): void;
+  setScissorTest(enabled: boolean): void;
+  getViewport(target: Vector4): Vector4;
+  readRenderTargetPixelsAsync(
+    target: RenderTarget,
+    x: number,
+    y: number,
+    width: number,
+    height: number,
+  ): Promise<ArrayBufferView>;
+}
+
+/** Everything a batch reads, sampled when the batch actually starts rather
+ *  than when it was requested — a parked request must draw the CURRENT
+ *  document, not the one that was on screen when the chrome asked. */
+interface PreviewContext {
+  doc: Doc3D;
+  layer: Layer3D | null;
+  /** The surface channels the graph's `source-*` nodes read, or null when the
+   *  layer, the model or the slot could not be resolved. */
+  source: MaterialGraphSourceChannels | null;
+  textures: ReturnType<typeof useShotTextures>;
+  gl: PreviewRenderer;
+  /** Master frame, or def-local under a component context — whichever the
+   *  stage's clock is on. Time-varying nodes read it. */
+  frame: number;
+}
+
+export function NodePreviews({
+  client,
+  doc,
+  layers,
+  target,
+}: {
+  client: StageClient;
+  doc: Doc3D;
+  /** The container the stage is drawing: the open definition under a component
+   *  context, the playhead's scene otherwise. The same value `StageTools` gets,
+   *  so the two can never disagree about which layer an id names. */
+  layers: { layersById: Readonly<Record<string, Layer3D>> } | null;
+  target: MaterialPreviewTarget;
+}): ReactElement {
+  const layer = layers?.layersById[target.layerId] ?? null;
+  const mesh = layer?.kind === "mesh" ? layer.mesh : undefined;
+  const resolveAsset = useStageAssetResolver();
+  const assetPath = mesh?.asset === undefined ? undefined : doc.assets[mesh.asset]?.path;
+  const url = assetPath === undefined ? null : resolveAsset(assetPath);
+  // A primitive has no source `Material` object to capture, so its channels are
+  // synthesised from the one colour it carries — exactly what `mesh.tsx` does
+  // for the specimen, so the thumbnail's `source-color` and the specimen's are
+  // the same value.
+  const primitiveSource = useMemo(
+    () =>
+      mesh === undefined || mesh.asset !== undefined
+        ? null
+        : primitiveMaterialGraphSource(
+            new Color().setStyle(mesh.color ?? DEFAULT_PRIMITIVE_COLOR),
+            FrontSide,
+          ),
+    [mesh],
+  );
+
+  if (url !== null) {
+    return (
+      <Suspense fallback={null}>
+        <ModelPreviewSource
+          client={client}
+          doc={doc}
+          layer={layer}
+          url={url}
+          slot={target.slot}
+        />
+      </Suspense>
+    );
+  }
+  return <PreviewRig client={client} doc={doc} layer={layer} source={primitiveSource} />;
+}
+
+/**
+ * A model layer's source channels, taken off the loaded glTF.
+ *
+ * `useLoader` answers from fiber's GLOBAL url cache, which the specimen mesh
+ * has already filled — so this suspends for one commit at most on a model that
+ * is already on screen, and never re-downloads. While it is suspended nothing
+ * registers a preview source and the bridge answers null, which is the same
+ * "not ready yet" the chrome already handles.
+ */
+function ModelPreviewSource({
+  client,
+  doc,
+  layer,
+  url,
+  slot,
+}: {
+  client: StageClient;
+  doc: Doc3D;
+  layer: Layer3D | null;
+  url: string;
+  slot: number | null;
+}): ReactElement {
+  const gltf = useLoader(GLTFLoader, url);
+  const source = useMemo(() => modelSlotSource(gltf, slot), [gltf, slot]);
+  return <PreviewRig client={client} doc={doc} layer={layer} source={source} />;
+}
+
+/**
+ * The glTF material a slot names, as channels.
+ *
+ * `parser.associations` is the LOADER's own mapping and the only authority on
+ * which source material is index N — a glTF material's name is optional and not
+ * unique, so a name match would be the implicit binding this schema refused.
+ * The layer-wide binding (`slot: null`) takes the lowest index, because a
+ * whole-model preview has no one submesh to speak for.
+ */
+function modelSlotSource(gltf: unknown, slot: number | null): MaterialGraphSourceChannels | null {
+  const associations = (
+    gltf as { parser?: { associations?: Map<object, { materials?: number }> } }
+  ).parser?.associations;
+  if (associations === undefined) return null;
+  let lowest: { index: number; material: Material } | null = null;
+  for (const [key, link] of associations) {
+    const index = link?.materials;
+    // Meshes and nodes ride the same map with their own keys, so "has a
+    // material index" is what identifies a material entry.
+    if (typeof index !== "number") continue;
+    const material = key as Material;
+    if (slot !== null) {
+      if (index === slot) return materialGraphSourceOf(material);
+      continue;
+    }
+    if (lowest === null || index < lowest.index) lowest = { index, material };
+  }
+  return lowest === null ? null : materialGraphSourceOf(lowest.material);
+}
+
+/** The registration and the atlas. Renders nothing: everything it owns lives in
+ *  three objects the React tree has no opinion about. */
+function PreviewRig({
+  client,
+  doc,
+  layer,
+  source,
+}: {
+  client: StageClient;
+  doc: Doc3D;
+  layer: Layer3D | null;
+  source: MaterialGraphSourceChannels | null;
+}): null {
+  const gl = useThree((state) => state.gl) as unknown as PreviewRenderer;
+  const textures = useShotTextures();
+
+  // Read through a REF so the registration below stays a mount-only effect.
+  // Re-registering on every commit would swap the source out from under a batch
+  // that is already awaiting its readback, and the stage commits on every seek.
+  const live = useRef<Omit<PreviewContext, "frame">>({ doc, layer, source, textures, gl });
+  live.current = { doc, layer, source, textures, gl };
+
+  const rig = useMemo(createNodePreviewRig, []);
+  useEffect(() => () => rig.dispose(), [rig]);
+
+  useLayoutEffect(() => {
+    // THE CLOCK, the playhead's one live answer: the stage's React state
+    // holds a frame only for an export's capture loop.
+    const context = (): PreviewContext => ({
+      ...live.current,
+      frame: client.clock.get().frame,
+    });
+    client.setNodePreviewSource((request) => rig.run(request, context));
+    return () => client.setNodePreviewSource(null);
+  }, [client, rig]);
+
+  return null;
+}
+
+interface NodePreviewRig {
+  run(
+    request: NodePreviewRequest,
+    context: () => PreviewContext,
+  ): Promise<NodePreviewBatch | null>;
+  dispose(): void;
+}
+
+/** One compiled node head, or the sentence saying why there is none. */
+type CellPlan =
+  | { nodeId: string; rig: CompiledMaterialGraphPreview; message?: undefined }
+  | { nodeId: string; rig?: undefined; message: string };
+
+function createNodePreviewRig(): NodePreviewRig {
+  const scene = new Scene();
+  // The unit quad exactly fills this frustum, so a cell's picture IS the whole
+  // UV domain with no border and no letterboxing to reason about.
+  const camera = new OrthographicCamera(-0.5, 0.5, 0.5, -0.5, 0, 1);
+  const geometry = createDesignPlaneGeometry(1, 1);
+  const quad = new Mesh(geometry);
+  // Off the near plane. At z = 0 the quad sits exactly ON it, which some drivers
+  // clip and some do not — a black thumbnail on one machine and not the next.
+  quad.position.z = -0.5;
+  quad.frustumCulled = false;
+  scene.add(quad);
+
+  // The orientation probe: grey ramping with uv.y, which the design-space quad
+  // puts at 0 along the TOP edge. See the header.
+  const probe = new MeshBasicNodeMaterial();
+  probe.colorNode = vec4(vec3(uv().y), 1);
+  probe.toneMapped = false;
+  probe.depthTest = false;
+  probe.depthWrite = false;
+
+  const atlas = new RenderTarget(1, 1, { depthBuffer: false });
+  const savedViewport = new Vector4();
+
+  /** Compiled heads, keyed by node and socket, valid for ONE semantic
+   *  signature. A literal drag or an override change moves a uniform, so it
+   *  answers from here; a topology change mints a new signature and every head
+   *  is disposed. */
+  const rigs = new Map<string, CompiledMaterialGraphPreview>();
+  let rigsFor: string | null = null;
+
+  /** Latest wins. At most one request is parked, and a batch whose successor
+   *  already exists resolves null rather than painting an answer the chrome is
+   *  about to replace. */
+  let running = false;
+  let parked: {
+    request: NodePreviewRequest;
+    context: () => PreviewContext;
+    resolve: (batch: NodePreviewBatch | null) => void;
+  } | null = null;
+
+  function disposeRigs(): void {
+    for (const head of rigs.values()) head.dispose();
+    rigs.clear();
+    rigsFor = null;
+  }
+
+  function key(nodeId: string, socket: string): string {
+    return `${nodeId}::${socket}`;
+  }
+
+  function headFor(
+    graph: MaterialGraph3D,
+    binding: MaterialGraphBinding,
+    source: MaterialGraphSourceChannels,
+    lighting: Doc3D["materialLighting"],
+    nodeId: string,
+  ): CellPlan {
+    const node = graph.nodesById[nodeId];
+    if (node === undefined || typeof node.kind !== "string") {
+      return { nodeId, message: `Node "${nodeId}" is not in this material graph.` };
+    }
+    if (node.scope !== undefined) {
+      return {
+        nodeId,
+        message: `Node "${nodeId}" sits inside a graph function, and a function body has no call-site arguments to bind, so it has no value to preview on its own.`,
+      };
+    }
+    const socket = materialGraphNodeSockets(graph, node).outputs[0];
+    if (socket === undefined) {
+      return { nodeId, message: `Node "${nodeId}" has no output socket to preview.` };
+    }
+    const cached = rigs.get(key(nodeId, socket.id));
+    if (cached !== undefined) return { nodeId, rig: cached };
+    try {
+      const head = compileMaterialGraphNodePreview({
+        graph,
+        binding,
+        source,
+        // No IBL: the preview head is an unlit `MeshBasicNodeMaterial` that
+        // paints the socket's value straight out, so an environment map would
+        // be built and bound for a material that cannot read it.
+        environment: null,
+        lighting,
+        nodeId,
+        socket: socket.id,
+      });
+      rigs.set(key(nodeId, socket.id), head);
+      return { nodeId, rig: head };
+    } catch (error) {
+      return { nodeId, message: messageOf(error) };
+    }
+  }
+
+  async function draw(
+    request: NodePreviewRequest,
+    context: PreviewContext,
+  ): Promise<NodePreviewBatch | null> {
+    const graph = context.doc.materials?.[request.materialId];
+    if (graph === undefined) return null;
+    const signature = materialGraphSemanticSignature(graph);
+    if (signature !== rigsFor) {
+      disposeRigs();
+      rigsFor = signature;
+    }
+
+    const size = request.size;
+    const refuse = (message: string): NodePreviewBatch => ({
+      materialId: request.materialId,
+      semanticSignature: signature,
+      size,
+      results: request.nodeIds.map((nodeId) => ({ nodeId, bitmap: null, message })),
+    });
+    const textures = context.textures;
+    if (textures === null) {
+      return refuse("The shot runtime's texture cache is not mounted, so this graph cannot be sampled.");
+    }
+    const source = context.source;
+    if (source === null) {
+      return refuse(
+        request.slot === null
+          ? "This layer has no source material, so its node previews have no channels to sample."
+          : `This model has no material slot ${request.slot}, so its node previews have no channels to sample.`,
+      );
+    }
+
+    const binding = bindingFor(context.layer, request);
+    const plans = request.nodeIds.map((nodeId) =>
+      headFor(graph, binding, source, context.doc.materialLighting, nodeId),
+    );
+    // Nothing compiled: every answer is already a sentence, so the GPU is never
+    // touched and the batch costs no stall at all.
+    if (plans.every((plan) => plan.rig === undefined)) {
+      return {
+        materialId: request.materialId,
+        semanticSignature: signature,
+        size,
+        results: plans.flatMap((plan) =>
+          plan.rig === undefined ? [{ nodeId: plan.nodeId, bitmap: null, message: plan.message }] : [],
+        ),
+      };
+    }
+
+    // Cell 0 is the probe, so the grid is sized for one more picture than the
+    // request asks for. Square-ish, because a single row of 65 cells at 256px
+    // is 16640 texels wide and past every device's texture limit.
+    const cells = plans.length + 1;
+    const cols = Math.ceil(Math.sqrt(cells));
+    const rows = Math.ceil(cells / cols);
+    const width = cols * size;
+    const height = rows * size;
+    atlas.setSize(width, height);
+
+    const frame: MaterialGraphFrame = {
+      graph,
+      binding,
+      sceneTimeSeconds: (context.frame * context.doc.fps.den) / context.doc.fps.num,
+      // Neutral, and deliberately: a thumbnail shows the NODE's value, so a
+      // layer's fade would make two identical branches read differently on two
+      // layers.
+      opacity: 1,
+      shading: context.layer?.kind === "mesh" ? context.layer.mesh.shading : undefined,
+      lighting: context.doc.materialLighting,
+      // A PRIMITIVE's base colour is a live uniform (its `source-color` node
+      // reads this every write), so a colour drag repaints the thumbnails
+      // without recompiling. A model's source colours are frozen at load.
+      ...(context.layer?.kind === "mesh" && context.layer.mesh.asset === undefined
+        ? { sourceColor: source.color }
+        : {}),
+      textures,
+    };
+
+    const gl = context.gl;
+    gl.getViewport(savedViewport);
+    gl.setRenderTarget(atlas);
+    gl.setScissorTest(true);
+    // Visual row-major with row 0 at the TOP; three normalises `setViewport` to
+    // a bottom-left origin on both backends, so the y flip is here and only
+    // here. Where each cell lands in the READBACK is decided after the probe
+    // has been read.
+    const drawCell = (index: number, material: Material): void => {
+      const x = (index % cols) * size;
+      const y = height - (Math.floor(index / cols) + 1) * size;
+      gl.setViewport(x, y, size, size);
+      gl.setScissor(x, y, size, size);
+      quad.material = material;
+      gl.render(scene, camera);
+    };
+    drawCell(0, probe);
+    plans.forEach((plan, i) => {
+      if (plan.rig === undefined) return;
+      plan.rig.write(frame);
+      drawCell(i + 1, plan.rig.material);
+    });
+    gl.setScissorTest(false);
+    // The presenter sets a viewport per pane every frame, but only while it is
+    // mounted — a camera-only layout leaves whatever was set last standing, so
+    // a batch that walked away with a 96px viewport would draw the next frame
+    // into one corner of the canvas.
+    gl.setViewport(savedViewport.x, savedViewport.y, savedViewport.z, savedViewport.w);
+    // LAST, so nothing between here and the next frame can draw into the atlas.
+    gl.setRenderTarget(null);
+
+    const data = await gl.readRenderTargetPixelsAsync(atlas, 0, 0, width, height);
+    const bytes = new Uint8ClampedArray(data.buffer, data.byteOffset, data.byteLength);
+    const image = new ImageData(bytes, width, height);
+    const flipY = readsBottomUp(bytes, width, height, size);
+
+    const results: NodePreviewResult[] = [];
+    for (const [i, plan] of plans.entries()) {
+      if (plan.rig === undefined) {
+        results.push({ nodeId: plan.nodeId, bitmap: null, message: plan.message });
+        continue;
+      }
+      const cell = i + 1;
+      const sx = (cell % cols) * size;
+      const row = Math.floor(cell / cols);
+      const sy = flipY ? height - (row + 1) * size : row * size;
+      results.push({
+        nodeId: plan.nodeId,
+        bitmap: await createImageBitmap(image, sx, sy, size, size, {
+          imageOrientation: flipY ? "flipY" : "none",
+          // The atlas is opaque (every head writes alpha 1), so premultiplying
+          // would be a no-op that costs a copy.
+          premultiplyAlpha: "none",
+        }),
+      });
+    }
+    return { materialId: request.materialId, semanticSignature: signature, size, results };
+  }
+
+  async function drain(
+    request: NodePreviewRequest,
+    context: () => PreviewContext,
+    resolve: (batch: NodePreviewBatch | null) => void,
+  ): Promise<void> {
+    let batch: NodePreviewBatch | null = null;
+    try {
+      batch = await draw(request, context());
+    } catch (error) {
+      // A batch that throws is a diagnostic, never a stage failure: the author
+      // is mid-edit and the thumbnails are the least important thing on screen.
+      console.warn("[shot] a node preview batch failed", error);
+    }
+    const next = parked;
+    parked = null;
+    if (next === null) {
+      running = false;
+      resolve(batch);
+      return;
+    }
+    // Superseded while it was drawing, so this answer is already stale. Close
+    // the bitmaps rather than hand back pictures nobody will paint.
+    closeBatch(batch);
+    resolve(null);
+    void drain(next.request, next.context, next.resolve);
+  }
+
+  return {
+    run(request, context) {
+      return new Promise<NodePreviewBatch | null>((resolve) => {
+        if (running) {
+          parked?.resolve(null);
+          parked = { request, context, resolve };
+          return;
+        }
+        running = true;
+        void drain(request, context, resolve);
+      });
+    },
+    dispose() {
+      parked?.resolve(null);
+      parked = null;
+      disposeRigs();
+      atlas.dispose();
+      geometry.dispose();
+      probe.dispose();
+    },
+  };
+}
+
+/**
+ * Which way the readback runs, measured off cell 0's ramp.
+ *
+ * The probe is painted into the atlas's visual TOP-LEFT cell, so it lands in
+ * buffer rows [0, size) when the backend hands rows back top-down, and in
+ * [height - size, height) when it hands them back bottom-up. Both candidate
+ * slots are sampled and the one holding the steeper dark-to-light ramp is the
+ * probe. Ties fall to top-down, which is the WebGPU answer.
+ *
+ * ponytail: a node whose own thumbnail happens to be a perfect inverted ramp in
+ * the opposite candidate slot could out-vote the probe. Fixing that needs a
+ * second probe cell reserved at the grid's other end; the failure mode is one
+ * upside-down batch, so it is not worth the layout it would cost.
+ */
+function readsBottomUp(
+  bytes: Uint8ClampedArray,
+  width: number,
+  height: number,
+  size: number,
+): boolean {
+  const x = Math.floor(size / 2);
+  const red = (row: number): number => bytes[(row * width + x) * 4] ?? 0;
+  const topDown = red(size - 1) - red(0);
+  const bottomUp = red(height - size) - red(height - 1);
+  return bottomUp > topDown;
+}
+
+/**
+ * The binding the request's slot renders through.
+ *
+ * Falls back to a bare binding on the requested graph whenever the layer no
+ * longer names it — a request can outlive a rebind by one round trip, and
+ * overrides are keyed by the parameter ids of the graph they were authored
+ * against, so carrying the old ones over would write a stale value into an
+ * unrelated uniform.
+ */
+function bindingFor(layer: Layer3D | null, request: NodePreviewRequest): MaterialGraphBinding {
+  const material = layer?.kind === "mesh" ? layer.mesh.material : undefined;
+  if (!isGraphMaterial(material)) return { kind: "graph", materialId: request.materialId };
+  const bound = materialBindingForSlot(material, request.slot);
+  return bound.materialId === request.materialId
+    ? bound
+    : { kind: "graph", materialId: request.materialId };
+}
+
+function closeBatch(batch: NodePreviewBatch | null): void {
+  if (batch === null) return;
+  for (const result of batch.results) result.bitmap?.close();
+}
+
+function messageOf(error: unknown): string {
+  return error instanceof Error ? error.message : String(error);
+}