@volter/editor-blender 0.1.150 → 0.1.152

package/contributions/blender-runtime-skin.ts

@@ -1,61 +1,31 @@
-/**
- * THE TIMELINE'S ANIMATION — three.js PLAYS Blender's animation; Blender holds it
- * as DATA (owner rule, 2026-09-20: "we visualize with three.js, not Blender").
- *
- * ## Why this exists at all
- *
- * The obvious Timeline asks Blender for frame N, lets the depsgraph evaluate
- * it and re-exports the mesh columns. That is the wrong architecture: it puts
- * a WASM depsgraph evaluation and a full geometry round trip between the
- * person's pointer and the picture, and it moves `scene.frame_current` sixty
- * times a second under every bpy reader in the session. So the skin travels in the
- * frame itself: the export door ships each Armature-deformed mesh's weights per
- * exported vertex (`bpy_web_export.cc` `write_skin`), the frame names every bone's
- * rest, channels, constraints and pose and each armature's NLA stack, and the presenter
- * builds the skeletons (`blender-runtime-skeleton.ts`). This director poses EVERY armature
- * at the scene frame with Blender's own evaluation (`blender-pose.ts`): its NLA tracks and
- * active action, blended as Blender blends them, and its Damped Track constraints. Every scrub
- * and every played frame costs ZERO calls into Blender; an action is baked again exactly when
- * the frame says its revision moved.
- *
- * ## The bind pose is the EXPORT pose, and that is the whole trick
- *
- * The export door runs with `evaluate: True`, so the columns the presenter
- * holds are the mesh ALREADY DEFORMED at whatever frame Blender sits on. Bind
- * a skeleton whose bones are in that same pose and the skinning is an identity
- * there — `skinMatrix = Σ wᵢ·Bᵢ·Bᵢ⁻¹ = I` — so the picture at the bind frame is
- * byte-for-byte the frame Blender presented, and every other frame is three's
- * own evaluation of the same skin over the same columns. Nothing has to move
- * Blender's frame, ever.
- *
- * ## The fidelity check, and what is not played
- *
- * At Blender's own frame, standing still, the picture IS Blender's pose (the frame's), and the
- * evaluator's answer for that frame is compared with it bone by bone (`poseDivergence`). A bone
- * further off than a tenth of a degree is said in the console with what would explain it, so a
- * difference between this Timeline (and a game, which uses the same evaluator) and Blender is
- * never silent. What the evaluator does not play — a constraint other than Damped Track, a
- * driver, non-default parenting — is named once as well. Anything Blender's depsgraph does to
- * GEOMETRY per frame (shape keys, a Displace or Cloth modifier) does not follow the bones.
- */
+/** The Timeline poses Blender-sampled animation on the scene’s three.js mixer. Exported weights bind against the presented export pose, making skinning an identity at that frame. Each action is sampled again only when its revision changes; scrubs make no engine calls. Compare the mixed pose against Blender’s presented pose and report divergence and unsupported evaluation features. */
 
-import type { BlenderActionClip } from '@volter/blender-engine/browser/rna';
+import { bytesFromBase64 } from '@volter/blender-engine/browser/three/blender-base64';
+import type { BlenderActionClip, BlenderSceneMovie } from '@volter/blender-engine/browser/rna';
 import type { BlenderArmature } from '@volter/blender-engine/browser/three/blender-runtime-armature';
 import type { BlenderRuntimeView } from '@volter/blender-engine/browser/three/blender-runtime-view';
 import { editorHost, type StageTransportHandle } from '@volter/sdk/host';
 import { blenderRnaSet } from '../host/blender-runtime-host';
-import { actionLayer, ArmaturePose, nlaLayers, poseClip, poseDivergence, type PoseClip, type PoseLayer } from './blender-pose';
+import { remedy } from './blender-remedy';
+import { actionLayer, nlaLayers, type PoseLayer } from './blender-pose';
+import { MixerPose, mixerClip, poseDivergence, type MixerClip } from './blender-mixer-pose';
+import { playMovie, type PlayMovie } from './blender-playback-movie';
+import { sceneMixer } from './blender-scene-mixer';
+import { invalidateStages } from '@volter/sdk/stage-invalidation';
 
 /** How the director reads the engine: the Timeline's subject (its keys, the scene's range) and
  *  one action baked for one armature. */
 export interface SkinReader {
-  clip(armature: string): Promise<BlenderActionClip | null>;
+  /** The scene's clock and range, and `armature`'s action when there is one (null: the scene alone). */
+  clip(armature: string | null): Promise<BlenderActionClip | null>;
   bake(armature: string, action: string): Promise<BlenderActionClip | null>;
+  /** Animated objects, cameras and marker cuts sampled at each scene frame for Timeline playback. */
+  movie(): Promise<BlenderSceneMovie | null>;
 }
 
 /** An armature's layers at a scene frame: its NLA, then its active action over them. */
 export function sceneLayers(armature: BlenderArmature, frame: number,
-  clips: (action: string) => PoseClip | null | undefined): { layers: PoseLayer[]; waiting: boolean; skipped: string[] } {
+  clips: (action: string) => MixerClip | null | undefined): { layers: PoseLayer[]; waiting: boolean; skipped: string[] } {
   const stack = nlaLayers(armature.animation, frame, clips);
   const action = armature.action ?? null;
   if (action) {
@@ -67,17 +37,7 @@ export function sceneLayers(armature: BlenderArmature, frame: number,
   return stack;
 }
 
-/** What to do about what plays only in Blender, by kind. */
-export function remedy(unsupported: readonly string[]): string {
-  const out: string[] = [];
-  if (unsupported.some((thing) => thing.includes(' constraint ') || thing.includes('drivers')))
-    out.push('Bake a constraint or driver into the action (Pose ▸ Animation ▸ Bake Action, Visual Keying), or aim with a Damped Track.');
-  if (unsupported.some((thing) => thing.includes('curves on ')))
-    out.push("Key the motion on a bone (the root bone) instead of the armature object, or move the object from the game.");
-  if (unsupported.some((thing) => thing.includes('interpolation') || thing.includes(' modifier')))
-    out.push('Use Bezier, Linear or Constant keys and the Cycles modifier only, or bake the curves (Key ▸ Bake Keyframes).');
-  return out.length ? ` ${out.join(' ')}` : '';
-}
+export { remedy } from './blender-remedy';
 
 /** True when an armature has anything to evaluate: an action, or NLA strips. */
 export function animated(armature: BlenderArmature): boolean {
@@ -95,14 +55,21 @@ export class BlenderSkinDirector {
   /** The presented view whose skeletons are posed (`sync`). */
   #view: BlenderRuntimeView | null = null;
   #reader: SkinReader | null = null;
+  /** THE SCENE'S MOVIE on the document's view (`docs/SCENE-ANIMATION.md`, step 2): its objects and
+   *  cameras posed at every Timeline frame beside the armatures, and its camera cuts. */
+  #movie: PlayMovie | null = null;
+  #movieKey: string | null = null;
+  #moviePosed: number | null = null;
+  /** What the last movie read held (null before one): said in `state()`, so an empty movie says why. */
+  #movieRead: { objects: number; materials: number; shapes: number } | null = null;
   /** The Timeline's subject: the armature its keys and header describe, and what its header was
    *  read for. Read again only when that changes. */
   #armature: string | null = null;
   #clipKey: string | null = null;
   #clip: BlenderActionClip | null = null;
   /** Each posed armature's evaluator, and every action baked for it, at the revision baked. */
-  #poses = new Map<string, ArmaturePose>();
-  #baked = new Map<string, { revision: number | null; clip: PoseClip | null | undefined }>();
+  #poses = new Map<string, MixerPose>();
+  #baked = new Map<string, { revision: number | null; clip: MixerClip | null | undefined }>();
   /** The frame the last SEEK asked for — see {@link frame}. */
   #seeked: number | null = null;
   /** The stage transport this skin is attached to, or null before `attachTo`.
@@ -114,8 +81,7 @@ export class BlenderSkinDirector {
   #warnings: string[] = [];
   /** What was already said in the console, so each finding is said once. */
   #said = new Set<string>();
-  /** Engine calls this director has made, counted so a walk can prove that a
-   *  scrub costs none (the acceptance the owner named). */
+  /** Engine-call count exposed in the Timeline state. */
   #engineCalls = 0;
 
   get version(): number {
@@ -169,6 +135,12 @@ export class BlenderSkinDirector {
      *  They agree after a pause or a scrub-end, and they are deliberately
      *  allowed to differ in between. */
     blenderFrame: number | null;
+    /** The scene's movie on the Timeline: how many objects it moves, the camera it cuts to now, the
+     *  frame it last posed (null: posed by Blender, standing at its own frame), and the camera the
+     *  camera view looks through. */
+    movie: { objects: number; camera: string | null; posed: number | null; viewCamera: string | null } | null;
+    /** What the last read of the scene's movie held: objects moved, materials and meshes keyed. */
+    movieRead: { objects: number; materials: number; shapes: number } | null;
   } {
     const clip = this.#clip;
     return {
@@ -189,6 +161,8 @@ export class BlenderSkinDirector {
       warnings: this.#warnings,
       engineCalls: this.#engineCalls,
       blenderFrame: clip?.frameCurrent ?? null,
+      movieRead: this.#movieRead,
+      movie: this.#movie ? { objects: this.#movie.objects.length, camera: this.#moviePosed === null ? null : this.#movie.cameraAt(this.#moviePosed), posed: this.#moviePosed, viewCamera: this.#view?.cameraViewCamera() ?? null } : null,
     };
   }
 
@@ -229,15 +203,16 @@ export class BlenderSkinDirector {
           ? names.find((name) => facts.armatures[name]!.action)! : null);
     // EVERY ARMATURE'S EVALUATOR, over the rig the presenter holds for it now.
     for (const [name, pose] of [...this.#poses])
-      if (!facts.armatures[name] || view.skeletons.rig(name) !== pose.rig) this.#poses.delete(name);
+      if (!facts.armatures[name] || view.skeletons.rig(name) !== pose.rig) { pose.dispose(); this.#poses.delete(name); }
     for (const [name, entry] of Object.entries(facts.armatures)) {
       const rig = view.skeletons.rig(name);
       if (!rig || !animated(entry)) {
+        this.#poses.get(name)?.dispose();
         this.#poses.delete(name);
         continue;
       }
       let pose = this.#poses.get(name);
-      if (!pose) this.#poses.set(name, pose = new ArmaturePose(rig));
+      if (!pose) this.#poses.set(name, pose = new MixerPose(rig, sceneMixer(view).mixer));
       pose.facts(entry);
     }
     // A BAKE IS GOOD FOR ITS ACTION'S REVISION; a new revision bakes it again when next needed.
@@ -250,12 +225,11 @@ export class BlenderSkinDirector {
     if (key !== this.#clipKey) {
       this.#clipKey = key;
       this.#armature = armature;
-      let clip: BlenderActionClip | null = null;
-      if (armature) {
-        this.#engineCalls++;
-        clip = await read.clip(armature);
-        if (clip?.action && clip.armature) this.#baked.set(`${clip.armature}\u0000${clip.action}`, { revision: facts.actions[clip.action] ?? null, clip: poseClip(clip) });
-      }
+      // THE SCENE'S CLOCK IS READ WITH OR WITHOUT AN ARMATURE (`docs/SCENE-ANIMATION.md`, step 1): a
+      // film of keyed cameras and props has a range, a frame and markers, and the Timeline plays it
+      this.#engineCalls++;
+      const clip: BlenderActionClip | null = await read.clip(armature);
+      if (clip?.action && clip.armature) this.#baked.set(`${clip.armature}\u0000${clip.action}`, { revision: facts.actions[clip.action] ?? null, clip: mixerClip(clip) });
       const previous = this.#clip;
       const rangeChanged = clip?.frameStart !== previous?.frameStart ||
         clip?.frameEnd !== previous?.frameEnd || clip?.fps !== previous?.fps;
@@ -269,23 +243,36 @@ export class BlenderSkinDirector {
         if (this.#transport && clip.fps > 0) this.#transport.seek(clip.frameCurrent / clip.fps);
       }
     }
+    // THE MOVIE IS READ AGAIN WHEN AN ACTION OR THE SCENE'S CLOCK MOVED (a key, a marker, the range), OR
+    // ITS CAMERAS DID: the scene camera switched, a camera's lens or placement changed unkeyed (the movie
+    // carries the scene camera and its settings as they were read)
+    const movieKey = JSON.stringify([facts.actions, facts.clock, this.#clip?.frameStart, this.#clip?.frameEnd, facts.cameras]);
+    if (movieKey !== this.#movieKey) {
+      this.#movieKey = movieKey;
+      this.#engineCalls++;
+      const data = await read.movie();
+      this.#movieRead = data ? { objects: data.objects.length, materials: data.materials.length, shapes: data.shapes.length } : null;
+      this.#movie?.dispose();
+      this.#movie = data && (data.objects.length || data.materials.length || data.shapes.length) ? playMovie(view, data, sceneMixer(view).mixer) : null;
+      for (const warning of this.#movie?.warnings ?? []) this.#say(`movie:${warning}`, warning);
+    }
     this.#pose();
     this.#publish();
   }
 
   /** One action of one armature: baked, or being baked now (once), or null when it plays nothing. */
-  #clipOf(armature: string, action: string): PoseClip | null | undefined {
+  #clipOf(armature: string, action: string): MixerClip | null | undefined {
     const key = `${armature}\u0000${action}`;
     const held = this.#baked.get(key);
     if (held) return held.clip;
     const reader = this.#reader;
     const revision = this.#view?.animationFacts().actions[action] ?? null;
     if (!reader) return null;
-    const entry: { revision: number | null; clip: PoseClip | null | undefined } = { revision, clip: undefined };
+    const entry: { revision: number | null; clip: MixerClip | null | undefined } = { revision, clip: undefined };
     this.#baked.set(key, entry);
     this.#engineCalls++;
     void reader.bake(armature, action).then((baked) => {
-      entry.clip = baked ? poseClip(baked) : null;
+      entry.clip = baked ? mixerClip(baked) : null;
       if (this.#baked.get(key) !== entry) return;
       this.#pose();
       this.#publish();
@@ -312,47 +299,70 @@ export class BlenderSkinDirector {
     const playing = this.#transport?.snapshot().playbackState === 'playing';
     const warnings: string[] = this.#clip?.reason ? [this.#clip.reason] : [];
     const world = { object: (name: string) => view.objectForBlenderName(name) };
+    const scene = sceneMixer(view);
+    // ONE FRAME, THREE STEPS (`blender-scene-mixer.ts`): every player places its actions, the scene's
+    // mixer evaluates them all at once, then each settles (constraints, the fidelity check).
+    // OBJECTS AND CAMERAS (an armature may ride on one): standing still at Blender's own frame the
+    // picture is Blender's, as for the armatures (an object moved by hand and not keyed stays where it
+    // was put until the frame changes, as in Blender), so the movie holds off the mixer there.
+    const movieHere = !!this.#movie && !(atBlender && !playing);
+    // (the drawn geometry is Blender's evaluation at its own frame: a shape key plays relative to it)
+    this.#movie?.place(movieHere ? frame : null, facts.frame);
+    this.#moviePosed = movieHere ? frame : null;
+    const placed: { name: string; pose: MixerPose; armature: BlenderArmature; check: boolean; unsupported: string[] }[] = [];
     for (const [name, pose] of this.#poses) {
       const armature = facts.armatures[name];
       if (!armature) continue;
       const unsupported = [...pose.unsupported(), ...this.#clipsOf(name).flatMap((clip) => clip.unsupported)];
       if (unsupported.length) {
-        const said = `${name} plays ${unsupported.join(', ')} only in Blender, not in the Timeline or a game.${remedy(unsupported)}`;
+        const said = `${name} plays ${unsupported.join(', ')} only in Blender, not in the Timeline preview.${remedy(unsupported)}`;
         warnings.push(said);
         this.#say(`unsupported:${name}:${unsupported.join('|')}`, said);
       }
       const stack = sceneLayers(armature, frame, (action) => this.#clipOf(name, action));
-      for (const skipped of stack.skipped) this.#say(`skipped:${name}:${skipped}`, `${name}: ${skipped} plays only in Blender, not in the Timeline or a game.`);
+      for (const skipped of stack.skipped) this.#say(`skipped:${name}:${skipped}`, `${name}: ${skipped} plays only in Blender, not in the Timeline preview.`);
       if (stack.waiting) continue;
-      if (atBlender && !playing) {
-        view.skeletons.restorePose(name);
-        pose.rig.object.updateMatrixWorld(true);
-        this.#check(name, pose, armature, stack.layers, world, unsupported);
-      } else {
-        pose.apply(stack.layers, world);
-      }
+      pose.place(stack.layers);
+      placed.push({ name, pose, armature, check: atBlender && !playing, unsupported });
+    }
+    scene.evaluate();
+    this.#movie?.settle();
+    if (this.#movie) {
+      // THE CUT ALWAYS FOLLOWS THE MARKERS, at Blender's own frame too: Blender switches `scene.camera`
+      // at bound markers as it plays, which a frame written back to it does not do
+      const cut = this.#movie.cameraAt(frame);
+      view.setPlaybackCamera(cut, cut === null ? null : this.#movie.cameraData(cut));
+      // (a scrub that moved only objects and cameras asks the stage for the frame it now shows)
+      invalidateStages();
+    } else view.setPlaybackCamera(null);
+    for (const one of placed) {
+      const posed = one.pose.settle(world);
+      if (!one.check) continue;
+      // AT BLENDER'S OWN FRAME the picture is Blender's pose, and the evaluator's answer is checked against it
+      this.#check(one.name, posed, one.armature, one.unsupported);
+      view.skeletons.restorePose(one.name);
+      one.pose.rig.object.updateMatrixWorld(true);
     }
     this.#warnings = warnings;
   }
 
   /** The clips baked so far for one armature. */
-  #clipsOf(armature: string): PoseClip[] {
-    const out: PoseClip[] = [];
+  #clipsOf(armature: string): MixerClip[] {
+    const out: MixerClip[] = [];
     for (const [key, entry] of this.#baked) if (key.startsWith(`${armature}`) && entry.clip) out.push(entry.clip);
     return out;
   }
 
-  /** THE FIDELITY CHECK at Blender's own frame: the evaluator's pose against Blender's. */
-  #check(name: string, pose: ArmaturePose, armature: BlenderArmature, layers: readonly PoseLayer[],
-    world: { object(name: string): ReturnType<BlenderRuntimeView['objectForBlenderName']> }, unsupported: readonly string[]): void {
-    const off = poseDivergence(pose.apply(layers, world, false), armature);
+  /** THE FIDELITY CHECK at Blender's own frame: the evaluator's pose (as settled) against Blender's. */
+  #check(name: string, posed: ReadonlyMap<string, import('three').Matrix4>, armature: BlenderArmature, unsupported: readonly string[]): void {
+    const off = poseDivergence(posed, armature);
     if (!off) return;
     const frame = this.#clip?.frameCurrent ?? this.frame();
     const why = unsupported.length
       ? `What Blender plays here and the evaluator does not: ${unsupported.join(', ')}.`
       : 'Nothing the evaluator knows of explains it; this is a defect in the presenter, report it with the file.';
     this.#say(`divergence:${name}:${frame}:${off.bone}:${Math.round(off.degrees)}`,
-      `${name} at frame ${frame}: bone "${off.bone}" is ${off.degrees.toFixed(1)}° and ${(off.offset * 100).toFixed(1)}% of its length off Blender's own pose, so the Timeline and a game show it differently from Blender. ${why}`);
+      `${name} at frame ${frame}: bone "${off.bone}" is ${off.degrees.toFixed(1)}° and ${(off.offset * 100).toFixed(1)}% of its length off Blender's own pose, so the Timeline preview differs from Blender. ${why}`);
   }
 
   #say(key: string, message: string): void {
@@ -386,9 +396,7 @@ export class BlenderSkinDirector {
   channels(): { bone: string; keys: number[] }[] {
     const bones = new Map<string, Set<number>>();
     for (const track of this.#clip?.tracks ?? []) {
-      const raw = atob(track.keysBase64);
-      const bytes = new Uint8Array(raw.length);
-      for (let i = 0; i < raw.length; i++) bytes[i] = raw.charCodeAt(i);
+      const bytes = bytesFromBase64(track.keysBase64);
       const keys = new Float32Array(bytes.buffer, 0, bytes.length >> 2);
       let frames = bones.get(track.bone);
       if (!frames) bones.set(track.bone, frames = new Set());
@@ -417,14 +425,13 @@ export class BlenderSkinDirector {
    *  session; writing it sixty times a second would be the architecture this
    *  module exists to avoid. The transport's `onSettled` is what calls this —
    *  a pause, or the quiet at the end of a scrub — so a slider drag writes
-   *  once. Moved verbatim from the Timeline look, which is no longer where a
-   *  playhead write belongs. */
+   *  once. */
   async writeBookmark(): Promise<number> {
     const frame = Math.round(this.frame());
     const clip = this.#clip;
     if (!clip) return frame;
     const scene = clip.scene;
-    if (!scene || clip.frameCurrent === frame) return frame;
+    if (clip.frameCurrent === frame) return frame;
     this.#engineCalls++;
     // `rna_set` refuses `bpy.context.scene` — the scene must be addressed by
     // name through `bpy.data.scenes[...]`.
@@ -506,8 +513,10 @@ export class BlenderSkinDirector {
     this.#detach?.();
     this.#clip = null;
     this.#seeked = null;
-    for (const name of this.#poses.keys()) this.#view?.skeletons.restorePose(name);
+    for (const [name, pose] of this.#poses) { pose.dispose(); this.#view?.skeletons.restorePose(name); }
     this.#poses.clear();
+    this.#movie?.dispose();
+    this.#movie = null;
     this.#baked.clear();
     this.#armature = null;
     this.#clipKey = null;
@@ -517,12 +526,7 @@ export class BlenderSkinDirector {
 /** THE ONE DIRECTOR — see the class comment for why it is module-scoped. */
 export const blenderSkin = new BlenderSkinDirector();
 
-/** STABLE FUNCTION IDENTITIES for `useSyncExternalStore`, and they are not
- *  ceremony: passing `blenderSkin.subscribe.bind(blenderSkin)` inline mints a
- *  NEW subscribe and a NEW getSnapshot on every render, so React tears the
- *  subscription down and rebuilds it each time — measured on the Timeline's
- *  first walk, where the view bound its rig and its published `drawn` never
- *  moved again because no re-render ever arrived. */
+/** Stable subscribe and snapshot function identities preserve useSyncExternalStore subscriptions across renders. */
 export function subscribeBlenderSkin(listener: () => void): () => void {
   return blenderSkin.subscribe(listener);
 }