@volter/editor-blender 0.1.149 → 0.1.151

package/contributions/blender-playback-movie.ts

@@ -0,0 +1,409 @@
+/**
+ * THE SCENE'S MOVIE, on three.js's own `AnimationMixer`: what Blender's playback shows (every animated object and
+ * camera, and the camera cuts at the Timeline's markers), played on a view of the model at a scene frame: the
+ * Timeline's (`blender-runtime-skin.ts`), and the glTF export's clips (`blender-export-gltf.ts`).
+ *
+ * - BLENDER SAMPLES, THREE.JS PLAYS. Blender evaluates every animated object's transform at every
+ *   scene frame (`session.py`'s `_sample_object`: its channels through `FCurve.evaluate`, with its
+ *   delta transform, placed in its parent's space), and the movie is one `THREE.AnimationClip` of
+ *   position, quaternion, scale (and visibility, where it is keyed) tracks on the copy's objects,
+ *   played on the scene's one mixer beside the armatures (`blender-scene-mixer.ts`): `place` sets
+ *   the clips' time, the scene evaluates, `settle` places a sequence and turns the aimed objects.
+ * - A CAMERA'S PROJECTION is sampled the same way (lens, sensor, shift, clip range) and fitted to
+ *   the preview's own frame when it is looked through.
+ * - THE CAMERA AT A FRAME is the camera of the last marker at or before it that is bound to one
+ *   (and not disabled in renders), else the earliest such marker's, else the scene's camera.
+ * - Track To and Damped Track on an object (a camera aimed at a subject) turn it after the mixer,
+ *   toward the target as it stands this frame.
+ */
+import { bytesFromBase64 } from '@volter/blender-engine/browser/three/blender-base64';
+import * as THREE from 'three';
+import type { BlenderClipColumn, BlenderMovieObject, BlenderSceneMovie } from '@volter/blender-engine/browser/rna';
+import type { BlenderRuntimeView } from '@volter/blender-engine/browser/three/blender-runtime-view';
+
+/** One Timeline marker, by frame, with the camera Blender cuts to there. */
+interface MovieMarker {
+  readonly frame: number;
+  readonly camera: string | null;
+}
+
+/** The scene's movie on a presented view, for the Timeline and the glTF export. */
+export interface PlayMovie {
+  /** Every object it moves, by Blender name. */
+  readonly objects: readonly string[];
+  /** Set the frame on the mixer (nothing moves until it evaluates); null holds the movie off it, so
+   *  an evaluation for others leaves its objects where they stand (its materials still take the
+   *  `presented` frame: the presenter does not follow a material's animation). `presented` is the
+   *  frame the drawn geometry was evaluated at (Blender's own): a shape key
+   *  plays relative to the value it already has there. */
+  place(frame: number | null, presented: number): void;
+  /** After the mixer: the moved objects' matrices composed, and the aimed objects turned. */
+  settle(): void;
+  /** Its actions off the mixer. */
+  dispose(): void;
+  /** Every clip it plays (objects and cameras, materials, shape keys), as three.js tracks on the
+   *  presented objects: what an export writes (`blender-export-gltf`). */
+  clips(): readonly THREE.AnimationClip[];
+  /** The camera Blender's playback looks through at `frame`; null when the scene has none. */
+  cameraAt(frame: number): string | null;
+  /** A camera as it stands at the last posed frame, in the shape of the frame's camera data (its
+   *  world matrix in Blender's space, by rows, and its sampled projection): what the editor's
+   *  camera view draws through while the Timeline plays. Null when the movie has no entry for that camera. */
+  cameraData(name: string): ({ name: string; type: string; sensor_fit: string; matrix: [number, number, number, number][] } & CameraProps) | null;
+  /** Animation behavior omitted by the three.js preview, each condition named once. */
+  readonly warnings: readonly string[];
+}
+
+const CAMERA_PROPS = ['lens', 'sensor_width', 'sensor_height', 'ortho_scale', 'shift_x', 'shift_y', 'clip_start', 'clip_end'] as const;
+type CameraProps = Record<(typeof CAMERA_PROPS)[number], number>;
+
+function float32Of(column: BlenderClipColumn): Float32Array {
+  const bytes = bytesFromBase64(column.base64);
+  return new Float32Array(bytes.buffer, bytes.byteOffset, bytes.byteLength >> 2);
+}
+
+const AXES: Record<string, readonly [number, number]> = {
+  TRACK_X: [0, 1], TRACK_Y: [1, 1], TRACK_Z: [2, 1], TRACK_NEGATIVE_X: [0, -1], TRACK_NEGATIVE_Y: [1, -1], TRACK_NEGATIVE_Z: [2, -1],
+  UP_X: [0, 1], UP_Y: [1, 1], UP_Z: [2, 1],
+};
+
+/** The rotation that points `trackAxis` (signed) along `forward` and `upAxis` as near `up` as it can. */
+function aimed(forward: THREE.Vector3, up: THREE.Vector3, trackAxis: string, upAxis: string): THREE.Quaternion | null {
+  const [track, sign] = AXES[trackAxis] ?? AXES['TRACK_NEGATIVE_Z']!;
+  const [upIndex] = AXES[upAxis] ?? AXES['UP_Y']!;
+  if (upIndex === track) return null;
+  const f = forward.clone().normalize();
+  const u = up.clone().sub(f.clone().multiplyScalar(up.dot(f)));
+  if (u.lengthSq() < 1e-10) return null;
+  u.normalize();
+  const columns: THREE.Vector3[] = [new THREE.Vector3(), new THREE.Vector3(), new THREE.Vector3()];
+  columns[track] = f.multiplyScalar(sign);
+  columns[upIndex] = u;
+  const third = 3 - track - upIndex;
+  columns[third] = new THREE.Vector3().crossVectors(columns[(third + 1) % 3]!, columns[(third + 2) % 3]!);
+  return new THREE.Quaternion().setFromRotationMatrix(new THREE.Matrix4().makeBasis(columns[0]!, columns[1]!, columns[2]!));
+}
+
+interface MovieObject {
+  readonly data: BlenderMovieObject;
+  readonly object: THREE.Object3D;
+  /** Its sampled transform, played on the mixer; null when it does not move. */
+  readonly action: THREE.AnimationAction | null;
+  /** The camera's sampled projection, by property. */
+  readonly projection: Partial<Record<(typeof CAMERA_PROPS)[number], Float32Array>>;
+  readonly projectionStart: number;
+}
+
+/** Bind the scene’s movie to its caller’s mixer. */
+export function playMovie(view: BlenderRuntimeView, data: BlenderSceneMovie, mixer: THREE.AnimationMixer): PlayMovie {
+  const warnings: string[] = [];
+  const objects: MovieObject[] = [];
+  const { start, end, fps } = data.scene;
+  const frames = Math.max(1, end - start + 1);
+  const times = Float32Array.from({ length: frames }, (_, i) => i / fps);
+  const unbound: string[] = [];
+  for (const entry of data.objects) {
+    const object = view.objectForBlenderName(entry.name);
+    // Report animated objects whose named presentation object is absent.
+    if (!object) { if (entry.sampled || entry.camera?.sampled) unbound.push(entry.name); continue; }
+    for (const thing of entry.unsupported ?? []) warnings.push(`${entry.name}: ${thing} plays only in Blender, not in this animation preview.`);
+    const sampled = entry.sampled;
+    const tracks: THREE.KeyframeTrack[] = [];
+    if (sampled) {
+      const column = (c: BlenderClipColumn, stride: number): [Float32Array, Float32Array] => {
+        const values = float32Of(c);
+        return [values.length === stride ? new Float32Array([0]) : times.slice(0, values.length / stride), values];
+      };
+      tracks.push(new THREE.VectorKeyframeTrack(`${object.uuid}.position`, ...column(sampled.position, 3)));
+      tracks.push(new THREE.QuaternionKeyframeTrack(`${object.uuid}.quaternion`, ...column(sampled.quaternion, 4)));
+      tracks.push(new THREE.VectorKeyframeTrack(`${object.uuid}.scale`, ...column(sampled.scale, 3)));
+      if (sampled.hidden) {
+        const [at, hidden] = column(sampled.hidden, 1);
+        tracks.push(new THREE.BooleanKeyframeTrack(`${object.uuid}.visible`, at, Array.from(hidden, (one) => one < 0.5)));
+      }
+    }
+    const projection: MovieObject['projection'] = {};
+    const lens = entry.camera?.sampled;
+    if (lens) for (const prop of CAMERA_PROPS) projection[prop] = float32Of(lens[prop]);
+    let action: THREE.AnimationAction | null = null;
+    if (tracks.length) {
+      action = mixer.clipAction(new THREE.AnimationClip(`movie:${entry.name}`, (frames - 1) / fps, tracks));
+      action.setLoop(THREE.LoopOnce, 1);
+      action.clampWhenFinished = true;
+      action.timeScale = 0;
+    }
+    objects.push({ data: entry, object, action, projection, projectionStart: lens?.start ?? start });
+  }
+  if (unbound.length) {
+    warnings.push(`The movie animates ${unbound.length} object(s) this copy does not draw by that name, so they hold still: ${unbound.slice(0, 12).join(', ')}${unbound.length > 12 ? ', …' : ''}.`);
+  }
+  // MATERIALS AND SHAPE KEYS (`docs/SCENE-ANIMATION.md` step 5): more clips on the same mixer, placed
+  // with the objects'. A material's keyed inputs drive the presented material of its name (through one
+  // mesh that wears it: the material is one object, shared); a mesh's keyed shape keys become morph
+  // targets on its drawn geometry (each drawn vertex takes its Blender vertex's offset, `blenderVertex`,
+  // and the normals the offset gives it), driven by the value Blender gives each key there minus the
+  // value the drawn geometry already holds (its value at the presented frame, subtracted in `settle`).
+  const materialActions: THREE.AnimationAction[] = [];
+  const shapeActions: THREE.AnimationAction[] = [];
+  const shapeMeshes: { mesh: THREE.Mesh; keys: Float32Array[] }[] = [];
+  const extraClip = (name: string, tracks: THREE.KeyframeTrack[], into: THREE.AnimationAction[]): void => {
+    if (!tracks.length) return;
+    const action = mixer.clipAction(new THREE.AnimationClip(name, (frames - 1) / fps, tracks));
+    action.setLoop(THREE.LoopOnce, 1);
+    action.clampWhenFinished = true;
+    action.timeScale = 0;
+    into.push(action);
+  };
+  const extras = (): THREE.AnimationAction[] => [...materialActions, ...shapeActions];
+  /** A key's value at a scene frame, from its sampled column. */
+  const valueAt = (values: Float32Array, frame: number): number =>
+    values.length <= 1 ? values[0] ?? 0 : values[Math.max(0, Math.min(values.length - 1, Math.round(frame - start)))]!;
+  const timesOf = (values: Float32Array, stride: number): Float32Array =>
+    values.length === stride ? new Float32Array([0]) : times.slice(0, values.length / stride);
+  for (const entry of data.materials) {
+    let path: string | null = null;
+    let material: THREE.Material | null = null;
+    view.root.traverse((object) => {
+      const mesh = object as THREE.Mesh;
+      if (path || !mesh.isMesh) return;
+      const list = Array.isArray(mesh.material) ? mesh.material : [mesh.material];
+      const index = list.findIndex((one) => one?.name === entry.name);
+      if (index < 0) return;
+      path = Array.isArray(mesh.material) ? `${mesh.uuid}.material[${index}]` : `${mesh.uuid}.material`;
+      material = list[index]!;
+    });
+    const worn = material as THREE.Material | null;
+    if (!path || !worn) { warnings.push(`The keyed material "${entry.name}" is not drawn here, so its animation does not play.`); continue; }
+    const tracks: THREE.KeyframeTrack[] = [];
+    if (entry.color) { const values = float32Of(entry.color); tracks.push(new THREE.ColorKeyframeTrack(`${path}.color`, timesOf(values, 3), values)); }
+    if (entry.emissive) {
+      const values = float32Of(entry.emissive);
+      tracks.push(new THREE.ColorKeyframeTrack(`${path}.emissive`, timesOf(values, 3), values));
+      if ('emissiveIntensity' in worn) (worn as THREE.MeshStandardMaterial).emissiveIntensity = 1;
+    }
+    if (entry.opacity) {
+      const values = float32Of(entry.opacity);
+      tracks.push(new THREE.NumberKeyframeTrack(`${path}.opacity`, timesOf(values, 1), values));
+      worn.transparent = true;
+      worn.needsUpdate = true;
+    }
+    extraClip(`movie-material:${entry.name}`, tracks, materialActions);
+  }
+  for (const entry of data.shapes) {
+    const owner = view.objectForBlenderName(entry.object);
+    if (!owner) continue;
+    if (!entry.mapped) { warnings.push(`${entry.object}: its shape keys play only in Blender, not in this animation preview: its modifiers change its vertices.`); continue; }
+    const meshes: THREE.Mesh[] = [];
+    owner.traverse((object) => { const mesh = object as THREE.Mesh; if (mesh.isMesh && mesh.geometry.getAttribute('blenderVertex')) meshes.push(mesh); });
+    for (const mesh of meshes) {
+      const geometry = mesh.geometry;
+      const source = geometry.getAttribute('blenderVertex') as THREE.BufferAttribute;
+      const drawn = geometry.getAttribute('position').count;
+      const morphs: THREE.BufferAttribute[] = [];
+      const normals: THREE.BufferAttribute[] = [];
+      const dictionary: Record<string, number> = {};
+      const tracks: THREE.KeyframeTrack[] = [];
+      const keys: Float32Array[] = [];
+      const position = geometry.getAttribute('position') as THREE.BufferAttribute;
+      const normal = geometry.getAttribute('normal') as THREE.BufferAttribute | undefined;
+      let fits = true;
+      entry.keys.forEach((key, k) => {
+        const delta = float32Of(key.delta);
+        const offsets = new Float32Array(drawn * 3);
+        for (let j = 0; j < drawn; j++) {
+          const from = source.getX(j);
+          if (from >= entry.vertices) { fits = false; continue; }
+          offsets[j * 3] = delta[from * 3]!;
+          offsets[j * 3 + 1] = delta[from * 3 + 1]!;
+          offsets[j * 3 + 2] = delta[from * 3 + 2]!;
+        }
+        const morph = new THREE.BufferAttribute(offsets, 3);
+        morph.name = key.name;
+        morphs.push(morph);
+        // THE NORMALS THE OFFSET GIVES: the shape's own, less the drawn ones (smooth over the drawn faces)
+        if (normal) {
+          const shaped = new THREE.BufferGeometry();
+          shaped.setAttribute('position', new THREE.BufferAttribute(Float32Array.from(position.array as Float32Array, (v, i) => v + offsets[i]!), 3));
+          if (geometry.index) shaped.setIndex(geometry.index);
+          shaped.computeVertexNormals();
+          const turned = shaped.getAttribute('normal') as THREE.BufferAttribute;
+          normals.push(new THREE.BufferAttribute(Float32Array.from(turned.array as Float32Array, (v, i) => v - (normal.array as Float32Array)[i]!), 3));
+          shaped.dispose();
+        }
+        dictionary[key.name] = k;
+        const values = float32Of(key.values);
+        keys.push(values);
+        // (by the key's name, through the mesh's dictionary: an exporter writes the target by it)
+        tracks.push(new THREE.NumberKeyframeTrack(`${mesh.uuid}.morphTargetInfluences[${key.name}]`, timesOf(values, 1), values));
+      });
+      if (!fits) { warnings.push(`${entry.object}: its drawn vertices do not match its mesh, so its shape keys play only in Blender.`); continue; }
+      geometry.morphAttributes['position'] = morphs;
+      if (normal && normals.length === morphs.length) geometry.morphAttributes['normal'] = normals;
+      geometry.morphTargetsRelative = true;
+      mesh.updateMorphTargets();
+      mesh.morphTargetDictionary = dictionary;
+      for (const one of Array.isArray(mesh.material) ? mesh.material : [mesh.material]) one.needsUpdate = true;
+      shapeMeshes.push({ mesh, keys });
+      extraClip(`movie-shapes:${entry.object}:${mesh.uuid}`, tracks, shapeActions);
+    }
+  }
+  // PARENTS BEFORE CHILDREN, so an object aimed by a constraint is turned from its parent's pose.
+  const depth = (object: THREE.Object3D): number => { let d = 0; for (let at = object.parent; at; at = at.parent) d += 1; return d; };
+  const aiming = objects
+    .filter((one) => one.data.constraints.some((c) => c.enabled && c.influence > 0 && c.target && (c.type === 'TRACK_TO' || c.type === 'DAMPED_TRACK')))
+    .sort((a, b) => depth(a.object) - depth(b.object));
+  const byName = new Map(objects.map((one) => [one.data.name, one]));
+  const markers = data.markers.map((marker) => ({ frame: marker.frame, camera: marker.camera }));
+  const cutMarkers = markers.filter((marker) => marker.camera !== null && !(byName.get(marker.camera)?.data.camera?.renderHidden ?? false));
+
+  let frameNow = data.scene.current;
+  /** Whether the last `place` put the movie on the mixer (false: held off it), for `settle`. */
+  let placed = false;
+  /** The frame the drawn geometry was evaluated at: Blender's own (the director says), else the read's. */
+  let presentedFrame = data.scene.current;
+  /** Track To and Damped Track, in Blender's space (the copy's root holds the turn to Y up). */
+  const constrain = (): void => {
+    if (aiming.length === 0) return;
+    view.root.updateMatrixWorld(true);
+    const inverseRoot = view.root.matrixWorld.clone().invert();
+    const [position, rotation, size] = [new THREE.Vector3(), new THREE.Quaternion(), new THREE.Vector3()];
+    for (const entry of aiming) {
+      for (const constraint of entry.data.constraints) {
+        if (!constraint.enabled || constraint.influence <= 0 || !constraint.target) continue;
+        const target = view.objectForBlenderName(constraint.target);
+        if (!target) continue;
+        target.updateWorldMatrix(true, false);
+        entry.object.updateWorldMatrix(true, false);
+        const own = inverseRoot.clone().multiply(entry.object.matrixWorld);
+        const aim = inverseRoot.clone().multiply(target.matrixWorld);
+        own.decompose(position, rotation, size);
+        const forward = new THREE.Vector3().setFromMatrixPosition(aim).sub(position);
+        if (forward.lengthSq() === 0) continue;
+        let turned: THREE.Quaternion | null;
+        if (constraint.type === 'TRACK_TO') {
+          const up = constraint.useTargetZ ? new THREE.Vector3(0, 0, 1).transformDirection(aim) : new THREE.Vector3(0, 0, 1);
+          turned = aimed(forward, up, constraint.trackAxis!, constraint.upAxis!);
+        } else {
+          const [index, sign] = AXES[constraint.trackAxis!]!;
+          const axis = new THREE.Vector3().setComponent(index, sign).applyQuaternion(rotation);
+          turned = new THREE.Quaternion().setFromUnitVectors(axis, forward.normalize()).multiply(rotation);
+        }
+        if (!turned) continue;
+        const world = new THREE.Matrix4().compose(position, rotation.clone().slerp(turned, Math.min(1, constraint.influence)), size);
+        const parentWorld = entry.object.parent ? inverseRoot.clone().multiply(entry.object.parent.matrixWorld) : new THREE.Matrix4();
+        parentWorld.invert().multiply(world).decompose(entry.object.position, entry.object.quaternion, entry.object.scale);
+        entry.object.updateMatrixWorld(true);
+      }
+    }
+  };
+
+  /** A camera's projection at the posed frame, interpolated between Blender's samples. */
+  const projectionOf = (entry: MovieObject): CameraProps => {
+    const own = entry.data.camera!;
+    const props = Object.fromEntries(CAMERA_PROPS.map((prop) => [prop, own[prop]])) as CameraProps;
+    const at = frameNow - entry.projectionStart;
+    for (const prop of CAMERA_PROPS) {
+      const column = entry.projection[prop];
+      if (!column) continue;
+      if (column.length === 1) { props[prop] = column[0]!; continue; }
+      const i = Math.max(0, Math.min(column.length - 1, Math.floor(at)));
+      const j = Math.min(column.length - 1, i + 1);
+      const t = Math.max(0, Math.min(1, at - i));
+      props[prop] = column[i]! + (column[j]! - column[i]!) * t;
+    }
+    return props;
+  };
+
+  return {
+    objects: objects.filter((one) => one.data.sampled).map((one) => one.data.name),
+    warnings,
+    place(frame, presented) {
+      presentedFrame = presented;
+      if (frame === null) {
+        for (const entry of objects) if (entry.action) entry.action.enabled = false;
+        for (const action of shapeActions) action.enabled = false;
+        for (const { mesh } of shapeMeshes) mesh.morphTargetInfluences?.fill(0);
+        // (the presenter draws a material's file value, not its animation: the movie keeps it)
+        const held = Math.max(0, Math.min(end, presentedFrame) - start) / fps;
+        for (const action of materialActions) {
+          action.enabled = true;
+          if (!action.isRunning()) action.play();
+          action.paused = false;
+          action.time = held;
+        }
+        placed = false;
+        return;
+      }
+      frameNow = frame;
+      placed = true;
+      const time = Math.max(0, Math.min(end, frame) - start) / fps;
+      for (const entry of objects) {
+        if (!entry.action) continue;
+        entry.action.enabled = true;
+        if (!entry.action.isRunning()) entry.action.play();
+        entry.action.paused = false;
+        entry.action.time = time;
+      }
+      for (const action of extras()) {
+        action.enabled = true;
+        if (!action.isRunning()) action.play();
+        action.paused = false;
+        action.time = time;
+      }
+    },
+    settle() {
+      if (!placed) return;
+      // THE MOVING OBJECTS' MATRICES, composed here from what the mixer wrote: the presenter states each
+      // object's matrix outright (`matrixAutoUpdate = false`, again at every Blender redraw), so the
+      // mixer's position, rotation and scale reach no drawn matrix by themselves.
+      for (const entry of objects) if (entry.action && entry.action.enabled) entry.object.updateMatrix();
+      // SHAPE KEYS relative to what the drawn geometry already holds (its value at the presented frame)
+      for (const { mesh, keys } of shapeMeshes) {
+        const influences = mesh.morphTargetInfluences;
+        if (!influences) continue;
+        keys.forEach((values, k) => { influences[k] = (influences[k] ?? 0) - valueAt(values, presentedFrame); });
+      }
+      constrain();
+    },
+    clips() {
+      return [...objects.flatMap((entry) => (entry.action ? [entry.action.getClip()] : [])), ...extras().map((action) => action.getClip())];
+    },
+    dispose() {
+      for (const entry of objects) {
+        if (!entry.action) continue;
+        const clip = entry.action.getClip();
+        entry.action.stop();
+        mixer.uncacheAction(clip);
+        mixer.uncacheClip(clip);
+      }
+      for (const action of extras()) {
+        const clip = action.getClip();
+        action.stop();
+        mixer.uncacheAction(clip);
+        mixer.uncacheClip(clip);
+      }
+      placed = false;
+    },
+    cameraAt(frame) {
+      // Compared with the whole frame, as Blender's playback compares it.
+      const now = Math.floor(frame);
+      let best: MovieMarker | null = null;
+      let first: MovieMarker | null = null;
+      for (const marker of cutMarkers) {
+        if (marker.frame <= now && (!best || marker.frame > best.frame)) best = marker;
+        if (!first || marker.frame < first.frame) first = marker;
+      }
+      return (best ?? first)?.camera ?? data.scene.camera;
+    },
+    cameraData(name) {
+      const entry = byName.get(name);
+      if (!entry?.data.camera) return null;
+      view.root.updateMatrixWorld(true);
+      entry.object.updateWorldMatrix(true, false);
+      const e = view.root.matrixWorld.clone().invert().multiply(entry.object.matrixWorld).elements;
+      const matrix = [0, 1, 2, 3].map((r) => [e[r]!, e[4 + r]!, e[8 + r]!, e[12 + r]!] as [number, number, number, number]);
+      const own = entry.data.camera;
+      return { name, type: own.type, sensor_fit: own.sensor_fit, matrix, ...projectionOf(entry) };
+    },
+  };
+}