@gtanczyk/diorama 0.0.0-stage → 0.2.0

package/app/src/meta/voice.ts

@@ -0,0 +1,103 @@
+// Muse Voice Transcribe realtime transport. File uploads use voice-file.ts through speech/transcribe.ts.
+import { apiKey } from './http.ts';
+import { inputHash, MUSE_VOICE, type Memo } from './memo.ts';
+
+export const VOICE_TRANSCRIBE = 'muse-voice-transcribe-1.0';
+const URL_ASR = 'wss://api.meta.ai/v1/asr/realtime';
+/** 80 ms of 16-bit mono PCM at 24 kHz. */
+const FRAME = 3840;
+
+// One active ASR stream per process, including the retry cooldown. Voice Design and speech share it.
+let queue: Promise<unknown> = Promise.resolve();
+// "Ingress audio slower than real-time": the server drops a stream that falls behind the clock (a busy event loop
+// while a build paints and masks). It is the pace, not the audio: the same take goes again, on the clock.
+const transient = (error: unknown) => /backlog|busy|overload|socket error|timed out|closed 10(06|11|13)|slower than real-?time|real-?time/i.test(String(error));
+
+/** `pcm24k`: 16-bit mono PCM at 24 kHz. Never send faster than real time, even for legacy callers. */
+export function transcribe(memo: Memo, label: string, pcm24k: Buffer, pace = 1): Promise<string> {
+  return memo.remember('voice', label, inputHash(VOICE_TRANSCRIBE, pcm24k), 'txt', () => {
+    const run = queue.then(async () => {
+      // Real time exactly: the server refuses audio slower than real time (halving the pace on a retry turned every
+      // backlog into a sure "slower than real-time"), and faster than real time is never sent either.
+      const rate = Number.isFinite(pace) ? Math.max(0.25, Math.min(1, pace)) : 1;
+      for (let attempt = 0; ; attempt++) {
+        try { return await listen(label, pcm24k, rate); }
+        catch (error) {
+          if (!transient(error) || attempt === 3) throw error;
+          await new Promise((r) => setTimeout(r, 1000 * 2 ** attempt));
+        }
+      }
+    });
+    queue = run.catch(() => {});
+    return run.then((data) => ({ data, via: MUSE_VOICE }));
+  }).then((b) => b.toString('utf8'));
+}
+
+async function listen(label: string, pcm24k: Buffer, pace: number): Promise<Buffer> {
+  const ws = new WebSocket(URL_ASR);
+  ws.binaryType = 'arraybuffer';
+  const finals: string[] = [];
+  let partial = '';
+  // Budget includes transmission at the current rate, then 30 seconds for the final transcript.
+  const audio = Buffer.concat([pcm24k, Buffer.alloc(FRAME * 12)]);
+  const done = new Promise<string>((resolve, reject) => {
+    let stopped = false, opened = false, ended = false;
+    let sendTimer: ReturnType<typeof setTimeout> | undefined;
+    let wake: (() => void) | undefined;
+    const wait = (ms: number) => new Promise<void>((r) => { wake = r; sendTimer = setTimeout(() => { wake = undefined; r(); }, ms); });
+    const finish = (error?: Error) => {
+      if (stopped) return;
+      stopped = true;
+      clearTimeout(timer); clearTimeout(sendTimer); wake?.();
+      if (error) reject(error);
+      else resolve([...finals, partial].join(' ').replace(/\s+/g, ' ').trim());
+      if (ws.readyState < 2) ws.close(1000);
+    };
+    const fail = (reason: string) => finish(new Error(`transcribe ${label}: ${reason}`));
+    const timer = setTimeout(() => fail('timed out'), Math.max(60000, audio.length / 48000 / pace * 1000 + 30000));
+    const sendAudio = async () => {
+      // Each frame goes when the clock says, counted from the first: waiting a frame's length after each send added
+      // the timer's lateness every 80 ms, and a busy moment never caught up, so the stream drifted behind real time.
+      const t0 = Date.now();
+      for (let at = 0; at < audio.length; at += FRAME) {
+        while (!stopped && ws.bufferedAmount > FRAME * 4) await wait(20);
+        if (stopped || ws.readyState !== 1) return;
+        const frame = audio.subarray(at, at + FRAME);
+        ws.send(frame);
+        if (stopped) return;
+        const due = t0 + ((at + frame.length) / 48000 / pace) * 1000;
+        const ahead = due - Date.now();
+        if (ahead > 0) await wait(ahead);
+      }
+      if (!stopped && ws.readyState === 1) {
+        ended = true;
+        ws.send(JSON.stringify({ type: 'endStream' }));
+      }
+    };
+    ws.onerror = () => fail('socket error');
+    ws.onclose = (e) => {
+      if (e.code === 1000 && ended) finish();
+      else fail(`closed ${e.code} ${e.reason}${e.code === 1000 ? ' before endStream' : ''}`);
+    };
+    ws.onmessage = (e) => {
+      if (stopped) return;
+      try {
+        const msg = JSON.parse(String(e.data)) as { type?: string; sessionId?: string; transcript?: string; final?: boolean; message?: string };
+        if (!msg.type && msg.sessionId && !opened) {
+          opened = true;
+          void sendAudio().catch((error) => fail(`socket error: ${error}`));
+        } else if (msg.type === 'transcript' && msg.transcript !== undefined) {
+          if (msg.final) { finals.push(msg.transcript); partial = ''; }
+          else partial = msg.transcript;
+        } else if (msg.type === 'error') fail(String(msg.message).slice(0, 300));
+      } catch { fail('invalid server message'); }
+    };
+    ws.onopen = () => {
+      if (stopped) return;
+      try {
+        ws.send(JSON.stringify({ authorization: { accessToken: `Bearer ${apiKey()}` }, audioEncoding: 'PCM_24KHZ', model: VOICE_TRANSCRIBE, mode: 'PUSH_TO_TALK' }));
+      } catch (error) { fail(String(error)); }
+    };
+  });
+  return Buffer.from(await done, 'utf8');
+}

package/app/src/model/animation.ts

@@ -0,0 +1,38 @@
+// Per-character choices live outside generated model/ and sprites/ so regeneration keeps them.
+import fs from 'node:fs';
+import { z } from 'zod';
+import { hasModelTexture } from './glb.ts';
+import type { WorldFiles } from '../world/files.ts';
+
+const Texture = z.enum(['views', 'model']);
+const Frames = z.union([z.literal(8), z.literal(16), z.literal(32)]);
+const Settings = z.object({ texture: Texture, frames: Frames, coatCorrection: z.boolean().optional(), automatic: z.literal(true).optional() });
+export type AnimationSettings = z.infer<typeof Settings>;
+
+/** Existing choices win; new characters select UV/16 only when their actual model supports it. */
+export function animationSettings(files: WorldFiles, key: string, patch: Partial<AnimationSettings> = {}, candidate?: Buffer): AnimationSettings {
+  const file = files.path('cast', key, 'animation.json');
+  const saved = fs.existsSync(file) ? Settings.parse(files.readJson(file)) : undefined;
+  const modelFile = files.path('cast', key, 'model', 'model.json');
+  const model = fs.existsSync(modelFile) ? files.readJson(modelFile) as { texture?: string } : undefined;
+  const legacy = !saved && model?.texture !== 'model' && fs.existsSync(files.path('cast', key, 'sprites', 'sprites.json'));
+  const automatic = patch.texture === undefined && patch.frames === undefined && (!saved || saved.automatic) && !legacy;
+  const glbFile = files.path('cast', key, 'model', 'model.glb');
+  const glb = candidate ?? (fs.existsSync(glbFile) ? fs.readFileSync(glbFile) : undefined);
+  const texture = automatic ? glb && hasModelTexture(glb) ? 'model' : 'views' : patch.texture ?? saved?.texture ?? 'views';
+  const changedMode = patch.texture !== undefined && patch.texture !== saved?.texture;
+  const frames = automatic ? texture === 'model' ? 16 : 8 : patch.frames ?? (changedMode ? texture === 'model' ? 16 : 8 : saved?.frames ?? (texture === 'model' ? 16 : 8));
+  // Automatic UV starts with the rig's own weights. A historical coat override remains respected.
+  const coatCorrection = patch.coatCorrection ?? saved?.coatCorrection ?? (automatic && texture === 'model' ? false : undefined);
+  const settings = Settings.parse({ texture, frames, ...(coatCorrection !== undefined && { coatCorrection }), ...(automatic && { automatic: true }) });
+  if (texture === 'views' && frames !== 8) throw new Error('Painted views use 8 frames; use --texture model for 16 or 32 frames');
+  return settings;
+}
+
+export function animationOptions(texture: string, frames = '', coat = ''): Partial<AnimationSettings> {
+  return { ...(texture && { texture: Texture.parse(texture) }), ...(frames && { frames: Frames.parse(Number(frames)) }), ...(coat && { coatCorrection: z.enum(['on', 'off']).parse(coat) === 'on' }) };
+}
+
+export function saveAnimationSettings(files: WorldFiles, key: string, settings: AnimationSettings) {
+  files.write(files.path('cast', key, 'animation.json'), settings);
+}

package/app/src/model/blender/relax.py

@@ -0,0 +1,284 @@
+# Blender (headless) step for a rigged figure: puts it into a calm standing rest pose before it walks.
+#
+#   blender -b --factory-startup --python relax.py -- in.glb out.glb [report.json]
+#
+# TRELLIS rebuilds the figure in whatever pose Muse painted: arms often held up and out (a Y-pose), feet
+# wide apart. The auto-rig skins it in that pose, and the game then swings arms ~100 degrees down and walks
+# legs that are planted wide, so every frame shows the skinning's worst case: shoulders pinched, sleeves
+# glued to the chest, hands sticking out, a waddle. Here Blender poses arms down and feet under the hips
+# once, smooths the deformed shoulders and hips with a Corrective Smooth, and makes that the new rest pose.
+# The game's own walk then turns bones only a little from rest. The original positions travel along as the
+# _REST vertex attribute, so paintings projected in the original pose still land on the same surface.
+import bpy, sys, json, math
+from mathutils import Vector, Matrix
+
+argv = sys.argv[sys.argv.index('--') + 1:]
+src, dst = argv[0], argv[1]
+report_path = argv[2] if len(argv) > 2 else None
+report = {'ok': False}
+
+
+def bind_pose_glb(path, out):
+    """The glb with every joint node moved to its bind pose (mesh world x inverse bind matrix).
+
+    Make-It-Animatable saves its bones in its own T-pose (seated, leant back for a long coat) while the mesh
+    is in the pose TRELLIS built; three.js and the game bind with the inverse bind matrices, Blender takes
+    the nodes as rest. Moving the nodes makes both agree (the same as web/hero3d.ts toBindPose)."""
+    import struct, numpy as np
+    data = open(path, 'rb').read()
+    n = struct.unpack_from('<I', data, 12)[0]
+    gj = json.loads(data[20:20 + n])
+    rest = data[20 + n:]
+    blen = struct.unpack_from('<I', rest, 0)[0]
+    blob = rest[8:8 + blen]
+    nodes = gj['nodes']
+
+    def local(nd):
+        if 'matrix' in nd:
+            return np.array(nd['matrix'], dtype=float).reshape(4, 4).T
+        T = np.eye(4); T[:3, 3] = nd.get('translation', [0, 0, 0])
+        x, y, z, w = nd.get('rotation', [0, 0, 0, 1])
+        R = np.eye(4)
+        R[:3, :3] = [[1 - 2 * (y * y + z * z), 2 * (x * y - z * w), 2 * (x * z + y * w)],
+                     [2 * (x * y + z * w), 1 - 2 * (x * x + z * z), 2 * (y * z - x * w)],
+                     [2 * (x * z - y * w), 2 * (y * z + x * w), 1 - 2 * (x * x + y * y)]]
+        S = np.diag(list(nd.get('scale', [1, 1, 1])) + [1])
+        return T @ R @ S
+    parent = {c: i for i, nd in enumerate(nodes) for c in nd.get('children', [])}
+
+    def world(i):
+        M = local(nodes[i])
+        while i in parent:
+            i = parent[i]
+            M = local(nodes[i]) @ M
+        return M
+    mesh_node = next(i for i, nd in enumerate(nodes) if 'skin' in nd)
+    skin = gj['skins'][nodes[mesh_node]['skin']]
+    acc = gj['accessors'][skin['inverseBindMatrices']]
+    bv = gj['bufferViews'][acc['bufferView']]
+    off = bv.get('byteOffset', 0) + acc.get('byteOffset', 0)
+    ibm = np.frombuffer(blob[off:off + 64 * acc['count']], dtype='<f4').reshape(-1, 4, 4).transpose(0, 2, 1)
+    MW = world(mesh_node)
+    joints = skin['joints']
+    depth = lambda i: 0 if i not in parent else 1 + depth(parent[i])
+    for k in sorted(range(len(joints)), key=lambda k: depth(joints[k])):
+        j = joints[k]
+        bind = MW @ np.linalg.inv(ibm[k].astype(float))
+        pw = world(parent[j]) if j in parent else np.eye(4)
+        L = np.linalg.inv(pw) @ bind
+        for key in ('translation', 'rotation', 'scale'):
+            nodes[j].pop(key, None)
+        nodes[j]['matrix'] = [float(v) for v in L.T.reshape(-1)]
+    js = json.dumps(gj).encode()
+    js += b' ' * (-len(js) % 4)
+    body = struct.pack('<II', len(js), 0x4E4F534A) + js + rest
+    open(out, 'wb').write(struct.pack('<III', 0x46546C67, 2, 12 + len(body)) + body)
+
+
+import os, tempfile
+bound = os.path.join(tempfile.mkdtemp(), 'bind.glb')
+bind_pose_glb(src, bound)
+bpy.ops.wm.read_factory_settings(use_empty=True)
+# Seams merged: Corrective Smooth would otherwise smooth each UV island apart and open cracks.
+bpy.ops.import_scene.gltf(filepath=bound, merge_vertices=True)
+arm = next(o for o in bpy.data.objects if o.type == 'ARMATURE')
+meshes = [o for o in bpy.data.objects if o.type == 'MESH' and any(m.type == 'ARMATURE' for m in o.modifiers)]
+if not meshes:
+    raise SystemExit('no skinned mesh')
+mesh = max(meshes, key=lambda o: len(o.data.vertices))
+if mesh.data.shape_keys:
+    raise SystemExit('the mesh has shape keys (facial morphs): its pose is left as it is')
+# Some rigs turn the mesh node itself (Make-It-Animatable leans it ~30 degrees); glTF ignores a skinned
+# mesh's own node, and three.js binds with it, so it is baked into the vertices first.
+# The same for the nodes above the armature: every object ends at identity, so the exported inverse bind
+# matrices and the mesh agree whatever the reader does with node transforms.
+def flatten(o):
+    for x in bpy.context.selected_objects:
+        x.select_set(False)
+    o.select_set(True)
+    bpy.context.view_layer.objects.active = o
+    bpy.ops.object.parent_clear(type='CLEAR_KEEP_TRANSFORM')
+    bpy.ops.object.transform_apply(location=True, rotation=True, scale=True)
+flatten(mesh)
+flatten(arm)
+for o in list(bpy.data.objects):
+    if o.type == 'EMPTY':
+        bpy.data.objects.remove(o)
+mesh.parent = arm
+pb = {b.name.split(':')[-1]: b for b in arm.pose.bones}
+need = ['Hips', 'LeftArm', 'LeftForeArm', 'LeftHand', 'RightArm', 'RightForeArm', 'RightHand', 'LeftUpLeg', 'LeftFoot', 'RightUpLeg', 'RightFoot']
+missing = [n for n in need if n not in pb]
+if missing:
+    raise SystemExit('bones missing: ' + ', '.join(missing))
+
+bpy.context.view_layer.objects.active = arm
+for b in arm.pose.bones:
+    b.rotation_mode = 'QUATERNION'
+    b.matrix_basis = Matrix.Identity(4)
+bpy.context.view_layer.update()
+W = arm.matrix_world
+
+def head(n):
+    return W @ pb[n].head
+
+def rotate_world(n, R):
+    """Turn bone n (and so its children) by world rotation R about its head."""
+    b = pb[n]
+    h = head(n)
+    world = W @ b.matrix
+    world = Matrix.Translation(h) @ R.to_matrix().to_4x4() @ Matrix.Translation(-h) @ world
+    b.matrix = W.inverted() @ world
+    bpy.context.view_layer.update()
+
+# Which way is up, and which way the figure faces, in Blender's world (glTF +y up, +z facing -> Blender +z, -y).
+up = Vector((0, 0, 1))
+height = max((mesh.matrix_world @ v.co).z for v in mesh.data.vertices) - min((mesh.matrix_world @ v.co).z for v in mesh.data.vertices)
+report['height'] = height
+
+# Arms: shoulder -> wrist straight down and a little out, in the figure's own frontal plane.
+ARM_OUT = math.radians(5)
+arms = {}
+for side, s in (('Left', 1), ('Right', -1)):
+    a, w = head(f'{side}Arm'), head(f'{side}Hand')
+    cur = (w - a).normalized()
+    out = Vector((s, 0, 0))
+    want = (out * math.sin(ARM_OUT) - up * math.cos(ARM_OUT)).normalized()
+    arms[side] = math.degrees(cur.angle(want))
+    rotate_world(f'{side}Arm', cur.rotation_difference(want))
+    # Elbow: forearm in line with the upper arm, a slight natural bend forward.
+    e, w = head(f'{side}ForeArm'), head(f'{side}Hand')
+    cur = (w - e).normalized()
+    upper = (e - head(f'{side}Arm')).normalized()
+    bend = Matrix.Rotation(math.radians(8), 3, Vector((1, 0, 0)))
+    want = (bend @ upper).normalized()
+    if want.y > upper.y:  # bend toward the front (-y), never backwards
+        want = (bend.inverted() @ upper).normalized()
+    rotate_world(f'{side}ForeArm', cur.rotation_difference(want))
+    # Hand in line with the forearm (a held-up palm otherwise sticks out sideways).
+    hb = pb[f'{side}Hand']
+    tail = W @ hb.tail
+    cur = (tail - head(f'{side}Hand')).normalized()
+    fore = (head(f'{side}Hand') - head(f'{side}ForeArm')).normalized()
+    rotate_world(f'{side}Hand', cur.rotation_difference(fore))
+report['armTurn'] = arms
+
+# Palms toward the thighs. A raised hand is painted palm forward or down; turned down with the arm it faces
+# back or out, and the fingers splay away from the body like a zombie's. The palm's normal is the hand's
+# flattest direction (least spread of its vertices, posed); the hand turns about the forearm until it points
+# across the body.
+def posed_points(group_name):
+    g = mesh.vertex_groups.get(group_name)
+    if g is None:
+        return []
+    dg = bpy.context.evaluated_depsgraph_get()
+    ev = mesh.evaluated_get(dg).data
+    pts = []
+    for v in mesh.data.vertices:
+        for vg in v.groups:
+            if vg.group == g.index and vg.weight > 0.6:
+                pts.append(mesh.matrix_world @ ev.vertices[v.index].co)
+    return pts
+
+palms = {}
+for side, s in (('Left', 1), ('Right', -1)):
+    name = pb[f'{side}Hand'].name
+    pts = posed_points(name)
+    if len(pts) < 12:
+        continue
+    import numpy as np
+    P = np.array([tuple(p) for p in pts])
+    P -= P.mean(axis=0)
+    _, _, vt = np.linalg.svd(P, full_matrices=False)
+    normal = Vector(vt[2].tolist())
+    axis = (head(f'{side}Hand') - head(f'{side}ForeArm')).normalized()
+    n = (normal - axis * normal.dot(axis))
+    want = Vector((1, 0, 0)) - axis * axis.x
+    if n.length < 1e-6 or want.length < 1e-6:
+        continue
+    n.normalize(); want.normalize()
+    if n.dot(want) < 0:
+        n = -n  # a plane's normal has no sign: the smaller turn
+    angle = math.atan2(n.cross(want).dot(axis), n.dot(want))
+    palms[side] = math.degrees(angle)
+    if abs(angle) > math.radians(10):
+        rotate_world(f'{side}Hand', Matrix.Rotation(angle, 3, axis).to_quaternion())
+report['palmTurn'] = palms
+
+# Legs: ankle under the hip joint, turned only in the frontal plane; feet keep their world orientation (flat).
+legs = {}
+for side, s in (('Left', 1), ('Right', -1)):
+    foot_world = (W @ pb[f'{side}Foot'].matrix).to_quaternion()
+    h, f = head(f'{side}UpLeg'), head(f'{side}Foot')
+    cur = f - h
+    # Only the sideways part changes (ankle a quarter of the hip joint's offset out): same forward lean.
+    want = Vector(((h.x - head('Hips').x) * 0.25, cur.y, cur.z))
+    legs[side] = math.degrees(cur.angle(want))
+    if legs[side] < 25:  # more is a kick or a stride, not a stance
+        rotate_world(f'{side}UpLeg', cur.normalized().rotation_difference(want.normalized()))
+    now = (W @ pb[f'{side}Foot'].matrix).to_quaternion()
+    rotate_world(f'{side}Foot', foot_world @ now.inverted())
+report['legTurn'] = legs
+
+# Bake the pose into the mesh, smoothing what skinning pinched, then make it the rest pose.
+bpy.ops.object.mode_set(mode='OBJECT')
+orig = [v.co.copy() for v in mesh.data.vertices]
+armmod = next(m for m in mesh.modifiers if m.type == 'ARMATURE')
+# Corrective Smooth only where the pose moved the surface: elsewhere (the head, its hair strands) it still
+# shifts thin pieces a little, which drags hair across the face once the paintings are projected on it.
+bpy.context.view_layer.objects.active = mesh
+dg = bpy.context.evaluated_depsgraph_get()
+posed = [v.co.copy() for v in mesh.evaluated_get(dg).data.vertices]
+moved = mesh.vertex_groups.new(name='relax_moved')
+for i, (a, b) in enumerate(zip(posed, orig)):
+    w = min(1.0, (a - b).length / (0.03 * height))
+    if w > 0.02:
+        moved.add([i], w, 'REPLACE')
+cs = mesh.modifiers.new('relax', 'CORRECTIVE_SMOOTH')
+cs.vertex_group = moved.name
+cs.rest_source = 'ORCO'
+cs.smooth_type = 'LENGTH_WEIGHTED'
+cs.iterations = 8
+cs.factor = 0.5
+cs.use_only_smooth = False
+bpy.context.view_layer.objects.active = mesh
+dg = bpy.context.evaluated_depsgraph_get()
+ev = mesh.evaluated_get(dg)
+baked = [v.co.copy() for v in ev.data.vertices]
+if len(baked) != len(orig):
+    raise SystemExit('modifier changed vertex count')
+mesh.modifiers.remove(cs)
+mesh.vertex_groups.remove(moved)
+for v, p in zip(mesh.data.vertices, baked):
+    v.co = p
+bpy.context.view_layer.objects.active = arm
+bpy.ops.object.mode_set(mode='POSE')
+bpy.ops.pose.armature_apply(selected=False)
+bpy.ops.object.mode_set(mode='OBJECT')
+
+# The original surface, in glTF's axes (x, z, -y) and the mesh's own space, for projecting the paintings.
+attr = mesh.data.attributes.new('_REST', 'FLOAT_VECTOR', 'POINT')
+for i, p in enumerate(orig):
+    attr.data[i].vector = (p.x, p.z, -p.y)
+
+# Stray pieces (the 42-vertex marker sphere some rigs carry) stay out of the export.
+for o in list(bpy.data.objects):
+    if o.type == 'MESH' and o is not mesh:
+        bpy.data.objects.remove(o)
+
+# Smooth shading, or the exporter splits every triangle corner into its own vertex (6.9k -> 27.7k): the game
+# then works out faceted normals and per-corner visibility, and the painted face breaks up into blotches.
+for poly in mesh.data.polygons:
+    poly.use_smooth = True
+move = max((a - b).length for a, b in zip(baked, orig)) / height
+report['maxMove'] = move
+# The head must not move at all: a shifted strand of hair or cheek shows as a torn face in every sprite.
+head_groups = {g.index for g in mesh.vertex_groups if g.name.split(':')[-1] in ('Head', 'Neck', 'HeadTop_End')}
+on_head = [any(vg.group in head_groups and vg.weight > 0.5 for vg in v.groups) for v in mesh.data.vertices]
+report['headMove'] = max(((a - b).length for a, b, h in zip(baked, orig, on_head) if h), default=0) / height
+bpy.ops.export_scene.gltf(filepath=dst, export_format='GLB', export_attributes=True, export_skins=True,
+                          export_animations=False, export_yup=True, export_apply=False)
+report['ok'] = True
+if report_path:
+    with open(report_path, 'w') as f:
+        json.dump(report, f)
+print('RELAX', json.dumps(report))

package/app/src/model/blender.ts

@@ -0,0 +1,115 @@
+// Optional local Blender. When it is installed, a rigged figure is put into a calm rest pose (arms down, feet
+// under the hips) before the game walks it: src/model/blender/relax.py. Without Blender nothing changes; the
+// game turns the bones from the pose TRELLIS built, as before.
+import fs from 'node:fs';
+import os from 'node:os';
+import path from 'node:path';
+import { execFile } from 'node:child_process';
+import { promisify } from 'node:util';
+import { fileURLToPath } from 'node:url';
+import { configDir } from '../secrets.ts';
+
+const exec = promisify(execFile);
+const HERE = path.dirname(fileURLToPath(import.meta.url));
+export const RELAX_SCRIPT = path.join(HERE, 'blender', 'relax.py');
+
+/** Machine-wide choice from Setup: use Blender when it is found (default), or not. */
+export type BlenderSettings = { enabled: boolean };
+const settingsFile = () => path.join(configDir(), 'blender.json');
+
+export function readBlenderSettings(): BlenderSettings {
+  try {
+    return { enabled: (JSON.parse(fs.readFileSync(settingsFile(), 'utf8')) as Partial<BlenderSettings>).enabled !== false };
+  } catch {
+    return { enabled: true };
+  }
+}
+
+export function writeBlenderSettings(settings: BlenderSettings) {
+  fs.mkdirSync(path.dirname(settingsFile()), { recursive: true, mode: 0o700 });
+  fs.writeFileSync(settingsFile(), `${JSON.stringify({ enabled: settings.enabled }, null, 2)}\n`, { mode: 0o600 });
+  found = undefined;
+}
+
+/** DIORAMA_BLENDER as set: a path, 'off', or undefined. */
+export const blenderEnv = () => process.env.DIORAMA_BLENDER;
+const envOff = () => /^(0|off|no|false)$/i.test(blenderEnv() ?? '');
+
+/** Where Blender is looked for, in order: DIORAMA_BLENDER (off/0 disables it), PATH, the usual install places. */
+function candidates(): string[] {
+  const set = blenderEnv();
+  if (set !== undefined) return envOff() ? [] : [set];
+  const onPath = (process.env.PATH ?? '').split(path.delimiter).filter(Boolean).map((d) => path.join(d, process.platform === 'win32' ? 'blender.exe' : 'blender'));
+  return [
+    ...onPath,
+    '/Applications/Blender.app/Contents/MacOS/Blender',
+    path.join(os.homedir(), 'Applications/Blender.app/Contents/MacOS/Blender'),
+    '/snap/bin/blender',
+    '/usr/bin/blender',
+    '/usr/local/bin/blender',
+    'C:\\Program Files\\Blender Foundation\\Blender\\blender.exe',
+  ];
+}
+
+let found: Promise<{ bin: string; version: string } | null> | undefined;
+
+/** The Blender installed here and its version, whether or not Setup turned it off. Blender 3.6+ (glTF attributes, Corrective Smooth). */
+export function detectBlender(): Promise<{ bin: string; version: string } | null> {
+  found ??= (async () => {
+    for (const bin of candidates()) {
+      if (!fs.existsSync(bin)) continue;
+      try {
+        const { stdout } = await exec(bin, ['-b', '--factory-startup', '--version'], { timeout: 60_000 });
+        const version = /Blender\s+(\d+\.\d+(?:\.\d+)?)/.exec(stdout)?.[1];
+        if (!version) continue;
+        const [major, minor] = version.split('.').map(Number);
+        if (major < 3 || (major === 3 && minor < 6)) continue;
+        return { bin, version };
+      } catch {
+        continue;
+      }
+    }
+    return null;
+  })();
+  return found;
+}
+
+/** The Blender to use, or null: none installed, or turned off in Setup (or DIORAMA_BLENDER=off). */
+export async function findBlender(): Promise<{ bin: string; version: string } | null> {
+  if (envOff() || (blenderEnv() === undefined && !readBlenderSettings().enabled)) return null;
+  return detectBlender();
+}
+
+export async function blenderState() {
+  const settings = readBlenderSettings();
+  const installed = await detectBlender();
+  return { enabled: blenderEnv() !== undefined ? !envOff() : settings.enabled, installed, env: blenderEnv() ?? null,
+    active: !!(await findBlender()) };
+}
+
+export type RelaxReport = { ok: boolean; height?: number; armTurn?: Record<string, number>; legTurn?: Record<string, number>;
+  palmTurn?: Record<string, number>; maxMove?: number; headMove?: number };
+
+/**
+ * The rigged glb in a calm rest pose, or null when there is no Blender or it failed (the caller keeps the
+ * original). The original surface rides along as the _REST attribute, for the painted views' projection.
+ */
+export async function relaxPose(glb: Buffer, log: (line: string) => void = () => {}): Promise<{ glb: Buffer; report: RelaxReport } | null> {
+  const blender = await findBlender();
+  if (!blender) return null;
+  const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'diorama-relax-'));
+  try {
+    const [src, dst, rep] = ['in.glb', 'out.glb', 'report.json'].map((f) => path.join(dir, f));
+    fs.writeFileSync(src, glb);
+    await exec(blender.bin, ['-b', '--factory-startup', '--python-exit-code', '1', '--python', RELAX_SCRIPT, '--', src, dst, rep], { timeout: 300_000, maxBuffer: 1 << 26 });
+    const report = JSON.parse(fs.readFileSync(rep, 'utf8')) as RelaxReport;
+    if (!report.ok || !fs.existsSync(dst)) return null;
+    return { glb: fs.readFileSync(dst), report };
+  } catch (error) {
+    const e = error as { stderr?: string; message?: string };
+    log(`Blender could not relax the pose: ${(e.stderr || e.message || String(error)).trim().split('\n').slice(-3).join(' ')}`);
+    return null;
+  } finally {
+    fs.rmSync(dir, { recursive: true, force: true });
+  }
+}

package/app/src/model/build.ts

@@ -0,0 +1,50 @@
+// The hero as a 3D model: TRELLIS reconstructs a textured mesh from Muse's four views, then
+// Make-It-Animatable gives it a humanoid skeleton (Mixamo bone names) and skin weights. Both run as
+// public Hugging Face Spaces (MIT); their answers are kept in the world's memo like model calls.
+import { inputHash, type Memo } from '../meta/memo.ts';
+import { filesNamed, Space } from './space.ts';
+import { inLane } from '../world/pool.ts';
+
+/** Requests open at once on each Space: its GPU queue serves one at a time anyway, a second keeps it busy. */
+const SPACE_LANES = Math.max(1, Number(process.env.DIORAMA_SPACE_LANES) || 2);
+
+export const TRELLIS = 'trellis-community/TRELLIS';
+export const AUTORIG = 'jasongzy/Make-It-Animatable';
+
+/** Muse's views (PNG, front first) to a textured GLB, about 30 s of GPU. With the profiles too, TRELLIS
+ * builds the body as deep as it is painted instead of guessing the depth from the front. */
+export async function reconstruct(memo: Memo, key: string, views: Buffer[], seed = 1): Promise<Buffer> {
+  return memo.remember('space', `${key}-trellis`, inputHash(TRELLIS, String(seed), ...views), 'glb', () => inLane('trellis', SPACE_LANES, async () => {
+    const space = new Space(TRELLIS);
+    // The app keeps each session's files in its own folder, made when the page loads.
+    await space.call('start_session', []);
+    const uploads = await Promise.all(views.map((png, i) => space.upload(`view-${i}.png`, png)));
+    const [pre] = await space.call('preprocess_images', [uploads.map((image) => ({ image, caption: null }))]);
+    const cuts = await Promise.all((pre[0] as { image: { path: string; url?: string } }[]).map(async ({ image }, i) =>
+      space.upload(`cut-${i}.png`, await space.download(image))));
+    // Selecting the Multiple Images tab is what sets the session's multi-image flag.
+    if (cuts.length > 1) await space.call('lambda_1', []);
+    // image, multiimages, seed, ss guidance + steps, slat guidance + steps, multiimage algo, mesh simplify, texture size
+    const events = await space.call('generate_and_extract_glb', [cuts[0], cuts.map((image) => ({ image, caption: null })), seed, 7.5, 12, 3.0, 12, 'stochastic', 0.95, 1024]);
+    const [glb] = filesNamed(events, '.glb');
+    if (!glb) throw new Error('TRELLIS answered without a model');
+    return space.download(glb);
+  }));
+}
+
+/** Textured GLB to a skinned GLB in its own pose (no reset to a T-pose, no animation), about 15 s of GPU. */
+export async function autoRig(memo: Memo, key: string, model: Buffer): Promise<Buffer> {
+  return memo.remember('space', `${key}-rig`, inputHash(AUTORIG, 'keep-pose', model), 'glb', () => inLane('autorig', SPACE_LANES, async () => {
+    const space = new Space(AUTORIG);
+    const input = await space.upload(`${key}.glb`, model);
+    // input, no fingers, rest pose, rest parts, gaussian, opacity, input normals, weight fix, bone shown,
+    // reset to rest, animation file, retarget, in place
+    await space.call('pipeline', [input, true, 'No', [], false, 0.01, false, true, 'LeftArm', false, null, true, true]);
+    // The pipeline keeps joints and weights in the session; the Blender step after it writes the rigged model.
+    // reset to rest, no fingers, rest pose, rest parts, animation file, retarget, in place
+    const events = await space.call('vis_blender', [false, true, 'No', [], null, true, true]);
+    const rigged = filesNamed(events, `${key}.glb`)[0] ?? filesNamed(events, '.glb').find((f) => !/joints|normed|sample|bw|rest/.test(f.path));
+    if (!rigged) throw new Error('auto-rig answered without a rigged model');
+    return space.download(rigged);
+  }));
+}

package/app/src/model/face.ts

@@ -0,0 +1,21 @@
+import fs from 'node:fs';
+import path from 'node:path';
+import { facialControl, isViseme } from '../play/face3d.ts';
+
+export function modelFaceStatus(dir: string, paintedViews: string[] = []): { mouth: boolean; eyes: boolean; targets: number; method: 'morphs' | 'painted-surfaces' | null; views: string[]; reason: string | null } | null {
+  const info = path.join(dir, 'model.json');
+  if (!fs.existsSync(info)) return null;
+  try {
+    const model = JSON.parse(fs.readFileSync(info, 'utf8')) as { glb: string; texture?: string };
+    const data = fs.readFileSync(path.join(dir, model.glb));
+    if (data.readUInt32LE(0) !== 0x46546c67 || data.toString('ascii', 16, 20) !== 'JSON') throw new Error('invalid GLB');
+    const gltf = JSON.parse(data.toString('utf8', 20, 20 + data.readUInt32LE(12)).trim()) as { meshes?: Array<{ extras?: { targetNames?: string[] }; primitives?: Array<{ targets?: unknown[] }> }> };
+    const controls = (gltf.meshes ?? []).flatMap(m => m.extras?.targetNames ?? []).map(facialControl).filter(c => c !== undefined);
+    const mouth = controls.some(c => c === 'jaw' || (isViseme(c) && c !== 'sil'));
+    const eyes = controls.some(c => c.startsWith('blink'));
+    const targets = (gltf.meshes ?? []).reduce((sum, m) => sum + Math.max(0, ...(m.primitives ?? []).map(p => p.targets?.length ?? 0)), 0);
+    const surfaces = (model as {texture?:string}).texture !== 'model' ? paintedViews : [];
+    return { mouth: mouth || surfaces.length > 0, eyes: eyes || surfaces.length > 0, targets, method: mouth ? 'morphs' : surfaces.length ? 'painted-surfaces' : null, views: surfaces,
+      reason: mouth || surfaces.length ? null : 'The 3D model needs facial morphs or calibrated painted facial surfaces.' };
+  } catch { return { mouth: false, eyes: false, targets: 0, method:null, views:[], reason: 'The 3D facial controls could not be inspected.' }; }
+}