@gtanczyk/diorama 0.0.0-stage → 0.2.0

package/app/src/model/fit.ts

@@ -0,0 +1,181 @@
+// Where each painted view of the hero lands on the model: a scale and offset per image axis, found by
+// laying the model's silhouette over the figure in the painting. The runtime projects with the same
+// numbers, so the paint sits on the coat while it walks.
+import fs from 'node:fs';
+import path from 'node:path';
+import sharp from 'sharp';
+import { normalise, readShape, type Shape } from './glb.ts';
+
+/** Which way each view looks at the hero (who faces +z): the model axis that runs left to right in the picture, and its sign. */
+export const VIEWS = {
+  front: { axis: 0, sign: 1 },
+  back: { axis: 0, sign: -1 },
+  left: { axis: 2, sign: -1 }, // facing the picture's left: the camera stands at +x
+  right: { axis: 2, sign: 1 },
+} as const;
+export type ViewName = keyof typeof VIEWS;
+
+/**
+ * u = a * axis + b, v = c * y + d, in image units (0..1, v from the top), model in hero units. bands: [a, b] for
+ * each of BANDS slices of the height from the feet up, fitted on that slice alone (the runtime blends them
+ * between slice centres). Muse paints a profile's head, body and legs each a little off from the model, and a
+ * fit of the whole silhouette leaves the profile's face on the model's cheek, a second face beside the front's.
+ */
+export type ViewFit = { u: [number, number]; v: [number, number]; iou: number; bands?: [number, number][] };
+
+export const BANDS = 20;
+
+const R = 160;
+
+/** How wide the silhouette grid is across, in heights: 1 for a person, more for a creature wider than it is tall (a cat side on). */
+function across(shape: Shape): number {
+  let most = 0;
+  for (let i = 0; i < shape.positions.length; i += 3) most = Math.max(most, Math.abs(shape.positions[i]), Math.abs(shape.positions[i + 2]));
+  return Math.max(1, 2 * most + 0.04);
+}
+
+function silhouette(shape: Shape, view: ViewName, W: number): Uint8Array {
+  const { axis, sign } = VIEWS[view];
+  const p = shape.positions;
+  const px = (i: number) => (sign * p[i * 3 + axis] / W + 0.5) * R;
+  const py = (i: number) => (1 - p[i * 3 + 1]) * R;
+  const out = new Uint8Array(R * R);
+  const t = shape.triangles;
+  for (let k = 0; k < t.length; k += 3) {
+    const [x0, y0, x1, y1, x2, y2] = [px(t[k]), py(t[k]), px(t[k + 1]), py(t[k + 1]), px(t[k + 2]), py(t[k + 2])];
+    const area = (x1 - x0) * (y2 - y0) - (x2 - x0) * (y1 - y0);
+    if (Math.abs(area) < 1e-9) continue;
+    for (let y = Math.max(0, Math.floor(Math.min(y0, y1, y2))); y <= Math.min(R - 1, Math.ceil(Math.max(y0, y1, y2))); y++) {
+      for (let x = Math.max(0, Math.floor(Math.min(x0, x1, x2))); x <= Math.min(R - 1, Math.ceil(Math.max(x0, x1, x2))); x++) {
+        const [cx, cy] = [x + 0.5, y + 0.5];
+        const w0 = ((x1 - cx) * (y2 - cy) - (x2 - cx) * (y1 - cy)) / area;
+        const w1 = ((x2 - cx) * (y0 - cy) - (x0 - cx) * (y2 - cy)) / area;
+        if (w0 >= 0 && w1 >= 0 && w0 + w1 <= 1) out[y * R + x] = 1;
+      }
+    }
+  }
+  return out;
+}
+
+/** The figure in a painted view on its flat background, at R x R, plus the image's aspect (height / width). */
+async function figureMask(png: Buffer): Promise<{ mask: Uint8Array; aspect: number }> {
+  const meta = await sharp(png).metadata();
+  const { data } = await sharp(png).removeAlpha().resize(R, R, { fit: 'fill' }).raw().toBuffer({ resolveWithObject: true });
+  const border: number[][] = [];
+  for (let i = 0; i < R; i++) for (const [x, y] of [[i, 0], [i, R - 1], [0, i], [R - 1, i]]) border.push([...data.subarray((y * R + x) * 3, (y * R + x) * 3 + 3)]);
+  const bg = [0, 1, 2].map((c) => border.map((b) => b[c]).sort((a, b) => a - b)[border.length >> 1]);
+  const mask = new Uint8Array(R * R);
+  for (let i = 0; i < R * R; i++) mask[i] = Math.abs(data[i * 3] - bg[0]) + Math.abs(data[i * 3 + 1] - bg[1]) + Math.abs(data[i * 3 + 2] - bg[2]) > 40 ? 1 : 0;
+  // A pale shirt is close to the grey background, so the cut leaves holes in the figure. The background is
+  // what joins the border; everything else is figure.
+  const outside = new Uint8Array(R * R);
+  const queue: number[] = [];
+  for (let i = 0; i < R; i++) for (const at of [i, (R - 1) * R + i, i * R, i * R + R - 1]) if (!mask[at] && !outside[at]) { outside[at] = 1; queue.push(at); }
+  while (queue.length) {
+    const at = queue.pop()!;
+    const [x, y] = [at % R, Math.floor(at / R)];
+    for (const [nx, ny] of [[x - 1, y], [x + 1, y], [x, y - 1], [x, y + 1]]) {
+      const next = ny * R + nx;
+      if (nx >= 0 && nx < R && ny >= 0 && ny < R && !mask[next] && !outside[next]) { outside[next] = 1; queue.push(next); }
+    }
+  }
+  for (let i = 0; i < R * R; i++) mask[i] = outside[i] ? 0 : 1;
+  return { mask, aspect: (meta.height ?? 1) / (meta.width ?? 1) };
+}
+
+/**
+ * How well the painted figure lies over the model's silhouette. `bare` weighs model left unpainted against paint
+ * off the model: above 1, a fit that covers the model wins, since a bare edge is painted from another view
+ * (its nose, its mouth) while paint off the model shows nowhere.
+ */
+function iou(sil: Uint8Array, mask: Uint8Array, a: number, b: number, c: number, d: number, rows: [number, number] = [0, R], bare = 1, W = 1): number {
+  let both = 0;
+  let silOnly = 0;
+  let maskOnly = 0;
+  for (let y = rows[0]; y < rows[1]; y++) {
+    const v = c * (1 - (y + 0.5) / R) + d;
+    const my = Math.floor(v * R);
+    for (let x = 0; x < R; x++) {
+      const u = a * W * ((x + 0.5) / R - 0.5) + b;
+      const mx = Math.floor(u * R);
+      const m = mx >= 0 && mx < R && my >= 0 && my < R ? mask[my * R + mx] : 0;
+      const s = sil[y * R + x];
+      both += s & m;
+      silOnly += s & (1 - m);
+      maskOnly += m & (1 - s);
+    }
+  }
+  return both ? both / (both + bare * silOnly + maskOnly) : 0;
+}
+
+export async function fitView(shape: Shape, view: ViewName, png: Buffer): Promise<ViewFit> {
+  const W = across(shape);
+  const sil = silhouette(shape, view, W);
+  const { mask, aspect } = await figureMask(png);
+  let top = R;
+  let bottom = -1;
+  let sumX = 0;
+  let n = 0;
+  for (let y = 0; y < R; y++) for (let x = 0; x < R; x++) if (mask[y * R + x]) { top = Math.min(top, y); bottom = Math.max(bottom, y); sumX += x; n++; }
+  if (!n) throw new Error(`${view}: no figure found on the background`);
+  const [vt, vb, uc] = [top / R, (bottom + 1) / R, (sumX / n + 0.5) / R];
+  let best: ViewFit = { u: [0, 0], v: [0, 0], iou: -1 };
+  for (let ks = 0.9; ks <= 1.1001; ks += 0.01) {
+    const c = -(vb - vt) * ks;
+    const d = vb;
+    for (let kx = 0.8; kx <= 1.2501; kx += 0.015) {
+      const a = (vb - vt) * ks * aspect * kx;
+      for (let db = -0.04; db <= 0.0401; db += 0.005) {
+        const score = iou(sil, mask, a, uc + db, c, d, [0, R], 1, W);
+        if (score > best.iou) best = { u: [a, uc + db], v: [c, d], iou: score };
+      }
+    }
+  }
+  return { ...best, iou: Math.round(best.iou * 1000) / 1000, bands: fitBands(sil, mask, best, W) };
+}
+
+/** Each slice's own scale and offset across, near the whole fit, then smoothed with its neighbours. */
+function fitBands(sil: Uint8Array, mask: Uint8Array, whole: ViewFit, W: number): [number, number][] {
+  const [a0, b0] = whole.u;
+  const [c, d] = whole.v;
+  const raw = [...Array(BANDS).keys()].map((k): [number, number] => {
+    // Model heights k/BANDS..(k+1)/BANDS are silhouette rows from (1 - (k+1)/BANDS) R down.
+    const rows: [number, number] = [Math.max(0, Math.floor((1 - (k + 1) / BANDS) * R)), Math.min(R, Math.ceil((1 - k / BANDS) * R))];
+    let area = 0;
+    for (let y = rows[0]; y < rows[1]; y++) for (let x = 0; x < R; x++) area += sil[y * R + x];
+    if (area < 20) return [a0, b0];
+    let best: [number, number] = [a0, b0];
+    let top = -Infinity;
+    for (let ka = 0.85; ka <= 1.1501; ka += 0.015) {
+      for (let db = -0.08; db <= 0.0801; db += 0.004) {
+        // A slight pull to the whole fit, so a slice with little to go on stays with its neighbours.
+        const score = iou(sil, mask, a0 * ka, b0 + db, c, d, rows, 2, W) - 0.05 * (Math.abs(ka - 1) + 4 * Math.abs(db));
+        if (score > top) [top, best] = [score, [a0 * ka, b0 + db]];
+      }
+    }
+    return best;
+  });
+  return raw.map((_, k) => {
+    const near = [raw[Math.max(0, k - 1)], raw[k], raw[k], raw[Math.min(BANDS - 1, k + 1)]];
+    return [near.reduce((s, n) => s + n[0], 0) / 4, near.reduce((s, n) => s + n[1], 0) / 4];
+  });
+}
+
+/** Fits every view of a model; model.json's `views`. */
+export async function fitViews(glb: Buffer, views: Record<ViewName, Buffer>): Promise<Record<ViewName, { image: string } & ViewFit>> {
+  const shape = normalise(readShape(glb, true));
+  const fits = {} as Record<ViewName, { image: string } & ViewFit>;
+  for (const v of Object.keys(VIEWS) as ViewName[]) fits[v] = { image: `${v}.png`, ...(await fitView(shape, v, views[v])) };
+  return fits;
+}
+
+/** A model fitted before views were fitted slice by slice is fitted again (no Muse, no TRELLIS: its own files). */
+export async function refitIfOld(modelDir: string, always = false): Promise<boolean> {
+  const file = path.join(modelDir, 'model.json');
+  const info = JSON.parse(fs.readFileSync(file, 'utf8')) as { glb: string; views: Record<ViewName, { image: string } & ViewFit> };
+  if (!always && Object.values(info.views).every((v) => v.bands?.length === BANDS)) return false;
+  const views = Object.fromEntries(Object.entries(info.views).map(([k, v]) => [k, fs.readFileSync(path.join(modelDir, v.image))])) as Record<ViewName, Buffer>;
+  info.views = await fitViews(fs.readFileSync(path.join(modelDir, info.glb)), views);
+  fs.writeFileSync(file, JSON.stringify(info, null, 2));
+  return true;
+}

package/app/src/model/glb.ts

@@ -0,0 +1,241 @@
+// Reads the bind-pose shape of a skinned glTF binary: vertex positions in scene space and triangles.
+// Enough for measuring a silhouette; the runtime loads the file itself with three.js.
+import fs from 'node:fs';
+import path from 'node:path';
+import { shapeProblem } from './quality.ts';
+export { MIN_DEPTH } from './quality.ts';
+
+type Node = { mesh?: number; skin?: number; children?: number[]; matrix?: number[]; translation?: number[]; rotation?: number[]; scale?: number[] };
+type Accessor = { bufferView: number; byteOffset?: number; componentType: number; count: number; type: string };
+type Gltf = {
+  nodes: Node[];
+  skins?: { joints: number[]; inverseBindMatrices?: number }[];
+  meshes: { primitives: { attributes: Record<string, number>; indices?: number }[] }[];
+  accessors: Accessor[];
+  bufferViews: { byteOffset?: number; byteStride?: number; byteLength: number }[];
+};
+
+export type Shape = { positions: Float32Array; triangles: Uint32Array };
+
+type M4 = number[];
+const identity = (): M4 => [1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1];
+function multiply(a: M4, b: M4): M4 {
+  const o = new Array(16).fill(0);
+  for (let c = 0; c < 4; c++) for (let r = 0; r < 4; r++) for (let k = 0; k < 4; k++) o[c * 4 + r] += a[k * 4 + r] * b[c * 4 + k];
+  return o;
+}
+function local(n: Node): M4 {
+  if (n.matrix) return n.matrix;
+  const [x, y, z, w] = n.rotation ?? [0, 0, 0, 1];
+  const [sx, sy, sz] = n.scale ?? [1, 1, 1];
+  const [tx, ty, tz] = n.translation ?? [0, 0, 0];
+  // Column-major T * R * S.
+  return [
+    (1 - 2 * (y * y + z * z)) * sx, 2 * (x * y + z * w) * sx, 2 * (x * z - y * w) * sx, 0,
+    2 * (x * y - z * w) * sy, (1 - 2 * (x * x + z * z)) * sy, 2 * (y * z + x * w) * sy, 0,
+    2 * (x * z + y * w) * sz, 2 * (y * z - x * w) * sz, (1 - 2 * (x * x + y * y)) * sz, 0,
+    tx, ty, tz, 1,
+  ];
+}
+
+function readAccessor(gltf: Gltf, bin: Buffer, index: number): Float32Array | Uint32Array {
+  const a = gltf.accessors[index];
+  const view = gltf.bufferViews[a.bufferView];
+  const width = { SCALAR: 1, VEC2: 2, VEC3: 3, VEC4: 4, MAT4: 16 }[a.type] ?? 1;
+  const size = { 5126: 4, 5125: 4, 5123: 2, 5121: 1 }[a.componentType] ?? 4;
+  const stride = view.byteStride ?? width * size;
+  const start = (view.byteOffset ?? 0) + (a.byteOffset ?? 0);
+  const out = a.componentType === 5126 ? new Float32Array(a.count * width) : new Uint32Array(a.count * width);
+  for (let i = 0; i < a.count; i++) for (let j = 0; j < width; j++) {
+    const at = start + i * stride + j * size;
+    out[i * width + j] = a.componentType === 5126 ? bin.readFloatLE(at) : a.componentType === 5125 ? bin.readUInt32LE(at) : a.componentType === 5123 ? bin.readUInt16LE(at) : bin.readUInt8(at);
+  }
+  return out;
+}
+
+/** Native sprite rendering needs UVs and a self-contained base-colour image on every mesh primitive. */
+export function hasModelTexture(glb: Buffer): boolean {
+  try {
+    if (glb.readUInt32LE(0) !== 0x46546c67) return false;
+    const length = glb.readUInt32LE(12);
+    const g = JSON.parse(glb.subarray(20, 20 + length).toString('utf8'));
+    const primitives = (g.meshes ?? []).flatMap((m: { primitives?: unknown[] }) => m.primitives ?? []);
+    return primitives.length > 0 && primitives.every((p: { attributes?: Record<string, number>; material?: number }) => {
+      const map = g.materials?.[p.material!]?.pbrMetallicRoughness?.baseColorTexture;
+      const uv = g.accessors?.[p.attributes?.[`TEXCOORD_${map?.texCoord ?? 0}`]!];
+      const position = g.accessors?.[p.attributes?.POSITION!];
+      const image = g.images?.[g.textures?.[map?.index]?.source];
+      const embedded = image && (typeof image.uri === 'string' && /^data:image\/(png|jpeg);base64,/.test(image.uri)
+        || Number.isInteger(image.bufferView) && g.bufferViews?.[image.bufferView]?.byteLength > 0);
+      return !!(map && (map.texCoord ?? 0) === 0 && uv?.type === 'VEC2' && uv.count > 0 && uv.count === position?.count && embedded);
+    });
+  } catch { return false; }
+}
+
+/**
+ * The first mesh in the file (the skinned one when there is a skin), in scene space. `original`: the surface as
+ * TRELLIS built it, where Blender relaxed the pose (src/model/blender.ts keeps it as the _REST attribute); the
+ * painted views were fitted to that surface.
+ */
+export function readShape(glb: Buffer, original = false): Shape {
+  if (glb.readUInt32LE(0) !== 0x46546c67) throw new Error('not a glTF binary');
+  const jsonLength = glb.readUInt32LE(12);
+  const gltf = JSON.parse(glb.subarray(20, 20 + jsonLength).toString('utf8')) as Gltf;
+  const binStart = 20 + jsonLength;
+  const bin = glb.subarray(binStart + 8, binStart + 8 + glb.readUInt32LE(binStart));
+  const parent = new Map<number, number>();
+  gltf.nodes.forEach((n, i) => n.children?.forEach((c) => parent.set(c, i)));
+  const nodeIndex = gltf.nodes.findIndex((n) => n.mesh !== undefined && n.skin !== undefined) >= 0
+    ? gltf.nodes.findIndex((n) => n.mesh !== undefined && n.skin !== undefined)
+    : gltf.nodes.findIndex((n) => n.mesh !== undefined);
+  if (nodeIndex < 0) throw new Error('no mesh in the file');
+  let world = identity();
+  for (let i: number | undefined = nodeIndex; i !== undefined; i = parent.get(i)) world = multiply(local(gltf.nodes[i]), world);
+  const positions: number[] = [];
+  const triangles: number[] = [];
+  for (const prim of gltf.meshes[gltf.nodes[nodeIndex].mesh!].primitives) {
+    const base = positions.length / 3;
+    const p = readAccessor(gltf, bin, original && prim.attributes._REST !== undefined ? prim.attributes._REST : prim.attributes.POSITION) as Float32Array;
+    for (let i = 0; i < p.length; i += 3) {
+      const [x, y, z] = [p[i], p[i + 1], p[i + 2]];
+      positions.push(world[0] * x + world[4] * y + world[8] * z + world[12], world[1] * x + world[5] * y + world[9] * z + world[13], world[2] * x + world[6] * y + world[10] * z + world[14]);
+    }
+    const idx = prim.indices !== undefined ? readAccessor(gltf, bin, prim.indices) : Uint32Array.from({ length: p.length / 3 }, (_, i) => i);
+    for (const k of idx) triangles.push(base + k);
+  }
+  return { positions: Float32Array.from(positions), triangles: Uint32Array.from(triangles) };
+}
+
+/** Inverse of a column-major 4x4 matrix (cofactors). */
+function invert(m: M4): M4 {
+  const o = new Array(16).fill(0);
+  for (let c = 0; c < 4; c++) for (let r = 0; r < 4; r++) {
+    // Cofactor of m[row c][col r], transposed into o[col c][row r].
+    const rows = [0, 1, 2, 3].filter((i) => i !== c);
+    const cols = [0, 1, 2, 3].filter((i) => i !== r);
+    const at = (i: number, j: number) => m[cols[j] * 4 + rows[i]];
+    const det3 = at(0, 0) * (at(1, 1) * at(2, 2) - at(1, 2) * at(2, 1)) - at(0, 1) * (at(1, 0) * at(2, 2) - at(1, 2) * at(2, 0)) + at(0, 2) * (at(1, 0) * at(2, 1) - at(1, 1) * at(2, 0));
+    o[c * 4 + r] = ((r + c) % 2 ? -1 : 1) * det3;
+  }
+  const det = m[0] * o[0] + m[4] * o[1] + m[8] * o[2] + m[12] * o[3];
+  return o.map((v) => v / det);
+}
+
+/**
+ * Why the auto-rig's skeleton cannot walk this body, or null. Make-It-Animatable guesses the joints from the
+ * shape alone; on a floor-length robe it once put both feet at hip height and the knees below them, so every
+ * step tore the robe apart and pulled the shoes up into poles. Heights are shares of the body's height, at bind.
+ */
+export function rigProblem(glb: Buffer): string | null {
+  const jsonLength = glb.readUInt32LE(12);
+  const gltf = JSON.parse(glb.subarray(20, 20 + jsonLength).toString('utf8')) as Gltf;
+  const skin = gltf.skins?.[0];
+  if (!skin?.inverseBindMatrices && skin?.inverseBindMatrices !== 0) return null;
+  const binStart = 20 + jsonLength;
+  const bin = glb.subarray(binStart + 8, binStart + 8 + glb.readUInt32LE(binStart));
+  const parent = new Map<number, number>();
+  gltf.nodes.forEach((n, i) => n.children?.forEach((c) => parent.set(c, i)));
+  const meshNode = gltf.nodes.findIndex((n) => n.mesh !== undefined && n.skin === 0);
+  let world = identity();
+  for (let i: number | undefined = meshNode; i !== undefined && i >= 0; i = parent.get(i)) world = multiply(local(gltf.nodes[i]), world);
+  const ibm = readAccessor(gltf, bin, skin.inverseBindMatrices);
+  const y: Record<string, number> = {};
+  skin.joints.forEach((node, j) => {
+    const bind = multiply(world, invert(Array.from(ibm.subarray(j * 16, j * 16 + 16))));
+    y[((gltf.nodes[node] as { name?: string }).name ?? '').replace(/^mixamorig:?/, '')] = bind[13];
+  });
+  const shape = readShape(glb).positions;
+  let [lo, hi] = [Infinity, -Infinity];
+  for (let i = 1; i < shape.length; i += 3) { lo = Math.min(lo, shape[i]); hi = Math.max(hi, shape[i]); }
+  const at = (bone: string) => (y[bone] - lo) / (hi - lo);
+  const need = ['Hips', 'Neck', 'Head', 'LeftLeg', 'RightLeg', 'LeftFoot', 'RightFoot'];
+  if (need.some((b) => y[b] === undefined)) return null;
+  const pct = (v: number) => `${Math.round(v * 100)}%`;
+  if (!(at('Hips') < at('Neck') && at('Neck') < at('Head'))) return `skeleton upside down: hips at ${pct(at('Hips'))}, head at ${pct(at('Head'))} of the height`;
+  for (const side of ['Left', 'Right']) {
+    const [knee, foot] = [at(`${side}Leg`), at(`${side}Foot`)];
+    // Feet sit at 4-13% in every rig that walks; a kicking or stepping pose reaches about 35%.
+    if (foot > 0.4 || foot >= knee || knee >= at('Hips')) return `skeleton misplaced: ${side.toLowerCase()} foot at ${pct(foot)}, knee at ${pct(knee)}, hips at ${pct(at('Hips'))} of the height`;
+  }
+  return null;
+}
+
+/** Feet at y 0, height 1, centred on x and z: the hero-unit space the runtime projects views in. */
+export function normalise({ positions, triangles }: Shape): Shape {
+  const lo = [Infinity, Infinity, Infinity];
+  const hi = [-Infinity, -Infinity, -Infinity];
+  for (let i = 0; i < positions.length; i++) {
+    lo[i % 3] = Math.min(lo[i % 3], positions[i]);
+    hi[i % 3] = Math.max(hi[i % 3], positions[i]);
+  }
+  const h = hi[1] - lo[1];
+  const centre = [(lo[0] + hi[0]) / 2, lo[1], (lo[2] + hi[2]) / 2];
+  return { positions: positions.map((v, i) => (v - centre[i % 3]) / h), triangles };
+}
+
+/** Why a model cannot stand in for the character, or null. Shared with the browser's loader. */
+export function badModel(shape: Shape): string | null {
+  return shapeProblem(shape.positions);
+}
+
+/** Why cast/<key>/model/ cannot be used (a model kept before badModel was checked), or null. */
+export function modelProblem(modelDir: string): string | null {
+  const glb = path.join(modelDir, 'model.glb');
+  if (!fs.existsSync(glb)) return 'no model.glb';
+  try {
+    const file = fs.readFileSync(glb);
+    return badModel(readShape(file)) ?? rigProblem(file);
+  } catch (error) {
+    return error instanceof Error ? error.message : String(error);
+  }
+}
+
+/** How far the body leans: the slope of x and of z against height, over the body between shins and brow. */
+export function lean(shape: Shape): { x: number; z: number } {
+  const { positions: p } = normalise(shape);
+  let [n, sy, syy, sx, sxy, sz, szy] = [0, 0, 0, 0, 0, 0, 0];
+  for (let i = 0; i < p.length; i += 3) {
+    const y = p[i + 1];
+    if (y < 0.1 || y > 0.9) continue;
+    n++; sy += y; syy += y * y; sx += p[i]; sxy += p[i] * y; sz += p[i + 2]; szy += p[i + 2] * y;
+  }
+  const d = n * syy - sy * sy;
+  return d > 0 ? { x: (n * sxy - sx * sy) / d, z: (n * szy - sz * sy) / d } : { x: 0, z: 0 };
+}
+
+const qmul = ([ax, ay, az, aw]: number[], [bx, by, bz, bw]: number[]) => [
+  aw * bx + ax * bw + ay * bz - az * by, aw * by - ax * bz + ay * bw + az * bx, aw * bz + ax * by - ay * bx + az * bw, aw * bw - ax * bx - ay * by - az * bz,
+];
+
+/** The same file with its scene turned so the body stands straight up. A model reconstructed from paintings
+ * can come out pitched or rolled by a few degrees; the fit has no rotation, and a hero should not walk tilted. */
+export function standUp(glb: Buffer): Buffer {
+  const { x, z } = lean(readShape(glb));
+  // Turning about x by -atan(z) takes the forward lean out; about z by atan(x) the sideways one.
+  const [tx, tz] = [-Math.atan(z) / 2, Math.atan(x) / 2];
+  const fix = qmul([0, 0, Math.sin(tz), Math.cos(tz)], [Math.sin(tx), 0, 0, Math.cos(tx)]);
+  if (Math.abs(fix[3]) > Math.cos(0.5 * Math.PI / 180 / 2)) return glb;
+  const jsonLength = glb.readUInt32LE(12);
+  const gltf = JSON.parse(glb.subarray(20, 20 + jsonLength).toString('utf8')) as Gltf & { scenes?: { nodes: number[] }[]; scene?: number };
+  const children = new Set(gltf.nodes.flatMap((n) => n.children ?? []));
+  const roots = gltf.scenes?.[gltf.scene ?? 0]?.nodes ?? gltf.nodes.map((_, i) => i).filter((i) => !children.has(i));
+  const [fx, fy, fz, fw] = fix;
+  const turn = local({ rotation: fix });
+  for (const i of roots) {
+    const n = gltf.nodes[i];
+    if (n.matrix) { n.matrix = multiply(turn, n.matrix); continue; }
+    n.rotation = qmul([fx, fy, fz, fw], n.rotation ?? [0, 0, 0, 1]);
+    const [px, py, pz] = n.translation ?? [0, 0, 0];
+    if (px || py || pz) n.translation = [0, 1, 2].map((r) => turn[r] * px + turn[4 + r] * py + turn[8 + r] * pz);
+  }
+  let json = Buffer.from(JSON.stringify(gltf), 'utf8');
+  json = Buffer.concat([json, Buffer.alloc((4 - (json.length % 4)) % 4, ' ')]);
+  const rest = glb.subarray(20 + jsonLength);
+  const head = Buffer.alloc(20);
+  head.writeUInt32LE(0x46546c67, 0);
+  head.writeUInt32LE(glb.readUInt32LE(4), 4);
+  head.writeUInt32LE(20 + json.length + rest.length, 8);
+  head.writeUInt32LE(json.length, 12);
+  head.writeUInt32LE(0x4e4f534a, 16);
+  return Buffer.concat([head, json, rest]);
+}

package/app/src/model/plan.ts

@@ -0,0 +1,151 @@
+// What sprites each character needs, read from the world and its story, so nothing is drawn that the game
+// never shows: walk facings only where the hero's routes go, a talking sheet only for who speaks, in the
+// one facing each conversation is held in, with only the gestures its lines use.
+import fs from 'node:fs';
+import { askSparkJson } from '../meta/spark.ts';
+import type { Memo } from '../meta/memo.ts';
+import { GESTURES, gestureIndex } from '../play/gestures.ts';
+import { Geometry } from '../play/geometry.ts';
+import { WalkGrid, type Point } from '../play/path.ts';
+import type { Story } from '../story/schema.ts';
+import type { WorldFiles } from '../world/files.ts';
+import { readBodies } from '../scene/cast.ts';
+import { speakerOf } from '../story/engine.ts';
+import { FACINGS, SHEET, type Facing } from './sprites.ts';
+
+export type CharacterPlan = {
+  key: string;
+  /** Walk sheets: the facings the routes use (the left ones mirrored). Empty for someone who never walks. */
+  walk: Facing[];
+  /** Talking sheets by facing, each with its gestures (cell order; `listen` first). */
+  talk: Partial<Record<Facing, string[]>>;
+  /** Gesture for each line this character says, by its text. */
+  lines: Record<string, string>;
+  /** Not upright on two legs (cast/figures.json): a 3D model with no skeleton that hops and squashes (src/play/creature.ts), no sheets. */
+  creature?: boolean;
+};
+
+/** Facing for a yaw (0 faces the viewer, PI/2 screen right); left-facing yaws mirror onto the right ones. */
+export function facingOf(yaw: number): Facing {
+  const a = Math.abs(Math.atan2(Math.sin(yaw), Math.cos(yaw)));
+  return (Object.keys(FACINGS) as Facing[]).reduce((best, f) => (Math.abs(FACINGS[f] - a) < Math.abs(FACINGS[best] - a) ? f : best));
+}
+
+/** A share of the walked distance below this does not earn a sheet; the nearest drawn facing stands in. */
+const WALK_SHARE = 0.08;
+
+/** Walked distance per facing over every route between the start and the spots where things are used, in every place. */
+export function walkFacings(geometries: Geometry[]): Facing[] {
+  const length: Partial<Record<Facing, number>> = {};
+  for (const g of geometries) {
+    const grid = new WalkGrid(g);
+    const spots: Point[] = [[g.start[0], g.start[1]], ...g.things.map((t) => [t.stand[0], t.stand[1]] as Point)];
+    for (let i = 0; i < spots.length; i++) {
+      for (let j = i + 1; j < spots.length; j++) {
+        let at = spots[i];
+        for (const next of grid.route(spots[i], spots[j])) {
+          const [dx, dy] = [next[0] - at[0], next[1] - at[1]];
+          const f = facingOf(Math.atan2(dx, dy));
+          length[f] = (length[f] ?? 0) + Math.hypot(dx, dy);
+          at = next;
+        }
+      }
+    }
+  }
+  const total = Object.values(length).reduce((a, b) => a + b, 0);
+  const used = (Object.keys(FACINGS) as Facing[]).filter((f) => (length[f] ?? 0) >= total * WALK_SHARE);
+  return used.length ? used : ['right'];
+}
+
+/** Facings across a conversation: the hero at the NPC's stand spot, each turned to the other. */
+function conversationFacings(g: Geometry, npc: string): { hero: Facing; npc: Facing } {
+  const t = g.things.find((x) => x.key === npc);
+  if (!t) return { hero: 'right', npc: 'front-right' };
+  const feet: Point = [(t.box[0] + t.box[2]) / 2, t.box[3]];
+  const [dx, dy] = [feet[0] - t.stand[0], feet[1] - t.stand[1]];
+  // Standing right next to each other (or the spot is under the feet): side by side, the NPC a little towards the viewer.
+  if (Math.hypot(dx, dy) < 40) return { hero: 'right', npc: 'front-right' };
+  // Someone talked to from behind still turns their face to the viewer: a talk sheet never shows a back.
+  const faces = facingOf(Math.atan2(-dx, -dy));
+  return { hero: facingOf(Math.atan2(dx, dy)), npc: faces === 'back' ? 'front' : faces === 'back-right' ? 'front-right' : faces };
+}
+
+type SpokenLine = { speaker: string; text: string };
+
+/** Spark picks a gesture for each spoken line, from the gestures the model can do. */
+async function gesturesFor(memo: Memo, lines: SpokenLine[]): Promise<string[]> {
+  if (!lines.length) return [];
+  const menu = GESTURES.map((g) => `- ${g.name}: ${g.means}`).join('\n');
+  const asked = lines.map((l, i) => `${i}. ${l.speaker}: "${l.text}"`).join('\n');
+  const reply = await askSparkJson<{ gestures: string[] }>(memo, {
+    label: 'sprites-gestures',
+    prompt: `Characters in a point-and-click game say these lines. For each line pick the one gesture the speaker makes while saying it, from this list:\n${menu}\n\nLines:\n${asked}\n\n` +
+      'Use "listen" for plain lines; keep it small, the same few gestures across a character are better than many. Answer JSON only: {"gestures": ["<name for line 0>", ...]} with exactly one name per line.',
+  });
+  return lines.map((_, i) => (gestureIndex(reply.gestures?.[i] ?? '') >= 0 ? reply.gestures[i] : 'listen'));
+}
+
+/** Each character who needs sprites: the hero, and NPCs who speak (a conversation or their own talk lines). */
+export async function planSprites(files: WorldFiles, story: Story | undefined): Promise<CharacterPlan[]> {
+  const world = files.world();
+  const hero = world.hero.key;
+  const geometry = (place: string): Geometry | undefined => {
+    const at = files.place(place, 'geometry.json');
+    return fs.existsSync(at) ? Geometry.parse(JSON.parse(fs.readFileSync(at, 'utf8'))) : undefined;
+  };
+  const geometries = world.places.map((p) => geometry(p.key)).filter((g): g is Geometry => !!g);
+  const plans = new Map<string, CharacterPlan>([[hero, { key: hero, walk: walkFacings(geometries), talk: {}, lines: {} }]]);
+  const walkers = new Set([...(story?.patrols ?? []).map((p) => p.npc),
+    ...[...(story?.interactions ?? []).flatMap((i) => i.do), ...(story?.dialogues ?? []).flatMap((d) => d.lines.flatMap((l) => l.choices.flatMap((c) => c.do)))]
+      .flatMap((e) => 'moveNpc' in e ? [e.moveNpc.npc] : [])]);
+  for (const npc of walkers) {
+    // A patrol repeats in both directions; include their opposite depth facings as well.
+    const walk = new Set(walkFacings(geometries));
+    for (const f of [...walk]) walk.add(f === 'front' ? 'back' : f === 'back' ? 'front' : f === 'front-right' ? 'back-right' : f === 'back-right' ? 'front-right' : f);
+    plans.set(npc, { key: npc, walk: [...walk], talk: {}, lines: {} });
+  }
+
+  // Who says what, and facing which way while saying it.
+  const spoken: (SpokenLine & { facing: Facing })[] = [];
+  for (const d of story?.dialogues ?? []) {
+    const g = geometry(d.place);
+    const facing = g ? conversationFacings(g, d.npc) : { hero: 'right' as Facing, npc: 'front-right' as Facing };
+    if (!plans.has(d.npc)) plans.set(d.npc, { key: d.npc, walk: [], talk: {}, lines: {} });
+    for (const l of d.lines) {
+      spoken.push({ speaker: d.npc, text: l.says, facing: facing.npc });
+      for (const c of l.choices) spoken.push({ speaker: hero, text: c.text, facing: facing.hero });
+    }
+  }
+  // A person's own talk lines, without a conversation, are said to the hero the same way.
+  for (const it of story?.interactions ?? []) {
+    if (speakerOf(world, it) === hero) continue;
+    const g = geometry(it.place);
+    if (!plans.has(it.thing)) plans.set(it.thing, { key: it.thing, walk: [], talk: {}, lines: {} });
+    spoken.push({ speaker: it.thing, text: it.say, facing: (g ? conversationFacings(g, it.thing) : { npc: 'front-right' as Facing }).npc });
+  }
+  // Keep existing line/gesture choices when rerendering a character's sheets. Only new or
+  // invalid assignments need planning; a frame-count change must not depend on an AI service.
+  const saved = new Map<string, Record<string, string>>();
+  for (const speaker of new Set(spoken.map((l) => l.speaker))) {
+    const file = files.path('cast', speaker, 'sprites', 'sprites.json');
+    const info = fs.existsSync(file) ? files.readJson(file) as { lines?: Record<string, string> } : undefined;
+    saved.set(speaker, info?.lines ?? {});
+  }
+  const gestures = spoken.map((l) => {
+    const name = saved.get(l.speaker)?.[l.text];
+    return typeof name === 'string' && gestureIndex(name) >= 0 ? name : undefined;
+  });
+  const missing = spoken.filter((_, i) => gestures[i] === undefined);
+  const planned = await gesturesFor(files.memo, missing);
+  let next = 0;
+  for (let i = 0; i < gestures.length; i++) gestures[i] ??= planned[next++];
+  spoken.forEach((l, i) => {
+    const plan = plans.get(l.speaker)!;
+    plan.lines[l.text] = gestures[i]!;
+    const sheet = (plan.talk[l.facing] ??= ['listen']);
+    if (!sheet.includes(gestures[i]!) && sheet.length < SHEET.cols * SHEET.rows) sheet.push(gestures[i]!);
+  });
+  // A cat or a round eye has no person's poses to draw over: it gets a 3D model that moves as one body.
+  const bodies = readBodies(files, fs.existsSync);
+  return [...plans.values()].map((p) => (bodies[p.key]?.upright === false ? { ...p, creature: true } : p));
+}

package/app/src/model/quality.ts

@@ -0,0 +1,48 @@
+// Geometry checks shared by the build, saved-world loader and browser (no filesystem imports).
+export const MIN_DEPTH = 0.08;
+
+/** Reject degenerate meshes and paper-like bodies, including cards lying down or tilted. */
+export function shapeProblem(p: ArrayLike<number>): string | null {
+  if (p.length < 12 || p.length % 3) return 'invalid: too few vertices';
+  const axes: number[][] = [[], [], []];
+  for (let i = 0; i < p.length; i++) {
+    if (!Number.isFinite(p[i])) return 'invalid: non-finite vertex';
+    axes[i % 3].push(p[i]);
+  }
+  const spans = axes.map((a) => {
+    a.sort((x, y) => x - y);
+    return a[Math.floor(a.length * 0.98)] - a[Math.floor(a.length * 0.02)];
+  });
+  if (spans.some((s) => s <= 0)) return 'flat: zero body extent';
+  if (spans[2] / spans[1] < MIN_DEPTH) return `flat: ${(spans[2] / spans[1]).toFixed(2)} of its height deep`;
+
+  // Principal axes expose a thin sheet even when all three world-axis extents look substantial.
+  // Compare thickness to the middle axis, not the longest: a long animal can still have a round body.
+  const n = p.length / 3, mean = axes.map((a) => a.reduce((sum, x) => sum + x, 0) / n);
+  const scale = Math.max(...spans);
+  const cov = Array.from({ length: 3 }, () => [0, 0, 0]);
+  for (let i = 0; i < p.length; i += 3) {
+    const v = [0, 1, 2].map((a) => (p[i + a] - mean[a]) / scale);
+    for (let a = 0; a < 3; a++) for (let b = 0; b < 3; b++) cov[a][b] += v[a] * v[b] / n;
+  }
+  // Jacobi diagonalisation of the symmetric 3x3 covariance matrix.
+  for (let step = 0; step < 20; step++) {
+    let a = 0, b = 1;
+    for (const [x, y] of [[0, 2], [1, 2]]) if (Math.abs(cov[x][y]) > Math.abs(cov[a][b])) [a, b] = [x, y];
+    if (Math.abs(cov[a][b]) < 1e-12) break;
+    const angle = 0.5 * Math.atan2(2 * cov[a][b], cov[b][b] - cov[a][a]);
+    const c = Math.cos(angle), s = Math.sin(angle), aa = cov[a][a], bb = cov[b][b], ab = cov[a][b];
+    for (let k = 0; k < 3; k++) if (k !== a && k !== b) {
+      const ka = cov[k][a], kb = cov[k][b];
+      cov[k][a] = cov[a][k] = c * ka - s * kb;
+      cov[k][b] = cov[b][k] = s * ka + c * kb;
+    }
+    cov[a][a] = c * c * aa - 2 * c * s * ab + s * s * bb;
+    cov[b][b] = s * s * aa + 2 * c * s * ab + c * c * bb;
+    cov[a][b] = cov[b][a] = 0;
+  }
+  const variance = [cov[0][0], cov[1][1], cov[2][2]].sort((a, b) => a - b);
+  const thickness = Math.sqrt(Math.max(0, variance[0]) / variance[1]);
+  if (!Number.isFinite(thickness) || thickness < MIN_DEPTH) return `flat: body thickness ratio ${thickness.toFixed(3)} (minimum ${MIN_DEPTH})`;
+  return null;
+}