@gtanczyk/diorama 0.0.0-stage → 0.2.0

package/app/src/model/space.ts

@@ -0,0 +1,168 @@
+// A Gradio app on a Hugging Face Space, called over its HTTP API: upload files, call an endpoint, read
+// its event stream, download the files it answers with. ZeroGPU Spaces count GPU time against the
+// caller's Hugging Face account, so the HF token goes along (only to *.hf.space, Hugging Face's own host).
+
+export const HF_TOKEN_VARS = ['HUGGING_FACE_API_KEY', 'HF_TOKEN'] as const;
+
+/** HTTP answers of a busy or restarting Space: asking again later can work. */
+export const TRANSIENT = [408, 429, 500, 502, 503, 504];
+
+/** How long a call waits: a Space that sends nothing (not even Gradio's heartbeat, every 15 s) for `idleMs`, or
+ * has not answered after `totalMs` (a ZeroGPU queue that never moves), is given up on, so the caller can ask
+ * again or go on without it instead of a build hanging on one character. Tests shorten them. */
+export const SPACE_WAIT = { idleMs: 180_000, totalMs: 900_000 };
+
+/** A file argument, as Gradio expects it after an upload. */
+export type SpaceFile = { path: string; meta: { _type: 'gradio.FileData' }; orig_name?: string };
+
+export class Space {
+  readonly base: string;
+  private readonly headers: Record<string, string>;
+
+  constructor(readonly id: string) {
+    this.base = `https://${id.toLowerCase().replace(/[/_.]/g, '-')}.hf.space/gradio_api`;
+    const token = HF_TOKEN_VARS.map((name) => process.env[name]).find(Boolean);
+    this.headers = token ? { Authorization: `Bearer ${token}` } : {};
+  }
+
+  async upload(name: string, data: Buffer): Promise<SpaceFile> {
+    // Uploading the same bytes again is safe; queue/join can start GPU work and must not be retried blindly.
+    for (let attempt = 0; ; attempt++) {
+      const form = new FormData();
+      form.append('files', new Blob([new Uint8Array(data)]), name);
+      const res = await fetch(`${this.base}/upload`, { method: 'POST', headers: this.headers, body: form, signal: AbortSignal.timeout(60_000) });
+      if (res.ok) {
+        const [where] = (await res.json()) as string[];
+        if (typeof where !== 'string' || !where) throw new Error(`${this.id} upload: missing file path`);
+        return { path: where, meta: { _type: 'gradio.FileData' }, orig_name: name };
+      }
+      // Do not put proxy HTML in the build feed. The operation and status are enough to diagnose it.
+      await res.body?.cancel();
+      if (!TRANSIENT.includes(res.status) || attempt === 2) {
+        throw new Error(`${this.id} upload: HTTP ${res.status} (${attempt + 1} attempt(s))`);
+      }
+      const retryAfter = res.headers.get('retry-after');
+      const seconds = retryAfter === null ? NaN : Number(retryAfter);
+      const delay = Number.isFinite(seconds) ? seconds * 1000 : retryAfter ? Date.parse(retryAfter) - Date.now() : 0;
+      await new Promise((resolve) => setTimeout(resolve, Math.min(30_000, Math.max(1000 * 2 ** attempt, Number.isFinite(delay) ? delay : 0))));
+    }
+  }
+
+  /** One session for every call, so endpoints that keep files per session (TRELLIS) find them. */
+  private readonly session = Math.random().toString(36).slice(2, 13);
+  private endpoints?: Map<string, { fn: number; states: number[] }>;
+
+  /** Calls an endpoint; returns the data of every event it sent (a generator sends several), in order. */
+  async call(api: string, data: unknown[]): Promise<unknown[][]> {
+    const { fn, states } = await this.endpoint(api);
+    // The API leaves out the app's hidden state inputs; the queue wants them in place, as null.
+    data = [...data];
+    for (const at of states) data.splice(at, 0, null);
+    const join = await fetch(`${this.base}/queue/join`, {
+      method: 'POST',
+      headers: { ...this.headers, 'Content-Type': 'application/json' },
+      body: JSON.stringify({ data, fn_index: fn, session_hash: this.session, event_data: null, trigger_id: null }),
+      signal: AbortSignal.timeout(60_000),
+    });
+    if (!join.ok) throw new Error(`${this.id} /${api}: ${join.status} ${(await join.text()).slice(0, 200)}`);
+    const { event_id } = (await join.json()) as { event_id: string };
+    const started = Date.now();
+    const stop = new AbortController();
+    let quiet: ReturnType<typeof setTimeout> | undefined;
+    let why = '';
+    const wait = () => {
+      clearTimeout(quiet);
+      const ms = Math.min(SPACE_WAIT.idleMs, SPACE_WAIT.totalMs - (Date.now() - started));
+      quiet = setTimeout(() => {
+        why = Date.now() - started >= SPACE_WAIT.totalMs ? `no answer after ${Math.round(SPACE_WAIT.totalMs / 1000)} s` : `silent for ${Math.round(SPACE_WAIT.idleMs / 1000)} s`;
+        stop.abort();
+      }, Math.max(0, ms));
+    };
+    wait();
+    try {
+      return await this.listen(api, event_id, stop.signal, wait);
+    } catch (error) {
+      if (why) throw new Error(`${this.id} /${api}: ${why}`);
+      throw error;
+    } finally {
+      clearTimeout(quiet);
+      stop.abort();
+    }
+  }
+
+  /** Reads the queue's event stream until the call's own answer; `heard` is called on every chunk. */
+  private async listen(api: string, event_id: string, signal: AbortSignal, heard: () => void): Promise<unknown[][]> {
+    const stream = await fetch(`${this.base}/queue/data?session_hash=${this.session}`, { headers: this.headers, signal });
+    if (!stream.ok || !stream.body) throw new Error(`${this.id} /${api}: queue ${stream.status}`);
+    const events: unknown[][] = [];
+    const decoder = new TextDecoder();
+    let buffered = '';
+    for await (const chunk of stream.body as unknown as AsyncIterable<Uint8Array>) {
+      heard();
+      buffered += decoder.decode(chunk, { stream: true });
+      const lines = buffered.split('\n');
+      buffered = lines.pop() ?? '';
+      for (const line of lines) {
+        if (!line.startsWith('data:')) continue;
+        if (process.env.DIORAMA_SPACE_DEBUG) console.error(line.slice(0, 400));
+        const msg = JSON.parse(line.slice(5)) as { msg: string; event_id?: string; success?: boolean; output?: { data?: unknown[]; error?: unknown; title?: string } };
+        if (msg.event_id && msg.event_id !== event_id) continue;
+        if (msg.msg === 'process_generating' && msg.output?.data) events.push(msg.output.data);
+        if (msg.msg === 'process_completed') {
+          if (!msg.success || msg.output?.error !== undefined && msg.output.error !== null) {
+            throw new Error(`${this.id} /${api}: ${JSON.stringify(msg.output?.error ?? msg.output).slice(0, 300)}`);
+          }
+          if (msg.output?.data) events.push(msg.output.data);
+          stream.body.cancel().catch(() => {});
+          return events;
+        }
+      }
+    }
+    throw new Error(`${this.id} /${api}: the stream closed before an answer`);
+  }
+
+  /** The fn_index of a named endpoint and where its hidden state inputs go, from the app's config. */
+  private async endpoint(api: string): Promise<{ fn: number; states: number[] }> {
+    if (!this.endpoints) {
+      const res = await fetch(`${this.base.replace(/\/gradio_api$/, '')}/config`, { headers: this.headers, signal: AbortSignal.timeout(60_000) });
+      if (!res.ok) throw new Error(`${this.id} config: ${res.status}`);
+      type Config = { components: { id: number; type: string }[]; dependencies: { id?: number; api_name?: string | false; inputs: number[] }[] };
+      const config = (await res.json()) as Config;
+      const state = new Set(config.components.filter((c) => c.type === 'state').map((c) => c.id));
+      this.endpoints = new Map(config.dependencies.flatMap((d, i) => d.api_name
+        ? [[d.api_name, { fn: d.id ?? i, states: d.inputs.flatMap((c, at) => (state.has(c) ? [at] : [])) }] as const]
+        : []));
+    }
+    const found = this.endpoints.get(api);
+    if (!found) throw new Error(`${this.id} has no endpoint /${api}`);
+    return found;
+  }
+
+  async download(file: { path: string; url?: string }): Promise<Buffer> {
+    // A download only reads a file the Space already made, so a busy or restarting Space is asked again.
+    for (let attempt = 0; ; attempt++) {
+      const res = await fetch(file.url ?? `${this.base}/file=${file.path}`, { headers: this.headers, signal: AbortSignal.timeout(120_000) }).catch((error: Error) => error);
+      if (!(res instanceof Error) && res.ok) return Buffer.from(await res.arrayBuffer());
+      if (!(res instanceof Error)) await res.body?.cancel();
+      const status = res instanceof Error ? res.message : res.status;
+      if ((!(res instanceof Error) && !TRANSIENT.includes(res.status)) || attempt === 3) {
+        throw new Error(`${this.id} download ${file.path}: ${status}${attempt ? ` (${attempt + 1} attempts)` : ''}`);
+      }
+      await new Promise((resolve) => setTimeout(resolve, 2000 * 2 ** attempt));
+    }
+  }
+}
+
+/** Every file in an endpoint's events whose path ends with `suffix`, the last one sent first. */
+export function filesNamed(events: unknown[][], suffix: string): { path: string; url?: string }[] {
+  const found: { path: string; url?: string }[] = [];
+  const visit = (v: unknown): void => {
+    if (!v || typeof v !== 'object') return;
+    if (Array.isArray(v)) return v.forEach(visit);
+    const o = v as Record<string, unknown>;
+    if (typeof o.path === 'string' && o.path.endsWith(suffix)) found.push({ path: o.path, url: typeof o.url === 'string' ? o.url : undefined });
+    Object.values(o).forEach(visit);
+  };
+  events.forEach(visit);
+  return found.reverse();
+}

package/app/src/model/sprites.ts

@@ -0,0 +1,293 @@
+// Walk-cycle sprites of a figure: the rigged model (cast/<key>/model/) walks in a grid of frames for one
+// facing, Muse draws that sheet again by hand in the world's painted style, keeping every pose and
+// position, and each painted frame is cut out and lined up with the frame it was drawn over. The
+// model gives the poses; everything seen in the game is Muse's.
+import fs from 'node:fs';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { build } from 'esbuild';
+import sharp from 'sharp';
+import type { AnimationSettings } from './animation.ts';
+import { chromium } from '../commands/explore.ts';
+import { museEdit } from '../meta/muse.ts';
+import { askSparkJson } from '../meta/spark.ts';
+import type { Memo } from '../meta/memo.ts';
+import { emptyMask, maskArea, type Mask } from '../image/mask.ts';
+import { components, dilate, maskBox, maskIou } from '../image/raster.ts';
+
+const WEB = path.join(path.dirname(fileURLToPath(import.meta.url)), '../../web');
+
+/** The five facings drawn; the three facing left are these mirrored. yaw 0 faces the viewer, PI/2 screen right. */
+
+/** Below this mean pose IoU a walk take has frames facing the wrong way or legs that hardly walk. */
+export const POSE_OK = 0.85;
+export const FACINGS = { front: 0, 'front-right': Math.PI / 4, right: Math.PI / 2, 'back-right': (3 * Math.PI) / 4, back: Math.PI } as const;
+export type Facing = keyof typeof FACINGS;
+
+/** One sheet: 8 frames of a full walk cycle in 4 columns and 2 rows, Muse's own size. */
+export const SHEET = { cols: 4, rows: 2, width: 1200, height: 896, feetAt: 0.95, px: 385, elevation: 0.18 } as const;
+/** One extra cell is reserved for standing; talk uses the same cell dimensions and grid. */
+export function modelSheet(frames: number) {
+  if (![8, 16, 32].includes(frames)) throw new Error('Model sprites require 8, 16 or 32 walk frames');
+  const rows = Math.ceil((frames + 1) / SHEET.cols);
+  // Keep painted 8-cell guides unchanged. Direct UV sheets have no provider size limit:
+  // render each cell at 1.5x in both axes to retain finer facial and hand detail.
+  const scale = 1.5;
+  return { ...SHEET, rows, frames, width: SHEET.width * scale, height: rows * (SHEET.height / SHEET.rows) * scale, px: SHEET.px * scale };
+}
+
+const GREY = { r: 217, g: 217, b: 217 };
+
+type RoutePage = {
+  route(url: string, fn: (route: { request(): { url(): string }; fulfill(o: { status?: number; contentType?: string; body: string | Buffer }): Promise<void> }) => unknown): Promise<void>;
+  goto(url: string): Promise<unknown>;
+  evaluate<T>(js: string): Promise<T>;
+  on(event: 'pageerror', fn: (e: { message: string }) => void): void;
+};
+
+/**
+ * Mean `same` (cutSheet) above this means Muse gave the 3D guide back with a filter over it instead of
+ * drawing it again. On foghorn's walk sheets, sheets drawn again scored 0.22-0.37 and traced ones 0.60-0.78;
+ * pose IoU did not tell them apart (a drawn sheet can follow the pose as closely as a traced one).
+ */
+export const TRACED = 0.5;
+
+/**
+ * A walk cell whose painted figure is shorter than this share of the sheet's usual one is not a step (a
+ * crouch, a sit). Walk frames vary by under 6% (5 worlds, 18 sheets); lamp-quarter's crouches were 0.73-0.78.
+ */
+export const OFF_GAIT = 0.88;
+
+/**
+ * The model walking (or, given `gestures`, standing in those talking poses of src/play/gestures.ts, one
+ * per cell), one transparent sheet per facing, rendered in headless Chromium with WebGL.
+ */
+export async function renderGuides(modelDir: string, facings: Facing[], gestures?: number[], /** grey clay, no textures */ plain = false, animation?: AnimationSettings, /** model.json fields to override */ over: Record<string, unknown> = {}): Promise<Record<string, Buffer>> {
+  const info = { ...JSON.parse(fs.readFileSync(path.join(modelDir, 'model.json'), 'utf8')), ...over };
+  if (animation) {
+    info.texture = animation.texture;
+    if (animation.coatCorrection !== undefined) info.coatCorrection = animation.coatCorrection;
+  }
+  const js = (await build({ entryPoints: [path.join(WEB, 'sheet.ts')], bundle: true, format: 'esm', write: false, target: 'es2022' })).outputFiles[0].text;
+  const browser = await (await chromium()).launch({ args: ['--use-gl=angle', '--use-angle=swiftshader', '--enable-unsafe-swiftshader'] });
+  try {
+    const page = (await browser.newPage()) as unknown as RoutePage;
+    const errors: string[] = [];
+    page.on('pageerror', (e) => errors.push(e.message));
+    await page.route('http://sprites.local/**', async (route) => {
+      const file = new URL(route.request().url()).pathname.slice(1);
+      if (!file) return route.fulfill({ contentType: 'text/html', body: `<!doctype html><script type="module">${js}</script>` });
+      const at = path.join(modelDir, path.basename(file));
+      if (!fs.existsSync(at)) return route.fulfill({ status: 404, body: '' });
+      return route.fulfill({ contentType: file.endsWith('.png') ? 'image/png' : 'application/octet-stream', body: fs.readFileSync(at) });
+    });
+    await page.goto('http://sprites.local/');
+    const out: Record<string, Buffer> = {};
+    const direct = animation?.texture === 'model';
+    for (const facing of facings) {
+      const ask = { info, base: 'http://sprites.local', yaw: FACINGS[facing], gestures, plain, ...SHEET, ...(direct && modelSheet(animation.frames)), stand: direct && !gestures };
+      const url = await page.evaluate<string>(`window.renderSheet(${JSON.stringify(ask)})`);
+      if (errors.length) throw new Error(`sheet page: ${errors.join('; ')}`);
+      out[facing] = Buffer.from(url.split(',')[1], 'base64');
+    }
+    return out;
+  } finally {
+    await browser.close();
+  }
+}
+
+/** The guide on Muse's plain grey, which is what it is asked to keep. */
+export const onGrey = (guide: Buffer) => sharp(guide).flatten({ background: GREY }).png().toBuffer();
+
+/** The guide as flat colour blocks (median filter, 8 colours): the render's shading is gone, nothing to trace. */
+export const flatGuide = async (grey: Buffer) =>
+  sharp(await sharp(grey).median(9).png().toBuffer()).png({ palette: true, colours: 8, dither: 0 }).toBuffer();
+
+const SEEN: Record<Facing, string> = {
+  front: 'walking towards the viewer, face to the viewer',
+  'front-right': 'walking diagonally towards the viewer and to the right of the picture (three-quarter view), face visible',
+  right: 'walking to the right of the picture, seen exactly from the side in profile',
+  'back-right': 'walking diagonally away from the viewer and to the right of the picture, seen three-quarters from behind: back of the head, hardly any face',
+  back: 'walking straight away from the viewer, seen from behind: back of the head and back of the clothes, no face',
+};
+
+/**
+ * Muse draws the sheet again from scratch over the guide. `refs`: the character (Muse's painted view for
+ * that side) and, for consistency across facings, a sheet already painted. `style`: the world's style, named
+ * once the world was given another one than it was made in (style_set); else the painterly gouache it was made for.
+ */
+export function paintSheet(memo: Memo, key: string, facing: Facing, looks: string, guide: Buffer, refs: Buffer[], take = 0, talkFrames = 0, style?: string): Promise<Buffer> {
+  const talk = talkFrames > 0; // the sheet is always full: Muse redraws a half-empty grid as a few big figures
+  const what = talk
+    ? `a talking sprite sheet: eight frames of one character standing still and talking, ${SEEN[facing].replace(/^walking/, 'turned')}, gesturing with empty hands (holding nothing) and the head, mouth slightly open in some frames; where the guide's coat bulges around a hand, it is only the hand gesture`
+    : `a walk-cycle sprite sheet: eight frames of one character ${SEEN[facing]}`;
+  return museEdit(memo, {
+    label: `${key}-${talk ? 'talk' : 'sprites'}-${facing}`, take, size: `${SHEET.width}x${SHEET.height}`, images: [guide, ...refs],
+    prompt: `The first image is only a rough 3D pose guide for ${what}. ` +
+      `Draw this sheet again from scratch as a hand-painted 2D game sprite sheet of the character in the other pictures (${looks}): same face, hair, headwear, clothes, patches, boots and colours, ${style ? `in their style: ${style}` : 'in their painterly gouache style'}. ` +
+      `Each frame: exactly the same position, size, facing and body pose as the guide (same leg, arm and hand positions), but clean anatomy: one face, natural hands, natural boots that keep their shape, smooth readable clothing folds, no patchy textures, no 3D or CG look. ` +
+      // A thing the model's views left standing at the feet is fused to a boot in the guide: a plank or a case swinging with the leg.
+      'Anything in the guide that lies on the ground or sticks out flat from the feet (a slab, a plank, a box, a case) is not part of the character: leave it out. ' +
+      'Plain flat light grey background, no ground shadows, nothing on the ground, nothing else.',
+  });
+}
+
+type Rgba = { data: Buffer; width: number; height: number };
+const rgba = async (png: Buffer, size?: [number, number]): Promise<Rgba> => {
+  let img = sharp(png).ensureAlpha();
+  if (size) img = img.resize(size[0], size[1], { fit: 'fill' });
+  const { data, info } = await img.raw().toBuffer({ resolveWithObject: true });
+  return { data, width: info.width, height: info.height };
+};
+
+/** Background of a painted cell: pixels joined to its border that are close to the border's median colour. */
+function background(img: Rgba, [x0, y0, w, h]: number[], tolerance = 26): Mask {
+  const px = (x: number, y: number) => ((y0 + y) * img.width + x0 + x) * 4;
+  const border: number[][] = [];
+  for (let x = 0; x < w; x++) for (const y of [0, h - 1]) border.push([...img.data.subarray(px(x, y), px(x, y) + 3)]);
+  for (let y = 0; y < h; y++) for (const x of [0, w - 1]) border.push([...img.data.subarray(px(x, y), px(x, y) + 3)]);
+  const med = [0, 1, 2].map((c) => border.map((p) => p[c]).sort((a, b) => a - b)[border.length >> 1]);
+  const near = (i: number) => {
+    const o = px(i % w, Math.floor(i / w));
+    return Math.hypot(img.data[o] - med[0], img.data[o + 1] - med[1], img.data[o + 2] - med[2]) < tolerance;
+  };
+  const bg = emptyMask(w, h);
+  const stack: number[] = [];
+  const push = (i: number) => {
+    if (!bg.bits[i] && near(i)) { bg.bits[i] = 1; stack.push(i); }
+  };
+  for (let x = 0; x < w; x++) { push(x); push((h - 1) * w + x); }
+  for (let y = 0; y < h; y++) { push(y * w); push(y * w + w - 1); }
+  while (stack.length) {
+    const i = stack.pop()!;
+    const x = i % w;
+    if (x > 0) push(i - 1);
+    if (x < w - 1) push(i + 1);
+    if (i >= w) push(i - w);
+    if (i < w * (h - 1)) push(i + w);
+  }
+  return bg;
+}
+
+/**
+ * How closely the drawn figure keeps the guide's own shading: the correlation of the two images' edge
+ * strength (Sobel on luminance) inside both figures, away from the outline. A sheet drawn again has its
+ * own folds, hair and face, about 0.2-0.4; the guide with a filter over it keeps the render's edges, 0.6+.
+ * Colour alone does not tell them apart: Muse repaints a traced figure in other colours just as often.
+ */
+function sameEdges(guide: Buffer, drawn: Buffer, W: number, [x0, y0, cw, ch]: number[]): number {
+  const lum = (d: Buffer, x: number, y: number) => { const i = ((y0 + y) * W + x0 + x) * 4; return 0.3 * d[i] + 0.59 * d[i + 1] + 0.11 * d[i + 2]; };
+  const edge = (d: Buffer, x: number, y: number) => {
+    const p = (dx: number, dy: number) => lum(d, x + dx, y + dy);
+    const gx = p(1, -1) + 2 * p(1, 0) + p(1, 1) - p(-1, -1) - 2 * p(-1, 0) - p(-1, 1);
+    const gy = p(-1, 1) + 2 * p(0, 1) + p(1, 1) - p(-1, -1) - 2 * p(0, -1) - p(1, -1);
+    return Math.hypot(gx, gy);
+  };
+  const inside = (x: number, y: number) => {
+    const i = ((y0 + y) * W + x0 + x) * 4 + 3;
+    return guide[i] > 40 && drawn[i] > 0;
+  };
+  const a: number[] = [];
+  const b: number[] = [];
+  for (let y = 4; y < ch - 4; y++) for (let x = 4; x < cw - 4; x++) {
+    if (!inside(x, y) || !inside(x + 3, y) || !inside(x - 3, y) || !inside(x, y + 3) || !inside(x, y - 3)) continue;
+    a.push(edge(guide, x, y));
+    b.push(edge(drawn, x, y));
+  }
+  if (a.length < 50) return 0;
+  const [ma, mb] = [a.reduce((s, v) => s + v, 0) / a.length, b.reduce((s, v) => s + v, 0) / b.length];
+  let [ab, aa, bb] = [0, 0, 0];
+  for (let i = 0; i < a.length; i++) { ab += (a[i] - ma) * (b[i] - mb); aa += (a[i] - ma) ** 2; bb += (b[i] - mb) ** 2; }
+  return aa && bb ? ab / Math.sqrt(aa * bb) : 0;
+}
+
+/**
+ * Cuts each painted frame out of its grey and lines it up with its guide frame: same feet line, same
+ * centre, same height. Returns the transparent sheet and how well each frame kept the guide's pose (IoU)
+ * and its look (`same`, sameEdges).
+ */
+export async function cutSheet(guidePng: Buffer, paintedPng: Buffer): Promise<{ sheet: Buffer; iou: number[]; legs: number[]; same: number[]; stand: number; off: number[] }> {
+  const { cols, rows, width: W, height: H } = SHEET;
+  const guide = await rgba(guidePng, [W, H]);
+  const painted = await rgba(paintedPng, [W, H]);
+  const out = Buffer.alloc(W * H * 4);
+  const [cw, ch] = [W / cols, H / rows];
+  const iou: number[] = [];
+  const legs: number[] = [];
+  const same: number[] = [];
+  const spread: number[] = [];
+  const tall: number[] = [];
+  for (let i = 0; i < cols * rows; i++) {
+    const cell = [(i % cols) * cw, Math.floor(i / cols) * ch, cw, ch];
+    const [x0, y0] = cell;
+    const want = emptyMask(cw, ch);
+    for (let y = 0; y < ch; y++) for (let x = 0; x < cw; x++) want.bits[y * cw + x] = guide.data[((y0 + y) * W + x0 + x) * 4 + 3] > 40 ? 1 : 0;
+    const bg = background(painted, cell);
+    const fig = emptyMask(cw, ch);
+    for (let k = 0; k < fig.bits.length; k++) fig.bits[k] = 1 - bg.bits[k];
+    // Keep blobs that overlap where the guide figure is (a stray speck or a neighbour's boot does not).
+    const near = dilate(want, 14);
+    const kept = emptyMask(cw, ch);
+    for (const blob of components(fig)) {
+      if (maskArea(blob) < 30 || !blob.bits.some((b, k) => b && near.bits[k])) continue;
+      blob.bits.forEach((b, k) => { if (b) kept.bits[k] = 1; });
+    }
+    const gb = maskBox(want);
+    const pb = maskBox(kept);
+    if (!gb || !pb) { iou.push(0); legs.push(0); same.push(0); spread.push(Infinity); tall.push(0); continue; }
+    tall.push(pb[3] - pb[1]);
+    // Same height, same feet line, same centre across.
+    const s = Math.min(1.2, Math.max(0.8, (gb[3] - gb[1]) / (pb[3] - pb[1])));
+    const [gcx, pcx] = [(gb[0] + gb[2]) / 2, (pb[0] + pb[2]) / 2];
+    const moved = emptyMask(cw, ch);
+    for (let y = 0; y < ch; y++) for (let x = 0; x < cw; x++) {
+      const sx = Math.round((x - gcx) / s + pcx);
+      const sy = Math.round((y - gb[3]) / s + pb[3]);
+      if (sx < 0 || sy < 0 || sx >= cw || sy >= ch || !kept.bits[sy * cw + sx]) continue;
+      moved.bits[y * cw + x] = 1;
+      const from = ((y0 + sy) * W + x0 + sx) * 4;
+      const to = ((y0 + y) * W + x0 + x) * 4;
+      painted.data.copy(out, to, from, from + 3);
+      out[to + 3] = 255;
+    }
+    same.push(Number(sameEdges(guide.data, out, W, cell).toFixed(3)));
+    iou.push(Number(maskIou(want, moved).toFixed(3)));
+    // Muse sometimes draws the other leg forward: the legs alone (lowest third) catch that.
+    const below = Math.round(gb[3] - (gb[3] - gb[1]) / 3);
+    const low = (m: Mask) => ({ ...m, bits: m.bits.map((b, k) => (k >= below * cw ? b : 0)) });
+    legs.push(Number(maskIou(low(want), low(moved)).toFixed(3)));
+    const feet = maskBox(low(want));
+    spread.push(feet ? feet[2] - feet[0] : Infinity);
+  }
+  // Muse now and then paints one frame of a walk as a crouch (lamp-quarter: the courier set down his lantern
+  // in the last cell of 3 of 4 sheets). Each cell is scaled to the guide's height above, so the cut sheet
+  // hides it; the painted figure, a fifth shorter than the rest, does not. A walk steps over such a cell.
+  const usual = [...tall].sort((a, b) => a - b)[Math.floor(tall.length / 2)];
+  const off = tall.flatMap((t, i) => (t < OFF_GAIT * usual ? [i] : []));
+  // Standing still shows the frame whose feet are closest together (the legs passing each other).
+  const stand = spread.reduce((best, s, i) => (!off.includes(i) && (best < 0 || s < spread[best]) ? i : best), -1);
+  return { sheet: await sharp(out, { raw: { width: W, height: H, channels: 4 } }).png().toBuffer(), iou, legs, same, stand: Math.max(0, stand), off };
+}
+
+
+/**
+ * Whether the model wears the front painting on its face (web/hero3d.ts `face: 'views'`) instead of its own
+ * UV texture. TRELLIS often bakes the face to a blank smudge or a dark band over the eyes; the painting then
+ * gives the face Muse drew. But the painting is fitted only by silhouette and can land a second pair of eyes
+ * beside the model's, so Spark compares the two heads, as the sprites will show them, and picks.
+ */
+export async function uvFaceLost(memo: Memo, key: string, modelDir: string, animation: AnimationSettings): Promise<boolean> {
+  const head = async (face: 'views' | undefined) => {
+    const render = (await renderGuides(modelDir, ['front'], [0], false, animation, { face })).front;
+    const trimmed = await sharp(render).trim().png().toBuffer({ resolveWithObject: true });
+    return sharp(trimmed.data).extract({ left: 0, top: 0, width: trimmed.info.width, height: Math.max(1, Math.round(trimmed.info.height * 0.2)) })
+      .flatten({ background: GREY }).resize({ width: 512 }).png().toBuffer();
+  };
+  const a = await askSparkJson<{ why?: unknown; better?: unknown }>(memo, {
+    label: `uv-face:${key}`, images: [await head(undefined), await head('views'), fs.readFileSync(path.join(modelDir, 'front.png'))],
+    prompt: 'Pictures 1 and 2 are the same 3D game character\'s head, rendered two ways; picture 3 is the painting of that character. Which render has the better face: '
+      + 'two clear, separate eyes in the right place, one nose, one mouth, closest to the painting? A face with no eyes, a dark band or smudge over the eyes, '
+      + 'or eyes doubled, shifted or cut by a seam is worse. Reply with JSON only: {"why": "a few words", "better": 1 or 2}.',
+  });
+  return Number(a.better) === 2;
+}

package/app/src/model/views.ts

@@ -0,0 +1,163 @@
+// Muse paints the hero from four sides, standing still: the front view is what TRELLIS turns into a
+// model, and all four are projected back onto it, so the walking model looks painted from every side.
+import sharp from 'sharp';
+import { museEdit } from '../meta/muse.ts';
+import type { Memo } from '../meta/memo.ts';
+import type { ViewName } from './fit.ts';
+import { askSparkJson } from '../meta/spark.ts';
+import { refusedByFilter } from '../scene/words.ts';
+
+const SIZE = '1024x1024';
+/** Each view as it lands, so Studio shows them one by one while the rest are painted. */
+type Landed = (name: ViewName, png: Buffer) => void;
+const STAND = 'standing still in a clear rigging A-pose: both straight arms extended diagonally outwards 35 degrees from vertical, elbows straight, hands at least one hand-width away from the thighs, a wide uninterrupted background gap between each arm and the torso and hips, legs straight and shoulder-width apart, feet flat';
+// The arms in every view where the front has them: TRELLIS builds one body from all four, and each view is
+// projected back onto it, so a hand in a pocket in one view is painted on the hip as a second pair of hands.
+// Looks often say "hand in pocket"; this pose wins over it.
+// Empty hands make Muse put down what the painting had in them: a suitcase standing at the feet in one view
+// becomes a slab TRELLIS fuses to a boot, and it swings with the leg in every sprite. So it is left out, not put down.
+// A held flag left "out of the hands" ends up waving behind the figure; TRELLIS fuses it to the back as a
+// mangled sheet and the auto-rig skins it to the spine, so a held pole or flag is named and banned outright.
+const HANDS = 'Both hands empty and fully visible, well separated from the thighs, never touching clothing: not in pockets, not on the hips, not holding anything, not in front of the body. Whatever the description has in their hands (a suitcase, a basket, a tool) is left out of the picture altogether, never put down on the ground beside them. ' +
+  'A flag, banner, pole, staff, spear, broom, umbrella, sign or lantern they held is left out too: not behind them, not over the shoulder, not on the back, nowhere in the picture.';
+const SAME_HEAD = 'Whatever is on the head (hood up or down, hat, hair) is exactly as in the front picture, from this side too.';
+// Looks written from a photo may still say where the person sat or lay ("against a headboard"): a profile then
+// paints the headboard behind them, and TRELLIS grows it into the body as a slab around the model.
+const PLAIN = 'Full body from head to feet, centred, nothing cropped. Standing free: nothing behind or beside them, no bed, headboard, chair, wall or pillow, whatever the description says about where they were. Plain flat light grey background, soft even light, no shadow on the ground, no other objects: nothing standing or lying on the ground, nothing at their feet.';
+
+/** Where a carried thing sits on the body, from the character's own point of view. */
+export type Spot = 'front' | 'back' | 'left' | 'right';
+/** Something the character wears or carries that is not plain clothing: a bag, a satchel, a lantern, a sword. */
+export type Carried = { name: string; at: Spot; strap?: string };
+
+/**
+ * Where a thing at `at` (the character's own front, back, left or right) shows in a view. Views are straight on:
+ * front sees the character's left at the picture's right; the "left" profile faces the picture's left and shows
+ * the character's left side (src/model/fit.ts axes).
+ */
+export function spotIn(view: ViewName, at: Spot): string {
+  const table: Record<ViewName, Record<Spot, string>> = {
+    front: { front: 'on the front of the body, in the middle', back: 'behind the body, hidden (only a strap may show)', left: 'at the right edge of the figure in the picture (the character\'s left side)', right: 'at the left edge of the figure in the picture (the character\'s right side)' },
+    back: { front: 'in front of the body, hidden (only a strap may show over the shoulder or round the side)', back: 'on the back, in the middle, fully visible', left: 'at the LEFT edge of the figure in the picture (the character\'s left side, seen from behind)', right: 'at the RIGHT edge of the figure in the picture (the character\'s right side, seen from behind)' },
+    left: { front: 'in front of the body, at the LEFT edge of the figure, where the character faces', back: 'on the back, at the RIGHT edge of the figure, behind the character', left: 'on the near side, fully visible against the body', right: 'on the far side, hidden behind the body (only a strap may show)' },
+    right: { front: 'in front of the body, at the RIGHT edge of the figure, where the character faces', back: 'on the back, at the LEFT edge of the figure, behind the character', left: 'on the far side, hidden behind the body (only a strap may show)', right: 'on the near side, fully visible against the body' },
+  };
+  return table[view][at];
+}
+
+/** What the character carries and where, read once off the front view: the other views are told exactly where each thing goes. */
+export async function readCarried(memo: Memo, key: string, front: Buffer): Promise<Carried[]> {
+  const a = await askSparkJson<{ items?: { name?: unknown; at?: unknown; strap?: unknown }[] }>(memo, {
+    label: `views-carried:${key}`, images: [front],
+    prompt: 'This character is seen straight from the front (their left hand is on the right of the picture). List what they wear on the body that is not plain clothing and that could end up on the wrong side in a drawing from another side: bags, satchels, backpacks, pouches, a sword in a belt, a quiver, a scarf end, a badge. '
+      + 'Only things fastened to the body (a strap, a belt, a hook, worn): never something held in a hand or held up, and never a flag, banner, pole, staff, umbrella or lantern, even when one shows behind them (those do not belong in this picture). '
+      + 'For each: where its main part sits on the body from the character\'s own point of view, one of "front", "back", "left", "right" (a bag hanging at their left hip is "left", one on their belly is "front"), and the strap if any (e.g. "over the right shoulder, across the chest"). '
+      + 'Reply with JSON only: {"items": [{"name": "brown leather satchel", "at": "front", "strap": "over the right shoulder, across the chest"}]} or {"items": []}.',
+  });
+  const spots = ['front', 'back', 'left', 'right'];
+  return (a.items ?? []).flatMap((i) => (typeof i.name === 'string' && spots.includes(String(i.at)) ? [{ name: i.name, at: i.at as Spot, ...(typeof i.strap === 'string' && i.strap && { strap: i.strap }) }] : []));
+}
+
+/** The sentence a view's prompt gets: each carried thing, where it shows in this view. */
+export function carriedLine(view: ViewName, items: Carried[]): string {
+  if (!items.length) return '';
+  return `What they carry stays where it is on the body, it does not move round when we walk round them: ${items.map((i) => `the ${i.name}${i.strap ? ` (strap ${i.strap})` : ''} is ${spotIn(view, i.at)}`).join('; ')}.`;
+}
+
+/** A view at check size: where a bag hangs reads at 640 px; four full 1024 px views cost four times the tokens. */
+const small = (png: Buffer) => sharp(png).resize(640, 640, { fit: 'inside' }).png().toBuffer();
+
+/** The views that show a carried thing in the wrong place, or something on the ground at the feet, with why. */
+export async function checkCarried(memo: Memo, key: string, views: Record<ViewName, Buffer>, items: Carried[], round = 0): Promise<Partial<Record<ViewName, string>>> {
+  const names: ViewName[] = ['front', 'back', 'left', 'right'];
+  const expect = items.length ? names.map((v, n) => `Picture ${n + 1} (${v} view): ${items.map((i) => `the ${i.name} should be ${spotIn(v, i.at)}`).join('; ')}.`).join('\n') : 'They carry nothing that is not plain clothing.';
+  const a = await askSparkJson<Record<string, { ok?: unknown; why?: unknown }>>(memo, {
+    label: `views-carried-check:${key}#${round}`, images: await Promise.all(names.map((v) => small(views[v]))),
+    prompt: `Four pictures of one character, seen from the front, back, their left side (facing the picture's left) and their right side (facing the picture's right). Check only where the carried things are, and that nothing stands or lies on the ground; ignore everything else.\n${expect}\n\n`
+      + 'A thing that shows on the wrong side of the body, on the front when it should be on the back (or the other way round), twice, or missing where it should be fully visible, is wrong. '
+      + 'Anything standing or lying on the ground, at the feet or beside them (a suitcase, a bag, a box, a plank), or held in a hand, is wrong in every view, the front too; so is a flag, banner, pole, staff or umbrella anywhere in the picture, behind the figure or over the shoulder too. '
+      + `Reply with JSON only: {${names.map((v) => `"${v}": {"ok": true or false, "why": "a few words"}`).join(', ')}}`,
+  });
+  return Object.fromEntries(names.flatMap((v) => (a[v]?.ok === false ? [[v, String(a[v]?.why ?? 'carried thing in the wrong place')]] : [])));
+}
+
+/** Rounds of repainting the views a carried thing went wrong in. */
+const CARRIED_ROUNDS = 2;
+
+/** Front from the cast painting; back from the front; the profiles from front and back, so the hood or hat stays the same.
+ * What they carry (a satchel, a lantern) is read off the front and placed by name in every other view, then all four
+ * are checked together and a view that put it on the wrong side, or put something down at the feet, is painted again.
+ * A creature (not a person on two legs: a round eye, a frog, a living teapot) is painted as it is, with no A-pose and no hands. */
+// The image safety filter refuses a prompt on single words (a long list of things to leave out reads as a list of
+// weapons): the view is asked once more in calm words, with the looks left to the picture and the hands said plainly.
+const HANDS_CALM = 'Both hands open and empty, well apart from the body, nothing held and nothing else in the picture.';
+type EditReq = Parameters<typeof museEdit>[1];
+const viewEdit = (memo: Memo, looks: string) => (req: EditReq) => museEdit(memo, req).catch((error: unknown) => {
+  if (!refusedByFilter(error)) throw error;
+  console.log(`  ${req.label}: the image safety filter refused the view; asking in calm words`);
+  const prompt = req.prompt.split(HANDS).join(HANDS_CALM).split(` (${looks})`).join('');
+  return museEdit(memo, { ...req, label: `${req.label}-calm`, prompt });
+});
+
+export async function paintViews(memo: Memo, key: string, looks: string, painting: Buffer, take = 0, creature = false, onView?: Landed, again = true): Promise<Record<ViewName, Buffer>> {
+  if (creature) return paintCreatureViews(memo, key, looks, painting, take, onView);
+  const edit = viewEdit(memo, looks);
+  const front = await edit({
+    label: `${key}-view-front`, size: SIZE, take, images: [painting],
+    prompt: `The same character as in the picture (${looks}), same clothes, colours and painted style, now seen straight from the front, ${STAND}, face looking at the viewer. Anything they carry (a bag, a satchel) stays where it is on their body in the picture. ${HANDS} ${PLAIN}`,
+  });
+  onView?.('front', front);
+  const items = await readCarried(memo, key, front);
+  if (items.length) console.log(`  ${key}: carries ${items.map((i) => `${i.name} (${i.at})`).join(', ')}`);
+  const paintBack = (t: number) => edit({
+    label: `${key}-view-back`, size: SIZE, take: t, images: [front],
+    prompt: `The same character as in the picture (${looks}), same clothes, colours, patches and painted style, in exactly the same pose, seen straight from behind: we see the back of the head, the back of the clothes and the heels; the face is not visible. Left and right swap when seen from behind. ${carriedLine('back', items)} ${HANDS} ${SAME_HEAD} Same size and position in the frame as the picture. ${PLAIN}`,
+  });
+  const views: Record<ViewName, Buffer> = { front, back: await paintBack(take), left: front, right: front };
+  onView?.('back', views.back);
+  const side = (facing: 'left' | 'right', t: number) => edit({
+    label: `${key}-view-${facing}`, size: SIZE, take: t, images: [front, views.back],
+    prompt: `The same character as in both pictures (front and back views), same clothes, colours, patches and painted style, now seen exactly from the side in profile, facing to the ${facing} of the picture, so we see their ${facing} side (${looks}). ${carriedLine(facing, items)} ${SAME_HEAD} Keep exactly the same rigging A-pose as the front and back: straight arms angled out sideways 35 degrees from vertical, hands clear of the thighs, legs straight and shoulder-width apart. Do not lower the arms against the hips for this profile. ${HANDS} Same size and position in the frame as the pictures. ${PLAIN}`,
+  });
+  [views.left, views.right] = await Promise.all((['left', 'right'] as const).map((f) => side(f, take).then((png) => (onView?.(f, png), png))));
+  for (let round = 0; round < CARRIED_ROUNDS; round++) {
+    const wrong = await checkCarried(memo, key, views, items, round);
+    // The other three are painted from the front: a wrong front means all four again, once.
+    if (wrong.front && again) {
+      console.log(`  ${key}: front view: ${wrong.front}; painting all four again`);
+      return paintViews(memo, key, looks, painting, take + 1, false, onView, false);
+    }
+    delete wrong.front;
+    const redo = Object.keys(wrong) as ViewName[];
+    if (!redo.length) break;
+    console.log(`  ${key}: ${redo.map((v) => `${v} view: ${wrong[v]}`).join('; ')}; painting ${redo.length === 1 ? 'it' : 'them'} again`);
+    const t = take + 1 + round;
+    if (wrong.back) (views.back = await paintBack(t), onView?.('back', views.back));
+    await Promise.all((['left', 'right'] as const).filter((f) => wrong[f]).map(async (f) => { views[f] = await side(f, t); onView?.(f, views[f]); }));
+  }
+  return views;
+}
+
+// A creature keeps its own shape: whatever feet, fins or nothing it has, as in the painting. The whole body
+// in every view, the same size and position, so TRELLIS builds one body from all four.
+const REST = 'resting in its natural pose, exactly the body shape, proportions, feet, eyes and markings of the picture, nothing added (no arms, no legs, no clothes it does not have)';
+
+async function paintCreatureViews(memo: Memo, key: string, looks: string, painting: Buffer, take: number, onView?: Landed): Promise<Record<ViewName, Buffer>> {
+  const edit = viewEdit(memo, looks);
+  const front = await edit({
+    label: `${key}-view-front`, size: SIZE, take, images: [painting],
+    prompt: `The same creature as in the picture (${looks}), same colours and painted style, now seen straight from the front, ${REST}, its face (or its front) to the viewer. The whole creature, centred, filling about two thirds of the frame. ${PLAIN}`,
+  });
+  onView?.('front', front);
+  const back = await edit({
+    label: `${key}-view-back`, size: SIZE, take, images: [front],
+    prompt: `The same creature as in the picture (${looks}), same colours and painted style, in exactly the same pose, seen straight from behind: we see its back, its face is not visible. Same size and position in the frame as the picture. ${PLAIN}`,
+  });
+  onView?.('back', back);
+  const side = (facing: 'left' | 'right') => edit({
+    label: `${key}-view-${facing}`, size: SIZE, take, images: [front, back],
+    prompt: `The same creature as in both pictures (front and back views), same colours and painted style, now seen exactly from the side in profile, facing to the ${facing} of the picture (${looks}), ${REST}. Same size and position in the frame as the pictures. ${PLAIN}`,
+  }).then((png) => (onView?.(facing, png), png));
+  const [left, right] = await Promise.all([side('left'), side('right')]);
+  return { front, back, left, right };
+}