@gtanczyk/diorama 0.0.0-stage → 0.1.0

package/app/src/scene/critic.ts

@@ -0,0 +1,169 @@
+// The critic: Spark looks at the painting and names up to four regions to fix; Muse repaints
+// the picture once, and only those regions are taken from it (Muse redraws everything on an edit).
+import sharp from 'sharp';
+import { z } from 'zod';
+import type { Memo } from '../meta/memo.ts';
+import { museEdit } from '../meta/muse.ts';
+import { askSparkJson } from '../meta/spark.ts';
+import type { Place } from '../world/schema.ts';
+import { paintedAnimals, type Brief } from './brief.ts';
+import { FRAME } from './layout.ts';
+import { pasteAlpha } from './plates.ts';
+
+export const Critique = z.object({
+  issues: z.array(z.object({
+    key: z.string(),
+    problem: z.enum(['missing', 'wrong', 'flat', 'unwanted']),
+    region: z.tuple([z.number(), z.number(), z.number(), z.number()]).describe('x0, y0, x1, y1'),
+    fix: z.string(),
+  })).max(4),
+});
+export type Critique = z.infer<typeof Critique>;
+
+export async function critique(memo: Memo, place: Place, brief: Brief, painting: Buffer): Promise<Critique> {
+  const expected = place.things.filter((t) => t.role !== 'npc').map((t) => `- ${t.key}: ${brief.looks[t.key] ?? t.looks}`);
+  // The place's people stand in the game as figures of their own: one also painted into the background is a double.
+  const cast = place.things.filter((t) => t.role === 'npc').map((t) => t.looks ?? t.key.replace(/-/g, ' '));
+  const prompt = `This is a painted background (${FRAME.width}x${FRAME.height} px) for a 2D point-and-click adventure. It should show:
+${expected.join('\n')}
+The whole picture is meant to be: ${place.shows}
+
+List at most 4 problems a player would notice, worst first:
+- missing: a listed thing is not there, or a big part of the scene above is (a tall tower, a castle with two towers, a house on top): key "scene" for that
+- wrong: it is there but does not look like its description
+- flat: it is a plain block or smear instead of a painted object
+- unwanted: people (any person or figure at all${cast.length ? `, even one that looks like ${cast.join(' or ')}: they are added over the picture later, so a painted one would be a double` : ''}), ${paintedAnimals(brief).length ? 'animals other than the listed ones' : 'animals'}, text, letters or a border that should not be there
+For each give the region to repaint as [x0, y0, x1, y1] in pixels and one sentence on what to paint there. Never ask for text or letters. If the painting is fine, give no issues.
+
+Never ask to paint a person in: the game adds its people over the picture.
+
+Reply with JSON only: {"issues": [{"key": "...", "problem": "missing | wrong | flat | unwanted", "region": [0, 0, 0, 0], "fix": "..."}]}`;
+  const parsed = Critique.parse(await askSparkJson(memo, { schema: Critique, label: `critic:${place.key}`, prompt, images: [painting] }));
+  const found = { issues: parsed.issues.filter((i) => !addsPerson(i)) };
+  // The critic weighs the whole picture and lets a figure pass when the place's words name one ("a child floats
+  // candle boats"); people are asked about on their own, as every one of them is a double or a stranger.
+  const people = await paintedPeople(memo, place.key, painting);
+  const near = (a: number[], b: number[]) => Math.abs(a[0] - b[0]) + Math.abs(a[1] - b[1]) < 120;
+  const extra = people.filter((p) => !found.issues.some((i) => i.problem === 'unwanted' && near(i.region, p.box))).map((p) => ({
+    key: 'person', problem: 'unwanted' as const, region: pad(p.box, 24),
+    fix: `Paint out the ${p.what}: continue what is behind them (planks, water, wall), add nothing new`,
+  }));
+  return { issues: [...found.issues, ...extra].slice(0, 6) };
+}
+
+const PERSON = /\b(child|children|kid|boy|girl|man|woman|men|women|person|people|figure|keeper|seller|vendor|fisherman|sailor|someone|crowd|silhouette)s?\b/i;
+
+/**
+ * A fix that would paint a person in: the place's words often name its people ("a child sets candle boats on the
+ * water"), and the critic asks for them as missing, but the game draws its people over the picture itself.
+ */
+export const addsPerson = (i: { problem: string; fix: string }) => i.problem !== 'unwanted' && /\b(paint|add|show|draw)\b/i.test(i.fix) && PERSON.test(i.fix);
+
+const People = z.object({ people: z.array(z.object({ what: z.string(), box: z.tuple([z.number(), z.number(), z.number(), z.number()]) })).max(6) });
+
+/** Every person or figure painted in, with a box: none belongs in a background, the game adds its people itself. */
+export async function paintedPeople(memo: Memo, key: string, painting: Buffer): Promise<Array<{ what: string; box: [number, number, number, number] }>> {
+  const prompt = `This is a painted background (${FRAME.width}x${FRAME.height} px) for a 2D adventure. It must show no people at all.
+List every person or human figure painted in it: adults, children, sitting, crouching or far away, a silhouette in a window or a doorway. Not statues, scarecrows, posters or pictures of people.
+For each give what it is in a few words ("a crouching child in a grey hoodie") and its box [x0, y0, x1, y1] in pixels, tight around the figure.
+Reply with JSON only: {"people": [{"what": "...", "box": [0, 0, 0, 0]}]}`;
+  try {
+    const { people } = People.parse(await askSparkJson(memo, { schema: People, label: `people:${key}`, prompt, images: [painting] }));
+    return people.filter((p) => p.box[2] - p.box[0] >= 12 && p.box[3] - p.box[1] >= 20);
+  } catch {
+    return [];
+  }
+}
+
+const pad = (b: [number, number, number, number], n: number): [number, number, number, number] =>
+  [Math.max(0, b[0] - n), Math.max(0, b[1] - n), Math.min(FRAME.width, b[2] + n), Math.min(FRAME.height, b[3] + n)];
+
+/** A soft-edged alpha mask covering the regions, so patches blend into the painting. */
+function regionAlpha(regions: [number, number, number, number][], feather = 14): Promise<Buffer> {
+  const { width: W, height: H } = FRAME;
+  const clamp = (v: number, hi: number) => Math.max(0, Math.min(hi, v));
+  const rects = regions.map(([x0, y0, x1, y1]) => {
+    const [l, t, r, b] = [clamp(Math.min(x0, x1), W), clamp(Math.min(y0, y1), H), clamp(Math.max(x0, x1), W), clamp(Math.max(y0, y1), H)];
+    return `<rect x="${l}" y="${t}" width="${r - l}" height="${b - t}" fill="#fff"/>`;
+  });
+  return sharp(Buffer.from(`<svg xmlns="http://www.w3.org/2000/svg" width="${W}" height="${H}"><rect width="${W}" height="${H}" fill="#000"/>${rects.join('')}</svg>`))
+    .blur(feather).grayscale().raw().toBuffer();
+}
+
+/** Repaints the critique's regions. Returns the painting unchanged when there is nothing to fix. */
+export async function repaint(memo: Memo, place: Place, painting: Buffer, found: Critique): Promise<Buffer> {
+  if (found.issues.length === 0) return painting;
+  const fixes = found.issues.map((i) => `- around x ${Math.round((i.region[0] + i.region[2]) / 2)}, y ${Math.round((i.region[1] + i.region[3]) / 2)}: ${i.fix}`);
+  const prompt = `Repaint only these spots of this background and leave everything else exactly as it is, same style and light:
+${fixes.join('\n')}
+No people, no animals the fixes don't ask for, no text or letters.`;
+  const patched = await museEdit(memo, { label: `repaint:${place.key}`, prompt, images: [painting] });
+  // A painting out (people, a border) and a person's own repaint are pasted as asked; the critic's other fixes are
+  // grown to all Muse painted for them, or left out when that runs on past REGION_REACH and would be sliced.
+  const asAsked = (i: Critique['issues'][number]) => i.problem === 'unwanted' || i.key === 'person';
+  const own = found.issues.filter((i) => !asAsked(i));
+  const { kept, dropped } = await critiqueRegions(painting, patched, own.map((i) => i.region));
+  for (const r of dropped) {
+    const i = own.find((o) => o.region === r);
+    console.log(`  ${place.key}: left out the repaint of ${i?.key} at [${r.join(', ')}]: Muse painted it far past its region, and pasted it would be sliced at the edge`);
+  }
+  return pasteAlpha(painting, patched, await regionAlpha([...found.issues.filter(asAsked).map((i) => i.region), ...kept]));
+}
+
+/** A critic's regions grown to all Muse painted for them; one still cut at REGION_REACH is dropped, as it was asked. */
+export async function critiqueRegions(painting: Buffer, patched: Buffer, regions: [number, number, number, number][]): Promise<{ kept: [number, number, number, number][]; dropped: [number, number, number, number][] }> {
+  const grown = await grownRegions(painting, patched, regions);
+  return { kept: grown.filter((g) => !g.cut).map((g) => g.region), dropped: regions.filter((_, i) => grown[i].cut) };
+}
+
+/** `patched` over `painting` inside the regions only (feathered), everything else untouched. */
+export async function pasteRegions(painting: Buffer, patched: Buffer, regions: [number, number, number, number][]): Promise<Buffer> {
+  return pasteAlpha(painting, patched, await regionAlpha(await wholeRegions(painting, patched, regions)));
+}
+
+/** How far a region may grow to take in the rest of what Muse painted across its edge. */
+const REGION_REACH = 160;
+const REGION_STEP = 12;
+
+/**
+ * The regions grown, side by side, while what Muse painted runs on across that side: a boat it moved a little to
+ * the right was cut at the region's edge and faded into the sea. A side grows while the change just outside it is
+ * well above the change Muse made everywhere (it redraws the whole picture a little), up to REGION_REACH px.
+ */
+export async function wholeRegions(painting: Buffer, patched: Buffer, regions: [number, number, number, number][]): Promise<[number, number, number, number][]> {
+  return (await grownRegions(painting, patched, regions)).map((g) => g.region);
+}
+
+/** Each region grown as wholeRegions does, and whether Muse's change still runs on across a side at REGION_REACH. */
+async function grownRegions(painting: Buffer, patched: Buffer, regions: [number, number, number, number][]): Promise<{ region: [number, number, number, number]; cut: boolean }[]> {
+  const { width: W, height: H } = FRAME;
+  const read = (b: Buffer) => sharp(b).resize(W, H, { fit: 'fill' }).removeAlpha().blur(3).raw().toBuffer();
+  const [a, b] = await Promise.all([read(painting), read(patched)]);
+  const diff = new Float32Array(W * H);
+  for (let i = 0; i < W * H; i++) diff[i] = Math.abs(a[i * 3] - b[i * 3]) + Math.abs(a[i * 3 + 1] - b[i * 3 + 1]) + Math.abs(a[i * 3 + 2] - b[i * 3 + 2]);
+  const sorted = Float32Array.from(diff).sort();
+  const enough = Math.max(70, 3 * sorted[Math.floor(sorted.length / 2)]);
+  // The mean change over a strip of the frame.
+  const strip = (x0: number, y0: number, x1: number, y1: number) => {
+    let [s, n] = [0, 0];
+    for (let y = Math.max(0, y0); y < Math.min(H, y1); y++) for (let x = Math.max(0, x0); x < Math.min(W, x1); x++) [s, n] = [s + diff[y * W + x], n + 1];
+    return n ? s / n : 0;
+  };
+  return regions.map((r) => {
+    let [x0, y0, x1, y1] = [Math.min(r[0], r[2]), Math.min(r[1], r[3]), Math.max(r[0], r[2]), Math.max(r[1], r[3])];
+    const [l, t, rr, bb] = [x0, y0, x1, y1];
+    // Only a side the change crosses: changed just inside the region's own edge, and just outside it.
+    const crosses = { r: strip(x1 - REGION_STEP, y0, x1, y1) > enough, l: strip(x0, y0, x0 + REGION_STEP, y1) > enough, b: strip(x0, y1 - REGION_STEP, x1, y1) > enough, t: strip(x0, y0, x1, y0 + REGION_STEP) > enough };
+    for (let k = 0; k < REGION_REACH / REGION_STEP; k++) {
+      let grew = false;
+      if (crosses.r && x1 < W && x1 - rr < REGION_REACH && strip(x1, y0, x1 + REGION_STEP, y1) > enough) [x1, grew] = [x1 + REGION_STEP, true];
+      if (crosses.l && x0 > 0 && l - x0 < REGION_REACH && strip(x0 - REGION_STEP, y0, x0, y1) > enough) [x0, grew] = [x0 - REGION_STEP, true];
+      if (crosses.b && y1 < H && y1 - bb < REGION_REACH && strip(x0, y1, x1, y1 + REGION_STEP) > enough) [y1, grew] = [y1 + REGION_STEP, true];
+      if (crosses.t && y0 > 0 && t - y0 < REGION_REACH && strip(x0, y0 - REGION_STEP, x1, y0) > enough) [y0, grew] = [y0 - REGION_STEP, true];
+      if (!grew) break;
+    }
+    const cut = (crosses.r && x1 < W && strip(x1, y0, x1 + REGION_STEP, y1) > enough) || (crosses.l && x0 > 0 && strip(x0 - REGION_STEP, y0, x0, y1) > enough)
+      || (crosses.b && y1 < H && strip(x0, y1, x1, y1 + REGION_STEP) > enough) || (crosses.t && y0 > 0 && strip(x0, y0 - REGION_STEP, x1, y0) > enough);
+    return { region: [Math.max(0, x0), Math.max(0, y0), Math.min(W, x1), Math.min(H, y1)] as [number, number, number, number], cut };
+  });
+}

package/app/src/scene/depthmap.ts

@@ -0,0 +1,149 @@
+// depthmap: how far every pixel of the painting is, so a person's shadow can follow the steps, climb a wall
+// and drop off a quay instead of lying flat like a sticker. Depth Anything V2 Small (ONNX, on the CPU, ~1 s)
+// gives relative depth; it is pinned to the place's own floor, whose distance we already know from the
+// horizon: on the walk cells the floor is one plane, so its disparity grows with y - horizonY. A robust line
+// through the walk cells turns the model's numbers into "the floor row at the same distance" for every pixel.
+// The runtime unprojects that to 3D (camera height from scale.pxPerY) and casts shadows on it.
+import fs from 'node:fs';
+import path from 'node:path';
+import sharp from 'sharp';
+import type { Geometry } from '../play/geometry.ts';
+// depth.png is half the frame (the model sees ~518 px across, more pixels would only store blur) and holds
+// rows below the horizon * DEPTH_UNIT as R (high byte) and G (low byte).
+import { DEPTH_SCALE, DEPTH_UNIT } from '../play/shadow.ts';
+import { FRAME } from './layout.ts';
+
+const MODEL_URL = 'https://huggingface.co/onnx-community/depth-anything-v2-small/resolve/main/onnx/model_quantized.onnx';
+const MODEL_FILE = path.resolve('.diorama/models/depth-anything-v2-small-q8.onnx');
+/** The model's input: multiples of its 14 px patches, at the frame's aspect. */
+const IN_W = 518;
+const IN_H = 392;
+/** A fit that explains fewer walk cells than this is not the floor: no depth map, shadows stay flat. */
+const MIN_INLIERS = 0.35;
+
+export type DepthMap = { png: Buffer; inliers: number; cells: number };
+
+async function modelPath(): Promise<string> {
+  if (fs.existsSync(MODEL_FILE)) return MODEL_FILE;
+  const res = await fetch(MODEL_URL);
+  if (!res.ok) throw new Error(`depth model: HTTP ${res.status} from ${MODEL_URL}`);
+  fs.mkdirSync(path.dirname(MODEL_FILE), { recursive: true });
+  fs.writeFileSync(`${MODEL_FILE}.part`, Buffer.from(await res.arrayBuffer()));
+  fs.renameSync(`${MODEL_FILE}.part`, MODEL_FILE);
+  return MODEL_FILE;
+}
+
+const disparities = new WeakMap<Buffer, Promise<Float32Array>>();
+
+/** Relative inverse depth (bigger = nearer) at the model's resolution; once per painting buffer. */
+function disparity(painting: Buffer): Promise<Float32Array> {
+  let d = disparities.get(painting);
+  if (!d) disparities.set(painting, (d = runModel(painting)));
+  return d;
+}
+
+async function runModel(painting: Buffer): Promise<Float32Array> {
+  // The WASM build runs in Node with nothing to install natively (no CUDA download on Linux).
+  const ort = await import('onnxruntime-web');
+  const raw = await sharp(painting).resize(IN_W, IN_H, { fit: 'fill' }).removeAlpha().raw().toBuffer();
+  const mean = [0.485, 0.456, 0.406];
+  const std = [0.229, 0.224, 0.225];
+  const n = IN_W * IN_H;
+  const input = new Float32Array(3 * n);
+  for (let i = 0; i < n; i++) for (let c = 0; c < 3; c++) input[c * n + i] = (raw[i * 3 + c] / 255 - mean[c]) / std[c];
+  const session = await ort.InferenceSession.create(new Uint8Array(fs.readFileSync(await modelPath())));
+  const out = await session.run({ [session.inputNames[0]]: new ort.Tensor('float32', input, [1, 3, IN_H, IN_W]) });
+  return out[session.outputNames[0]].data as Float32Array;
+}
+
+/** Bilinear sample of the model's grid at a frame pixel. */
+function sampleAt(d: Float32Array, x: number, y: number): number {
+  const fx = Math.min(IN_W - 1.001, Math.max(0, ((x + 0.5) * IN_W) / FRAME.width - 0.5));
+  const fy = Math.min(IN_H - 1.001, Math.max(0, ((y + 0.5) * IN_H) / FRAME.height - 0.5));
+  const [x0, y0] = [Math.floor(fx), Math.floor(fy)];
+  const [u, v] = [fx - x0, fy - y0];
+  const at = (xx: number, yy: number) => d[yy * IN_W + xx];
+  return (at(x0, y0) * (1 - u) + at(x0 + 1, y0) * u) * (1 - v) + (at(x0, y0 + 1) * (1 - u) + at(x0 + 1, y0 + 1) * u) * v;
+}
+
+/**
+ * The line d = a * (y - horizonY) + b that most walk cells agree with (RANSAC, then least squares on its
+ * inliers). Cells on raised floors or stairs disagree and are left out, so the main floor sets the scale.
+ */
+export function fitFloor(samples: Array<[number, number]>, seed = 1): { a: number; b: number; inliers: number } | undefined {
+  if (samples.length < 8) return undefined;
+  let rnd = seed;
+  const random = () => ((rnd = (rnd * 16807) % 2147483647) / 2147483647);
+  const ds = samples.map((s) => s[1]).sort((p, q) => p - q);
+  const tol = 0.05 * (ds[Math.floor(ds.length * 0.95)] - ds[Math.floor(ds.length * 0.05)] || 1);
+  let best: number[] = [];
+  for (let it = 0; it < 400; it++) {
+    const [p, q] = [samples[Math.floor(random() * samples.length)], samples[Math.floor(random() * samples.length)]];
+    if (Math.abs(p[0] - q[0]) < 40) continue;
+    const a = (q[1] - p[1]) / (q[0] - p[0]);
+    if (a <= 0) continue;
+    const b = p[1] - a * p[0];
+    const inl = samples.flatMap((s, i) => (Math.abs(s[1] - (a * s[0] + b)) < tol ? [i] : []));
+    if (inl.length > best.length) best = inl;
+  }
+  if (best.length < 2) return undefined;
+  let [sx, sy, sxx, sxy] = [0, 0, 0, 0];
+  for (const i of best) {
+    const [x, y] = samples[i];
+    sx += x; sy += y; sxx += x * x; sxy += x * y;
+  }
+  const n = best.length;
+  const a = (n * sxy - sx * sy) / (n * sxx - sx * sx);
+  const b = (sy - a * sx) / n;
+  return a > 0 ? { a, b, inliers: n / samples.length } : undefined;
+}
+
+/** Below this share of the floor's own depth gradient (per screen row), a surface stands upright: a wall, a bridge's side. */
+export const UPRIGHT_SHARE = 0.2;
+
+/**
+ * Where the painting stands upright rather than lies flat, from the depth model alone (before the scale is known):
+ * on a floor the depth changes steadily down the screen, on a wall it hardly does. The floor's own gradient is
+ * read off the floor masks. Derive never lays a walking path across an upright surface (the side of a bridge,
+ * the wall down into an underpass): from above it is thin air.
+ */
+export async function uprightTest(painting: Buffer, floor: { bits: Uint8Array; width: number; height: number }): Promise<(x: number, y: number) => boolean> {
+  const d = await disparity(painting);
+  const S = 24;
+  const grad = (x: number, y: number) => (sampleAt(d, x, y + S / 2) - sampleAt(d, x, y - S / 2)) / S;
+  const at = (x: number, y: number) => floor.bits[Math.min(floor.height - 1, Math.max(0, Math.round(y))) * floor.width + Math.min(floor.width - 1, Math.max(0, Math.round(x)))];
+  const samples: number[] = [];
+  for (let y = S; y < FRAME.height - S; y += 16) for (let x = 8; x < FRAME.width; x += 16) if (at(x, y - S / 2) && at(x, y + S / 2)) samples.push(grad(x, y));
+  if (samples.length < 20) return () => false;
+  samples.sort((a, b) => a - b);
+  const floorGrad = samples[Math.floor(samples.length / 2)];
+  if (!(floorGrad > 0)) return () => false;
+  return (x, y) => {
+    const g = [grad(x - 4, y), grad(x, y), grad(x + 4, y)].sort((a, b) => a - b)[1];
+    return g < UPRIGHT_SHARE * floorGrad;
+  };
+}
+
+/** depth.png for a derived place, or undefined (with the reason) when the model's depth doesn't fit its floor. */
+export async function depthMap(painting: Buffer, g: Geometry): Promise<DepthMap | { reason: string }> {
+  const d = await disparity(painting);
+  const hor = g.scale.horizonY;
+  const samples: Array<[number, number]> = [];
+  g.walk.forEach((row, r) => [...row].forEach((ch, c) => {
+    const [x, y] = [(c + 0.5) * g.cell, (r + 0.5) * g.cell];
+    if (ch === '#' && y > hor + 8) samples.push([y - hor, sampleAt(d, x, y)]);
+  }));
+  const fit = fitFloor(samples);
+  if (!fit || fit.inliers < MIN_INLIERS) return { reason: `depth fits the floor on ${Math.round((fit?.inliers ?? 0) * 100)}% of walk cells` };
+  const [w, h] = [FRAME.width / DEPTH_SCALE, FRAME.height / DEPTH_SCALE];
+  const rgb = Buffer.alloc(w * h * 3);
+  for (let y = 0; y < h; y++) for (let x = 0; x < w; x++) {
+    // The floor row (below the horizon) at this pixel's distance; the sky and the far bank sit just under it.
+    const rows = (sampleAt(d, x * DEPTH_SCALE + 0.5, y * DEPTH_SCALE + 0.5) - fit.b) / fit.a;
+    const v = Math.round(Math.min(65535, Math.max(1, rows * DEPTH_UNIT)));
+    rgb[(y * w + x) * 3] = v >> 8;
+    rgb[(y * w + x) * 3 + 1] = v & 255;
+  }
+  const png = await sharp(rgb, { raw: { width: w, height: h, channels: 3 } }).png({ compressionLevel: 9 }).toBuffer();
+  return { png, inliers: fit.inliers, cells: samples.length };
+}