@gtanczyk/diorama 0.0.0-stage → 0.1.0

package/app/src/scene/backdrop.ts

@@ -0,0 +1,326 @@
+// backdrop: the painting with every card (each sharp standing thing) taken out, for parallax. When the
+// cards slide against the floor, a few px of what is behind them show; Muse paints what is there.
+// Muse repaints the whole picture on an edit, so only the cards' masks (grown, feathered) are taken from it.
+// A card SAM still finds on the backdrop gets a smeared fill of its surroundings instead: blurry beats a ghost.
+// Muse sometimes leaves a card's dark silhouette behind (a ghost SAM does not call by the card's name): that
+// is read from the pixels, the outline darker inside than out, and asked of Muse once more, then smeared.
+// Muse's repaint is brought to the painting's tone around each card before it is pasted, so it leaves no patch.
+import sharp from 'sharp';
+import { emptyMask, maskArea, type Mask } from '../image/mask.ts';
+import { components, dilate, intersect, maskBox, maskIou, subtract, unionAll } from '../image/raster.ts';
+import type { Memo } from '../meta/memo.ts';
+import { museEdit } from '../meta/muse.ts';
+import { samFind } from '../meta/sam.ts';
+import type { Place } from '../world/schema.ts';
+import { FRAME } from './layout.ts';
+import { pasteInside } from './plates.ts';
+
+/** Far more than a card ever slides against the backdrop behind it (see PARALLAX_PX). */
+export const BACKDROP_GROW = 14;
+
+/**
+ * Fills the masked pixels from their surroundings: a pyramid of averages of the known pixels,
+ * read back from the finest level that has any. Smooth, never sharp; fine for a strip of a few px.
+ * With `known`, only those pixels feed the fill.
+ */
+export function smearFill(rgb: Uint8Array, W: number, H: number, hole: Mask, known?: Mask): Uint8Array {
+  type Level = { w: number; h: number; sum: Float32Array; weight: Float32Array };
+  const base: Level = { w: W, h: H, sum: new Float32Array(W * H * 3), weight: new Float32Array(W * H) };
+  for (let i = 0; i < W * H; i++) {
+    // Known pixels: everything outside the hole, or only `known` (to grow a layer from its own pixels).
+    if (known ? !known.bits[i] : hole.bits[i]) continue;
+    base.weight[i] = 1;
+    for (let c = 0; c < 3; c++) base.sum[i * 3 + c] = rgb[i * 3 + c];
+  }
+  const levels = [base];
+  while (levels.at(-1)!.w > 1 || levels.at(-1)!.h > 1) {
+    const f = levels.at(-1)!;
+    const w = Math.max(1, Math.ceil(f.w / 2));
+    const h = Math.max(1, Math.ceil(f.h / 2));
+    const next: Level = { w, h, sum: new Float32Array(w * h * 3), weight: new Float32Array(w * h) };
+    for (let y = 0; y < f.h; y++) for (let x = 0; x < f.w; x++) {
+      const i = y * f.w + x;
+      const j = (y >> 1) * w + (x >> 1);
+      next.weight[j] += f.weight[i];
+      for (let c = 0; c < 3; c++) next.sum[j * 3 + c] += f.sum[i * 3 + c];
+    }
+    levels.push(next);
+  }
+  const out = rgb.slice();
+  for (let i = 0; i < W * H; i++) {
+    if (!hole.bits[i]) continue;
+    let [x, y] = [i % W, Math.floor(i / W)];
+    for (const l of levels) {
+      const j = y * l.w + x;
+      if (l.weight[j] > 0) {
+        for (let c = 0; c < 3; c++) out[i * 3 + c] = l.sum[j * 3 + c] / l.weight[j];
+        break;
+      }
+      [x, y] = [x >> 1, y >> 1];
+    }
+  }
+  return out;
+}
+
+export type Card = { key: string; phrase: string; mask: Mask };
+/** cut: the card leaves a painted part of its thing on the backdrop (see seamOf): the longest run, where. */
+export type Backdrop = { png: Buffer; report: Record<string, { stillThere: boolean; iou: number; ghost?: boolean; /** Taken out on a close crop after the whole-picture edit left it. */ close?: boolean; cut?: { px: number; box: [number, number, number, number] | null } }> };
+
+/** Darker than this (in luma, 0..255) across its outline, a removed card left its silhouette. */
+const GHOST_STEP = 12;
+/** ...and this share of its inside is darker than what Muse painted around it. */
+const GHOST_SHARE = 0.45;
+
+const luma = (rgb: Uint8Array, i: number) => 0.3 * rgb[i * 3] + 0.59 * rgb[i * 3 + 1] + 0.11 * rgb[i * 3 + 2];
+
+/**
+ * Did the removal leave a dark patch the card's shape (its silhouette, or Muse's repaint a shade darker than the
+ * painting around it)? A background that continues does not follow the outline of what was pasted: a ghost
+ * does, darker just inside the pasted area than just outside, all the way round, and most of it darker than
+ * the painting around it. `area`: what was pasted for one card (its mask, grown).
+ */
+export function ghostOf(rgb: Uint8Array, W: number, H: number, area: Mask): { step: number; share: number; ghost: boolean } {
+  const inner = subtract(dilate(area, -2), dilate(area, -8));
+  const outer = subtract(dilate(area, 12), dilate(area, 5));
+  const core = dilate(area, -4);
+  const mean = (m: Mask) => {
+    let [sum, n] = [0, 0];
+    for (let i = 0; i < W * H; i++) if (m.bits[i]) [sum, n] = [sum + luma(rgb, i), n + 1];
+    return n ? sum / n : NaN;
+  };
+  const around: number[] = [];
+  for (let i = 0; i < W * H; i++) if (outer.bits[i]) around.push(luma(rgb, i));
+  around.sort((a, b) => a - b);
+  const median = around[around.length >> 1] ?? 0;
+  let [dark, n] = [0, 0];
+  for (let i = 0; i < W * H; i++) {
+    if (!core.bits[i]) continue;
+    n++;
+    if (luma(rgb, i) < median - 20) dark++;
+  }
+  const step = mean(outer) - mean(inner);
+  const share = n ? dark / n : 0;
+  return { step: Math.round(step), share: Math.round(share * 100) / 100, ghost: n > 200 && step > GHOST_STEP && share > GHOST_SHARE };
+}
+
+/**
+ * Muse repaints the whole picture on an edit, often a shade darker or bluer, and not evenly: pasted back
+ * inside a card's area that shift shows as a patch the card's shape. Where both show the same thing (a ring
+ * just outside the area) the difference is read, carried smoothly into the area, and added to Muse's pixels,
+ * so the repaint meets the painting with no seam.
+ */
+export async function matchTone(edited: Buffer, painting: Buffer, area: Mask): Promise<Buffer> {
+  const { width: W, height: H } = FRAME;
+  const [e, p] = await Promise.all([edited, painting].map(async (b) => new Uint8Array(await sharp(b).resize(W, H, { fit: 'fill' }).removeAlpha().raw().toBuffer())));
+  const ring = subtract(dilate(area, 10), area);
+  // The difference, halved and centred on 128 so it fits a byte; the smear fills it in from the ring.
+  const diff = new Uint8Array(W * H * 3).fill(128);
+  for (let i = 0; i < W * H; i++) if (ring.bits[i]) for (let c = 0; c < 3; c++) diff[i * 3 + c] = 128 + ((p[i * 3 + c] - e[i * 3 + c]) >> 1);
+  // Blurred, as the smear comes in blocks deep inside a big area.
+  const field = new Uint8Array(await sharp(smearFill(diff, W, H, area, ring), { raw: { width: W, height: H, channels: 3 } }).blur(8).raw().toBuffer());
+  const out = Uint8Array.from(e);
+  const reach = dilate(area, 10);
+  for (let i = 0; i < W * H; i++) {
+    if (!reach.bits[i]) continue;
+    for (let c = 0; c < 3; c++) out[i * 3 + c] = Math.max(0, Math.min(255, e[i * 3 + c] + 2 * (field[i * 3 + c] - 128)));
+  }
+  return sharp(out, { raw: { width: W, height: H, channels: 3 } }).png().toBuffer();
+}
+
+/** Outline px of a cut where what is painted outside does not go on inside: as wide a step as this (RGB distance, blurred)... */
+const SEAM_STEP = 28;
+/** ...along a run this long (px of outline), or this share of the whole outline. */
+const SEAM_RUN = 24;
+const SEAM_SHARE = 0.06;
+
+/** px: the longest run cut through, share: of the whole outline; seam: every outline px that steps (to draw). */
+export type Seam = { px: number; share: number; box: [number, number, number, number] | null; cut: boolean; seam: Mask };
+
+/**
+ * Was a thing cut short? When its mask leaves out a painted part of it (fish heads over a bucket's rim), Muse
+ * takes the whole thing out, but the backdrop is pasted only up to the mask (grown) and cuts that part through:
+ * just outside the pasted area the painting shows the part, just inside the backdrop shows what is behind.
+ * Read along the outline of `area` (what was pasted for one card), an outline px is cut where (1) the step
+ * across it in the backdrop, less the step the painting itself takes further out (texture that changes anyway),
+ * is large, and (2) Muse's edit took out what the painting shows just outside too: it was part of what Muse
+ * removed. (2) tells a cut part from a wall Muse continued a little out of line. Outline px inside `pasted` (other
+ * cards' areas) are no outline; below `floorY` (where the thing meets the floor) it is not read: parts stick out
+ * at the top and the sides, and a floor continued a little differently steps there with nothing cut.
+ */
+export function seamOf(
+  img: { painting: Uint8Array; backdrop: Uint8Array; edited: Uint8Array },
+  W: number,
+  H: number,
+  area: Mask,
+  { pasted, floorY = H }: { pasted?: Mask; floorY?: number } = {},
+): Seam {
+  // Colour, not luma: a dark barrel on dark cobbles differs in hue.
+  const channels = (rgb: Uint8Array) => [0, 1, 2].map((c) => {
+    const out = new Float32Array(W * H);
+    for (let i = 0; i < W * H; i++) out[i] = rgb[i * 3 + c];
+    return blurred(out, W, H, 3);
+  });
+  const [P, B, E] = [channels(img.painting), channels(img.backdrop), channels(img.edited)];
+  // Which way is in, from the area blurred: its gradient points into it.
+  const a = new Float32Array(W * H);
+  for (let i = 0; i < W * H; i++) a[i] = area.bits[i];
+  const soft = blurred(a, W, H, 4);
+  const edge = subtract(dilate(area, 1), area);
+  const idx = (x: number, y: number) => Math.min(H - 1, Math.max(0, Math.round(y))) * W + Math.min(W - 1, Math.max(0, Math.round(x)));
+  const dist = (p: Float32Array[], i: number, q: Float32Array[], j: number) => Math.hypot(p[0][i] - q[0][j], p[1][i] - q[1][j], p[2][i] - q[2][j]);
+  const seam = emptyMask(W, H);
+  let outline = 0;
+  for (let i = 0; i < W * H; i++) {
+    if (!edge.bits[i] || pasted?.bits[i]) continue;
+    const [x, y] = [i % W, Math.floor(i / W)];
+    if (x < 2 || y < 2 || x >= W - 2 || y >= H - 2 || y > floorY) continue;
+    let [nx, ny] = [soft[idx(x + 1, y)] - soft[idx(x - 1, y)], soft[idx(x, y + 1)] - soft[idx(x, y - 1)]];
+    const len = Math.hypot(nx, ny);
+    if (len < 1e-4) continue;
+    [nx, ny] = [nx / len, ny / len];
+    outline++;
+    const inside = idx(x + 12 * nx, y + 12 * ny);
+    const out = idx(x - 4 * nx, y - 4 * ny);
+    const further = idx(x - 14 * nx, y - 14 * ny);
+    // A lamp's painted glow round a thing is light, not a part of it: where the painting just outside is bright
+    // and warm, a step there is the glow Muse took out with the lamp, and brushing it onto the card left a disc.
+    const glow = 0.3 * P[0][out] + 0.59 * P[1][out] + 0.11 * P[2][out] > 150 && P[0][out] - P[2][out] > 50;
+    if (!glow && dist(B, inside, P, out) - 0.5 * dist(P, further, P, out) > SEAM_STEP && dist(E, out, P, out) > SEAM_STEP) seam.bits[i] = 1;
+  }
+  // Runs along the outline: seam px a few apart joined (a fish tail between two heads still steps a little).
+  const runs = components(dilate(seam, 5)).map((c) => ({ c, px: maskArea(intersect(c, seam)) }));
+  const longest = runs.reduce((best, r) => (r.px > (best?.px ?? 0) ? r : best), undefined as (typeof runs)[number] | undefined);
+  const px = longest?.px ?? 0;
+  const share = outline ? px / outline : 0;
+  return { px, share: Math.round(share * 100) / 100, box: longest ? maskBox(longest.c) ?? null : null, cut: px >= SEAM_RUN && share >= SEAM_SHARE, seam };
+}
+
+/** A box blur three times over (close to a gaussian of about `r`). */
+function blurred(img: Float32Array, W: number, H: number, r: number): Float32Array {
+  let cur = img;
+  for (let pass = 0; pass < 3; pass++) {
+    for (const along of ['x', 'y'] as const) {
+      const out = new Float32Array(W * H);
+      const [lines, len, step, lineStep] = along === 'x' ? [H, W, 1, W] : [W, H, W, 1];
+      for (let l = 0; l < lines; l++) {
+        const base = l * lineStep;
+        let sum = 0;
+        for (let i = -r; i <= r; i++) sum += cur[base + Math.min(len - 1, Math.max(0, i)) * step];
+        for (let i = 0; i < len; i++) {
+          out[base + i * step] = sum / (2 * r + 1);
+          sum += cur[base + Math.min(len - 1, i + r + 1) * step] - cur[base + Math.max(0, i - r) * step];
+        }
+      }
+      cur = out;
+    }
+  }
+  return cur;
+}
+
+async function ghostsIn(png: Buffer, cards: Card[]): Promise<Card[]> {
+  const { width: W, height: H } = FRAME;
+  const rgb = new Uint8Array(await sharp(png).resize(W, H, { fit: 'fill' }).removeAlpha().raw().toBuffer());
+  return cards.filter((card) => ghostOf(rgb, W, H, dilate(card.mask, BACKDROP_GROW)).ghost);
+}
+
+export async function backdropFor(memo: Memo, place: Place, cards: Card[], painting: Buffer): Promise<Backdrop> {
+  const { width: W, height: H } = FRAME;
+  const area = dilate(unionAll(W, H, cards.map((c) => c.mask)), BACKDROP_GROW);
+  const names = cards.map((c) => `the ${c.phrase}`);
+  const list = names.length > 1 ? `${names.slice(0, -1).join(', ')} and ${names.at(-1)}` : names[0];
+  const prompt = `Remove ${list} from this painting. Paint what is behind ${cards.length > 1 ? 'them' : 'it'} where ${cards.length > 1 ? 'they were' : 'it was'}: continue the floor, the walls and the background, in the same style and light. Leave everything else exactly as it is. No people, no text.`;
+  const edited = await museEdit(memo, { label: `backdrop:${place.key}`, prompt, images: [painting] });
+  let png = await pasteInside(painting, await matchTone(edited, painting, area), area);
+  const report: Backdrop['report'] = {};
+  const leftover: Mask[] = [];
+  // Is each still there? SAM by the same phrase, inside the area the backdrop replaced; all cards at once.
+  const seen = await Promise.all(cards.map((card) => samFind(memo, { label: `backdrop-check:${place.key}/${card.key}`, image: png, prompt: card.phrase })));
+  cards.forEach((card, i) => {
+    const found = seen[i];
+    const iou = Math.max(0, ...found.map((f) => maskIou(intersect(f.mask, area), card.mask)));
+    const left = found.reduce((px, f) => px + maskArea(intersect(f.mask, card.mask)), 0);
+    const stillThere = iou > 0.4 || left > 0.4 * maskArea(card.mask);
+    report[card.key] = { stillThere, iou: Math.round(iou * 100) / 100 };
+    if (stillThere) leftover.push(dilate(card.mask, BACKDROP_GROW));
+  });
+  // A card Muse left in: asked once more on a close crop round it (Muse sees it big, and the rest of the picture
+  // cannot pull it back in), pasted inside its area; only what SAM still finds after that is smeared.
+  const stuck = cards.filter((c) => report[c.key].stillThere);
+  if (stuck.length) {
+    leftover.length = 0;
+    for (const card of stuck) {
+      const fixed = await removeClose(memo, place, card, png);
+      if (fixed) {
+        const grown = dilate(card.mask, BACKDROP_GROW);
+        const candidate = await pasteInside(png, await matchTone(fixed, png, grown), grown);
+        const again = await samFind(memo, { label: `backdrop-check-close:${place.key}/${card.key}`, image: candidate, prompt: card.phrase });
+        const iou = Math.max(0, ...again.map((f) => maskIou(intersect(f.mask, area), card.mask)));
+        const left = again.reduce((px, f) => px + maskArea(intersect(f.mask, card.mask)), 0);
+        if (!(iou > 0.4 || left > 0.4 * maskArea(card.mask))) {
+          png = candidate;
+          report[card.key] = { stillThere: false, iou: Math.round(iou * 100) / 100, close: true };
+          continue;
+        }
+      }
+      leftover.push(dilate(card.mask, BACKDROP_GROW));
+    }
+  }
+  // A ghost left where a card was taken out: Muse once more on those, then the smear for any still there.
+  let ghosts = await ghostsIn(png, cards.filter((c) => !report[c.key].stillThere));
+  if (ghosts.length) {
+    const named = ghosts.map((c) => `the ${c.phrase}`).join(', ');
+    const again = await museEdit(memo, {
+      label: `backdrop-ghost:${place.key}`,
+      prompt: `A dark silhouette is left in this painting where ${named} stood. Paint over it what is behind: continue the background, the water, the floor and the light around it, with no shape, shadow or outline of ${ghosts.length > 1 ? 'them' : 'it'}. Leave everything else exactly as it is. No people, no text.`,
+      images: [png],
+    }).catch(() => undefined);
+    const redo = dilate(unionAll(W, H, ghosts.map((c) => c.mask)), BACKDROP_GROW);
+    if (again) png = await pasteInside(png, await matchTone(again, png, redo), redo);
+    const still = new Set((await ghostsIn(png, ghosts)).map((c) => c.key));
+    for (const c of ghosts) report[c.key].ghost = true;
+    ghosts = ghosts.filter((c) => still.has(c.key));
+    for (const c of ghosts) leftover.push(dilate(c.mask, BACKDROP_GROW));
+  }
+  if (leftover.length) png = await pasteInside(png, await smearedBackdrop(cards, painting), unionAll(W, H, leftover), 2);
+  // The gate: a card whose outline cuts through a part of its thing (its mask left that part out).
+  const raw = async (b: Buffer) => new Uint8Array(await sharp(b).resize(W, H, { fit: 'fill' }).removeAlpha().raw().toBuffer());
+  const img = { painting: await raw(painting), backdrop: await raw(png), edited: await raw(edited) };
+  for (const card of cards) {
+    const box = maskBox(card.mask);
+    if (!box) continue;
+    const pasted = unionAll(W, H, cards.filter((c) => c !== card).map((c) => dilate(c.mask, BACKDROP_GROW)));
+    const seam = seamOf(img, W, H, dilate(card.mask, BACKDROP_GROW), { pasted, floorY: box[3] - 0.15 * (box[3] - box[1]) });
+    if (seam.cut) report[card.key].cut = { px: seam.px, box: seam.box };
+  }
+  return { png, report };
+}
+
+/**
+ * One card taken out on a close crop: the card's box with room round it, scaled up to Muse's size, edited, and
+ * scaled back into the whole picture. Undefined when Muse fails (the smear stays the fallback).
+ */
+async function removeClose(memo: Memo, place: Place, card: Card, png: Buffer): Promise<Buffer | undefined> {
+  const { width: W, height: H } = FRAME;
+  const box = maskBox(card.mask);
+  if (!box) return undefined;
+  const [x0, y0, x1, y1] = box;
+  const side = Math.min(Math.max(W, H), Math.max(x1 - x0, y1 - y0) * 2.2);
+  const w = Math.round(Math.min(W, side * 1.34)), h = Math.round(Math.min(H, side));
+  const left = Math.round(Math.min(W - w, Math.max(0, (x0 + x1) / 2 - w / 2))), top = Math.round(Math.min(H - h, Math.max(0, (y0 + y1) / 2 - h / 2)));
+  const crop = await sharp(png).resize(W, H, { fit: 'fill' }).extract({ left, top, width: w, height: h }).resize(1200, 896, { fit: 'fill' }).png().toBuffer();
+  const edited = await museEdit(memo, {
+    label: `backdrop-close:${place.key}/${card.key}`, images: [crop],
+    prompt: `Remove the ${card.phrase} from this picture completely. Paint what is behind it where it was: continue the water, the planks, the stones, the walls and the light around it, with no trace, shadow, reflection or outline of it. Leave everything else exactly as it is, same painted style. No people, no text.`,
+  }).catch(() => undefined);
+  if (!edited) return undefined;
+  const back = await sharp(edited).resize(w, h, { fit: 'fill' }).png().toBuffer();
+  return sharp(png).resize(W, H, { fit: 'fill' }).composite([{ input: back, left, top }]).png().toBuffer();
+}
+
+/** The smeared fill alone, for a place without a key or when Muse is not wanted. */
+export async function smearedBackdrop(cards: Card[], painting: Buffer): Promise<Buffer> {
+  const { width: W, height: H } = FRAME;
+  const area = dilate(unionAll(W, H, cards.map((c) => c.mask)), BACKDROP_GROW);
+  const rgb = new Uint8Array(await sharp(painting).resize(W, H, { fit: 'fill' }).removeAlpha().raw().toBuffer());
+  return sharp(smearFill(rgb, W, H, area), { raw: { width: W, height: H, channels: 3 } }).png().toBuffer();
+}

package/app/src/scene/blockin.ts

@@ -0,0 +1,189 @@
+// blockin.png: the layout as a shaded sketch. Muse keeps where things are and paints what they are.
+// Flat soft boxes told Muse only "something goes here", so it moved and resized them freely: a stage floor
+// with a vanishing grid, a silhouette per kind of thing with its top lit and its shadow on the floor say
+// how deep each thing stands and what it roughly is (ootb measure 10-28: median IoU of painted things
+// against the plan was 0.25 with boxes).
+import sharp from 'sharp';
+import type { Place } from '../world/schema.ts';
+
+type Thing = Place['things'][number];
+import { FRAME, type Layout, type Shape } from './layout.ts';
+
+type Box = Layout['things'][number];
+
+const WORDS: [Shape, RegExp][] = [
+  ['steps', /\b(steps?|stairs?|staircase|stairway|ladder)(e?s)?\b/],
+  ['flat', /\b(rug|carpet|water|pond|puddle|pool|tracks?|rails?|path|road|mat|stream|canal|river)(e?s)?\b/],
+  ['arch', /\b(door|doorway|gate|arch|archway|tunnel|portal|entrance|passage|window)(e?s)?\b/],
+  ['tree', /\b(tree|willow|oak|pine|birch|palm|grove|orchard)(e?s)?\b/],
+  ['post', /\b(lamp|lantern post|street ?light|post|pole|sign ?post|mast|torch|column|pillar|chimney)(e?s)?\b/],
+  ['round', /\b(barrel|pot|jar|urn|bucket|basket|rock|boulder|bush|shrub|reeds?|well|cauldron|sack|haystack|pumpkin|crock)(e?s)?\b/],
+  ['low', /\b(bench|boat|cart|bed|sofa|couch|table|desk|trough|log|bridge|crate|chest|box)(e?s)?\b/],
+];
+
+/**
+ * Layouts laid out before Spark named shapes (or a shape it left out) get one from the thing's words:
+ * its key first ("curving-tram-tracks" lie flat though their looks run "into a dark arch"), then its looks.
+ */
+export function shapeOf(box: Box, thing?: Thing): Shape {
+  if (box.shape) return box.shape;
+  for (const words of [box.key.replace(/-/g, ' '), (thing?.looks ?? '').toLowerCase()])
+    for (const [shape, re] of WORDS) if (re.test(words)) return shape;
+  return box.height > 2.5 * box.width ? 'post' : box.width > 2 * box.height ? 'low' : 'block';
+}
+
+const hex = (c: string) => [1, 3, 5].map((i) => parseInt(c.slice(i, i + 2), 16));
+/** k > 0 towards white, k < 0 towards black. */
+export function shade(colour: string, k: number): string {
+  const to = k > 0 ? 255 : 0;
+  return `#${hex(colour).map((v) => Math.round(v + (to - v) * Math.abs(k)).toString(16).padStart(2, '0')).join('')}`;
+}
+
+/** How much of a thing's top the stage camera sees: none at the horizon, more the nearer it stands. */
+const topOf = (layout: Layout, b: Box) => Math.max(4, Math.min(0.35 * b.height, 0.22 * b.width * ((b.footY - layout.horizon) / (FRAME.height - layout.horizon))));
+
+/** One thing as a silhouette standing on the floor: body, lit top, a dark rim, its shadow first. */
+function thingSvg(layout: Layout, b: Box, shape: Shape, i: number): string {
+  const { x, footY: y, width: w, height: h, colour: c } = b;
+  const l = x - w / 2, r = x + w / 2, t = y - h;
+  const rim = `stroke="${shade(c, -0.55)}" stroke-width="3" stroke-linejoin="round"`;
+  const body = `fill="url(#b${i})" ${rim}`;
+  const lit = `fill="${shade(c, 0.32)}" ${rim}`;
+  const dark = shade(c, -0.6);
+  const grad = `<linearGradient id="b${i}" x1="0" y1="0" x2="0" y2="1"><stop offset="0" stop-color="${shade(c, 0.12)}"/><stop offset="1" stop-color="${shade(c, -0.28)}"/></linearGradient>`;
+  const shadow = shape === 'flat' ? '' : `<ellipse cx="${x + w * 0.06}" cy="${y}" rx="${w * 0.62}" ry="${Math.max(6, w * 0.11)}" fill="#000" fill-opacity="0.38" filter="url(#soft)"/>`;
+  const top = topOf(layout, b);
+  const parts: string[] = [];
+  switch (shape) {
+    case 'flat': {
+      // Lies on the floor: a squashed oval, seen from above.
+      const d = Math.max(10, Math.min(h, w * 0.35));
+      parts.push(`<ellipse cx="${x}" cy="${y - d / 2}" rx="${w / 2}" ry="${d / 2}" fill="${c}" ${rim}/>`, `<ellipse cx="${x - w * 0.08}" cy="${y - d * 0.62}" rx="${w * 0.3}" ry="${d * 0.18}" fill="${shade(c, 0.25)}"/>`);
+      break;
+    }
+    case 'post': {
+      const pole = Math.max(8, w * 0.22), head = Math.min(w, h * 0.3);
+      parts.push(`<rect x="${x - pole / 2}" y="${t + head * 0.7}" width="${pole}" height="${h - head * 0.7}" ${body}/>`,
+        `<rect x="${x - pole}" y="${y - pole * 0.8}" width="${pole * 2}" height="${pole * 0.8}" ${body}/>`,
+        `<rect x="${x - head / 2}" y="${t}" width="${head}" height="${head * 0.9}" rx="${head * 0.2}" ${lit}/>`);
+      break;
+    }
+    case 'tree': {
+      const trunk = Math.max(10, w * 0.16);
+      parts.push(`<rect x="${x - trunk / 2}" y="${t + h * 0.45}" width="${trunk}" height="${h * 0.55}" fill="${dark}" ${rim}/>`,
+        `<ellipse cx="${x}" cy="${t + h * 0.33}" rx="${w / 2}" ry="${h * 0.33}" ${body}/>`,
+        `<ellipse cx="${x - w * 0.12}" cy="${t + h * 0.22}" rx="${w * 0.26}" ry="${h * 0.14}" fill="${shade(c, 0.25)}"/>`);
+      break;
+    }
+    case 'arch': {
+      // A frame with a dark opening: a door, gate or passage leads somewhere.
+      const ro = w / 2, fw = Math.max(8, w * 0.16);
+      const ri = ro - fw;
+      parts.push(`<path d="M${l},${y} L${l},${t + ro} A${ro},${ro} 0 0 1 ${r},${t + ro} L${r},${y} Z" ${body}/>`,
+        `<path d="M${l + fw},${y} L${l + fw},${t + ro + fw * 0.4} A${ri},${ri} 0 0 1 ${r - fw},${t + ro + fw * 0.4} L${r - fw},${y} Z" fill="${dark}"/>`);
+      break;
+    }
+    case 'round': {
+      const e = Math.max(6, Math.min(top, w * 0.18));
+      parts.push(`<path d="M${l},${t + e} L${l},${y - e} A${w / 2},${e} 0 0 0 ${r},${y - e} L${r},${t + e} Z" ${body}/>`,
+        `<ellipse cx="${x}" cy="${t + e}" rx="${w / 2}" ry="${e}" ${lit}/>`);
+      break;
+    }
+    case 'steps': {
+      const n = 4, rise = h / n;
+      for (let s = 0; s < n; s++) {
+        const inset = (w * 0.12 * s) / n;
+        const sy = y - (s + 1) * rise;
+        parts.push(`<rect x="${l + inset}" y="${sy}" width="${w - 2 * inset}" height="${rise}" fill="${shade(c, -0.25 + 0.12 * s)}" ${rim}/>`,
+          `<rect x="${l + inset + 3}" y="${sy + 2}" width="${w - 2 * inset - 6}" height="${Math.max(3, rise * 0.25)}" fill="${shade(c, 0.3)}"/>`);
+      }
+      break;
+    }
+    default: {
+      // block and low: a box with its top seen from a little above.
+      const slant = top * 0.5;
+      parts.push(`<rect x="${l}" y="${t + top}" width="${w}" height="${h - top}" ${body}/>`,
+        `<path d="M${l},${t + top} L${l + slant},${t} L${r - slant},${t} L${r},${t + top} Z" ${lit}/>`);
+    }
+  }
+  return `<defs>${grad}</defs>${shadow}${parts.join('')}`;
+}
+
+/** A floor tiled towards the vanishing point, so it reads as ground going back, not a coloured band. */
+function floorSvg(layout: Layout, f: Layout['floors'][number], i: number): string {
+  const { width: W, height: H } = FRAME;
+  const vx = W / 2, vy = layout.horizon;
+  const pts = f.outline.map((p) => p.join(',')).join(' ');
+  const lines: string[] = [];
+  for (let k = -8; k <= 8; k++) lines.push(`M${vx},${vy} L${vx + k * W * 0.32},${H + 400}`);
+  // Rows close up towards the horizon as perspective does.
+  for (let d = 1; d <= 14; d++) {
+    const y = vy + (H - vy) * (1 / (1 + 0.45 * (14 - d))) ;
+    lines.push(`M0,${y} L${W},${y}`);
+  }
+  return `<clipPath id="f${i}"><polygon points="${pts}"/></clipPath>
+<polygon points="${pts}" fill="${f.colour}"/>
+<path d="${lines.join(' ')}" stroke="${shade(f.colour, -0.35)}" stroke-width="2" stroke-opacity="0.35" fill="none" clip-path="url(#f${i})"/>
+<polygon points="${pts}" fill="none" stroke="${shade(f.colour, 0.3)}" stroke-width="3" stroke-opacity="0.8"/>`;
+}
+
+/** NPCs get a spot in the layout but no mass: a character painted into the background can't move. */
+export function blockinSvg(layout: Layout, place: Place): string {
+  const { width: W, height: H } = FRAME;
+  const npcs = new Set(place.things.filter((t) => t.role === 'npc').map((t) => t.key));
+  const things = layout.things
+    .filter((t) => !npcs.has(t.key))
+    .sort((a, b) => a.footY - b.footY) // far first, near on top
+    .map((t, i) => thingSvg(layout, t, shapeOf(t, place.things.find((p) => p.key === t.key)), i));
+  const glows = layout.lights.map((l, i) => `<radialGradient id="g${i}"><stop offset="0" stop-color="${l.colour}" stop-opacity="0.9"/><stop offset="0.25" stop-color="${l.colour}" stop-opacity="0.5"/><stop offset="1" stop-color="${l.colour}" stop-opacity="0"/></radialGradient><circle cx="${l.x}" cy="${l.y}" r="80" fill="url(#g${i})"/><circle cx="${l.x}" cy="${l.y}" r="9" fill="#fffbe8"/>`);
+  return `<svg xmlns="http://www.w3.org/2000/svg" width="${W}" height="${H}">
+<defs><filter id="soft"><feGaussianBlur stdDeviation="6"/></filter>
+<linearGradient id="sky" x1="0" y1="0" x2="0" y2="1"><stop offset="0" stop-color="${shade(layout.sky, -0.25)}"/><stop offset="1" stop-color="${shade(layout.sky, 0.18)}"/></linearGradient>
+<linearGradient id="ground" x1="0" y1="0" x2="0" y2="1"><stop offset="0" stop-color="${shade(layout.ground, 0.12)}"/><stop offset="1" stop-color="${shade(layout.ground, -0.3)}"/></linearGradient></defs>
+<rect width="${W}" height="${layout.horizon}" fill="url(#sky)"/>
+<rect y="${layout.horizon}" width="${W}" height="${H - layout.horizon}" fill="url(#ground)"/>
+<path d="M0,${layout.horizon} L${W},${layout.horizon}" stroke="${shade(layout.ground, -0.45)}" stroke-width="3"/>
+${[...layout.floors.map((f, i) => floorSvg(layout, f, i)), ...things, ...glows].join('\n')}
+</svg>`;
+}
+
+/** Barely softened: the silhouettes are meant to be followed, only their hard pixel edges are not. */
+export function renderBlockin(layout: Layout, place: Place): Promise<Buffer> {
+  return sharp(Buffer.from(blockinSvg(layout, place))).blur(1.5).png().toBuffer();
+}
+
+const esc = (s: string) => s.replace(/&/g, '&amp;').replace(/</g, '&lt;').replace(/>/g, '&gt;');
+
+/**
+ * plan.png, for people: the block-in with every thing and floor named, as a plan sheet reads. Muse never sees
+ * it (it would paint the words); Studio and the film show it, so the shapes read as a deliberate plan.
+ */
+export function planSvg(layout: Layout, place: Place, names: Record<string, string> = {}): string {
+  const { width: W, height: H } = FRAME;
+  const name = (key: string) => esc(names[key] ?? key.replace(/-/g, ' '));
+  const tag = (x: number, y: number, text: string, big: boolean) => {
+    const size = big ? 26 : 21, w = text.length * size * 0.56 + 24;
+    const cx = Math.min(W - w / 2 - 6, Math.max(w / 2 + 6, x));
+    return `<g font-family="Inter, Helvetica, Arial, sans-serif" font-size="${size}" font-weight="${big ? 600 : 500}">
+<rect x="${cx - w / 2}" y="${y - size - 4}" width="${w}" height="${size + 14}" rx="${(size + 14) / 2}" fill="${big ? '#fffaf0' : '#1d1b22'}" fill-opacity="${big ? 0.94 : 0.72}"/>
+<text x="${cx}" y="${y + 1}" text-anchor="middle" fill="${big ? '#2a2420' : '#f4efe6'}"${big ? '' : ' font-style="italic"'}>${text}</text></g>`;
+  };
+  const npcs = new Set(place.things.filter((t) => t.role === 'npc').map((t) => t.key));
+  const things = layout.things.map((t) => {
+    if (npcs.has(t.key)) {
+      // A person is painted later, on their own: a dashed spot where they will stand.
+      return `<ellipse cx="${t.x}" cy="${t.footY}" rx="${t.width / 2}" ry="${Math.max(8, t.width * 0.14)}" fill="none" stroke="#fffaf0" stroke-width="3" stroke-dasharray="8 7"/>${tag(t.x, t.footY - 14, name(t.key), true)}`;
+    }
+    const top = shapeOf(t, place.things.find((p) => p.key === t.key)) === 'flat' ? t.footY - Math.max(10, Math.min(t.height, t.width * 0.35)) : t.footY - t.height;
+    return `<line x1="${t.x}" y1="${top - 6}" x2="${t.x}" y2="${top + 10}" stroke="#fffaf0" stroke-width="2"/>${tag(t.x, Math.max(30, top - 12), name(t.key), true)}`;
+  });
+  const floors = layout.floors.map((f) => {
+    const xs = f.outline.map(([x]) => x), ys = f.outline.map(([, y]) => y);
+    return tag((Math.min(...xs) + Math.max(...xs)) / 2, Math.max(...ys) - 18, name(f.key), false);
+  });
+  return `<svg xmlns="http://www.w3.org/2000/svg" width="${W}" height="${H}">${[...floors, ...things].join('\n')}</svg>`;
+}
+
+export function renderPlan(layout: Layout, place: Place, blockin: Buffer, names?: Record<string, string>): Promise<Buffer> {
+  return sharp(blockin).resize(FRAME.width, FRAME.height, { fit: 'fill' }).composite([{ input: Buffer.from(planSvg(layout, place, names)) }]).png().toBuffer();
+}

package/app/src/scene/brief.ts

@@ -0,0 +1,74 @@
+// brief.json: Spark describes how each thing and floor should look before Muse paints it.
+// Muse turns undescribed masses into flat walls and posts, so every mass gets a concrete look.
+import { z } from 'zod';
+import type { Memo } from '../meta/memo.ts';
+import { askSparkJson } from '../meta/spark.ts';
+import type { Place, World } from '../world/schema.ts';
+import type { Layout } from './layout.ts';
+import { SAFE_WORDS } from './words.ts';
+
+export const Brief = z.object({
+  looks: z.record(z.string(), z.string().min(10)),
+  /** Things that move in the game (water, fire, smoke, animals); painted still, animated later. */
+  alive: z.array(z.object({ key: z.string(), as: z.enum(['water', 'fire', 'smoke', 'animal', 'cloth', 'light']) })),
+});
+export type Brief = z.infer<typeof Brief>;
+
+/** Animals that are things of the place: painted into the background, so the painter must not leave them out. */
+export const paintedAnimals = (brief?: Brief) => (brief?.alive ?? []).filter((a) => a.as === 'animal').map((a) => a.key);
+
+export function briefIssues(brief: Brief, place: Place, layout: Layout): string[] {
+  const keys = [...place.things.filter((t) => t.role !== 'npc').map((t) => t.key), ...layout.floors.map((f) => f.key)];
+  return keys.filter((k) => !brief.looks[k]).map((k) => `no look for ${k}`);
+}
+
+/** The last word of a cast member's key names what it is ("harbour-cat" → cat, "old-ferryman" → ferryman). */
+const castWord = (key: string) => key.split('-').filter((w) => w.length >= 3).pop();
+
+/**
+ * Looks that paint one of the place's cast into a thing ("a basket holding a curled-up ginger cat" with a cat
+ * in the cast): the game draws its cast over the picture, so that one would be a double. Only when the thing's
+ * own words did not name it (its key does not count: a cat-basket is the cat's bed, not a cat).
+ */
+export function castInLooks(brief: Brief, place: Place): string[] {
+  const cast = place.things.filter((t) => t.role === 'npc').map((t) => castWord(t.key)).filter((w): w is string => !!w);
+  const out: string[] = [];
+  for (const t of place.things.filter((t) => t.role !== 'npc')) {
+    const look = brief.looks[t.key];
+    if (!look) continue;
+    for (const w of cast) {
+      const named = new RegExp(`\\b${w}s?\\b`, 'i');
+      if (named.test(look) && !named.test(t.looks ?? '')) out.push(`${t.key}'s look paints the ${w}, who is in the cast: leave the ${w} out of it`);
+    }
+  }
+  return out;
+}
+
+export async function writeBrief(memo: Memo, world: World, place: Place, layout: Layout): Promise<Brief> {
+  const cast = place.things.filter((t) => t.role === 'npc').map((t) => `${t.key}${t.looks ? ` (${t.looks})` : ''}`);
+  const subjects = [
+    ...place.things.filter((t) => t.role !== 'npc').map((t) => `${t.key}: ${t.looks} (${t.role})`),
+    ...layout.floors.map((f) => `${f.key}: ${f.is} (floor)`),
+  ];
+  const prompt = `You brief a painter for one background of a 2D point-and-click adventure.
+
+Style: ${world.style}
+Place: ${place.shows}
+Mood: ${place.mood}
+
+For each subject below write its look in one sentence: shape, material, surface texture, wear, colour in this light. Be concrete enough that nobody could paint it as a plain block.
+${subjects.map((s) => `- ${s}`).join('\n')}
+
+${SAFE_WORDS}
+
+Also list which subjects move in the game: water that flows, fire, smoke, animals, cloth in wind, flickering light. They are painted still and animated later, so their look must describe them at rest. An animal among the subjects is painted in, standing still, lying or grazing: describe the animal itself, never an empty spot (nobody else will draw it).
+
+${cast.length ? `The game draws these characters over the picture itself, walking and talking: ${cast.join('; ')}. Never paint them, nor one like them, into any subject's look (no cat curled in a basket that is in this list), even where the place's words put them there: the subject is shown empty, ready for them.\n\n` : ''}Reply with JSON only: {"looks": {"<key>": "..."}, "alive": [{"key": "<key>", "as": "water | fire | smoke | animal | cloth | light"}]}`;
+  let brief = Brief.parse(await askSparkJson(memo, { label: `brief:${place.key}`, prompt }));
+  // A cast member painted into a thing anyway: asked once more, saying which.
+  const doubles = castInLooks(brief, place);
+  if (doubles.length) brief = Brief.parse(await askSparkJson(memo, { label: `brief:${place.key}:no-cast`, prompt: `${prompt}\n\nAn earlier answer did this, which is wrong: ${doubles.join('; ')}.` }));
+  const issues = briefIssues(brief, place, layout);
+  if (issues.length) throw new Error(`brief for ${place.key}: ${issues.join('; ')}`);
+  return brief;
+}