@gtanczyk/diorama 0.0.0-stage → 0.2.0

package/app/src/scene/paint-layout.ts

@@ -0,0 +1,135 @@
+// Check composition before SAM/derive can turn a moved path into plausible but wrong geometry.
+import sharp from 'sharp';
+import { z } from 'zod';
+import type { Memo } from '../meta/memo.ts';
+import { inputHash } from '../meta/memo.ts';
+import { museEdit } from '../meta/muse.ts';
+import { askSparkJson } from '../meta/spark.ts';
+import type { Place } from '../world/schema.ts';
+import type { Brief } from './brief.ts';
+import { FRAME, type Layout } from './layout.ts';
+
+const Box = z.tuple([z.number().min(0).max(FRAME.width), z.number().min(0).max(FRAME.height), z.number().min(0).max(FRAME.width), z.number().min(0).max(FRAME.height)])
+  .refine(([l, t, r, b]) => r > l && b > t, 'box must have positive area');
+export const PaintedLayout = z.object({
+  horizon: z.number().min(0).max(FRAME.height),
+  people: z.boolean(),
+  things: z.array(z.object({ key: z.string(), box: Box.nullable(), approach: z.boolean(), why: z.string() })),
+  floors: z.array(z.object({ key: z.string(), coverage: z.number().min(0).max(1), connected: z.boolean(), reachesEdges: z.boolean(), why: z.string() })),
+});
+export type PaintedLayout = z.infer<typeof PaintedLayout>;
+
+export const paintingLayoutHash = (place: Place, layout: Layout, blockin: Buffer, painting: Buffer) =>
+  // Removing a decorative click target without painting it out is a supported masks-only edit.
+  // Its planned geometry and the pixels stay checked; changes to story targets still invalidate the report.
+  inputHash('paint-layout-v1', JSON.stringify({ key: place.key, shows: place.shows, mood: place.mood, targets: place.things.filter((t) => t.role !== 'prop') }), JSON.stringify(layout), blockin, painting);
+
+/** Numeric plan, shared by painting, comparison and repair. Decorative silhouette details may change. */
+export function layoutContract(place: Place, layout: Layout): string {
+  const boxes = layout.things.filter((b) => place.things.some((t) => t.key === b.key && t.role !== 'npc'))
+    .map((b) => `- ${b.key}: box [${Math.round(b.x - b.width / 2)}, ${Math.round(b.footY - b.height)}, ${Math.round(b.x + b.width / 2)}, ${Math.round(b.footY)}]`);
+  const floors = layout.floors.map((f) => `- ${f.key}: ${f.is}; polygon ${JSON.stringify(f.outline)}`);
+  return `Composition contract (${FRAME.width}x${FRAME.height} pixels, x right, y down):
+Horizon y=${layout.horizon}. Keep this camera and perspective, not just the scene's subject.
+Object boxes [left, top, right, bottom]:
+${boxes.join('\n')}
+Walkable floor polygons:
+${floors.join('\n')}
+Keep the connected walking surfaces, their joins and their frame-edge exits in these positions. A floor must remain solid usable ground, never water, a drop, a railing or an obstacle. Leave an approach on connected ground beside every item and exit. Do not replace a nearby interactive object with a distant decorative copy. Do not invent extra bridges to compensate for moving objects. The first image controls composition; other references control appearance only.`;
+}
+
+/**
+ * Spark's yes/no and its note disagree now and then: approach=false with "Continuous planks lead to it" rejected a
+ * pier six times over. A no counts when its note names what is in the way; a no whose note names nothing in the way
+ * is kept as drift (soft), and derive and verify measure the reach on the painting itself (SPOT_UNREACHABLE,
+ * FLOOR_UNREACHABLE) and send real breaks to repair.
+ */
+const BLOCKED = /\b(no|not|never|without|cannot|can't|cant|isn't|doesn't|don't|nothing)\b|\bn't\b|water|sea\b|river|gap|drop|cliff|ledge|railing|fence|wall|barrier|block|obstruct|cut off|cut-off|sever|broken|break|missing|absent|unreach|inaccessib|isolat|separat|float|detach|disconnect|across|beyond|away from|moved|shifted|displaced|only (by|via|through)|too (steep|far|high|small|narrow)/i;
+export const CONTRADICTED = "(Spark's note names nothing in the way)";
+const saysBlocked = (why: string) => BLOCKED.test(why);
+
+/** Fail closed on incomplete observations: an omitted floor is not evidence that it is connected. */
+export function paintedLayoutIssues(place: Place, layout: Layout, seen: PaintedLayout): string[] {
+  const issues: string[] = [];
+  if (seen.people) issues.push('unwanted people painted into the background; the game adds its cast separately');
+  if (Math.abs(seen.horizon - layout.horizon) > 70) issues.push(`camera horizon moved from ${layout.horizon} to ${Math.round(seen.horizon)}`);
+  for (const t of place.things.filter((t) => t.role !== 'npc')) {
+    const planned = layout.things.find((b) => b.key === t.key);
+    if (!planned) { issues.push(`${t.key}: absent from plan`); continue; }
+    const matches = seen.things.filter((s) => s.key === t.key);
+    if (matches.length !== 1) { issues.push(`${t.key}: comparison must locate it exactly once`); continue; }
+    const s = matches[0];
+    if (!s.box) { issues.push(`${t.key}: not painted; ${s.why}`); continue; }
+    const [l, top, r, bottom] = s.box;
+    if (Math.abs((l + r) / 2 - planned.x) > Math.max(60, planned.width * 0.35)
+      || Math.abs(bottom - planned.footY) > Math.max(55, planned.height * 0.4)) issues.push(`${t.key}: moved from its planned box to [${s.box.join(', ')}]; ${s.why}`);
+    const w = (r - l) / planned.width, h = (bottom - top) / planned.height;
+    if (w < 0.45 || w > 2.2 || h < 0.45 || h > 2.2) issues.push(`${t.key}: painted size differs too much from its planned box; ${s.why}`);
+    if (t.role !== 'prop' && !s.approach) issues.push(`${t.key}: no usable approach on connected ground; ${s.why}${saysBlocked(s.why) ? '' : ` ${CONTRADICTED}`}`);
+  }
+  for (const f of layout.floors) {
+    const matches = seen.floors.filter((s) => s.key === f.key);
+    if (matches.length !== 1) { issues.push(`${f.key}: comparison must inspect this floor exactly once`); continue; }
+    const s = matches[0];
+    if (s.coverage < 0.6) issues.push(`${f.key}: only ${Math.round(s.coverage * 100)}% of its planned area is walkable ground; ${s.why}`);
+    if (!s.connected) issues.push(`${f.key}: its planned connection to the main floor is broken; ${s.why}${saysBlocked(s.why) ? '' : ` ${CONTRADICTED}`}`);
+    const edge = f.outline.some(([x, y]) => x <= 1 || x >= FRAME.width - 1 || y <= 1 || y >= FRAME.height - 1);
+    if (edge && !s.reachesEdges) issues.push(`${f.key}: the planned route no longer reaches its frame edge; ${s.why}${saysBlocked(s.why) ? '' : ` ${CONTRADICTED}`}`);
+  }
+  return issues;
+}
+
+export async function inspectPaintedLayout(memo: Memo, place: Place, layout: Layout, blockin: Buffer, painting: Buffer, brief: Brief): Promise<PaintedLayout> {
+  const prompt = `Compare image 1 (authoritative block-in plan) with image 2 (painted game background). Measure image 2, not image 1. Both are ${FRAME.width}x${FRAME.height}.
+${layoutContract(place, layout)}
+Descriptions: ${JSON.stringify(place.things.filter((t) => t.role !== 'npc').map((t) => ({ key: t.key, role: t.role, looks: brief.looks[t.key] ?? t.looks })))}
+Return every listed non-NPC thing and every planned floor exactly once, with these exact keys.
+For each thing: its actual tight painted box or null if missing; approach=true only if the player can walk up to it along visibly continuous solid ground from the main floor (props may be decorative); why describes the evidence, including water, railings or gaps that block it. Do not use the nearest unrelated floor or a remote decorative copy as an approach.
+For each floor: coverage is the fraction of its PLANNED polygon still covered by the intended walkable surface in image 2 (not overall floor area elsewhere). connected=true only if it visibly joins the first floor via solid walkable surfaces. reachesEdges=true if it still reaches every frame edge its planned polygon touched; true when no frame edges were planned. Small painterly edge differences are fine; displaced or severed routes are not.
+Horizon is the camera's perspective horizon in image 2, not the skyline or top stair. people=true if image 2 contains any painted person or human figure: the game adds its cast separately, so a composition repair must not add people.
+Return JSON {"horizon":300,"people":false,"things":[{"key":"...","box":[0,0,100,100],"approach":true,"why":"..."}],"floors":[{"key":"...","coverage":0.95,"connected":true,"reachesEdges":true,"why":"..."}]}`;
+  return PaintedLayout.parse(await askSparkJson(memo, { schema: PaintedLayout, label: `paint-layout:${place.key}`, prompt, images: [blockin, painting] }));
+}
+
+const REPAIRS = 1;
+/**
+ * Drift the game copes with: masks, depth and walk are measured from the painting itself, so a camera a little
+ * higher, a thing a little off its box or a size off its block-in is no reason to repaint (Muse does it on almost
+ * every painting; failing on it left new worlds with no painting at all). What breaks play still repaints: a
+ * floor gone, cut off or not reaching its exit, a story thing missing or out of reach, people painted in.
+ */
+export const softIssue = (issue: string) => /^camera horizon moved|: moved from its planned box|: painted size differs too much/.test(issue) || issue.endsWith(CONTRADICTED);
+
+export type LayoutAttempt = { seen: PaintedLayout; issues: string[] };
+export class PaintingLayoutError extends Error {
+  constructor(readonly attempts: LayoutAttempt[]) {
+    super(`PAINT_LAYOUT_MISMATCH: ${attempts.at(-1)!.issues.join('; ')}. A composition repair was tried. Read place_status.paintingLayout; use paint_retry for a new take or layout_fix for a simpler connected plan (after 2 rejected plans in a row, paint_retry comes first). No floor has been invented to hide this mismatch.`);
+  }
+}
+
+/** Full-frame repair: a moved camera/path cannot be fixed by pasting a little rectangle. Never publish a rejected candidate. */
+export async function alignPainting(memo: Memo, place: Place, layout: Layout, blockin: Buffer, painting: Buffer, brief: Brief, take: number, fixes: string[] = []): Promise<{ painting: Buffer; attempts: LayoutAttempt[] }> {
+  const attempts: LayoutAttempt[] = [];
+  // One repair round: each is a full Muse repaint plus a Spark check (minutes per place); a second rarely saved
+  // a painting the first could not, and the place then goes to the agent (layout_fix, paint_retry) anyway.
+  for (let n = 0; n <= REPAIRS; n++) {
+    const seen = await inspectPaintedLayout(memo, place, layout, blockin, painting, brief);
+    const issues = paintedLayoutIssues(place, layout, seen);
+    attempts.push({ seen, issues });
+    if (!issues.some((i) => !softIssue(i))) {
+      if (issues.length) console.log(`  ${place.key}: composition drift kept (the game measures the painting itself): ${issues.join('; ')}`);
+      return { painting, attempts };
+    }
+    if (n === REPAIRS) throw new PaintingLayoutError(attempts);
+    console.log(`  ${place.key}: composition repair ${n + 1}/${REPAIRS}: ${issues.join('; ')}`);
+    const prompt = `Restore this game background to its planned composition. Image 1 is the authoritative spatial plan. Image 2 supplies ONLY the finished appearance, materials, palette and lighting. Repaint the whole composition where necessary, not a small patch. Preserve object identities and art style from image 2, but restore the camera, sizes, walking surfaces, joins and positions from image 1.
+${layoutContract(place, layout)}
+Object appearances: ${JSON.stringify(brief.looks)}
+Measured problems to correct:\n${issues.map((i) => `- ${i}`).join('\n')}
+Retain these requested details where compatible with the plan:\n${fixes.map((f) => `- ${f}`).join('\n')}
+No people, no text, labels, diagram lines or borders. Do not turn the plan's flat colours or tiles into the final art. Do not add unplanned boardwalks over water to hide a displaced object. Repair round ${n + 1}.`;
+    const candidate = await museEdit(memo, { label: `paint-align:${place.key}`, prompt, images: [blockin, painting], take });
+    painting = await sharp(candidate).resize(FRAME.width, FRAME.height, { fit: 'fill' }).png().toBuffer();
+  }
+  throw new PaintingLayoutError(attempts);
+}

package/app/src/scene/paint.ts

@@ -0,0 +1,81 @@
+import { STYLE_REFERENCE_PROMPT } from '../world/style-reference.ts';
+// paint: blockin.png becomes painting.png through one Muse edit.
+import sharp from 'sharp';
+import type { Memo } from '../meta/memo.ts';
+import { museEdit } from '../meta/muse.ts';
+import type { Place, World } from '../world/schema.ts';
+import { fallsIn } from '../play/weather.ts';
+import { paintedAnimals, type Brief } from './brief.ts';
+import { shapeOf } from './blockin.ts';
+import { FRAME, type Layout, type Shape } from './layout.ts';
+import { layoutContract } from './paint-layout.ts';
+
+/** Animals listed as masses are painted (a cow in its stall); any other animal is not. */
+const noAnimals = (brief?: Brief) => {
+  const keep = paintedAnimals(brief);
+  return keep.length ? `no animals other than the ${keep.join(', ')} listed above (paint ${keep.length > 1 ? 'those' : 'that'} in, at rest)` : 'no animals';
+};
+
+/** The place's cast is drawn by the game: the place's words may put one in the scene ("a basket where a cat curls up"). */
+const castNote = (place: Place) => {
+  const cast = place.things.filter((t) => t.role === 'npc').map((t) => t.key.replace(/-/g, ' '));
+  return cast.length ? ` The game draws ${cast.join(', ')} over the picture itself: do not paint ${cast.length > 1 ? 'them' : 'it'}, even where the place's words say ${cast.length > 1 ? 'they are' : 'it is'} there; leave that spot empty.` : '';
+};
+
+const NOUN: Record<Shape, string> = { block: 'box', low: 'low wide box', post: 'thin post', tree: 'tree shape', arch: 'arch', round: 'round shape', steps: 'steps', flat: 'flat oval on the floor' };
+
+/** Where a mass is, in the words a painter uses: across (fifths) and how far back it stands. */
+export function whereIs(layout: Layout, box: Layout['things'][number]): string {
+  const across = ['on the far left', 'left of centre', 'in the middle', 'right of centre', 'on the far right'][Math.min(4, Math.max(0, Math.floor((box.x / FRAME.width) * 5)))];
+  const back = (box.footY - layout.horizon) / (FRAME.height - layout.horizon);
+  return `${back < 0.3 ? 'at the back' : back < 0.65 ? 'halfway back' : 'at the front'}, ${across}`;
+}
+
+export function paintPrompt(world: World, place: Place, layout: Layout, brief?: Brief): string {
+  const look = (key: string, fallback: string) => brief?.looks[key] ?? fallback;
+  const masses = place.things
+    .filter((t) => t.role !== 'npc')
+    .map((t) => {
+      const box = layout.things.find((b) => b.key === t.key)!;
+      return `- the ${NOUN[shapeOf(box, t)]} ${whereIs(layout, box)}: ${look(t.key, t.looks)}`;
+    });
+  const floors = layout.floors.map((f) => `- the tiled floor area${layout.floors.length > 1 ? ` ${whereFloor(layout, f)}` : ''}: ${look(f.key, f.is)}, open floor to walk on`);
+  return `Paint this sketch as a finished background of a 2D point-and-click adventure game. The sketch is the plan: every shape stays where it is, at its size, and becomes what is listed; its shadow is where it stands on the floor. Keep the horizon and the camera of the sketch. The tiles on the floors only show the perspective: paint the real ground there, not tiles.
+
+Place: ${place.shows}
+Mood: ${place.mood}
+Style: ${world.style}
+
+The shapes:
+${[...masses, ...floors].join('\n')}
+
+${layoutContract(place, layout)}
+
+Seen a little from above, like a stage. Keep the floor areas clear to walk on. No people, ${noAnimals(brief)}, no text or letters, no border.${castNote(place)}${fallsIn(`${place.shows} ${place.mood}`) ? '\nThe rain or snow itself is animated over the picture later: paint no falling streaks, drops or flakes in the air, only what the weather leaves (wet sheen, puddles, snow lying on the ground).' : ''}`;
+}
+
+function whereFloor(layout: Layout, f: Layout['floors'][number]): string {
+  const ys = f.outline.map(([, y]) => y), xs = f.outline.map(([x]) => x);
+  const mid = { key: f.key, colour: '', x: (Math.min(...xs) + Math.max(...xs)) / 2, footY: (Math.min(...ys) + Math.max(...ys)) / 2, width: 0, height: 0, metres: 1 };
+  return whereIs(layout, mid);
+}
+
+/** Children's drawings of the place or of things in it, as more images after the block-in. */
+export type Drawing = { key: string; image: Buffer };
+
+export function drawingsPrompt(place: Place, drawings: Drawing[]): string {
+  if (drawings.length === 0) return '';
+  const which = drawings.map((d, i) => `image ${i + 2} is ${d.key === place.key ? 'this place' : `the ${d.key}`}`);
+  return `
+
+The other images are children's drawings on paper: ${which.join(', ')}. Paint what they drew so the children know it at once: their colours, shapes and ideas. A drawing fills its paper, but its thing does not fill the place: paint each at its real size beside the others (a hat is smaller than a log, a person comes up to a door's top). The composition still comes only from the first image; leave out the paper, the table and any written names.`;
+}
+
+/** One Muse edit, scaled back to the layout frame (Muse answers larger than asked). */
+export async function paintPlace(memo: Memo, world: World, place: Place, layout: Layout, blockin: Buffer, brief?: Brief, take = 0, drawings: Drawing[] = [], concept?: Buffer, styleReference?: Buffer): Promise<Buffer> {
+  const prompt = paintPrompt(world, place, layout, brief) + drawingsPrompt(place, drawings)
+    + (styleReference ? `\nImage ${drawings.length + 2}: ${STYLE_REFERENCE_PROMPT}` : '')
+    + (concept ? '\nThe LAST image is the creator-approved concept for this place. Preserve its distinctive architecture, palette, materials and visual identity. The FIRST image remains the authoritative layout: adapt the approved appearance to those positions and clear walking areas. Do not add characters or extra objects from the concept.' : '');
+  const painted = await museEdit(memo, { label: `paint:${place.key}`, prompt, images: [blockin, ...drawings.map((d) => d.image), ...(styleReference ? [styleReference] : []), ...(concept ? [concept] : [])], take });
+  return sharp(painted).resize(FRAME.width, FRAME.height, { fit: 'fill' }).png().toBuffer();
+}

package/app/src/scene/photo.ts

@@ -0,0 +1,123 @@
+import { STYLE_REFERENCE_PROMPT } from '../world/style-reference.ts';
+// A real photo as a place: crop it to the game frame, let Spark read the room into things,
+// and let Muse repaint it in the world's style without moving anything.
+import sharp from 'sharp';
+import { z } from 'zod';
+import type { Memo } from '../meta/memo.ts';
+import { museEdit } from '../meta/muse.ts';
+import { askSparkJson } from '../meta/spark.ts';
+import { Place, World } from '../world/schema.ts';
+import type { Brief } from './brief.ts';
+import { FRAME } from './layout.ts';
+
+/** Where the game frame sits in the original photo, so a film can line the real and painted frames up. */
+export const PhotoCrop = z.object({
+  source: z.object({ width: z.number(), height: z.number() }),
+  crop: z.object({ x: z.number(), y: z.number(), width: z.number(), height: z.number() }),
+  frame: z.object({ width: z.number(), height: z.number() }),
+});
+export type PhotoCrop = z.infer<typeof PhotoCrop>;
+
+/** The largest centred crop with the frame's aspect, scaled to the frame. EXIF rotation is applied first. */
+export async function framePhoto(photo: Buffer): Promise<{ image: Buffer; crop: PhotoCrop }> {
+  const upright = await sharp(photo).rotate().toBuffer();
+  const { width = 0, height = 0 } = await sharp(upright).metadata();
+  const aspect = FRAME.width / FRAME.height;
+  const w = Math.min(width, Math.round(height * aspect));
+  const h = Math.min(height, Math.round(w / aspect));
+  const crop = { x: Math.round((width - w) / 2), y: Math.round((height - h) / 2), width: w, height: h };
+  const image = await sharp(upright).extract({ left: crop.x, top: crop.y, width: w, height: h })
+    .resize(FRAME.width, FRAME.height, { fit: 'fill' }).png().toBuffer();
+  return { image, crop: { source: { width, height }, crop, frame: { ...FRAME } } };
+}
+
+/** Spark reads the photo into a place: what it shows and the things in it, the way out as an exit. */
+export async function describePhoto(memo: Memo, image: Buffer, idea: string): Promise<Place> {
+  const prompt = `This photo of a real room becomes the first place of a 2D point-and-click adventure. The game idea:
+"""
+${idea}
+"""
+
+Describe the room as it is; invent nothing that is not in the photo, and leave out any person in it.
+- key: a kebab-case name for the room
+- shows: what the photo shows, front to back
+- mood: the light and time of day in the photo, and the feeling it should have in the game
+- things: 3 to 7 things big enough to click, each with role prop, item or exit; the door is an exit, one small loose object the hero could pick up is an item
+
+Reply with JSON only:
+{"key": "...", "shows": "...", "mood": "...", "things": [{"key": "kebab-key", "role": "prop | item | exit", "looks": "a few words"}]}`;
+  let ask = prompt;
+  for (let attempt = 1; attempt <= 3; attempt++) {
+    const answer = await askSparkJson(memo, { label: `photo-place#${attempt}`, prompt: ask, images: [image], personal: true });
+    const parsed = Place.omit({ exits: true }).safeParse(answer);
+    const issues = parsed.success ? (parsed.data.things.some((t) => t.role === 'exit') ? [] : ['no thing has role exit; the door is the exit']) : parsed.error.issues.map((i) => `${i.path.join('.')}: ${i.message}`);
+    if (parsed.success && issues.length === 0) return { ...parsed.data, exits: [] };
+    console.log(`  photo place try ${attempt}: ${issues.join('; ')}`);
+    ask = `${prompt}\n\nYour last answer was:\n${JSON.stringify(answer)}\n\nFix: ${issues.join('; ')}`;
+  }
+  throw new Error('Spark could not read the photo into a place in 3 tries');
+}
+
+export function photoPaintPrompt(world: World, place: Place, brief?: Brief): string {
+  const looks = place.things.filter((t) => t.role !== 'npc').map((t) => `- ${t.key}: ${brief?.looks[t.key] ?? t.looks}`);
+  return `Repaint this photo as a finished background of a 2D point-and-click adventure game. Keep the camera and every object exactly where and as big as it is in the photo: same walls, same furniture, same door, same floor. Only the medium changes.
+
+Style: ${world.style}
+Mood: ${place.mood}
+Objects to keep recognisable:
+${looks.join('\n')}
+
+Remove every person; paint the furniture and floor behind them. Keep the floor clear to walk on. No text or letters, no border.`;
+}
+
+export async function paintPhoto(memo: Memo, world: World, place: Place, photo: Buffer, brief?: Brief, take = 0, concept?: Buffer, styleReference?: Buffer): Promise<Buffer> {
+  const prompt = photoPaintPrompt(world, place, brief) + (styleReference ? '\nThe LAST image: ' + STYLE_REFERENCE_PROMPT : '') + (concept ? '\nThe second image is the creator-approved visual concept: preserve its palette and treatment. The first photo is authoritative for the real room, furniture, perspective and layout.' : '');
+  const painted = await museEdit(memo, { label: `paint:${place.key}`, prompt, images: [photo, ...(concept ? [concept] : []), ...(styleReference ? [styleReference] : [])], take });
+  return sharp(painted).resize(FRAME.width, FRAME.height, { fit: 'fill' }).png().toBuffer();
+}
+
+// Looks feed every later painting (cast, the four model views, sprites), so they describe the person and not
+// the photo: "lying against a headboard" in looks puts a headboard behind the figure in every view, and
+// "legs not visible" leaves the painters to guess. This catches what the photo leaked into them.
+const POSE_WORDS = /\b(not visible|out of (the )?frame|cropped|cut off|(from|at) the (chest|waist|shoulders)|chest[- ]up|waist[- ]up|lying|lies|reclin\w*|sitting|sits|seated|leaning|propped|headboard|pillows?|bed|sofa|couch|armchair|chair|desk|blanket|duvet|selfie|in the photo|camera)\b/gi;
+
+/** The words in looks that are about the photo (pose, furniture, framing) rather than the person. */
+export function poseWords(looks: string): string[] {
+  return [...new Set([...looks.matchAll(POSE_WORDS)].map((m) => m[0].toLowerCase()))];
+}
+
+/** Spark describes the person in a photo head to toe, so world.json's hero is them, standing, whatever the photo shows. */
+export async function describePerson(memo: Memo, photo: Buffer): Promise<{ key: string; looks: string }> {
+  const prompt = `This photo shows the real person who will be the hero of a 2D point-and-click adventure, where they will stand and walk. Describe them head to toe as a game character standing on their feet: build, hair, face, beard or glasses, every piece of clothing with its colour, trousers or skirt, shoes. No name, no guesses about who they are.
+
+Describe the person only, never the photo: not how they sit or lie, not the bed, chair or pillow, not how the photo frames them. When the photo does not show part of them (legs, trousers, shoes), choose plain ones that go with the rest and describe those as if seen; never write that something is not visible.
+
+The key names who they are in the game (like the-developer), not where they are in the photo.
+
+Reply with JSON only: {"key": "kebab-key like the-developer", "looks": "..."}`;
+  let ask = prompt;
+  let hero: { key: string; looks: string } | undefined;
+  for (let attempt = 1; attempt <= 3; attempt++) {
+    const answer = await askSparkJson<{ key: string; looks: string }>(memo, { label: `hero-photo#${attempt}`, prompt: ask, images: [photo], personal: true });
+    hero = World.shape.hero.parse(answer);
+    const leaked = poseWords(`${hero.key.replace(/-/g, ' ')} ${hero.looks}`);
+    if (!leaked.length) return hero;
+    console.log(`  hero photo try ${attempt}: looks speak of the photo (${leaked.join(', ')})`);
+    ask = `${prompt}\n\nYour last answer was:\n${JSON.stringify(answer)}\n\nFix: it describes the photo, not the person (${leaked.join(', ')}). Describe them standing, head to toe, with every piece of clothing as if seen.`;
+  }
+  return hero!;
+}
+
+/** Whether the photo shows the person standing with the whole body in frame, so the hero can be painted in that very pose. */
+export async function photoStanding(memo: Memo, photo: Buffer): Promise<boolean> {
+  const answer = await askSparkJson<{ standing?: unknown }>(memo, {
+    label: 'hero-photo-pose', images: [photo], personal: true,
+    prompt: 'Does this photo show the person standing upright on their feet, with the whole body from the top of the head down to both feet inside the frame? Sitting, lying, leaning back, kneeling, or any part of the legs or feet cut off by the frame or hidden is no.\n\nReply with JSON only: {"standing": true or false}',
+  });
+  return answer.standing === true;
+}
+
+/** Upright, at most 1600 px on the long side, PNG: what Spark and Muse get. */
+export function normalisePhoto(photo: Buffer): Promise<Buffer> {
+  return sharp(photo).rotate().resize(1600, 1600, { fit: 'inside', withoutEnlargement: true }).png().toBuffer();
+}

package/app/src/scene/plates.ts

@@ -0,0 +1,256 @@
+// plates: for things the hero carries off, the spot where the thing was, painted empty. Muse edits a close crop
+// around the thing; only the thing's mask (grown a little, feathered, matched in tone) is taken from it.
+// SAM then looks for the thing again on the plate: if it is still there, the plate is flagged.
+import sharp from 'sharp';
+import { emptyMask, maskArea, maskToPng, type Mask } from '../image/mask.ts';
+import { clampRect, components, dilate, growRect, intersect, maskBox, maskIou, rectMask, subtract, union, unionAll } from '../image/raster.ts';
+import type { Memo } from '../meta/memo.ts';
+import { museEdit } from '../meta/muse.ts';
+import { samFind } from '../meta/sam.ts';
+import type { Place } from '../world/schema.ts';
+import { FRAME } from './layout.ts';
+
+export const PLATE_GROW = 12;
+/** Mean difference (0-255) from the painting just around a patch above which the seam shows. */
+export const SEAM_SHOWS = 25;
+
+/** Pastes `patch` over `painting` inside `mask`, with a soft edge. */
+export async function pasteInside(painting: Buffer, patch: Buffer, mask: Mask, feather = 4): Promise<Buffer> {
+  const { width: W, height: H } = FRAME;
+  const alpha = await sharp(await maskToPng(mask)).blur(feather).toColourspace('b-w').raw().toBuffer();
+  return pasteAlpha(painting, patch, alpha);
+}
+
+/**
+ * `patch` over `painting` through `alpha` (raw, one byte per pixel, FRAME size); everything at alpha 0 is left as it
+ * was. The RGBA patch is built by hand: sharp's joinChannel with a raw alpha let the whole patch through, so every
+ * edit used to replace the whole painting.
+ */
+export async function pasteAlpha(painting: Buffer, patch: Buffer, alpha: Buffer): Promise<Buffer> {
+  const { width: W, height: H } = FRAME;
+  const rgb = await sharp(patch).resize(W, H, { fit: 'fill' }).removeAlpha().raw().toBuffer();
+  const rgba = Buffer.alloc(W * H * 4);
+  for (let i = 0; i < W * H; i++) {
+    rgba[i * 4] = rgb[i * 3];
+    rgba[i * 4 + 1] = rgb[i * 3 + 1];
+    rgba[i * 4 + 2] = rgb[i * 3 + 2];
+    rgba[i * 4 + 3] = alpha[i];
+  }
+  const cut = await sharp(rgba, { raw: { width: W, height: H, channels: 4 } }).png().toBuffer();
+  return sharp(painting).resize(W, H, { fit: 'fill' }).composite([{ input: cut }]).png().toBuffer();
+}
+
+/**
+ * Muse's edit inside `area` only, transparent elsewhere, with a soft edge. A Muse edit repaints the whole picture and
+ * often shifts its tone (darker, wetter, another light), so the patch is first matched, channel by channel, to the
+ * painting in a ring just outside the area: mean and spread of the edit's ring are mapped onto the painting's.
+ */
+export async function patchInside(painting: Buffer, edited: Buffer, area: Mask, feather = 4): Promise<{ png: Buffer; ringError: number; stain: number }> {
+  const { width: W, height: H } = FRAME;
+  const [paint, edit] = await Promise.all([painting, edited].map((b) => sharp(b).resize(W, H, { fit: 'fill' }).removeAlpha().raw().toBuffer()));
+  const ring = subtract(dilate(area, 18), area);
+  const stats = (px: Buffer) => [0, 1, 2].map((c) => {
+    let n = 0, sum = 0, sq = 0;
+    for (let i = 0; i < W * H; i++) if (ring.bits[i]) (n++, sum += px[i * 3 + c], sq += px[i * 3 + c] ** 2);
+    const mean = n ? sum / n : 0;
+    return { mean, sd: n ? Math.sqrt(Math.max(1, sq / n - mean * mean)) : 1 };
+  });
+  const [want, have] = [stats(paint), stats(edit)];
+  // How far the matched edit still is from the painting just around the patch: a Muse edit that moved things
+  // (it repaints the whole picture) leaves a seam however good its fill looks, and scores high here.
+  let err = 0, n = 0;
+  for (let i = 0; i < W * H; i++) {
+    if (!ring.bits[i]) continue;
+    for (let c = 0; c < 3; c++) {
+      const k = Math.min(1.6, Math.max(0.6, want[c].sd / have[c].sd));
+      err += Math.abs(want[c].mean + (edit[i * 3 + c] - have[c].mean) * k - paint[i * 3 + c]);
+    }
+    n += 3;
+  }
+  const alpha = await sharp(await maskToPng(area)).blur(feather).extractChannel(0).raw().toBuffer();
+  const out = Buffer.alloc(W * H * 4);
+  for (let i = 0; i < W * H; i++) {
+    if (!alpha[i]) continue;
+    for (let c = 0; c < 3; c++) {
+      // The spread is matched only part way, so a flat ring cannot flatten the fill to one colour.
+      const k = Math.min(1.6, Math.max(0.6, want[c].sd / have[c].sd));
+      out[i * 4 + c] = Math.max(0, Math.min(255, Math.round(want[c].mean + (edit[i * 3 + c] - have[c].mean) * k)));
+    }
+    out[i * 4 + 3] = alpha[i];
+  }
+  // A stain: the share of the patch darker or lighter than nearly all of the ground around it (a hole, a puddle,
+  // a smear Muse painted where the thing was).
+  const luma = (px: Buffer, i: number, step: number) => 0.3 * px[i * step] + 0.59 * px[i * step + 1] + 0.11 * px[i * step + 2];
+  const around: number[] = [];
+  for (let i = 0; i < W * H; i++) if (ring.bits[i]) around.push(luma(paint, i, 3));
+  around.sort((a, b) => a - b);
+  const [lo, hi] = around.length ? [around[Math.floor(around.length * 0.02)] - 20, around[Math.floor(around.length * 0.98)] + 20] : [0, 255];
+  let inside = 0, off = 0;
+  for (let i = 0; i < W * H; i++) {
+    if (!area.bits[i]) continue;
+    inside++;
+    const l = luma(out, i, 4);
+    if (l < lo || l > hi) off++;
+  }
+  return { png: await sharp(out, { raw: { width: W, height: H, channels: 4 } }).png().toBuffer(), ringError: n ? err / n : 0, stain: inside ? off / inside : 0 };
+}
+
+/**
+ * A Muse edit of the part of the painting around a small thing, blown up to the full frame, then put back at its place
+ * in a copy of the painting; null for a thing too large for a close view. Seen whole, a small thing is a few pixels
+ * Muse may fill with whatever it likes; seen close it may redraw the ground with other lines. Neither wins always.
+ */
+async function editAround(memo: Memo, label: string, prompt: string, painting: Buffer, mask: Mask, take = 0): Promise<Buffer | null> {
+  const { width: W, height: H } = FRAME;
+  const box = maskBox(mask);
+  if (!box) return null;
+  // Three times the thing each way, in the frame's shape, at least a quarter of the frame wide.
+  const w = Math.min(W, Math.max(W / 4, (box[2] - box[0]) * 3, ((box[3] - box[1]) * 3 * W) / H));
+  if (w > W * 0.6) return null;
+  const h = (w * H) / W;
+  const left = Math.round(Math.max(0, Math.min(W - w, (box[0] + box[2]) / 2 - w / 2)));
+  const top = Math.round(Math.max(0, Math.min(H - h, (box[1] + box[3]) / 2 - h / 2)));
+  const crop = { left, top, width: Math.min(W - left, Math.round(w)), height: Math.min(H - top, Math.round(h)) };
+  const full = await sharp(painting).resize(W, H, { fit: 'fill' }).png().toBuffer();
+  const close = await sharp(full).extract(crop).resize(W, H, { kernel: 'lanczos3' }).png().toBuffer();
+  const edited = await museEdit(memo, { label: `${label}:close`, prompt, images: [close], take });
+  const back = await sharp(edited).resize(crop.width, crop.height, { fit: 'fill' }).png().toBuffer();
+  return sharp(full).composite([{ input: back, left: crop.left, top: crop.top }]).png().toBuffer();
+}
+
+/** How far from the thing's mask a part SAM left out (a rope, a handle, a strap) may reach. */
+export const ATTACHED_REACH = 72;
+
+/**
+ * SAM's mask often leaves out a part of the thing (the rope tied to a bucket), which then lies on the floor once the
+ * thing is gone. Muse, asked to remove the thing, removes those parts too: where the edit differs strongly from the
+ * painting just outside the area, in blobs touching it, the area grows to take them in. A shifted or retoned edit
+ * changes everything a little, and Muse adds things of its own; only clear changes towards the ground count, and at most as much again as the thing itself.
+ */
+export async function withAttached(painting: Buffer, edited: Buffer, mask: Mask, area: Mask): Promise<Mask> {
+  const { width: W, height: H } = FRAME;
+  const [paint, edit] = await Promise.all([painting, edited].map((b) => sharp(b).resize(W, H, { fit: 'fill' }).removeAlpha().raw().toBuffer()));
+  const near = subtract(dilate(mask, ATTACHED_REACH), area);
+  // The edit's overall shift in tone, read on the ring around the area, is taken out first.
+  const ring = subtract(dilate(area, 10), area);
+  const shift = [0, 1, 2].map((c) => {
+    const d: number[] = [];
+    for (let i = 0; i < W * H; i++) if (ring.bits[i]) d.push(edit[i * 3 + c] - paint[i * 3 + c]);
+    return d.length ? d.sort((a, b) => a - b)[d.length >> 1] : 0;
+  });
+  // The ground, as the painting has it around the area.
+  const ground = [0, 1, 2].map((c) => {
+    const v: number[] = [];
+    for (let i = 0; i < W * H; i++) if (ring.bits[i]) v.push(paint[i * 3 + c]);
+    return v.length ? v.sort((a, b) => a - b)[v.length >> 1] : 0;
+  });
+  const fromGround = (px: Buffer, i: number, off = [0, 0, 0]) => Math.hypot(...[0, 1, 2].map((c) => px[i * 3 + c] - off[c] - ground[c]));
+  const changed = emptyMask(W, H);
+  for (let i = 0; i < W * H; i++) {
+    if (!near.bits[i]) continue;
+    const d = Math.hypot(...[0, 1, 2].map((c) => edit[i * 3 + c] - shift[c] - paint[i * 3 + c]));
+    // Removed means the edit turned it into ground: something Muse added (a stain, a shadow) is not part of the thing.
+    if (d > 60 && fromGround(edit, i, shift) < fromGround(paint, i) - 40) changed.bits[i] = 1;
+  }
+  const touching = dilate(area, 2);
+  let budget = maskArea(mask);
+  const parts: Mask[] = [];
+  for (const blob of components(dilate(changed, 1))) {
+    const size = maskArea(blob);
+    if (size < 40 || size > budget || maskArea(intersect(blob, touching)) === 0) continue;
+    parts.push(blob);
+    budget -= size;
+  }
+  return parts.length ? union(area, dilate(unionAll(W, H, parts), 4)) : area;
+}
+
+/** How bad a patch looks, in seam units: its seam, and a stain over a tenth of it as bad as a seam that shows. */
+const STAIN_WEIGHT = SEAM_SHOWS * 10;
+const flaw = (p: { ringError: number; stain: number }) => p.ringError + STAIN_WEIGHT * p.stain;
+
+/** ringError: mean difference (0-255) from the painting just around the patch (see SEAM_SHOWS). */
+export type Plate = { thing: string; png: Buffer; stillThere: boolean; iou: number; ringError: number; stain: number };
+
+export async function plateFor(memo: Memo, place: Place, thing: { key: string; phrase: string; mask: Mask }, painting: Buffer, ground: string): Promise<Plate> {
+  const prompt = `Remove the ${thing.phrase} from this picture. Where it was, paint only ${ground}, continuing the surroundings exactly: the same flat colours, texture, ink lines and light as right next to it. Add nothing new: no objects, stones, debris, stains, puddles or shadows. Leave everything else exactly as it is. No people, no text.`;
+  const label = `plate:${place.key}/${thing.key}`;
+  const area = dilate(thing.mask, PLATE_GROW);
+  // Two edits, the whole painting and a close view; the one that meets the painting around the patch best is kept.
+  const takes = async (take: number) => {
+    const edits = (await Promise.all([museEdit(memo, { label, prompt, images: [painting], take }), editAround(memo, label, prompt, painting, thing.mask, take)]))
+      .filter((e): e is Buffer => e !== null);
+    // The plate is only the patch where the thing was: drawn over the painting (or the backdrop), nothing else can differ.
+    return Promise.all(edits.map(async (e) => patchInside(painting, e, await withAttached(painting, e, thing.mask, area))));
+  };
+  let patches = await takes(0);
+  // A seam or a stain that shows: one more take of each, and the best of all four.
+  if (Math.min(...patches.map(flaw)) > SEAM_SHOWS) patches = [...patches, ...(await takes(1))];
+  const { png, ringError, stain } = patches.sort((a, b) => flaw(a) - flaw(b))[0];
+  // Is it still there? SAM by the same phrase on the painting with the patch on, inside the area the plate replaced.
+  const patched = await sharp(painting).resize(FRAME.width, FRAME.height, { fit: 'fill' }).composite([{ input: png }]).png().toBuffer();
+  const found = await samFind(memo, { label: `plate-check:${place.key}/${thing.key}`, image: patched, prompt: thing.phrase });
+  const iou = Math.max(0, ...found.map((f) => maskIou(intersect(f.mask, area), thing.mask)));
+  const left = found.reduce((px, f) => px + maskArea(intersect(f.mask, thing.mask)), 0);
+  return { thing: thing.key, png, iou, ringError, stain, stillThere: iou > 0.4 || left > 0.4 * maskArea(thing.mask) };
+}
+
+/** How far around a takeable thing a change keeps off: Muse shifts what it redraws by a few dozen px. */
+export const CHANGE_KEEP_OFF = 24;
+
+/**
+ * Where a change plate shows: the thing grown wide (light spills), soft-edged, but never over the `others`
+ * (things the hero can take, grown by CHANGE_KEEP_OFF) outside the changed thing itself. They keep the
+ * painting's pixels, so their hover outline and their plate still match what is seen.
+ */
+export async function changeAlpha(mask: Mask, others: Mask[]): Promise<Buffer> {
+  const alpha = await sharp(await maskToPng(dilate(mask, 48))).blur(14).extractChannel(0).raw().toBuffer();
+  if (!others.length) return alpha;
+  let keep: Mask | undefined;
+  for (const o of others) keep = keep ? union(keep, o) : o;
+  const off = await sharp(await maskToPng(dilate(keep!, CHANGE_KEEP_OFF))).blur(6).extractChannel(0).raw().toBuffer();
+  for (let i = 0; i < alpha.length; i++) if (!mask.bits[i]) alpha[i] = Math.round((alpha[i] * (255 - off[i])) / 255);
+  return alpha;
+}
+
+/**
+ * The changed thing's own shape in its change plate: Muse redraws all it sees, and a boat that gets a lantern may come
+ * back longer or lower than the one painted, with the lantern standing up out of it. SAM by how it looks now
+ * (`looks`: what was found near the painted thing, all of it), else by its box grown for that, on the plate; undefined
+ * when what it finds is not the thing (little of the painted one, or far bigger). The runtime outlines this while
+ * the change holds, so the hover line follows the thing as it is drawn then.
+ */
+export async function changeShape(memo: Memo, place: Place, thing: { key: string; mask: Mask }, plate: Buffer, looks: string): Promise<Mask | undefined> {
+  const { width: W, height: H } = FRAME;
+  const box = maskBox(thing.mask);
+  if (!box) return undefined;
+  const near = clampRect(growRect(box, 1.6), W, H);
+  const rgba = await sharp(plate).ensureAlpha().raw().toBuffer();
+  const flat = await sharp(rgba, { raw: { width: W, height: H, channels: 4 } }).removeAlpha().png().toBuffer();
+  const px = maskArea(thing.mask);
+  // Only where the plate shows: elsewhere the painting is drawn, not the change.
+  const shown = (m: Mask): Mask => ({ ...m, bits: m.bits.map((b, i) => (b && rgba[i * 4 + 3] > 96 ? 1 : 0)) });
+  const fits = (m: Mask) => maskArea(intersect(m, thing.mask)) >= 0.6 * px && maskArea(m) <= 3 * px;
+  const nearBox = rectMask(W, H, near);
+  const named = await samFind(memo, { label: `change-shape:${place.key}/${thing.key}:${looks}`, image: flat, prompt: looks });
+  const parts = named.filter((f) => maskArea(intersect(f.mask, nearBox)) >= 0.5 * maskArea(f.mask)).map((f) => f.mask);
+  if (parts.length) {
+    const shape = shown(unionAll(W, H, parts));
+    if (fits(shape)) return shape;
+  }
+  const [left, top, right, bottom] = near;
+  const [boxed] = await samFind(memo, { label: `change-shape-box:${place.key}/${thing.key}`, image: flat, prompt: [{ left, top, right, bottom }] });
+  const shape = boxed && shown(boxed.mask);
+  return shape && fits(shape) ? shape : undefined;
+}
+
+/** For a thing the story changes (a beacon lit): the thing as it looks after, transparent elsewhere, so several
+ * changes in one place stack. Light may spill, so the area is wide. */
+export async function changePlate(memo: Memo, place: Place, thing: { key: string; phrase: string; mask: Mask }, painting: Buffer, looks: string, others: Mask[] = []): Promise<Buffer> {
+  const prompt = `Change only the ${thing.phrase} in this painting: it is now ${looks}. Add the light it casts on what is right around it. Leave everything else exactly as it is and where it is, same style. No people, no animals, no text.`;
+  const edited = await museEdit(memo, { label: `change:${place.key}/${thing.key}:${looks}`, prompt, images: [painting] });
+  const { width: W, height: H } = FRAME;
+  const alpha = await changeAlpha(thing.mask, others);
+  const rgba = await sharp(edited).resize(W, H, { fit: 'fill' }).ensureAlpha().raw().toBuffer();
+  for (let i = 0; i < W * H; i++) rgba[i * 4 + 3] = alpha[i];
+  return sharp(rgba, { raw: { width: W, height: H, channels: 4 } }).png().toBuffer();
+}