@gtanczyk/diorama 0.0.0-stage → 0.1.0

package/app/src/scene/masks.ts

@@ -0,0 +1,392 @@
+// masks: one SAM 3.1 mask per painted thing and per floor, checked against where Spark saw it.
+// Things: SAM by phrase, the instances near Spark's box; a mask that fails the sanity gate is
+// asked again with the box; if that fails too the box itself is the mask, marked coarse.
+// Floors: SAM by name and by the box of Spark's cells; the candidate that agrees best with the
+// cells wins; with none close enough the cells are the floor, marked coarse.
+import sharp from 'sharp';
+import { emptyMask, maskArea, type Mask } from '../image/mask.ts';
+import { clampRect, components, dilate, fillHoles, framedOpening, growRect, intersect, union, subtract, maskBox, maskIou, polygonMask, rectArea, rectIou, rectMask, unionAll, withoutSpecks, type Rect } from '../image/raster.ts';
+import type { Memo } from '../meta/memo.ts';
+import { SAM_MAX_BOXES, samFind, type Box, type Found } from '../meta/sam.ts';
+import type { Place } from '../world/schema.ts';
+import { FRAME, type Layout } from './layout.ts';
+import { cellsBox, cellsMask, GRID, type Kind, type Spots } from './locate.ts';
+
+export type MaskSource = 'sam-text' | 'sam-box' | 'colour-cut' | 'sketch-colour' | 'spark-box' | 'layout-box' | 'spark-cells' | 'layout';
+
+/** whole: px added by the whole-thing pass (parts SAM by name left out: what sits in or on it). */
+export type ThingMask = { key: string; kind: Kind; phrase: string; source: MaskSource; coarse: boolean; px: number; box: Rect; mask: Mask; whole?: number };
+export type FloorMask = { key: string; source: MaskSource; coarse: boolean; px: number; iouCells: number; iouLayout: number; mask: Mask };
+
+const { width: W, height: H } = FRAME;
+const toBox = ([left, top, right, bottom]: Rect): Box => ({ left, top, right, bottom });
+const fromBox = (b: Box): Rect => [b.left, b.top, b.right, b.bottom];
+
+/** Where the layout put a thing, for things Spark could not find. */
+export function layoutBox(layout: Layout, key: string): Rect | undefined {
+  const t = layout.things.find((b) => b.key === key);
+  return t && clampRect([t.x - t.width / 2, t.footY - t.height, t.x + t.width / 2, t.footY], W, H);
+}
+
+/**
+ * The mask-sanity gate for a thing against the box it should fill. Empty is not found; much bigger
+ * than the box while barely overlapping it is a different object (SAM takes every instance it sees).
+ */
+export function thingMaskFault(mask: Mask, prior: Rect): string | undefined {
+  const px = maskArea(mask);
+  if (px < 150) return px === 0 ? 'empty' : `only ${px} px`;
+  const box = maskBox(mask)!;
+  const iou = rectIou(box, prior);
+  if (px > 3 * rectArea(prior) && iou < 0.3) return `${px} px is over 3x its box and overlaps it ${iou.toFixed(2)}`;
+  if (iou < 0.1) return `lies outside its box (overlap ${iou.toFixed(2)})`;
+}
+
+/**
+ * A box prompt sometimes returns what surrounds the thing (the puddle around a can) instead of it.
+ * Such a mask leaves the middle of the box empty.
+ */
+export function hollowFault(mask: Mask, [x0, y0, x1, y1]: Rect): string | undefined {
+  const [w, h] = [x1 - x0, y1 - y0];
+  const middle = rectMask(mask.width, mask.height, [x0 + w / 3, y0 + h / 3, x1 - w / 3, y1 - h / 3]);
+  const area = maskArea(middle);
+  const covered = area > 0 ? maskArea(intersect(mask, middle)) / area : 1;
+  if (covered < 0.25) return `covers only ${Math.round(covered * 100)}% of the middle of its box`;
+}
+
+/**
+ * A way out set in a wall (a doorway, an arch, a gate) asked of SAM often comes back as its frame: posts and lintel,
+ * the middle empty (hollowFault). The way out is the gap the frame holds, open at the bottom where the floor runs
+ * through. Undefined when the frame holds no gap worth the name (a closed door, a wall).
+ */
+export function openingOf(frame: Mask): Mask | undefined {
+  const box = maskBox(frame);
+  if (!box) return undefined;
+  // Slivers where a post was cut ragged are not the opening.
+  const gap = components(dilate(dilate(framedOpening(frame, box), -3), 3))[0];
+  if (!gap) return undefined;
+  const filled = fillHoles(gap);
+  return maskArea(filled) >= 0.2 * rectArea(box) && !hollowFault(filled, maskBox(filled)!) ? filled : undefined;
+}
+
+/**
+ * The last try before a plain box: inside the box, the pixels whose colour stands out from a ring
+ * just outside it (the floor or wall around the thing), as one solid blob.
+ */
+export function colourCut(rgb: Buffer, box: Rect, ring = 12): Mask | undefined {
+  const [x0, y0, x1, y1] = clampRect(box, W, H);
+  const [ox0, oy0, ox1, oy1] = clampRect([x0 - ring, y0 - ring, x1 + ring, y1 + ring], W, H);
+  const sum = [0, 0, 0];
+  const sq = [0, 0, 0];
+  let n = 0;
+  for (let y = oy0; y < oy1; y++) {
+    for (let x = ox0; x < ox1; x++) {
+      if (x >= x0 && x < x1 && y >= y0 && y < y1) continue;
+      for (let c = 0; c < 3; c++) {
+        const v = rgb[(y * W + x) * 3 + c];
+        sum[c] += v;
+        sq[c] += v * v;
+      }
+      n++;
+    }
+  }
+  if (n === 0) return undefined;
+  const mean = sum.map((v) => v / n);
+  const spread = Math.sqrt(sq.reduce((t, v, c) => t + v / n - mean[c] ** 2, 0));
+  const limit = Math.max(40, 2 * spread);
+  const raw = emptyMask(W, H);
+  for (let y = y0; y < y1; y++) {
+    for (let x = x0; x < x1; x++) {
+      const i = y * W + x;
+      const d = Math.hypot(rgb[i * 3] - mean[0], rgb[i * 3 + 1] - mean[1], rgb[i * 3 + 2] - mean[2]);
+      if (d > limit) raw.bits[i] = 1;
+    }
+  }
+  const blob = components(dilate(dilate(raw, 2), -2))[0];
+  if (!blob) return undefined;
+  // Fill holes: whatever inside the box the outside can't reach is part of the thing.
+  const outside = emptyMask(W, H);
+  const stack: number[] = [];
+  for (let x = x0; x < x1; x++) stack.push(y0 * W + x, (y1 - 1) * W + x);
+  for (let y = y0; y < y1; y++) stack.push(y * W + x0, y * W + x1 - 1);
+  while (stack.length) {
+    const i = stack.pop()!;
+    const x = i % W;
+    const y = Math.floor(i / W);
+    if (x < x0 || x >= x1 || y < y0 || y >= y1 || outside.bits[i] || blob.bits[i]) continue;
+    outside.bits[i] = 1;
+    stack.push(i - 1, i + 1, i - W, i + W);
+  }
+  const solid = rectMask(W, H, [x0, y0, x1, y1]);
+  const filled = subtract(solid, outside);
+  const share = maskArea(filled) / Math.max(1, rectArea([x0, y0, x1, y1]));
+  return share > 0.15 && share < 0.92 ? filled : undefined;
+}
+
+/**
+ * After the colour cut, before a plain box: inside the box, the pixels of the hue the layout gave the thing (the
+ * sketch Muse painted over), as they lie: walls, a hedge, a row of shelves, not one solid blob. A box as the mask
+ * draws its hover outline as a rectangle over everything in it. Undefined when that hue is not the thing's: too
+ * little or nearly all of the box has it.
+ */
+export function layoutColourCut(rgb: Buffer, box: Rect, colour: string): Mask | undefined {
+  const hex = /^#?([0-9a-f]{6})$/i.exec(colour.trim())?.[1];
+  if (!hex) return undefined;
+  const want = hsv(...([0, 2, 4].map((i) => parseInt(hex.slice(i, i + 2), 16)) as [number, number, number]));
+  // A grey or cream sketch colour names no hue to find.
+  if (want.s < 0.25) return undefined;
+  const near = (i: number) => {
+    const c = hsv(rgb[i * 3], rgb[i * 3 + 1], rgb[i * 3 + 2]);
+    // Painted light and shadow change brightness and some saturation, not hue.
+    return c.s > 0.18 && c.v > 0.12 && Math.min(Math.abs(c.h - want.h), 360 - Math.abs(c.h - want.h)) < 16;
+  };
+  const [x0, y0, x1, y1] = clampRect(box, W, H);
+  const raw = emptyMask(W, H);
+  for (let y = y0; y < y1; y++) for (let x = x0; x < x1; x++) if (near(y * W + x)) raw.bits[y * W + x] = 1;
+  const area = rectArea([x0, y0, x1, y1]);
+  const kept = withoutSpecks(dilate(dilate(raw, 2), -2), Math.max(80, area * 0.002));
+  const share = maskArea(kept) / Math.max(1, area);
+  return share > 0.05 && share < 0.85 ? kept : undefined;
+}
+
+function hsv(r: number, g: number, b: number): { h: number; s: number; v: number } {
+  const max = Math.max(r, g, b);
+  const d = max - Math.min(r, g, b);
+  const h = d === 0 ? 0 : max === r ? ((g - b) / d + 6) % 6 : max === g ? (b - r) / d + 2 : (r - g) / d + 4;
+  return { h: h * 60, s: max ? d / max : 0, v: max / 255 };
+}
+
+/** A single SAM match near a misplaced box: similar size, centre within twice the box's size. */
+export function loneMatch(found: Rect, box: Rect): boolean {
+  const ratio = rectArea(found) / Math.max(1, rectArea(box));
+  const size = Math.max(box[2] - box[0], box[3] - box[1]);
+  const dist = Math.hypot((found[0] + found[2] - box[0] - box[2]) / 2, (found[1] + found[3] - box[1] - box[3]) / 2);
+  return ratio > 0.25 && ratio < 4 && dist < 2 * size;
+}
+
+/** The instances that belong to the thing: the one nearest its box, plus any others mostly inside it. */
+export function pickInstances(found: Found[], prior: Rect, passage = false): Mask | undefined {
+  if (found.length === 0) return undefined;
+  const scored = found.map((f) => {
+    const box = fromBox(f.box);
+    const inside = rectArea([Math.max(box[0], prior[0]), Math.max(box[1], prior[1]), Math.min(box[2], prior[2]), Math.min(box[3], prior[3])]);
+    return { f, iou: rectIou(box, prior), inside: rectArea(box) > 0 ? inside / rectArea(box) : 0 };
+  });
+  const best = scored.reduce((a, b) => (b.iou > a.iou ? b : a));
+  if (best.iou === 0) return undefined;
+  const keep = scored.filter((s) => s === best || s.inside > 0.6);
+  const mask = unionAll(W, H, keep.map((s) => s.f.mask));
+  // Stray specks of a lossy mask would each become an occluder later.
+  return withoutSpecks(mask, passage ? 8 : Math.max(60, maskArea(mask) * 0.03));
+}
+
+/**
+ * The box SAM is asked by for the whole thing: its mask's box joined with Spark's (Spark boxes a "bucket of fish
+ * heads" with the fish), grown on the top and the sides, where contents stick out; barely below, that is floor.
+ */
+export function wholeBox(mask: Rect, spark: Rect | undefined): Rect {
+  const [x0, y0, x1, y1] = spark ? [Math.min(mask[0], spark[0]), Math.min(mask[1], spark[1]), Math.max(mask[2], spark[2]), Math.max(mask[3], spark[3])] : mask;
+  const g = 0.15 * Math.max(x1 - x0, y1 - y0);
+  return clampRect([x0 - g, y0 - 2 * g, x1 + g, y1 + 0.2 * g], W, H);
+}
+
+/**
+ * The thing as one piece: SAM by name (`named`) joined with the parts SAM by the whole box (`boxed`) finds attached
+ * to it. Refused when the box found something else (it covers little of the named mask) or took in its surroundings
+ * (it adds over twice the thing itself); `others` (the other things) are never taken. Specks and slivers along the
+ * outline are not parts. Returns the mask and the px it gained (0: as it was).
+ */
+export function wholeThing(named: Mask, boxed: Mask | undefined, others: Mask, spark?: Rect): { mask: Mask; added: number; why?: string } {
+  const px = maskArea(named);
+  if (!boxed || px === 0) return { mask: named, added: 0 };
+  const cover = maskArea(intersect(boxed, named)) / px;
+  if (cover < 0.6) return { mask: named, added: 0, why: `the box found something else (covers ${Math.round(cover * 100)}% of it)` };
+  // Spark boxed the whole thing as it saw it: what SAM's grown box finds past that (wall, ceiling) is not the thing.
+  const within = spark ? intersect(boxed, rectMask(named.width, named.height, clampRect(growRect(spark, 1.1), named.width, named.height))) : boxed;
+  const extra = subtract(subtract(within, dilate(named, 2)), others);
+  const touch = dilate(named, 4);
+  const parts = components(extra).filter((c) => maskArea(c) >= Math.max(80, 0.01 * px) && maskArea(intersect(c, touch)) > 0);
+  const added = parts.reduce((n, c) => n + maskArea(c), 0);
+  if (added < 0.03 * px) return { mask: named, added: 0 };
+  // Contents can be as big as their container (a heap of fish over a small bucket), not much bigger. SAM by name can
+  // also take a sliver of the thing (the boards of a shelf without its frame and tins): then Spark's box says how big
+  // the thing is, and what fills it is the thing.
+  const size = spark ? Math.max(px, 0.5 * rectArea(spark)) : px;
+  if (added > 2 * size) return { mask: named, added: 0, why: `the box took in ${added} px around it, over twice the thing` };
+  // The 2 px seam left between the mask and its parts closes with them.
+  const joined = unionAll(named.width, named.height, [named, ...parts]);
+  const closed = subtract(dilate(dilate(joined, 3), -3), others);
+  return { mask: closed, added: maskArea(closed) - px };
+}
+
+async function samBoxes(memo: Memo, label: string, painting: Buffer, boxes: Rect[]): Promise<(Mask | undefined)[]> {
+  const out: (Mask | undefined)[] = [];
+  for (let i = 0; i < boxes.length; i += SAM_MAX_BOXES) {
+    const chunk = boxes.slice(i, i + SAM_MAX_BOXES);
+    const found = await samFind(memo, { label: `${label}#${i / SAM_MAX_BOXES}`, image: painting, prompt: chunk.map(toBox) });
+    // One object per box; the parser keeps the prompt's order as the object id.
+    chunk.forEach((_, j) => out.push(found.find((f) => f.id === String(j))?.mask));
+  }
+  return out;
+}
+
+export async function maskThings(memo: Memo, place: Place, layout: Layout, spots: Spots, painting: Buffer, log: (line: string) => void): Promise<ThingMask[]> {
+  const painted = place.things.filter((t) => t.role !== 'npc');
+  const tried = await Promise.all(painted.map(async (t) => {
+    const spot = spots[t.key];
+    const sparkBox = spot.box && clampRect(spot.box, W, H);
+    let prior = sparkBox && rectArea(sparkBox) > 0 ? sparkBox : layoutBox(layout, t.key)!;
+    // The phrase, Spark's other name for it, then the head noun: SAM often knows "can" when it misses "brass oil can".
+    const passage = t.role === 'exit' && (spot.kind === 'walk-on' || spot.kind === 'flat');
+    const phrases = [...new Set([...(passage ? [`entire ${spot.phrase}, all visible steps and landings up to the edge of the image`] : []), spot.phrase, spot.also, spot.phrase.trim().split(/\s+/).at(-1)].filter((p): p is string => !!p?.trim()))];
+    let mask: Mask | undefined;
+    let lone: Found | undefined;
+    for (const [i, phrase] of phrases.entries()) {
+      const found = await samFind(memo, { label: `mask:${place.key}/${t.key}${i ? `~${phrase}` : ''}`, image: painting, prompt: phrase });
+      mask = pickInstances(found, prior, passage);
+      if (mask) break;
+      if (found.length === 1 && !lone) lone = found[0];
+    }
+    // Spark's box can sit beside the thing. One SAM match of the right size close by is the thing.
+    if (!mask && lone && sparkBox && loneMatch(fromBox(lone.box), sparkBox)) {
+      log(`${t.key}: Spark's box misses it; took SAM's only match nearby`);
+      mask = withoutSpecks(lone.mask, Math.max(60, maskArea(lone.mask) * 0.03));
+      prior = fromBox(lone.box);
+    }
+    const opening = t.role === 'exit' && spot.kind === 'wall';
+    if (mask && opening && hollowFault(mask, maskBox(mask)!)) {
+      const gap = openingOf(mask);
+      if (gap) {
+        log(`${t.key}: SAM by name gave the frame of the way out; took the opening it holds`);
+        mask = gap;
+      }
+    }
+    const fault = mask ? thingMaskFault(mask, prior) : `SAM found no "${phrases.join('" or "')}"`;
+    return { t, spot, prior, opening, fromSpark: !!sparkBox && prior !== layoutBox(layout, t.key), mask, fault };
+  }));
+
+  const retry = tried.filter((r) => r.fault);
+  const pixels = await sharp(painting).resize(W, H, { fit: 'fill' }).removeAlpha().raw().toBuffer();
+  const boxed = await samBoxes(memo, `mask-box:${place.key}`, painting, retry.map((r) => clampRect(r.prior, W, H)));
+  const masked = tried.map((r) => {
+    const base = { key: r.t.key, kind: r.spot.kind, phrase: r.spot.phrase };
+    let mask = r.mask;
+    let source: MaskSource = 'sam-text';
+    if (r.fault) {
+      log(`${r.t.key}: ${r.fault}, asking SAM with the box`);
+      let byBox = boxed[retry.indexOf(r)];
+      // The box around a doorway brings back its frame; the opening it holds is the way out.
+      const gap = r.opening && byBox && hollowFault(byBox, r.prior) ? openingOf(byBox) : undefined;
+      if (gap) {
+        log(`${r.t.key}: SAM by the box gave the frame of the way out; took the opening it holds`);
+        byBox = gap;
+      }
+      const boxFault = byBox ? thingMaskFault(byBox, r.prior) ?? hollowFault(byBox, r.prior) : 'SAM box found nothing';
+      if (byBox && !boxFault) {
+        mask = withoutSpecks(byBox, Math.max(60, maskArea(byBox) * 0.03));
+        source = 'sam-box';
+      } else {
+        const cut = colourCut(pixels, r.prior);
+        const cutFault = cut ? thingMaskFault(cut, r.prior) ?? hollowFault(cut, r.prior) : 'nothing stands out from its surroundings';
+        const planned = layout.things.find((b) => b.key === r.t.key)?.colour;
+        const byColour = cut && !cutFault ? undefined : planned ? layoutColourCut(pixels, r.prior, planned) : undefined;
+        if (cut && !cutFault) {
+          mask = cut;
+          source = 'colour-cut';
+          log(`${r.t.key}: ${boxFault}; cut it out of its box by colour`);
+        } else if (byColour) {
+          mask = byColour;
+          source = 'sketch-colour';
+          log(`${r.t.key}: ${boxFault}; colour cut: ${cutFault}; took the pixels of its sketch colour ${planned} in its box`);
+        } else {
+          mask = rectMask(W, H, r.prior);
+          source = r.fromSpark ? 'spark-box' : 'layout-box';
+          log(`${r.t.key}: ${boxFault}; colour cut: ${cutFault}; the ${source} is its mask (coarse)`);
+        }
+      }
+    }
+    return { ...base, source, coarse: source === 'spark-box' || source === 'layout-box', px: maskArea(mask!), box: maskBox(mask!)!, mask: mask!, prior: r.spot.box ? clampRect(r.spot.box, W, H) : undefined };
+  });
+  return wholeThings(memo, place, painting, masked, log);
+}
+
+/** Did the parts found reach the edge of the box asked by (its top or a side, inside the frame)? Then there is more of it. */
+export function reachesEdge(found: Rect, asked: Rect): boolean {
+  const near = 6;
+  return (asked[1] > 0 && found[1] - asked[1] < near) || (asked[0] > 0 && found[0] - asked[0] < near) || (asked[2] < W && asked[2] - found[2] < near);
+}
+
+/**
+ * Things that stand or sit (not floors laid flat): a batch of box prompts per place, see wholeThing. Parts that
+ * reach the edge of the box are asked once more by a box grown around them (fish piled high over a rim).
+ */
+async function wholeThings(memo: Memo, place: Place, painting: Buffer, masked: Array<ThingMask & { prior?: Rect }>, log: (line: string) => void): Promise<ThingMask[]> {
+  const out: ThingMask[] = masked.map(({ prior: _, ...t }) => t);
+  // A way out set in a wall (a door, an arch, a gate) has nothing in or on it: SAM by its grown box finds the house
+  // around it, and the door, joined with part of the house, then reads far taller than a person can pass.
+  const opening = (t: ThingMask) => t.kind === 'wall' && place.things.find((p) => p.key === t.key)?.role === 'exit';
+  let asked: { key: string; box: Rect; spark?: Rect }[] = masked.filter((t) => t.source === 'sam-text' && t.kind !== 'walk-on' && t.kind !== 'flat' && !opening(t)).map((t) => ({ key: t.key, box: wholeBox(t.box, t.prior), spark: t.prior }));
+  for (let round = 0; round < 2 && asked.length; round++) {
+    const boxed = await samBoxes(memo, `mask-whole${round ? round + 1 : ''}:${place.key}`, painting, asked.map((a) => a.box));
+    const again: typeof asked = [];
+    asked.forEach((a, i) => {
+      const k = out.findIndex((t) => t.key === a.key);
+      const t = out[k];
+      const others = unionAll(W, H, out.filter((o) => o.key !== t.key).map((o) => o.mask));
+      const whole = wholeThing(t.mask, boxed[i], others, a.spark);
+      if (whole.why) console.log(`  ${place.key}: ${t.key}: kept its mask, ${whole.why}`);
+      if (!whole.added) return;
+      const box = maskBox(whole.mask)!;
+      out[k] = { ...t, mask: whole.mask, px: maskArea(whole.mask), box, whole: (t.whole ?? 0) + whole.added };
+      if (reachesEdge(box, a.box)) again.push({ key: t.key, box: wholeBox(box, undefined), spark: undefined });
+    });
+    asked = again;
+  }
+  for (const t of out) if (t.whole) log(`${t.key}: SAM by name left parts of it out (what is in or on it); ${t.whole} px joined from SAM by its whole box`);
+  return out;
+}
+
+/**
+ * A floor as one surface: SAM cuts around everything lying on it (wood shavings, pebbles, leaves) and frays where
+ * the floor fades into fog or shadow. Gaps of a few px close, and holes it closes on every side up to 1.5% of the
+ * frame are filled: what lies there is walked over. Bigger holes stay (a pond, the floor under a wide table);
+ * standing things are cut out of the walk by their own masks later.
+ */
+export function solidFloor(m: Mask): Mask {
+  const closed = union(m, dilate(dilate(m, 3), -3));
+  return withoutSpecks(fillHoles(closed, 0.015 * m.width * m.height), 200);
+}
+
+/** Keeps a floor candidate to within one grid cell of Spark's marking and scores the agreement. */
+export function scoreFloor(candidate: Mask, marked: Mask): { mask: Mask; iou: number } {
+  const mask = intersect(candidate, dilate(marked, GRID));
+  return { mask, iou: maskIou(mask, marked) };
+}
+
+export async function maskFloors(memo: Memo, place: Place, layout: Layout, marked: Record<string, string[]>, painting: Buffer, log: (line: string) => void): Promise<FloorMask[]> {
+  const boxes = layout.floors.map((f) => cellsBox(marked[f.key]));
+  const withBox = boxes.flatMap((b, i) => (b ? [{ i, box: clampRect([b[0] - GRID / 2, b[1] - GRID / 2, b[2] + GRID / 2, b[3] + GRID / 2], W, H) }] : []));
+  const byBox = await samBoxes(memo, `floor-box:${place.key}`, painting, withBox.map((b) => b.box));
+
+  return Promise.all(layout.floors.map(async (floor, i) => {
+    const planned = polygonMask(W, H, floor.outline);
+    const cells = cellsMask(marked[floor.key]);
+    if (maskArea(cells) === 0) {
+      log(`floor ${floor.key}: Spark marked no cells; using the layout outline (coarse)`);
+      return { key: floor.key, source: 'layout' as const, coarse: true, px: maskArea(planned), iouCells: 0, iouLayout: 1, mask: planned };
+    }
+    const found = await samFind(memo, { label: `floor:${place.key}/${floor.key}`, image: painting, prompt: floor.key.replace(/-/g, ' ') });
+    const candidates: { source: MaskSource; mask: Mask }[] = [
+      ...found.map((f) => ({ source: 'sam-text' as const, mask: f.mask })),
+      ...(found.length > 1 ? [{ source: 'sam-text' as const, mask: unionAll(W, H, found.map((f) => f.mask)) }] : []),
+    ];
+    const boxed = byBox[withBox.findIndex((b) => b.i === i)];
+    if (boxed) candidates.push({ source: 'sam-box', mask: boxed });
+    const scored = candidates.map((c) => ({ ...c, ...scoreFloor(solidFloor(c.mask), cells) })).sort((a, b) => b.iou - a.iou);
+    const best = scored[0];
+    if (best && best.iou >= 0.3) {
+      return { key: floor.key, source: best.source, coarse: false, px: maskArea(best.mask), iouCells: best.iou, iouLayout: maskIou(best.mask, planned), mask: best.mask };
+    }
+    log(`floor ${floor.key}: best SAM candidate agrees ${(best?.iou ?? 0).toFixed(2)} with Spark's cells; using the cells (coarse)`);
+    return { key: floor.key, source: 'spark-cells' as const, coarse: true, px: maskArea(cells), iouCells: 1, iouLayout: maskIou(cells, planned), mask: cells };
+  }));
+}

package/app/src/scene/occluders.ts

@@ -0,0 +1,123 @@
+// Where a thing meets the floor. A standing thing hides the hero when his feet are behind its floor
+// line, decided per column, so a slanted boat or a table seen from the corner hides him correctly.
+import { maskArea, type Mask } from '../image/mask.ts';
+import { bottomRows, components, maskBox, type Rect } from '../image/raster.ts';
+
+export type XY = [number, number];
+
+/** Blobs smaller than this don't hide anyone; they are specks or tiny details. */
+export const MIN_OCCLUDER_PX = 900;
+
+/**
+ * The chain of the lowest points that bulges downward (y grows down): the floor contact of a
+ * blob's bottom edge, bridging gaps between legs. Points must be sorted by x.
+ */
+export function lowerHull(points: XY[]): XY[] {
+  const hull: XY[] = [];
+  for (const p of points) {
+    // Pop the last point while it lies above (smaller y than) the line from the one before it to p.
+    while (hull.length >= 2) {
+      const [ax, ay] = hull[hull.length - 2];
+      const [bx, by] = hull[hull.length - 1];
+      const cross = (bx - ax) * (p[1] - ay) - (by - ay) * (p[0] - ax);
+      if (cross >= 0) hull.pop();
+      else break;
+    }
+    hull.push(p);
+  }
+  return hull;
+}
+
+/** Drops steep ends of the hull: they come from overhangs (a lantern arm, a roof) and not from the floor. */
+export function trimSteepEnds(line: XY[]): XY[] {
+  const steep = (a: XY, b: XY) => Math.abs(b[1] - a[1]) > 2 * Math.abs(b[0] - a[0]);
+  let from = 0;
+  let to = line.length - 1;
+  while (to - from >= 1 && steep(line[from], line[from + 1])) from++;
+  while (to - from >= 1 && steep(line[to - 1], line[to])) to--;
+  return line.slice(from, to + 1);
+}
+
+/** The floor line of a blob: per-column bottoms, lower hull, steep ends trimmed. */
+export function floorLine(blob: Mask): XY[] {
+  const rows = bottomRows(blob);
+  const points: XY[] = [];
+  rows.forEach((y, x) => {
+    if (y >= 0) points.push([x, y + 1]);
+  });
+  return trimSteepEnds(lowerHull(points));
+}
+
+export function lineYAt(line: XY[], x: number): number {
+  if (x <= line[0][0]) return line[0][1];
+  for (let i = 1; i < line.length; i++) {
+    const [x1, y1] = line[i];
+    if (x <= x1) {
+      const [x0, y0] = line[i - 1];
+      return x1 === x0 ? y1 : y0 + ((y1 - y0) * (x - x0)) / (x1 - x0);
+    }
+  }
+  return line[line.length - 1][1];
+}
+
+export type Blob = {
+  /** "barrel" for a single blob, "barrel#2" for the second largest of several. */
+  key: string;
+  thing: string;
+  mask: Mask;
+  box: Rect;
+  line: XY[];
+  baseY: number;
+  /** Floor rows covered by the thing's foot, every few columns: [x, back, front]. */
+  foot: [number, number, number][];
+  /** Spans the whole frame height: always in front of the hero. */
+  front: boolean;
+};
+
+/** The footprint is as deep as a third of the thing's width, at least 12 px. */
+export const footDepth = (width: number) => Math.max(12, 0.35 * width);
+
+export function blobsOf(thing: string, mask: Mask, step = 4): Blob[] {
+  const blobs = components(mask).filter((b) => maskArea(b) >= MIN_OCCLUDER_PX);
+  return blobs.map((b, i) => {
+    const box = maskBox(b)!;
+    const line = floorLine(b);
+    const x0 = line[0][0];
+    const x1 = line[line.length - 1][0];
+    const depth = footDepth(x1 - x0);
+    const foot: [number, number, number][] = [];
+    for (let x = x0; x <= x1; x += step) {
+      const front = Math.round(lineYAt(line, x));
+      foot.push([x, Math.max(box[1], front - Math.round(depth)), front]);
+    }
+    return {
+      key: blobs.length === 1 ? thing : `${thing}#${i + 1}`,
+      thing,
+      mask: b,
+      box,
+      line,
+      baseY: Math.max(...line.map(([, y]) => y)),
+      foot,
+      front: box[1] <= 3 && box[3] >= mask.height - 4,
+    };
+  });
+}
+
+/** Fewer points for the same line: keeps the corners, drops points within `tolerance` px of a straight run. */
+export function simplifyLine(line: XY[], tolerance = 1.5): XY[] {
+  if (line.length <= 2) return line;
+  const out: XY[] = [line[0]];
+  let anchor = 0;
+  for (let i = 2; i < line.length; i++) {
+    const [ax, ay] = line[anchor];
+    const [bx, by] = line[i];
+    const len = Math.hypot(bx - ax, by - ay) || 1;
+    const off = line.slice(anchor + 1, i).some(([px, py]) => Math.abs((bx - ax) * (ay - py) - (ax - px) * (by - ay)) / len > tolerance);
+    if (off) {
+      anchor = i - 1;
+      out.push(line[anchor]);
+    }
+  }
+  out.push(line[line.length - 1]);
+  return out;
+}

package/app/src/scene/occlusion-check.ts

@@ -0,0 +1,138 @@
+// occlusion-check: what the depth map will do to people standing on this place's floor, before anyone plays
+// it (src/play/depth-occlusion.ts does it at play time). Read by verify: no depth map means only named
+// things can hide anyone; floor cells on an upright face put people behind it; a spot where the painting
+// covers nearly all of a person means they vanish there.
+import type { Geometry } from '../play/geometry.ts';
+import { personPx, stepAt } from '../play/geometry.ts';
+import { contactRows, depthAt, frontMask, hiddenShare, MAX_HIDDEN, PERSON_WIDTH } from '../play/depth-occlusion.ts';
+import { UPRIGHT_SHARE } from './depthmap.ts';
+import type { Depth } from '../play/shadow.ts';
+import type { Finding } from './verify.ts';
+import { gridOf } from './verify.ts';
+import { nearestRoomy, reach, roomy } from './walk.ts';
+
+/** Every how many walk cells (each way) a person is stood to check. */
+const STRIDE = 2;
+
+export type OcclusionCheck = {
+  /** Walk cells checked, cells on an upright face (feet over a wall or thin air), cells where the depth map would hide most of a person and so hides nothing. */
+  checked: number;
+  onFace: Array<[number, number]>;
+  unsure: Array<[number, number]>;
+  /** Cells where something unmasked (no card) hides part of a person: the depth map's own work. */
+  hiddenByUnmasked: number;
+};
+
+const boxOf = (pts: Array<[number, number]>): [number, number, number, number] => [
+  Math.min(...pts.map((p) => p[0])), Math.min(...pts.map((p) => p[1])), Math.max(...pts.map((p) => p[0])), Math.max(...pts.map((p) => p[1])),
+];
+
+/**
+ * Feet at (x, y) on an upright face: under most of their width the depth hardly changes down the screen (a floor's
+ * changes a row per row): the side of a bridge, the wall down to an underpass, a floor mask or a joining path laid on
+ * them. A card's thing in front of the feet is left out, and a post or a can in front of one foot is not most of it.
+ */
+export function onFace(g: Geometry, d: Depth, x: number, y: number, owned?: Uint8Array): boolean {
+  const h = personPx(g, y);
+  const w = Math.max(4, Math.round(h * PERSON_WIDTH));
+  let face = 0;
+  let feet = 0;
+  const span = Math.max(16, 0.25 * h);
+  for (let fx = x - w / 2; fx <= x + w / 2; fx += 2) {
+    const k = Math.min(d.w - 1, Math.max(0, Math.floor(fx / 2))) + Math.min(d.h - 1, Math.floor((y - 1) / 2)) * d.w;
+    if (owned?.[k]) continue;
+    feet++;
+    if ((depthAt(d, fx, y + span / 2) - depthAt(d, fx, y - span / 2)) / span < UPRIGHT_SHARE) face++;
+  }
+  return feet > 0 && face > 0.6 * feet;
+}
+
+/** Floor cut off by more than this share of the walk cells stays: a big part means the face test is wrong there. */
+const FACE_CUT_MAX = 0.15;
+
+/**
+ * The walk with its cells on an upright face closed (the front of a bridge's planks, a bank's edge down to a ledge),
+ * and the floor only they led to (a ledge of rock plates under the bridge's end): feet there stood over thin air.
+ * Stairs are upright by nature, risers and all, and keep their cells. Undefined when nothing closes, when no floor is
+ * near the start, or when it would cut off a big part of the place.
+ */
+export function closeFaces(g: Geometry, d: Depth, owned?: Uint8Array): { walk: string[]; zone: string[]; start: [number, number]; closed: number; beyond: number } | undefined {
+  const grid = gridOf(g);
+  const all = grid.count();
+  const faces = [...grid.open.keys()].filter((i) => grid.open[i] && !stepAt(g, ...grid.centre(i)) && onFace(g, d, ...grid.centre(i), owned));
+  if (!faces.length) return undefined;
+  faces.forEach((i) => (grid.open[i] = 0));
+  // A start on the face (the front of the planks) steps back onto the floor.
+  const cells = roomy(grid);
+  const start = nearestRoomy(grid, ...g.start, cells);
+  if (start === undefined || Math.hypot(...grid.centre(start).map((v, k) => v - g.start[k]) as [number, number]) > 8 * g.cell) return undefined;
+  const keep = reach(grid, start, cells);
+  for (const i of [...keep.keys()].filter((k) => keep[k])) for (const n of grid.neighbours(i)) if (grid.open[n] && !cells[n]) keep[n] = 1;
+  const beyond = [...grid.open.keys()].filter((i) => grid.open[i] && !keep[i]);
+  if (beyond.length > FACE_CUT_MAX * all) return undefined;
+  for (const i of [...faces, ...beyond]) [grid.open[i], grid.zone[i]] = [0, -1];
+  return { ...grid.rowsOut(), start: cells[grid.cellOf(...g.start)] ? g.start : grid.centre(start), closed: faces.length, beyond: beyond.length };
+}
+
+export function checkOcclusion(g: Geometry, d: Depth, owned?: Uint8Array): OcclusionCheck {
+  const grid = gridOf(g);
+  const out: OcclusionCheck = { checked: 0, onFace: [], unsure: [], hiddenByUnmasked: 0 };
+  const contact = contactRows(g, d);
+  for (let r = 0; r < grid.rows; r += STRIDE) for (let c = 0; c < grid.cols; c += STRIDE) {
+    const i = grid.at(c, r);
+    if (!grid.open[i]) continue;
+    const [x, y] = grid.centre(i);
+    out.checked++;
+    const h = personPx(g, y);
+    const w = Math.max(4, Math.round(h * PERSON_WIDTH));
+    if (onFace(g, d, x, y, owned)) out.onFace.push([x, y]);
+    if (hiddenShare(g, d, x, y, owned) > MAX_HIDDEN) {
+      out.unsure.push([x, y]);
+      continue;
+    }
+    const rect: [number, number, number, number] = [Math.round(x - w / 2), Math.round(y - h), Math.round(x + w / 2), Math.round(y - Math.max(3, h * 0.04))];
+    if (frontMask(g, d, rect, x, y, owned)) out.hiddenByUnmasked++;
+  }
+  return out;
+}
+
+/** verify's findings from the check (all flags: the place plays, a person should look). */
+export function occlusionFindings(g: Geometry, check: OcclusionCheck | null): Finding[] {
+  if (!check) return g.depth ? [] : [{ code: 'NO_DEPTH', detail: 'no depth map: only named things with masks can hide someone; a lamp post or railing nobody named will be drawn behind them' }];
+  const out: Finding[] = [];
+  // A few cells on a face are the edge of a kerb or a step; many are a floor mask laid on a wall or a bridge's side.
+  if (check.onFace.length >= Math.max(10, 0.02 * check.checked)) {
+    out.push({ code: 'WALK_ON_FACE', detail: `${check.onFace.length} of ${check.checked} checked floor cells lie on an upright face (a bridge's side, a wall down to a lower floor): feet there are over a wall or thin air. Look at walk.png; floor_cells closes them`, box: boxOf(check.onFace) });
+  }
+  if (check.unsure.length >= Math.max(10, 0.05 * check.checked)) {
+    out.push({ code: 'DEPTH_UNSURE', detail: `${check.unsure.length} of ${check.checked} checked floor cells have over ${Math.round(MAX_HIDDEN * 100)}% of a person in front of them by the depth map (stairs, a slope, or floor where there is none): nothing unmasked hides anyone there`, box: boxOf(check.unsure) });
+  }
+  return out;
+}
+
+/** Reads depth.png and the cards' masks of a place folder (node only), and runs the check. Null without a depth map. */
+export async function checkPlaceOcclusion(dir: string, g: Geometry): Promise<OcclusionCheck | null> {
+  const read = await placeDepth(dir, g);
+  return read && checkOcclusion(g, read.d, read.owned);
+}
+
+/** depth.png of a place folder and, at its size, the pixels the cards look after (node only). Null without a depth map. */
+export async function placeDepth(dir: string, g: Geometry): Promise<{ d: Depth; owned: Uint8Array } | null> {
+  if (!g.depth) return null;
+  const fs = await import('node:fs');
+  const path = await import('node:path');
+  const { default: sharp } = await import('sharp');
+  const { decodeDepth } = await import('../play/shadow.ts');
+  const file = path.join(dir, g.depth);
+  if (!fs.existsSync(file)) return null;
+  const { data, info } = await sharp(file).ensureAlpha().raw().toBuffer({ resolveWithObject: true });
+  const d = decodeDepth(data, info.width, info.height);
+  const owned = new Uint8Array(d.w * d.h);
+  for (const mask of new Set(g.occluders.map((o) => o.mask).filter((m): m is string => !!m))) {
+    const at = path.join(dir, mask);
+    if (!fs.existsSync(at)) continue;
+    const m = await sharp(at).resize(d.w, d.h, { fit: 'fill' }).greyscale().raw().toBuffer();
+    for (let i = 0; i < m.length; i++) if (m[i] > 127) owned[i] = 1;
+  }
+  return { d, owned };
+}