@gtanczyk/diorama 0.0.0-stage → 0.2.0

package/app/src/scene/held.ts

@@ -0,0 +1,109 @@
+// What a character holds or carries as part of who they are (a giant slice of toast over their head, a lantern, a
+// net of fish): named from their description, found by SAM on its own and kept on the cut-out. SAM's "person" is
+// only the body, and the plain-ground heuristics in warp.ts (silhouette, companions) drop anything bigger than the
+// figure as the scene behind it, so a prop larger than its holder needs its own mask.
+import sharp from 'sharp';
+import { emptyMask, maskArea, type Mask } from '../image/mask.ts';
+import { components, dilate, maskBox, subtract, union } from '../image/raster.ts';
+import { fillHoles } from './warp.ts';
+import type { Memo } from '../meta/memo.ts';
+import { samFind } from '../meta/sam.ts';
+import { askSparkJson } from '../meta/spark.ts';
+
+/** Words that say the description has something in hand; without one no Spark call is made. */
+const HOLDING = /\b(hold|holds|holding|held|carr(y|ies|ying|ied)|clutch\w*|grip\w*|grasp\w*|wield\w*|lift\w*|rais\w*|balanc\w*|hug\w*|cradl\w*|bear(s|ing)?|tow\w*|drag\w*|push\w*|pull\w*|swing\w*|brandish\w*|in (his|her|their|its|one|both) (hand|hands|arms|paws|claws|wings|beak|mouth))\b/i;
+
+/**
+ * The things the character holds, carries or lifts, as short phrases SAM can search ("slice of toast", "lantern").
+ * `holds` (cast/fixes.json, set by the person) wins; otherwise one memoized Spark call on the description, and none
+ * at all when it says nothing about holding. Clothing and what is worn is never in the list: SAM's person has it.
+ */
+export async function heldThings(memo: Memo, who: { key: string; looks: string; holds?: string[] }): Promise<string[]> {
+  if (who.holds) return who.holds.map((h) => h.trim()).filter(Boolean);
+  if (!HOLDING.test(who.looks)) return [];
+  const a = await askSparkJson<{ held?: unknown }>(memo, {
+    label: `figure-held:${who.key}`,
+    prompt: `A character of a 2D adventure game is described like this:\n"${who.looks}"\n\nList the separate objects the character holds, carries, lifts or balances (in hands, arms, paws, beak or on the head as a load), as short concrete noun phrases of 1 to 4 words that an image segmentation model can search for, e.g. "slice of toast", "lantern", "fishing net". Never list clothing, a hat, shoes, a bag strap worn on the body, hair, body parts, or a face or pattern drawn on the object itself. If they hold nothing, the list is empty.\n\nReply with JSON only: {"held": ["..."]}`,
+  });
+  return Array.isArray(a.held) ? a.held.filter((h): h is string => typeof h === 'string' && !!h.trim()).map((h) => h.trim()).slice(0, 3) : [];
+}
+
+/**
+ * Every instance SAM finds of the held things that touches the figure (or comes within a few percent of its height),
+ * outside SAM's person, as one mask; undefined when there is none. One or two SAM calls per phrase, memoized with the
+ * image; when SAM finds none of them, what stands out from the plain ground and touches the body (plainHeld).
+ */
+export async function heldMask(memo: Memo, key: string, image: Buffer, person: Mask, phrases: string[]): Promise<Mask | undefined> {
+  const box = maskBox(person);
+  if (!box || !phrases.length) return undefined;
+  const near = dilate(person, Math.max(3, Math.round(0.04 * (box[3] - box[1]))));
+  let out: Mask | undefined;
+  for (const phrase of phrases) {
+    // "slice of toast" may find nothing where "toast" does: the head noun is asked next.
+    const head = /\bof\b/i.test(phrase) ? phrase.split(/\bof\b/i).pop()!.trim() : phrase.trim().split(/\s+/).pop()!;
+    for (const prompt of [...new Set([phrase, head])]) {
+      let hit = false;
+      for (const found of await samFind(memo, { label: `figure:${key}:held`, image, prompt })) {
+        const own = subtract(found.mask, person);
+        if (maskArea(own) < 0.01 * maskArea(person)) continue;
+        if (!own.bits.some((b, i) => b && near.bits[i])) continue;
+        out = union(out ?? emptyMask(person.width, person.height), own);
+        hit = true;
+      }
+      if (hit) break;
+    }
+  }
+  if (out) return out;
+  const { data } = await sharp(image).resize(person.width, person.height, { fit: 'fill' }).ensureAlpha().raw().toBuffer({ resolveWithObject: true });
+  return plainHeld(data, person);
+}
+
+/**
+ * Without SAM: the figure is painted alone on a plain ground, so what stands out from that ground and touches the
+ * body (not the picture's edge) is what they hold. Used when the description says they hold something SAM did not find.
+ */
+export function plainHeld(rgba: Buffer, person: Mask, tolerance = 64): Mask | undefined {
+  const { width: W, height: H } = person;
+  const box = maskBox(person);
+  if (!box) return undefined;
+  const [, py0, , py1] = box;
+  const ground = groundColour(rgba, W, H);
+  const ink = emptyMask(W, H);
+  for (let i = 0; i < W * H; i++) ink.bits[i] = !person.bits[i] && Math.hypot(rgba[i * 4] - ground[0], rgba[i * 4 + 1] - ground[1], rgba[i * 4 + 2] - ground[2]) > tolerance ? 1 : 0;
+  // The outline drawn round a thing is ink too, so the thing and its outline join into one piece.
+  const near = dilate(person, 3);
+  let out: Mask | undefined;
+  for (const c of components(fillHoles(ink))) {
+    const b = maskBox(c)!;
+    if (b[0] <= 1 || b[1] <= 1 || b[2] >= W - 1 || b[3] >= H - 1) continue;
+    if (maskArea(c) < 0.05 * maskArea(person) || !c.bits.some((v, i) => v && near.bits[i])) continue;
+    // A patch of floor or shadow at the feet is not held.
+    if ((b[1] + b[3]) / 2 > py0 + 0.85 * (py1 - py0)) continue;
+    out = union(out ?? emptyMask(W, H), subtract(c, person));
+  }
+  return out;
+}
+
+/** The patches of plain ground (the picture's border colour) inside `whole` that are no part of `body`, bigger than a speck. */
+export function groundGaps(rgba: Buffer, whole: Mask, body: Mask, tolerance = 40): Mask[] {
+  const { width: W, height: H } = whole;
+  const ground = groundColour(rgba, W, H);
+  const gap = emptyMask(W, H);
+  for (let i = 0; i < W * H; i++) gap.bits[i] = whole.bits[i] && !body.bits[i] && Math.hypot(rgba[i * 4] - ground[0], rgba[i * 4 + 1] - ground[1], rgba[i * 4 + 2] - ground[2]) <= tolerance ? 1 : 0;
+  return components(gap).filter((c) => maskArea(c) > 0.002 * maskArea(body));
+}
+
+const groundColour = (rgba: Buffer, W: number, H: number) => {
+  const border: number[][] = [];
+  for (let x = 0; x < W; x += 4) for (const y of [0, H - 1]) border.push([...rgba.subarray((y * W + x) * 4, (y * W + x) * 4 + 3)]);
+  for (let y = 0; y < H; y += 4) for (const x of [0, W - 1]) border.push([...rgba.subarray((y * W + x) * 4, (y * W + x) * 4 + 3)]);
+  return [0, 1, 2].map((c) => border.map((p) => p[c]).sort((a, b) => a - b)[border.length >> 1]);
+};
+
+/** The share of the cut-out's height the body takes, its feet at the bottom: a prop held over the head makes it less than 1. */
+export function bodyShare(person: Mask, whole: Mask): number {
+  const p = maskBox(person);
+  const w = maskBox(whole);
+  if (!p || !w) return 1;
+  return Math.min(1, Math.round(((p[3] - p[1]) / Math.max(1, w[3] - w[1])) * 1000) / 1000);
+}

package/app/src/scene/icons.ts

@@ -0,0 +1,185 @@
+// icons: a picture per inventory item, and only for the items the story has. An item the hero picks up in a
+// place is cut out of that place's painting with the thing's SAM mask (no model call); one that is too small
+// there, or that only a person hands over, is painted by Muse in the world's style and cut out by SAM.
+import fs from 'node:fs';
+import sharp from 'sharp';
+import { z } from 'zod';
+import { emptyMask, imageSize, maskArea, maskFromPng, maskToPng, type Mask } from '../image/mask.ts';
+import { clampRect, components, dilate, intersect, maskBox, subtract } from '../image/raster.ts';
+import type { Memo } from '../meta/memo.ts';
+import { museEdit, musePaint } from '../meta/muse.ts';
+import { samFind } from '../meta/sam.ts';
+import { carriesOff } from '../story/engine.ts';
+import type { Story } from '../story/schema.ts';
+import type { WorldFiles } from '../world/files.ts';
+import type { World } from '../world/schema.ts';
+
+/** Icons are square, this many px a side; the item fills ICON_FILL of it. */
+export const ICON_PX = 192;
+const ICON_FILL = 168;
+/** A cut-out whose longer side is shorter than this in the painting looks blurry blown up: Muse redraws it. */
+export const MIN_CUT_PX = 56;
+
+/**
+ * Where the hero gets each item: picked up from a thing in a place (the thing of the same key, the one the painting
+ * shows), or handed over (a talk, a use, a choice, or found in something else, like a fish taken from the stalls).
+ */
+export function itemSources(story: Story): Record<string, { place: string; thing: string } | null> {
+  return Object.fromEntries(story.items.map((item) => {
+    const take = story.interactions.find((it) => it.thing === item.key && carriesOff(it));
+    return [item.key, take ? { place: take.place, thing: take.thing } : null];
+  }));
+}
+
+export const IconFixes = z.record(z.string(), z.object({
+  /** cut: from the painting; paint: Muse draws it. Unset, the step decides. */
+  how: z.enum(['cut', 'paint']).optional(),
+  /** The person's words on how it should look, for Muse. */
+  hint: z.string().min(3).optional(),
+  take: z.number().int().min(0).optional(),
+}));
+export type IconFixes = z.infer<typeof IconFixes>;
+
+export const iconFixesFile = (files: WorldFiles) => files.path('items', 'fixes.json');
+
+export function readIconFixes(files: WorldFiles): IconFixes {
+  const file = iconFixesFile(files);
+  return fs.existsSync(file) ? IconFixes.parse(files.readJson(file)) : {};
+}
+
+export const ItemsInfo = z.record(z.string(), z.object({
+  icon: z.string(),
+  how: z.enum(['cut', 'redraw', 'paint']),
+  from: z.object({ place: z.string(), thing: z.string() }).nullable(),
+}));
+export type ItemsInfo = z.infer<typeof ItemsInfo>;
+
+/** The image inside `mask`, transparent elsewhere, trimmed and centred on a square icon. */
+export async function cutIcon(image: Buffer, mask: Mask, pad = 3): Promise<Buffer> {
+  const box = maskBox(mask);
+  if (!box) throw new Error('empty mask');
+  const [x0, y0, x1, y1] = clampRect([box[0] - pad, box[1] - pad, box[2] + pad, box[3] + pad], mask.width, mask.height);
+  const alpha = await sharp(await maskToPng(mask)).blur(0.8).extractChannel(0).raw().toBuffer();
+  const rgba = await sharp(image).resize(mask.width, mask.height, { fit: 'fill' }).ensureAlpha().raw().toBuffer();
+  for (let i = 0; i < mask.width * mask.height; i++) rgba[i * 4 + 3] = alpha[i];
+  const piece = await sharp(rgba, { raw: { width: mask.width, height: mask.height, channels: 4 } })
+    .extract({ left: x0, top: y0, width: x1 - x0, height: y1 - y0 }).png().toBuffer();
+  const fitted = await sharp(piece).resize(ICON_FILL, ICON_FILL, { fit: 'inside', kernel: 'lanczos3' }).png().toBuffer();
+  return sharp({ create: { width: ICON_PX, height: ICON_PX, channels: 4, background: { r: 0, g: 0, b: 0, alpha: 0 } } })
+    .composite([{ input: fitted, gravity: 'centre' }]).png().toBuffer();
+}
+
+/**
+ * Cuts the item out of Muse's picture: SAM by its name, else as "object" (a name like "stale buns" may mean nothing to
+ * it), else everything that stands out from the plain ground Muse was asked for, read from the picture's border.
+ */
+async function samCut(memo: Memo, label: string, picture: Buffer, phrase: string): Promise<Buffer> {
+  const mask = await offGround(picture);
+  for (const prompt of [phrase, 'object']) {
+    const found = await samFind(memo, { label, image: picture, prompt });
+    const best = found.sort((a, b) => maskArea(b.mask) - maskArea(a.mask))[0];
+    if (!best) continue;
+    // SAM may name a part ("hot tea" is the tea in the cup): when what stands out from the ground holds SAM's find
+    // and is much bigger, the item is all of that.
+    const whole = mask && maskArea(intersect(mask, best.mask)) > 0.8 * maskArea(best.mask) && maskArea(mask) > 1.5 * maskArea(best.mask);
+    return cutIcon(picture, await peelGround(picture, whole ? mask : best.mask));
+  }
+  if (!mask) throw new Error(`${label}: SAM did not find "${phrase}" on Muse's picture, and nothing stands out from its ground`);
+  return cutIcon(picture, await peelGround(picture, mask));
+}
+
+/**
+ * SAM's mask of a thing Muse painted often takes in the wash or shadow it stands on, a halo in the bag. Pixels at the
+ * mask's edge coloured like what lies just outside it are peeled off, a few px at a time; the thing keeps its holes filled.
+ */
+export async function peelGround(picture: Buffer, mask: Mask, rounds = 6): Promise<Mask> {
+  const { width: W, height: H } = mask;
+  const px = await sharp(picture).removeAlpha().resize(W, H, { fit: 'fill' }).raw().toBuffer();
+  const near = (i: number, c: number[]) => Math.hypot(px[i * 3] - c[0], px[i * 3 + 1] - c[1], px[i * 3 + 2] - c[2]) < 36;
+  let m = mask;
+  for (let r = 0; r < rounds; r++) {
+    const ring = subtract(dilate(m, 4), m);
+    const v: number[][] = [[], [], []];
+    for (let i = 0; i < W * H; i++) if (ring.bits[i]) for (let c = 0; c < 3; c++) v[c].push(px[i * 3 + c]);
+    if (!v[0].length) break;
+    const ground = v.map((a) => a.sort((x, y) => x - y)[a.length >> 1]);
+    const edge = subtract(m, dilate(m, -4));
+    const next = { ...m, bits: m.bits.slice() };
+    let peeled = 0;
+    for (let i = 0; i < W * H; i++) if (edge.bits[i] && near(i, ground)) (next.bits[i] = 0, peeled++);
+    if (!peeled) break;
+    const blob = components(next)[0];
+    // Never peel the thing itself away: stop where most of it would go.
+    if (!blob || maskArea(blob) < maskArea(mask) * 0.4) break;
+    m = blob;
+  }
+  return fillHoles(m);
+}
+
+function fillHoles(m: Mask): Mask {
+  const { width: W, height: H } = m;
+  const out = { ...m, bits: m.bits.slice() };
+  const outside = { ...m, bits: m.bits.map((b) => 1 - b) };
+  for (const hole of components(outside)) {
+    const box = maskBox(hole)!;
+    if (box[0] > 0 && box[1] > 0 && box[2] < W && box[3] < H) for (let i = 0; i < W * H; i++) if (hole.bits[i]) out.bits[i] = 1;
+  }
+  return out;
+}
+
+/** The largest blob whose colour differs from the picture's border (the plain ground), or null. */
+export async function offGround(picture: Buffer): Promise<Mask | null> {
+  const { data, info } = await sharp(picture).removeAlpha().resize(400, 400, { fit: 'inside' }).raw().toBuffer({ resolveWithObject: true });
+  const { width: w, height: h } = info;
+  const border: number[][] = [[], [], []];
+  for (let x = 0; x < w; x++) for (const y of [0, 1, h - 2, h - 1]) for (let c = 0; c < 3; c++) border[c].push(data[(y * w + x) * 3 + c]);
+  for (let y = 0; y < h; y++) for (const x of [0, 1, w - 2, w - 1]) for (let c = 0; c < 3; c++) border[c].push(data[(y * w + x) * 3 + c]);
+  const ground = border.map((v) => v.sort((a, b) => a - b)[v.length >> 1]);
+  const dist = new Float32Array(w * h);
+  for (let i = 0; i < w * h; i++) dist[i] = Math.hypot(data[i * 3] - ground[0], data[i * 3 + 1] - ground[1], data[i * 3 + 2] - ground[2]);
+  // Muse's "plain" ground is often a wash with its own shades: the cut is where the distances split in two (Otsu),
+  // never closer to the ground than 40.
+  const cut = Math.max(40, otsu(dist));
+  const small = emptyMask(w, h);
+  for (let i = 0; i < w * h; i++) if (dist[i] > cut) small.bits[i] = 1;
+  const blob = components(small).sort((a, b) => maskArea(b) - maskArea(a))[0];
+  if (!blob || maskArea(blob) < w * h * 0.01) return null;
+  const full = await imageSize(picture);
+  return maskFromPng(await sharp(await maskToPng(blob)).resize(full.width, full.height, { fit: 'fill' }).png().toBuffer());
+}
+
+/** The threshold that best splits `values` (0-442) into two groups. */
+function otsu(values: Float32Array): number {
+  const bins = new Float64Array(443);
+  for (const v of values) bins[Math.min(442, Math.round(v))]++;
+  let total = 0, sum = 0;
+  for (let t = 0; t < bins.length; t++) (total += bins[t], sum += t * bins[t]);
+  let best = 0, at = 0, n0 = 0, s0 = 0;
+  for (let t = 0; t < bins.length; t++) {
+    n0 += bins[t];
+    s0 += t * bins[t];
+    const n1 = total - n0;
+    if (!n0 || !n1) continue;
+    const between = n0 * n1 * (s0 / n0 - (sum - s0) / n1) ** 2;
+    if (between > best) (best = between, at = t);
+  }
+  return at;
+}
+
+const look = (name: string, hint?: string) => (hint ? `${name} (${hint})` : name);
+
+/** Muse redraws a small cut-out larger: same thing, same style, alone on a plain ground. */
+export async function redrawIcon(memo: Memo, world: World, item: { key: string; name: string }, crop: Buffer, hint?: string, take = 0): Promise<Buffer> {
+  const prompt = `This is a small ${item.name} cut from a painting. Paint the same ${look(item.name, hint)} large, whole and centred, alone on a plain flat pale background with no shadow or ground under it, `
+    + `as an inventory icon for a point-and-click adventure game. Keep its shape, colours and details, in this style: ${world.style}. No hands, no people, no text, no frame.`;
+  const picture = await museEdit(memo, { label: `icon:${item.key}:redraw`, prompt, images: [crop], take });
+  return samCut(memo, `icon:${item.key}`, picture, item.name);
+}
+
+/** Muse paints an item nobody can see in any place. */
+export async function paintIcon(memo: Memo, world: World, item: { key: string; name: string }, hint?: string, take = 0): Promise<Buffer> {
+  const prompt = `A single ${look(item.name, hint)}, whole and centred, alone on a plain flat pale background with no shadow or ground under it, as an inventory icon for a point-and-click adventure game `
+    + `set in "${world.title}". Style: ${world.style}. No hands, no people, no text, no frame.`;
+  const picture = await musePaint(memo, { label: `icon:${item.key}:paint`, prompt, take });
+  return samCut(memo, `icon:${item.key}`, picture, item.name);
+}