@gtanczyk/diorama 0.0.0-stage → 0.1.0

package/app/src/image/mask.ts

@@ -0,0 +1,29 @@
+// A mask is one byte per pixel (0 or 1) at the size of the image it belongs to.
+import sharp from 'sharp';
+
+export type Mask = { width: number; height: number; bits: Uint8Array };
+
+export const emptyMask = (width: number, height: number): Mask => ({ width, height, bits: new Uint8Array(width * height) });
+
+export const maskArea = (m: Mask) => m.bits.reduce((sum, b) => sum + b, 0);
+
+/** Writes a mask as an 8-bit grayscale PNG (0 / 255). */
+export function maskToPng(m: Mask): Promise<Buffer> {
+  const gray = Buffer.alloc(m.bits.length);
+  for (let i = 0; i < gray.length; i++) gray[i] = m.bits[i] ? 255 : 0;
+  return sharp(gray, { raw: { width: m.width, height: m.height, channels: 1 } }).png().toBuffer();
+}
+
+/** Reads any image as a mask: a pixel is set when its gray value is above the midpoint. */
+export async function maskFromPng(png: Buffer): Promise<Mask> {
+  const { data, info } = await sharp(png).grayscale().raw().toBuffer({ resolveWithObject: true });
+  const bits = new Uint8Array(info.width * info.height);
+  for (let i = 0; i < bits.length; i++) bits[i] = data[i * info.channels] > 127 ? 1 : 0;
+  return { width: info.width, height: info.height, bits };
+}
+
+export async function imageSize(image: Buffer): Promise<{ width: number; height: number }> {
+  const { width, height } = await sharp(image).metadata();
+  if (!width || !height) throw new Error('unreadable image');
+  return { width, height };
+}

package/app/src/image/overlay.ts

@@ -0,0 +1,38 @@
+// Review pictures: masks tinted over the painting, so a person sees at a glance what a step found.
+import sharp from 'sharp';
+import type { Mask } from './mask.ts';
+
+export type Tint = { mask: Mask; rgb: [number, number, number]; alpha?: number };
+
+/** Distinct, readable colours for up to a dozen layers; they repeat after that. */
+export const PALETTE: [number, number, number][] = [
+  [255, 70, 70], [70, 200, 255], [255, 210, 40], [170, 90, 255], [60, 230, 120], [255, 130, 200],
+  [255, 140, 30], [40, 120, 255], [200, 255, 60], [0, 200, 180], [230, 60, 160], [150, 150, 255],
+];
+
+/** Tints each mask over the painting; a mask's edge pixels are drawn solid. */
+export async function tintOver(painting: Buffer, tints: Tint[], extraSvg = ''): Promise<Buffer> {
+  const { data, info } = await sharp(painting).removeAlpha().raw().toBuffer({ resolveWithObject: true });
+  const { width: W, height: H } = info;
+  const out = Buffer.from(data);
+  for (const { mask, rgb, alpha = 0.45 } of tints) {
+    for (let y = 0; y < H; y++) {
+      for (let x = 0; x < W; x++) {
+        const i = y * W + x;
+        if (!mask.bits[i]) continue;
+        const edge = x === 0 || y === 0 || x === W - 1 || y === H - 1 || !mask.bits[i - 1] || !mask.bits[i + 1] || !mask.bits[i - W] || !mask.bits[i + W];
+        const a = edge ? 1 : alpha;
+        for (let c = 0; c < 3; c++) out[i * 3 + c] = Math.round(out[i * 3 + c] * (1 - a) + rgb[c] * a);
+      }
+    }
+  }
+  const img = sharp(out, { raw: { width: W, height: H, channels: 3 } });
+  const svg = extraSvg && `<svg xmlns="http://www.w3.org/2000/svg" width="${W}" height="${H}">${extraSvg}</svg>`;
+  return (svg ? img.composite([{ input: Buffer.from(svg) }]) : img).png().toBuffer();
+}
+
+/** Labels for an overlay, drawn with a dark outline so they read on any painting. */
+export function svgLabel(x: number, y: number, text: string, rgb: [number, number, number] = [255, 255, 255]): string {
+  const safe = text.replace(/&/g, '&amp;').replace(/</g, '&lt;');
+  return `<text x="${x}" y="${y}" font-family="sans-serif" font-size="15" font-weight="bold" fill="rgb(${rgb.join(',')})" stroke="#000" stroke-width="3" paint-order="stroke">${safe}</text>`;
+}

package/app/src/image/raster.ts

@@ -0,0 +1,224 @@
+// Pixel arithmetic on masks: boxes, overlaps, blobs, polygons. Pure code, no images on disk.
+import { emptyMask, maskArea, type Mask } from './mask.ts';
+
+/** [x0, y0, x1, y1] in pixels; x1 and y1 are exclusive. */
+export type Rect = [number, number, number, number];
+
+export const rectArea = ([x0, y0, x1, y1]: Rect) => Math.max(0, x1 - x0) * Math.max(0, y1 - y0);
+
+export function rectIou(a: Rect, b: Rect): number {
+  const inter = rectArea([Math.max(a[0], b[0]), Math.max(a[1], b[1]), Math.min(a[2], b[2]), Math.min(a[3], b[3])]);
+  const union = rectArea(a) + rectArea(b) - inter;
+  return union > 0 ? inter / union : 0;
+}
+
+/** Sorted corners, clamped to the frame and rounded to whole pixels. */
+export function clampRect([x0, y0, x1, y1]: Rect, width: number, height: number): Rect {
+  const cx = (v: number) => Math.round(Math.max(0, Math.min(width, v)));
+  const cy = (v: number) => Math.round(Math.max(0, Math.min(height, v)));
+  return [cx(Math.min(x0, x1)), cy(Math.min(y0, y1)), cx(Math.max(x0, x1)), cy(Math.max(y0, y1))];
+}
+
+export function growRect([x0, y0, x1, y1]: Rect, by: number): Rect {
+  const dx = ((x1 - x0) * (by - 1)) / 2;
+  const dy = ((y1 - y0) * (by - 1)) / 2;
+  return [x0 - dx, y0 - dy, x1 + dx, y1 + dy];
+}
+
+/** The smallest rect around the set pixels, or undefined for an empty mask. */
+export function maskBox(m: Mask): Rect | undefined {
+  let x0 = m.width, y0 = m.height, x1 = -1, y1 = -1;
+  for (let y = 0; y < m.height; y++) {
+    for (let x = 0; x < m.width; x++) {
+      if (!m.bits[y * m.width + x]) continue;
+      if (x < x0) x0 = x;
+      if (x > x1) x1 = x;
+      if (y < y0) y0 = y;
+      y1 = y;
+    }
+  }
+  return x1 < 0 ? undefined : [x0, y0, x1 + 1, y1 + 1];
+}
+
+export function rectMask(width: number, height: number, rect: Rect): Mask {
+  const m = emptyMask(width, height);
+  const [x0, y0, x1, y1] = clampRect(rect, width, height);
+  for (let y = y0; y < y1; y++) m.bits.fill(1, y * width + x0, y * width + x1);
+  return m;
+}
+
+/** Even-odd scanline fill of a polygon given as [x, y] points. */
+export function polygonMask(width: number, height: number, points: [number, number][]): Mask {
+  const m = emptyMask(width, height);
+  for (let y = 0; y < height; y++) {
+    const cy = y + 0.5;
+    const xs: number[] = [];
+    for (let i = 0; i < points.length; i++) {
+      const [ax, ay] = points[i];
+      const [bx, by] = points[(i + 1) % points.length];
+      if ((ay <= cy) !== (by <= cy)) xs.push(ax + ((cy - ay) / (by - ay)) * (bx - ax));
+    }
+    xs.sort((a, b) => a - b);
+    for (let i = 0; i + 1 < xs.length; i += 2) {
+      const from = Math.max(0, Math.ceil(xs[i] - 0.5));
+      const to = Math.min(width, Math.ceil(xs[i + 1] - 0.5));
+      if (to > from) m.bits.fill(1, y * width + from, y * width + to);
+    }
+  }
+  return m;
+}
+
+function combine(a: Mask, b: Mask, op: (x: number, y: number) => number): Mask {
+  if (a.width !== b.width || a.height !== b.height) throw new Error(`mask sizes differ: ${a.width}x${a.height} vs ${b.width}x${b.height}`);
+  const out = emptyMask(a.width, a.height);
+  for (let i = 0; i < out.bits.length; i++) out.bits[i] = op(a.bits[i], b.bits[i]);
+  return out;
+}
+
+export const union = (a: Mask, b: Mask) => combine(a, b, (x, y) => x | y);
+export const intersect = (a: Mask, b: Mask) => combine(a, b, (x, y) => x & y);
+export const subtract = (a: Mask, b: Mask) => combine(a, b, (x, y) => x & (1 - y));
+
+export function unionAll(width: number, height: number, masks: Mask[]): Mask {
+  return masks.reduce(union, emptyMask(width, height));
+}
+
+export function maskIou(a: Mask, b: Mask): number {
+  let inter = 0, either = 0;
+  for (let i = 0; i < a.bits.length; i++) {
+    inter += a.bits[i] & b.bits[i];
+    either += a.bits[i] | b.bits[i];
+  }
+  return either > 0 ? inter / either : 0;
+}
+
+/** Grows (r > 0) or shrinks (r < 0) a mask by a square of |r| px, as two separable passes of running counts. */
+export function dilate(m: Mask, r: number): Mask {
+  if (r === 0) return { ...m, bits: m.bits.slice() };
+  const grow = r > 0;
+  const k = Math.abs(r);
+  const { width: W, height: H } = m;
+  // One line at a time (a row, or a column with stride W): a pixel is set when any pixel within k is (growing),
+  // or when all 2k+1 are, inside the frame (shrinking: outside the frame counts as empty, so it eats in from the edges).
+  const pass = (src: Uint8Array, along: 'x' | 'y') => {
+    const out = new Uint8Array(src.length);
+    const [lines, len, step, lineStep] = along === 'x' ? [H, W, 1, W] : [W, H, W, 1];
+    const sum = new Int32Array(len + 1);
+    for (let l = 0; l < lines; l++) {
+      const base = l * lineStep;
+      for (let i = 0; i < len; i++) sum[i + 1] = sum[i] + src[base + i * step];
+      for (let i = 0; i < len; i++) {
+        const lo = i - k;
+        const hi = i + k;
+        const n = sum[Math.min(len, hi + 1)] - sum[Math.max(0, lo)];
+        out[base + i * step] = grow ? (n > 0 ? 1 : 0) : lo >= 0 && hi < len && n === 2 * k + 1 ? 1 : 0;
+      }
+    }
+    return out;
+  };
+  return { width: W, height: H, bits: pass(pass(m.bits, 'x'), 'y') };
+}
+
+/** 4-connected blobs, largest first. */
+export function components(m: Mask): Mask[] {
+  const { width: W, height: H } = m;
+  const label = new Int32Array(W * H).fill(-1);
+  const blobs: number[][] = [];
+  const stack: number[] = [];
+  for (let start = 0; start < m.bits.length; start++) {
+    if (!m.bits[start] || label[start] >= 0) continue;
+    const pixels: number[] = [];
+    label[start] = blobs.length;
+    stack.push(start);
+    while (stack.length) {
+      const i = stack.pop()!;
+      pixels.push(i);
+      const x = i % W;
+      for (const n of [x > 0 ? i - 1 : -1, x < W - 1 ? i + 1 : -1, i - W, i + W]) {
+        if (n >= 0 && n < W * H && m.bits[n] && label[n] < 0) {
+          label[n] = blobs.length;
+          stack.push(n);
+        }
+      }
+    }
+    blobs.push(pixels);
+  }
+  return blobs.sort((a, b) => b.length - a.length).map((pixels) => {
+    const blob = emptyMask(W, H);
+    for (const i of pixels) blob.bits[i] = 1;
+    return blob;
+  });
+}
+
+/** Drops blobs smaller than `minPx`. */
+export function withoutSpecks(m: Mask, minPx: number): Mask {
+  return unionAll(m.width, m.height, components(m).filter((c) => maskArea(c) >= minPx));
+}
+
+/** For each column of the mask, its lowest set row (-1 where the column is empty). */
+export function bottomRows(m: Mask): Int32Array {
+  const rows = new Int32Array(m.width).fill(-1);
+  for (let y = 0; y < m.height; y++) {
+    for (let x = 0; x < m.width; x++) if (m.bits[y * m.width + x]) rows[x] = y;
+  }
+  return rows;
+}
+
+/**
+ * Fills the holes of a mask: unset regions it closes on every side (they never reach the frame's edge) of at most
+ * `maxPx` each. Bigger holes are real gaps (the floor around a wide table), smaller ones are what lies on the surface.
+ */
+export function fillHoles(m: Mask, maxPx = Infinity): Mask {
+  const { width: W, height: H } = m;
+  const bits = m.bits.slice();
+  const seen = new Uint8Array(W * H);
+  const stack: number[] = [];
+  for (let start = 0; start < bits.length; start++) {
+    if (bits[start] || seen[start]) continue;
+    const pixels: number[] = [];
+    let open = false;
+    seen[start] = 1;
+    stack.push(start);
+    while (stack.length) {
+      const i = stack.pop()!;
+      pixels.push(i);
+      const x = i % W;
+      const y = (i - x) / W;
+      if (x === 0 || y === 0 || x === W - 1 || y === H - 1) open = true;
+      for (const n of [x > 0 ? i - 1 : -1, x < W - 1 ? i + 1 : -1, y > 0 ? i - W : -1, y < H - 1 ? i + W : -1]) {
+        if (n >= 0 && !bits[n] && !seen[n]) {
+          seen[n] = 1;
+          stack.push(n);
+        }
+      }
+    }
+    if (!open && pixels.length <= maxPx) for (const i of pixels) bits[i] = 1;
+  }
+  return { width: W, height: H, bits };
+}
+
+/**
+ * What a frame holds: the opening of a doorway, an arch or a gate, whose posts stand at the sides of its box and whose
+ * lintel spans the top. Row by row below the lintel, the gap between the inner edge of the left post and that of the
+ * right one; a post SAM cut short keeps its last edge down to the box's bottom, where the floor runs through.
+ */
+export function framedOpening(frame: Mask, box: Rect): Mask {
+  const { width: W, height: H } = frame;
+  const [x0, y0, x1, y1] = clampRect(box, W, H);
+  const out = emptyMask(W, H);
+  const third = Math.max(1, Math.floor((x1 - x0) / 3));
+  const mid = [x0 + third, x1 - third];
+  // The lintel: the lowest frame row in the top half of the box's middle third.
+  let top = -1;
+  for (let y = y0; y < y0 + (y1 - y0) / 2; y++) for (let x = mid[0]; x < mid[1]; x++) if (frame.bits[y * W + x]) top = y;
+  if (top < 0) return out;
+  let left = -1;
+  let right = -1;
+  for (let y = top + 1; y < y1; y++) {
+    for (let x = mid[0] - 1; x >= x0; x--) if (frame.bits[y * W + x]) { left = x; break; }
+    for (let x = mid[1]; x < x1; x++) if (frame.bits[y * W + x]) { right = x; break; }
+    if (left < 0 || right < 0) continue;
+    for (let x = left + 1; x < right; x++) if (!frame.bits[y * W + x]) out.bits[y * W + x] = 1;
+  }
+  return out;
+}

package/app/src/main.ts

@@ -0,0 +1,28 @@
+import './commands/index.ts';
+import { registry } from './registry.ts';
+import { loadSecrets, redact } from './secrets.ts';
+
+function help() {
+  console.log('diorama: a story prompt becomes painted scenes you can walk in\n\nusage: diorama <command> [args]\n');
+  for (const cmd of registry.values()) console.log(`  ${cmd.usage}`);
+  console.log('\nfirst time: diorama setup (keys, what is missing), then diorama studio. Worlds are written to ./worlds here.');
+}
+
+const [name, ...args] = process.argv.slice(2);
+const cmd = name ? registry.get(name) : undefined;
+if (!cmd) {
+  help();
+  process.exit(!name || name === 'help' ? 0 : 2);
+}
+if (args.includes('--help') || args.includes('-h')) {
+  console.log(`usage: diorama ${cmd.usage}`);
+  process.exit(0);
+}
+try {
+  // Keys kept by `diorama setup` fill in what the environment leaves unset.
+  if (name !== 'setup') await loadSecrets();
+  await cmd.run(args);
+} catch (error) {
+  console.error(redact(`diorama ${name}: ${error instanceof Error ? error.message : error}`));
+  process.exit(1);
+}

package/app/src/meta/choice.ts

@@ -0,0 +1,27 @@
+// Which models make a world, as picked in Studio Setup (~/.config/diorama/models.json). DIORAMA_SPARK_VIA and
+// DIORAMA_IMAGE_VIA, where the Studio or a command was started, win over the pick.
+import fs from 'node:fs';
+import path from 'node:path';
+import { configDir } from '../secrets.ts';
+
+export type ModelChoice = { spark?: string; image?: string };
+
+const file = () => path.join(configDir(), 'models.json');
+
+export function readModelChoice(): ModelChoice {
+  try {
+    const c = JSON.parse(fs.readFileSync(file(), 'utf8')) as ModelChoice;
+    return { spark: typeof c.spark === 'string' ? c.spark : undefined, image: typeof c.image === 'string' ? c.image : undefined };
+  } catch {
+    return {};
+  }
+}
+
+export function writeModelChoice(choice: ModelChoice) {
+  fs.mkdirSync(path.dirname(file()), { recursive: true, mode: 0o700 });
+  fs.writeFileSync(file(), `${JSON.stringify(choice, null, 2)}\n`);
+}
+
+/** The model for Spark's asks and for pictures: the environment first, then the Setup pick. */
+export const sparkChoice = () => process.env.DIORAMA_SPARK_VIA || readModelChoice().spark;
+export const imageChoice = () => process.env.DIORAMA_IMAGE_VIA || readModelChoice().image;

package/app/src/meta/http.ts

@@ -0,0 +1,99 @@
+// The Meta Model API: one key, a POST that retries what is worth retrying.
+export const META_API = 'https://api.meta.ai/v1';
+
+/** The first of these that is set holds the Meta Model API key. */
+export const KEY_VARS = ['META_API_KEY', 'META_MUSE_API_KEY', 'META_MODEL_API_KEY'] as const;
+
+export function apiKey(): string {
+  const key = KEY_VARS.map((name) => process.env[name]).find(Boolean);
+  if (!key) throw new Error(`no Meta Model API key: set one of ${KEY_VARS.join(', ')}`);
+  return key;
+}
+
+export class MetaHttpError extends Error {
+  constructor(readonly status: number, message: string) {
+    super(message);
+  }
+}
+
+const TRANSIENT = new Set([408, 409, 429, 500, 502, 503, 504]);
+const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms));
+/** Tries for a request whose connection dropped: 2+4+8+16+30+30+30+30 s of waiting. */
+const OFFLINE_TRIES = 9;
+
+/**
+ * How long one try may take before it is given up and tried again. Without a limit a stalled request waits for the
+ * HTTP client's own 300 s: on the Mac one backdrop edit took 343 s where it takes 10 to 20. Muse paints in 10-25 s,
+ * SAM answers in 1-6 s, Spark thinks for up to ~100 s.
+ */
+function patience(route: string, body: unknown): number {
+  // An edit from several pictures (a place painted from its sketch, concept and style) takes longer, more so with
+  // a few painting at once: 90 s cut every one of 8 such paints off three times, 270-300 s each where one patient
+  // try takes 40-120 s (lantern-stranger, lost-friends-maze). A dropped try is asked again from the back of the queue.
+  if (route.startsWith('/images/')) return ((body as { images?: unknown[] } | null)?.images?.length ?? 0) > 1 ? 180_000 : 90_000;
+  const model = (body as { model?: string } | null)?.model ?? '';
+  return model.startsWith('sam') ? 60_000 : 240_000;
+}
+
+/**
+ * How many requests each model may have open at once. The build runs places, cast and sound side by side
+ * (src/world/schedule.ts), so without a cap a four-place world can ask SAM for dozens of cut-outs at the same
+ * moment. A request over the cap waits its turn; DIORAMA_LANES="muse=6,spark=12,sam=16" sets them.
+ */
+// Muse at 6: eight multi-picture paints at once queued past every timeout on the Meta side.
+const LANES: Record<string, number> = { muse: 6, spark: 12, sam: 16 };
+for (const pair of (process.env.DIORAMA_LANES ?? '').split(',')) {
+  const [lane, n] = pair.split('=');
+  if (lane in LANES && Number(n) >= 1) LANES[lane] = Math.floor(Number(n));
+}
+const open: Record<string, number> = {};
+const waiting: Record<string, Array<() => void>> = {};
+const laneOf = (route: string, body: unknown) => (route.startsWith('/images/') ? 'muse' : ((body as { model?: string } | null)?.model ?? '').startsWith('sam') ? 'sam' : 'spark');
+
+async function inLane<T>(lane: string, work: () => Promise<T>): Promise<T> {
+  if ((open[lane] ?? 0) >= LANES[lane]) await new Promise<void>((go) => (waiting[lane] ??= []).push(go));
+  else open[lane] = (open[lane] ?? 0) + 1;
+  try {
+    return await work();
+  } finally {
+    // The slot passes straight to the next in line, or is given back.
+    const next = waiting[lane]?.shift();
+    if (next) next();
+    else open[lane]--;
+  }
+}
+
+/** POSTs JSON. Rate limits, server errors and stalls are retried with backoff (2, 4, 8 s); network drops for longer. */
+export async function metaPost(route: string, body: unknown, accept?: string): Promise<Response> {
+  const tries = 4;
+  const limit = patience(route, body);
+  for (let n = 1; ; n++) {
+    let failure: unknown;
+    try {
+      // The slot is held until the answer is read or turned down, not through the wait before a retry.
+      const res = await inLane(laneOf(route, body), async () => {
+        const res = await fetch(META_API + route, {
+          signal: AbortSignal.timeout(limit),
+          method: 'POST',
+          headers: { Authorization: `Bearer ${apiKey()}`, 'Content-Type': 'application/json', ...(accept && { Accept: accept }) },
+          body: JSON.stringify(body),
+        });
+        return res.ok ? res : new MetaHttpError(res.status, `POST ${route}: ${res.status} ${(await res.text()).slice(0, 300)}`);
+      });
+      if (!(res instanceof MetaHttpError)) return res;
+      failure = res;
+      if (!TRANSIENT.has(res.status)) throw failure;
+    } catch (error) {
+      if (error instanceof MetaHttpError && !TRANSIENT.has(error.status)) throw error;
+      failure = error;
+    }
+    // A dropped connection (Wi-Fi gone, the laptop dozing: "fetch failed") gets longer: up to ~3 min in all, so a
+    // short outage does not cost a place its painting.
+    const offline = failure instanceof TypeError || (failure instanceof Error && /fetch failed|ECONNRESET|ENOTFOUND|EAI_AGAIN|socket/i.test(failure.message));
+    if (n >= (offline ? OFFLINE_TRIES : tries)) throw failure;
+    // Said in the job log, so a slow step shows why.
+    const why = failure instanceof Error && failure.name === 'TimeoutError' ? `no answer in ${limit / 1000} s` : failure instanceof Error ? failure.message.slice(0, 120) : String(failure);
+    console.log(`  POST ${route}: ${why}; trying again (${n + 1}/${tries})`);
+    await sleep(Math.min(30_000, 1000 * 2 ** n));
+  }
+}

package/app/src/meta/memo.ts

@@ -0,0 +1,91 @@
+// Model answers are stored by a hash of everything that went in, so a rerun with the same
+// inputs costs nothing, and any change to a prompt or image asks the model again.
+import { createHash } from 'node:crypto';
+import fs from 'node:fs';
+import os from 'node:os';
+import path from 'node:path';
+import { trackOperation } from '../studio/operation.ts';
+import { currentStep } from './step.ts';
+
+/** sound: ElevenLabs effects and music; music: MusicGen run locally; voice: speech and its Muse Voice transcript. */
+/** space: a Hugging Face Space (TRELLIS, auto-rig), not a Meta model; its answers are kept the same way. */
+export type ModelName = 'spark' | 'muse' | 'sam' | 'sound' | 'music' | 'voice' | 'space';
+
+const SHARED = new Set<ModelName>(['sound', 'music', 'voice']);
+const sharedFile = (model: ModelName, hash: string, ext: string) => path.join(process.env.DIORAMA_CACHE || path.join(os.homedir(), '.cache', 'diorama'), 'memo', model, `${hash}.${ext}`);
+
+export function inputHash(...parts: (string | Buffer)[]): string {
+  const h = createHash('sha256');
+  for (const part of parts) h.update(part).update('\u0000');
+  return h.digest('hex').slice(0, 32);
+}
+
+/** What a call cost, when the asker knows: dollars and which backend answered (e.g. "claude" for a dev-mode Spark). */
+export type Paid = { data: Buffer; usd?: number; via?: string };
+
+export class Memo {
+  /** Calls actually sent to each model in this process (cache hits are not counted). */
+  readonly sent: Record<ModelName, number> = { spark: 0, muse: 0, sam: 0, sound: 0, music: 0, voice: 0, space: 0 };
+
+  constructor(readonly dir: string) {}
+
+  /** Asks on their way: the same question asked twice at once (two exits with one sound) is paid once. */
+  private readonly asking = new Map<string, Promise<Buffer>>();
+
+  async remember(model: ModelName, label: string, hash: string, ext: string, ask: () => Promise<Buffer | Paid>): Promise<Buffer> {
+    const id = `${model}/${hash}.${ext}`;
+    const pending = this.asking.get(id);
+    if (pending) {
+      const answer = await pending;
+      this.log({ model, label, hash, ms: 0, hit: true });
+      return answer;
+    }
+    const work = this.recall(model, label, hash, ext, ask);
+    this.asking.set(id, work);
+    try {
+      return await work;
+    } finally {
+      this.asking.delete(id);
+    }
+  }
+
+  private async recall(model: ModelName, label: string, hash: string, ext: string, ask: () => Promise<Buffer | Paid>): Promise<Buffer> {
+    const file = path.join(this.dir, model, `${hash}.${ext}`);
+    // Paid audio is also kept for every world on this machine: another world asking the same pays nothing.
+    const shared = SHARED.has(model) ? sharedFile(model, hash, ext) : undefined;
+    for (const f of [file, shared]) {
+      if (!f || !fs.existsSync(f)) continue;
+      this.log({ model, label, hash, ms: 0, hit: true });
+      const answer = fs.readFileSync(f);
+      if (f !== file) (fs.mkdirSync(path.dirname(file), { recursive: true }), fs.writeFileSync(file, answer));
+      return answer;
+    }
+    const started = Date.now();
+    const got = await trackOperation(model, label, ask);
+    const { data: answer, usd, via } = Buffer.isBuffer(got) ? { data: got } as Paid : got;
+    for (const f of [file, shared]) if (f) (fs.mkdirSync(path.dirname(f), { recursive: true }), fs.writeFileSync(f, answer));
+    this.sent[model]++;
+    this.log({ model, label, hash, ms: Date.now() - started, hit: false, ...(usd !== undefined && { usd: Math.round(usd * 1e6) / 1e6 }), ...(via && { via }) });
+    return answer;
+  }
+
+  /** Whether an answer is kept, here or in the cache every world shares. */
+  has(model: ModelName, hash: string, ext: string): boolean {
+    return fs.existsSync(path.join(this.dir, model, `${hash}.${ext}`)) || (SHARED.has(model) && fs.existsSync(sharedFile(model, hash, ext)));
+  }
+
+  /**
+   * Forgets one answer, e.g. a reply that failed its gate and must be asked again. Also the copy every world shares:
+   * left there, the next run would read the rejected take back from it and never ask again.
+   */
+  forget(model: ModelName, hash: string, ext: string) {
+    fs.rmSync(path.join(this.dir, model, `${hash}.${ext}`), { force: true });
+    if (SHARED.has(model)) fs.rmSync(sharedFile(model, hash, ext), { force: true });
+  }
+
+  private log(entry: { model: ModelName; label: string; hash: string; ms: number; hit: boolean; usd?: number; via?: string }) {
+    fs.mkdirSync(this.dir, { recursive: true });
+    const step = currentStep();
+    fs.appendFileSync(path.join(this.dir, 'calls.jsonl'), `${JSON.stringify({ at: new Date().toISOString(), ...entry, ...(step && { step }) })}\n`);
+  }
+}

package/app/src/meta/muse.ts

@@ -0,0 +1,70 @@
+// Muse Image: paints from a prompt, or from reference images whose layout it keeps.
+import { metaPost } from './http.ts';
+import { inputHash, type Memo } from './memo.ts';
+import { imageChoice } from './choice.ts';
+import { IMAGE_MODELS, paint } from './openrouter.ts';
+import { PRICES } from './prices.ts';
+
+export const MUSE = 'muse-image-1.0';
+
+/**
+ * Which model paints (Studio Setup, or DIORAMA_IMAGE_VIA): "meta" (Muse Image, the default), a short name from IMAGE_MODELS
+ * (gemini-flash, gemini-lite, nano-banana, gemini-pro) or any OpenRouter image model id. Another model's
+ * pictures are kept under their own memo keys, so switching back finds the Muse ones again.
+ */
+export function imageVia(): string | null {
+  const via = imageChoice();
+  return !via || via === 'meta' ? null : (IMAGE_MODELS[via] ?? via);
+}
+
+type MuseAsk = { label: string; prompt: string; size?: string; /** Changes only the memo key, to get a new take. */ take?: number };
+
+/** The picture behind a URL answer, its download retried like the request was (a dropped connection, a 5xx). */
+async function fetchImage(url: string): Promise<Buffer> {
+  for (let n = 1; ; n++) {
+    try {
+      const res = await fetch(url, { signal: AbortSignal.timeout(60_000) });
+      if (res.ok) return Buffer.from(await res.arrayBuffer());
+      if (res.status < 500 || n >= 6) throw new Error(`image download: ${res.status}`);
+    } catch (error) {
+      if (n >= 6 || (error instanceof Error && error.message.startsWith('image download:'))) throw error;
+    }
+    await new Promise((r) => setTimeout(r, Math.min(30_000, 1000 * 2 ** n)));
+  }
+}
+
+async function firstImage(res: Response, label: string): Promise<Buffer> {
+  const body = (await res.json()) as { data?: { b64_json?: string; url?: string }[] };
+  const item = body.data?.[0];
+  if (item?.b64_json) return Buffer.from(item.b64_json, 'base64');
+  if (item?.url) return fetchImage(item.url);
+  throw new Error(`muse ${label}: no image in the answer`);
+}
+
+// Without output_format the API answers WebP; everything downstream reads PNG.
+const PNG_OUT = { n: 1, response_format: 'b64_json', output_format: 'png' } as const;
+
+export async function musePaint(memo: Memo, { label, prompt, size = '1200x896', take = 0 }: MuseAsk): Promise<Buffer> {
+  const via = imageVia();
+  if (via) return memo.remember('muse', label, inputHash(via, 'paint', size, String(take), prompt), 'png', () => paint(label, via, prompt, size));
+  return memo.remember('muse', label, inputHash(MUSE, 'paint', size, String(take), prompt), 'png', async () =>
+    ({ data: await firstImage(await metaPost('/images/generations', { model: MUSE, prompt, size, ...PNG_OUT }), label), usd: PRICES.museImage }));
+}
+
+export async function museEdit(memo: Memo, { label, prompt, size = '1200x896', take = 0, images }: MuseAsk & { images: Buffer[] }): Promise<Buffer> {
+  if (images.length === 0) throw new Error(`muse ${label}: an edit needs an image`);
+  const via = imageVia();
+  if (via) return memo.remember('muse', label, inputHash(via, 'edit', size, String(take), prompt, ...images), 'png', () => paint(label, via, prompt, size, images));
+  return memo.remember('muse', label, inputHash(MUSE, 'edit', size, String(take), prompt, ...images), 'png', async () => ({
+    usd: PRICES.museImage,
+    data: await firstImage(await metaPost('/images/edits', {
+      model: MUSE,
+      prompt,
+      size,
+      images: images.map((image) => ({ image_url: `data:image/png;base64,${image.toString('base64')}` })),
+      reasoning_strength: 'low',
+      tool_enablement: { enable_image_search: false, enable_web_search: false, enable_shell: false },
+      ...PNG_OUT,
+    }), label),
+  }));
+}