@gtanczyk/diorama 0.0.0-stage → 0.2.0

package/app/src/commands/faces.ts

@@ -0,0 +1,80 @@
+import fs from 'node:fs';
+import { castLanes, mapLimit } from '../world/pool.ts';
+import { CalibrationTarget, setFaceCalibration } from '../scene/face-calibration.ts';
+import { defineCommand, takeOption } from '../registry.ts';
+import { WorldFiles } from '../world/files.ts';
+import { players } from '../world/drawings.ts';
+import { prepareFace, repairFaceFrame, calibrateFaceFrame } from '../scene/faces.ts';
+import { registeredFace } from '../scene/faces-data.ts';
+import { FacePreparationError, LipBox, MouthName } from '../scene/face-work.ts';
+import { prepareLipSync } from '../sound/lipsync-make.ts';
+
+export function faceCharacters(files: WorldFiles): string[] {
+  const world = files.world();
+  return [...new Set([world.hero.key, ...world.places.flatMap(p => p.things.filter(t => t.role === 'npc').map(t => t.key)), ...players(files).map(p => p.key)])];
+}
+
+defineCommand({
+  name: 'faces',
+  usage: 'faces <slug> [key...]    prepare speaking faces for paintings, 2D rigs and talk sprites, and cached audio lip-sync [--audio-only] [--image-only] [--force] [--take N] [--rhubarb PATH]',
+  async run(args) {
+    let take = Number(takeOption(args, 'take', '0'));
+    let repairWorkId: string | undefined;
+    const executable = takeOption(args, 'rhubarb', '');
+    const repairId=takeOption(args,'repair-frame',''), calibrateId=takeOption(args,'calibrate-frame','');
+    const revision=takeOption(args,'revision',''), mouth=takeOption(args,'mouth',''), hint=takeOption(args,'hint','');
+    const box=takeOption(args,'box','');
+    const calibration=takeOption(args,'calibration',''), imageHash=takeOption(args,'image-hash',''), open=takeOption(args,'open','');
+    const flag = (name: string) => { const i = args.indexOf(`--${name}`); return i >= 0 ? (args.splice(i, 1), true) : false; };
+    const audioOnly = flag('audio-only'), imageOnly = flag('image-only'), force = flag('force');
+    const [slug, ...only] = args;
+    if (!slug) throw new Error('usage: diorama faces <slug> [key...]');
+    const files = new WorldFiles(slug), keys = faceCharacters(files);
+    for (const key of only) if (!keys.includes(key)) throw new Error(`${slug} has no character ${key}`);
+    if(calibration) {
+      if(only.length!==1 || repairId || calibrateId || !['true','false'].includes(open))throw new Error('Choose one inspected calibration');
+      await setFaceCalibration(files,only[0],CalibrationTarget.parse(JSON.parse(calibration)),revision,imageHash,LipBox.parse(JSON.parse(box)),open==='true');
+    }
+    if(repairId || calibrateId) {
+      if(only.length!==1 || repairId && calibrateId) throw new Error('Choose one character and one face correction');
+      const work = repairId ? await repairFaceFrame(files,only[0],repairId,revision,MouthName.parse(mouth),hint)
+        : calibrateFaceFrame(files,only[0],calibrateId,revision,MouthName.parse(mouth),LipBox.parse(JSON.parse(box)));
+      take = work.take; repairWorkId = work.id;
+    }
+    const statusFile = files.path('face-preparation.json');
+    const status = fs.existsSync(statusFile) ? files.readJson(statusFile) as Record<string, { face?: string | null; audio?: string[] }> : {};
+    for (const key of keys) if (!status[key]) {
+      const attempt=files.path('cast',key,'face-attempt.json');
+      if(fs.existsSync(attempt)) {
+        const previous=files.readJson(attempt) as {state?:string;reason?:string};
+        status[key]=previous.state==='failed'&&!registeredFace(files,key)?.info.imagePose?.lips?{face:previous.reason??'Face preparation failed'}:previous.state?{face:null}:{};
+      }
+    }
+    const failures: string[] = [];
+    // Each character's face is its own work (its own files under cast/<key>): a few side by side.
+    if (!audioOnly) await mapLimit(keys.filter(k => !only.length || only.includes(k)), castLanes(), async (key) => {
+      try { await prepareFace(files, key, { take, force, imageOnly, repairWorkId }); (status[key] ??= {}).face = null; }
+      catch (e) {
+        const reason = e instanceof Error ? e.message : String(e);
+        files.write(files.path('cast', key, 'face-attempt.json'), { at: new Date().toISOString(), state: 'failed', take, reason });
+        console.log(`${key}: face preparation failed; existing face retained: ${reason}`);
+        // A retry (new talk sheets, a new take) that fails leaves the working face in place: the character
+        // still speaks with moving lips, so the build is not short of faces. Asked for by name, it is reported.
+        if (registeredFace(files, key)?.info.imagePose?.lips && !only.length) { (status[key] ??= {}).face = null; return; }
+        failures.push(`${key}: ${reason}`);
+        (status[key] ??= {}).face = reason;
+      }
+    });
+    files.write(statusFile, status);
+    if (fs.existsSync(files.path('voices.json'))) {
+      const audio = await prepareLipSync(files, only, executable || undefined);
+      for (const key of [...new Set([...keys, ...Object.keys(status)])].filter(k => !only.length || only.includes(k))) delete (status[key] ??= {}).audio;
+      for (const failure of audio.failures ?? []) {
+        failures.push(`${failure.speaker}: ${failure.reason}`);
+        ((status[failure.speaker] ??= {}).audio ??= []).push(failure.reason);
+      }
+      files.write(statusFile, status);
+    }
+    if (failures.length) throw new FacePreparationError(failures);
+  },
+});

package/app/src/commands/icons.ts

@@ -0,0 +1,69 @@
+// diorama icons <slug> [item...]: items/<key>.png for every item in story.json, and items/items.json saying how each was made.
+import fs from 'node:fs';
+import sharp from 'sharp';
+import { maskFromPng } from '../image/mask.ts';
+import { maskBox } from '../image/raster.ts';
+import { defineCommand } from '../registry.ts';
+import { cutIcon, itemSources, ItemsInfo, MIN_CUT_PX, paintIcon, readIconFixes, redrawIcon } from '../scene/icons.ts';
+import { Story } from '../story/schema.ts';
+import { WorldFiles } from '../world/files.ts';
+
+defineCommand({
+  name: 'icons',
+  usage: 'icons <slug> [item...]  an inventory icon per story item: cut from the painting where the hero picks it up, else painted by Muse (items/fixes.json: how, hint, take)',
+  async run([slug, ...only]) {
+    if (!slug) throw new Error('usage: diorama icons <slug> [item...]');
+    const files = new WorldFiles(slug);
+    if (!fs.existsSync(files.path('story.json'))) {
+      console.log('no story.json: no items, no icons');
+      return;
+    }
+    const world = files.world();
+    const story = Story.parse(files.readJson(files.path('story.json')));
+    const sources = itemSources(story);
+    const fixes = readIconFixes(files);
+    const infoFile = files.path('items', 'items.json');
+    const info: ItemsInfo = fs.existsSync(infoFile) ? ItemsInfo.parse(files.readJson(infoFile)) : {};
+    // Items the story no longer has lose their icons.
+    for (const key of Object.keys(info)) if (!sources[key] && !story.items.some((i) => i.key === key)) delete info[key];
+    const items = story.items.filter((i) => only.length === 0 || only.includes(i.key));
+    const failed: string[] = [];
+    // An item without an icon shows its name in the bag, so one failure does not stop the build.
+    await Promise.all(items.map(async (item) => { try {
+      const from = sources[item.key];
+      const fix = fixes[item.key] ?? {};
+      let png: Buffer;
+      let how: ItemsInfo[string]['how'];
+      const maskFile = from && files.place(from.place, 'masks', `${from.thing}.png`);
+      if (from && maskFile && fs.existsSync(maskFile) && fix.how !== 'paint') {
+        const painting = fs.readFileSync(files.place(from.place, 'painting.png'));
+        const mask = await maskFromPng(fs.readFileSync(maskFile));
+        const box = maskBox(mask);
+        const side = box ? Math.max(box[2] - box[0], box[3] - box[1]) : 0;
+        // A hint from the person means the picture itself should change, so Muse redraws even a large cut-out.
+        if (fix.how === 'cut' || (side >= MIN_CUT_PX && !fix.hint)) {
+          png = await cutIcon(painting, mask);
+          how = 'cut';
+        } else {
+          // Too small to blow up, or the person asked for changes: Muse redraws the cut-out.
+          const crop = await sharp(await cutIcon(painting, mask, 12)).flatten({ background: '#f3ead8' }).png().toBuffer();
+          png = await redrawIcon(files.memo, world, item, crop, fix.hint, fix.take);
+          how = 'redraw';
+        }
+      } else {
+        if (from) console.log(`  ${item.key}: ${from.place}/${from.thing} has no mask, Muse paints it`);
+        png = await paintIcon(files.memo, world, item, fix.hint, fix.take);
+        how = 'paint';
+      }
+      files.write(files.path('items', `${item.key}.png`), png);
+      info[item.key] = { icon: `items/${item.key}.png`, how, from };
+      console.log(`${item.key}: ${how}${from ? ` from ${from.place}/${from.thing}` : ''}`);
+    } catch (error) {
+      failed.push(item.key);
+      console.log(`  ${item.key}: no icon (${error instanceof Error ? error.message : error}); the bag shows its name. icon_redo with a hint tries again`);
+    } }));
+    files.write(infoFile, info);
+    if (failed.length) console.log(`no icon for: ${failed.join(', ')}`);
+    console.log(`calls sent: muse ${files.memo.sent.muse}, sam ${files.memo.sent.sam}`);
+  },
+});

package/app/src/commands/index.ts

@@ -0,0 +1,37 @@
+// Command modules; each one calls defineCommand.
+import './doctor.ts';
+import './world.ts';
+import './extend.ts';
+import './story.ts';
+import './playthrough.ts';
+import './layout.ts';
+import './paint.ts';
+import './masks.ts';
+import './plates.ts';
+import './icons.ts';
+import './derive.ts';
+import './backdrop.ts';
+import './weather.ts';
+import './reactions.ts';
+import './sound.ts';
+import './voices.ts';
+import './faces.ts';
+import './voice-retry.ts';
+import './verify.ts';
+import './explore.ts';
+import './build.ts';
+import './cast.ts';
+import './model.ts';
+import './sprites.ts';
+import './style.ts';
+import './language.ts';
+import './play.ts';
+import './export.ts';
+import './publish.ts';
+import './reel.ts';
+import './clip.ts';
+import './setup.ts';
+import './browsers.ts';
+import './action.ts';
+import './studio.ts';
+import './mcp.ts';

package/app/src/commands/language.ts

@@ -0,0 +1,87 @@
+// diorama language <slug> <Language>: the world told in another language. Title, pitch, names and every line of
+// the story are translated in one Spark pass; the paintings, masks, sounds and puzzles stay as they are. Then the
+// voices are cast again for the new language (no accent of the old one) and every spoken line is read again,
+// and the talking faces follow the new takes.
+import fs from 'node:fs';
+import { defineCommand, registry } from '../registry.ts';
+import { castFixesFile, readCastFixes } from '../scene/cast-fixes.ts';
+import { SoundPlan } from '../sound/plan.ts';
+import { readStoryFixes, StoryFixes, storyFixesFile, textAt } from '../story/fixes.ts';
+import { Story } from '../story/schema.ts';
+import { WorldFiles } from '../world/files.ts';
+import { applyWorldTexts, languagePlan, planLine, relanguage, translateTexts, worldTexts } from '../world/language.ts';
+import { placeSources, soundPlanFrom } from './sound.ts';
+
+const run = (name: string, args: string[]) => registry.get(name)!.run(args);
+
+defineCommand({
+  name: 'language',
+  usage: 'language <slug> <Language>  tell the world in another language ("English", "Polish"): title, pitch, names and the story translated in one Spark pass, then the voices cast again for that language and every spoken line read again [--text-only: leave the voices for later] [--estimate: only say what it redoes and costs]',
+  async run(args) {
+    const estimate = args.includes('--estimate');
+    const textOnly = args.includes('--text-only');
+    const [slug, language] = args.filter((a) => !a.startsWith('--'));
+    if (!slug || !language) throw new Error('usage: diorama language <slug> <Language> [--text-only] [--estimate]');
+    const files = new WorldFiles(slug);
+    const world = files.world();
+    const storyFile = files.path('story.json');
+    const story = fs.existsSync(storyFile) ? Story.parse(files.readJson(storyFile)) : null;
+    const plan = languagePlan(files, world, story, language);
+    console.log(`language ${slug}: ${planLine(plan)}`);
+    if (estimate) return;
+
+    if (!plan.same) {
+      const texts = worldTexts(world, story);
+      const kids = world.audience === 'kids' || fs.existsSync(files.path('drawings'));
+      console.log(`translating ${Object.keys(texts).length} texts from ${plan.from} to ${plan.to}`);
+      const translated = await translateTexts(files.memo, texts, plan.from, plan.to, kids);
+      const now = applyWorldTexts(world, story, translated, plan.to);
+      // sound.json is planned from the world and story: the words changed, not what makes a sound, so the plan stays.
+      const soundFile = files.path('sound.json');
+      const sound = fs.existsSync(soundFile) ? SoundPlan.safeParse(files.readJson(soundFile)) : undefined;
+      const sources = sound?.success ? placeSources(files, world) : undefined;
+      const soundWasCurrent = sound?.success && sources && sound.data.from === soundPlanFrom(world, story, sources);
+
+      files.write(files.path('world.json'), { ...JSON.parse(fs.readFileSync(files.path('world.json'), 'utf8')), ...now.world, ...(now.world.language ? {} : { language: undefined }) });
+      if (now.story) {
+        files.write(storyFile, now.story);
+        // The person's reworded lines are kept across rebuilds: they are reworded in the new language too.
+        const fixes = readStoryFixes(files);
+        if (Object.keys(fixes.texts).length) {
+          for (const path of Object.keys(fixes.texts)) {
+            const text = textAt(now.story, path);
+            if (text) fixes.texts[path] = { ...fixes.texts[path], text };
+          }
+          files.write(storyFixesFile(files), StoryFixes.parse(fixes));
+        }
+      }
+      // A voice the person described for the old language ("Native Polish speaker") is asked for in the new one.
+      const castFixes = readCastFixes(files);
+      let castChanged = false;
+      for (const fix of Object.values(castFixes)) {
+        if (fix.voice?.design && fix.voice.design !== relanguage(fix.voice.design, plan.from, plan.to)) {
+          fix.voice.design = relanguage(fix.voice.design, plan.from, plan.to);
+          castChanged = true;
+        }
+      }
+      if (castChanged) files.write(castFixesFile(files), castFixes);
+      if (soundWasCurrent && sound?.success && sources) files.write(soundFile, { ...sound.data, from: soundPlanFrom(files.world(), now.story, sources) });
+      for (const id of ['world.title', 'world.pitch']) console.log(`  ${id}: ${translated[id]}`);
+      for (const [id, text] of Object.entries(translated).filter(([id]) => id.startsWith('names.'))) console.log(`  ${id.slice(6)}: ${texts[id]} -> ${text}`);
+      console.log(`world.json and story.json are in ${plan.to} now`);
+    }
+    if (textOnly) return void console.log('voices left for later: run_step voices, then faces');
+    if (!story) return;
+    console.log('\n# voices');
+    await run('voices', [slug]);
+    if (fs.existsSync(files.path('cast'))) {
+      console.log('\n# faces');
+      try {
+        await run('build', [slug, '--from', 'faces', '--to', 'faces']);
+      } catch (error) {
+        // Talking mouths are polish: the voices are in, the game plays without them.
+        console.log(`faces not made again (${error instanceof Error ? error.message : error}): run_step faces later`);
+      }
+    }
+  },
+});

package/app/src/commands/layout.ts

@@ -0,0 +1,31 @@
+// diorama layout <slug> [place...]: layout.json, blockin.png (for Muse) and plan.png (for people) for each place.
+import fs from 'node:fs';
+import { renderBlockin, renderPlan } from '../scene/blockin.ts';
+import { readCorrections } from '../scene/corrections.ts';
+import { layOut } from '../scene/layout.ts';
+import { defineCommand } from '../registry.ts';
+import { WorldFiles } from '../world/files.ts';
+import { approvedConcept } from '../world/sketch.ts';
+
+defineCommand({
+  name: 'layout',
+  usage: 'layout <slug> [place...] lay out places (layout.json, blockin.png) with Muse Spark',
+  async run([slug, ...only]) {
+    if (!slug) throw new Error('usage: diorama layout <slug> [place...]');
+    const files = new WorldFiles(slug);
+    const world = files.world();
+    const places = world.places.filter((p) => only.length === 0 || only.includes(p.key));
+    await Promise.all(places.map(async (place) => {
+      // A place made from a photo is measured from it, and the photo stands in for the block-in.
+      const photoFile = files.place(place.key, 'photo.png');
+      const photo = fs.existsSync(photoFile) ? fs.readFileSync(photoFile) : undefined;
+      const layout = await layOut(files.memo, world, place, photo, readCorrections(files.place(place.key)).layout?.guidance, approvedConcept(files, 'place', place.key));
+      files.write(files.place(place.key, 'layout.json'), layout);
+      const blockin = photo ?? await renderBlockin(layout, place);
+      files.write(files.place(place.key, 'blockin.png'), blockin);
+      files.write(files.place(place.key, 'plan.png'), await renderPlan(layout, place, blockin, world.names));
+      console.log(`${place.key}: ${layout.floors.length} floor(s), ${layout.things.length} things, horizon ${layout.horizon}`);
+    }));
+    console.log(`spark calls sent: ${files.memo.sent.spark}`);
+  },
+});

package/app/src/commands/masks.ts

@@ -0,0 +1,117 @@
+// diorama masks <slug> [place...]: masks/<thing>.png, masks/floors/<floor>.png and masks.json per place.
+import fs from 'node:fs';
+import { maskFromPng, maskToPng, type Mask } from '../image/mask.ts';
+import { PALETTE, svgLabel, tintOver } from '../image/overlay.ts';
+import { defineCommand } from '../registry.ts';
+import { Brief } from '../scene/brief.ts';
+import { carryFrom, ran, remakes, tilesLine } from '../scene/carry.ts';
+import { brushed, correctCells, correctSpots, readCorrections } from '../scene/corrections.ts';
+import { Layout } from '../scene/layout.ts';
+import { locateThings, markFloors } from '../scene/locate.ts';
+import { checkMasks, type MaskCheck } from '../scene/maskcheck.ts';
+import { maskFloors, maskThings } from '../scene/masks.ts';
+import { paintingLayoutHash } from '../scene/paint-layout.ts';
+import { maskBox, type Rect } from '../image/raster.ts';
+import { WorldFiles } from '../world/files.ts';
+
+defineCommand({
+  name: 'masks',
+  usage: 'masks <slug> [place...]  find each thing and floor in the painting with Spark and SAM 3.1',
+  async run([slug, ...only]) {
+    if (!slug) throw new Error('usage: diorama masks <slug> [place...]');
+    const files = new WorldFiles(slug);
+    const world = files.world();
+    const places = world.places.filter((p) => only.length === 0 || only.includes(p.key));
+    await Promise.all(places.map(async (place) => {
+      const at = (...parts: string[]) => files.place(place.key, ...parts);
+      fs.rmSync(at('verify.json'), { force: true });
+      const layout = Layout.parse(files.readJson(at('layout.json')));
+      const brief = Brief.parse(files.readJson(at('brief.json')));
+      const painting = fs.readFileSync(at('painting.png'));
+      // A rejected/stale checked painting must not be rescued by masks and invented floor strokes.
+      // Legacy worlds without a report remain editable; rebuilding from paint checks them too.
+      if (!fs.existsSync(at('photo.png')) && fs.existsSync(at('painting-layout.json'))) {
+        const report = files.readJson(at('painting-layout.json')) as { ok?: boolean; inputs?: string };
+        if (!report.ok || report.inputs !== paintingLayoutHash(place, layout, fs.readFileSync(at('blockin.png')), painting)) {
+          throw new Error(`PAINT_LAYOUT_MISMATCH ${place.key}: painting/layout changed or its composition check failed; rebuild this place from paint to inspect and repair it before masks`);
+        }
+      }
+      const flagged: string[] = [];
+      const log = (line: string) => {
+        flagged.push(line);
+        console.log(`  ${place.key}: ${line}`);
+      };
+
+      // After a small repaint, what it did not reach is found again on the old painting: the memo has those answers.
+      const carry = await carryFrom(at(), 'masks', painting);
+      if (carry) console.log(`  ${place.key}: ${tilesLine(carry)}; only what it reaches is asked again`);
+      const seen = carry?.old ?? painting;
+      const [found, marked] = await Promise.all([
+        locateThings(files.memo, place, brief, seen),
+        markFloors(files.memo, place, layout, brief, seen),
+      ]);
+      files.write(at('locate.json'), { things: found, floors: marked });
+      // The person's fixes go in before SAM is asked and their brush strokes after, every time.
+      const fixes = readCorrections(at());
+      const spots = correctSpots(found, fixes);
+      const cells = correctCells(marked, fixes);
+      const before = async (file: string): Promise<Mask | undefined> => (carry && fs.existsSync(at('masks', file)) ? maskFromPng(fs.readFileSync(at('masks', file))) : undefined);
+      // A thing or floor keeps its mask when the change does not reach it, or only grazes a big one.
+      const keptThing = new Set<string>();
+      const keptFloor = new Set<string>();
+      for (const t of place.things) {
+        const last = await before(`${t.key}.png`);
+        if (carry && last && !remakes(carry, last)) keptThing.add(t.key);
+      }
+      for (const f of layout.floors) {
+        const last = await before(`floors/${f.key}.png`);
+        if (carry && last && !remakes(carry, last)) keptFloor.add(f.key);
+      }
+      const split = <T>(all: T[], kept: (x: T) => boolean) => [all.filter(kept), all.filter((x) => !kept(x))];
+      const [oldThings, newThings] = split(place.things, (t) => keptThing.has(t.key));
+      const [oldFloors, newFloors] = split(layout.floors, (f) => keptFloor.has(f.key));
+      const [thingsOld, thingsNew, floorsOld, floorsNew] = await Promise.all([
+        oldThings.some((t) => t.role !== 'npc') ? maskThings(files.memo, { ...place, things: oldThings }, layout, spots, carry!.old, log) : [],
+        newThings.some((t) => t.role !== 'npc') ? maskThings(files.memo, { ...place, things: newThings }, layout, spots, painting, log) : [],
+        oldFloors.length ? maskFloors(files.memo, place, { ...layout, floors: oldFloors }, cells, carry!.old, log) : [],
+        newFloors.length ? maskFloors(files.memo, place, { ...layout, floors: newFloors }, cells, painting, log) : [],
+      ]);
+      const inOrder = <T extends { key: string }>(keys: string[], got: T[]) => keys.flatMap((k) => got.filter((g) => g.key === k));
+      const things = inOrder(place.things.map((t) => t.key), [...thingsOld, ...thingsNew]).map((t) => brushed(t, fixes.things[t.key]?.strokes));
+      const floors = inOrder(layout.floors.map((f) => f.key), [...floorsOld, ...floorsNew]).map((f) => brushed(f, fixes.floors[f.key]?.strokes));
+      if (carry) console.log(`  ${place.key}: kept ${thingsOld.length}/${things.length} things and ${floorsOld.length}/${floors.length} floors as they were; masked again: ${[...thingsNew, ...floorsNew].map((m) => m.key).join(', ') || 'nothing'}`);
+      const lastChecks = carry && fs.existsSync(at('masks.json')) ? ((files.readJson(at('masks.json')) as { things?: Record<string, { box?: Rect; check?: MaskCheck }> }).things ?? {}) : {};
+
+      fs.rmSync(at('masks'), { recursive: true, force: true });
+      for (const t of things) files.write(at('masks', `${t.key}.png`), await maskToPng(t.mask));
+      for (const f of floors) files.write(at('masks', 'floors', `${f.key}.png`), await maskToPng(f.mask));
+      const layers = [...floors.map((f) => ({ key: f.key, mask: f.mask, alpha: 0.25 })), ...things.map((t) => ({ key: t.key, mask: t.mask, alpha: 0.45 }))];
+      const tints = layers.map((l, i) => ({ mask: l.mask, alpha: l.alpha, rgb: PALETTE[i % PALETTE.length] }));
+      const labels = [...floors.map((f) => ({ key: f.key, box: maskBox(f.mask) })), ...things.map((t) => ({ key: t.key, box: t.box as Rect | undefined }))]
+        .map((l, i) => (l.box ? svgLabel(l.box[0] + 4, l.box[1] + 18, l.key, PALETTE[i % PALETTE.length]) : ''));
+      files.write(at('masks', 'overlay.png'), await tintOver(painting, tints, labels.join('')));
+      // A corrected box guides SAM; it does not prove what SAM cut out. Kept masks retain their check.
+      let checks: Record<string, MaskCheck> = Object.fromEntries(things.flatMap((t) => (keptThing.has(t.key) && lastChecks[t.key]?.check && JSON.stringify(t.box) === JSON.stringify(lastChecks[t.key].box) ? [[t.key, lastChecks[t.key].check!]] : [])));
+      try {
+        const look = things.filter((t) => !checks[t.key]);
+        // Asked by what the thing is (the world's words), not by how the brief dressed it: a brief that described
+        // "the empty stall the cow uses" let a mask on an empty stall pass.
+        if (look.length) checks = { ...checks, ...await checkMasks(files.memo, place.key, painting, look.map((t) => ({ key: t.key, looks: place.things.find((x) => x.key === t.key)?.looks ?? brief.looks[t.key] ?? t.key, box: t.box as Rect | undefined }))) };
+        for (const [key, c] of Object.entries(checks)) if (!c.ok) log(`${key}: Spark sees ${c.seen ?? 'something else'} in its mask`);
+      } catch (error) {
+        throw new Error(`mask check failed: ${error instanceof Error ? error.message : error}`);
+      }
+      const round = (v: number) => Math.round(v * 100) / 100;
+      files.write(at('masks.json'), {
+        things: Object.fromEntries(things.map(({ key, mask: _, ...t }) => [key, { ...t, ...(checks[key] && { check: checks[key] }) }])),
+        floors: Object.fromEntries(floors.map(({ key, mask: _, iouCells, iouLayout, ...f }) => [key, { ...f, iouCells: round(iouCells), iouLayout: round(iouLayout) }])),
+        flagged,
+      });
+      // After a carry the old painting stays the one the answers belong to, so a rerun on this painting asks nothing new.
+      ran(at(), 'masks', carry?.old ?? painting);
+      const coarse = [...things, ...floors].filter((m) => m.coarse).map((m) => m.key);
+      console.log(`${place.key}: ${things.length} things, ${floors.length} floors${coarse.length ? `, coarse: ${coarse.join(', ')}` : ''}`);
+    }));
+    console.log(`calls sent: spark ${files.memo.sent.spark}, sam ${files.memo.sent.sam}`);
+  },
+});

package/app/src/commands/mcp.ts

@@ -0,0 +1,52 @@
+// diorama mcp: the Studio's actions as an MCP server over stdio, for Muse Code (or any MCP client) started
+// by hand. With a Studio running for this folder it forwards every call there, so the page shows the agent's
+// work live; without one it runs the actions itself.
+import path from 'node:path';
+import readline from 'node:readline';
+import '../actions/index.ts';
+import { defineCommand } from '../registry.ts';
+import { Events } from '../studio/events.ts';
+import { snapshotBeforeJob } from '../studio/history.ts';
+import { Jobs } from '../studio/jobs.ts';
+import { handleMcp, type Reply } from '../studio/mcp.ts';
+import { readStudioFile } from '../studio/discovery.ts';
+
+defineCommand({
+  name: 'mcp',
+  usage: 'mcp                     serve the Studio actions as MCP tools on stdio (forwards to a running diorama studio)',
+  async run() {
+    const studio = readStudioFile(process.cwd());
+    const events = new Events();
+    const root = path.resolve('worlds');
+    const ctx = { by: 'agent' as const, root, events, jobs: new Jobs(events, { beforeStart: snapshotBeforeJob(root, events) }) };
+    const out = (reply: Reply | Reply[] | null) => reply && process.stdout.write(`${JSON.stringify(reply)}\n`);
+    // Logs go to stderr: stdout carries only protocol messages.
+    console.error(studio ? `diorama mcp: forwarding to the Studio at ${studio.url}` : 'diorama mcp: no Studio running here, running actions in this process');
+    const lines = readline.createInterface({ input: process.stdin });
+    for await (const line of lines) {
+      if (!line.trim()) continue;
+      let message: unknown;
+      try {
+        message = JSON.parse(line);
+      } catch {
+        out({ jsonrpc: '2.0', id: null, error: { code: -32700, message: 'parse error' } });
+        continue;
+      }
+      if (studio) {
+        const res = await fetch(`${studio.url}mcp`, {
+          method: 'POST',
+          headers: { authorization: `Bearer ${studio.agentToken}`, 'content-type': 'application/json' },
+          body: line,
+        }).catch((error: Error) => ({ ok: false, status: 0, text: async () => error.message }));
+        if (res.status === 202) continue;
+        const text = await res.text();
+        if (res.ok) process.stdout.write(`${text}\n`);
+        else out({ jsonrpc: '2.0', id: (message as { id?: string | number }).id ?? null, error: { code: -32603, message: `the Studio answered ${res.status}: ${text.slice(0, 200)}` } });
+        continue;
+      }
+      if (Array.isArray(message)) out((await Promise.all(message.map((m) => handleMcp(m, ctx)))).filter((r): r is Reply => !!r));
+      else out(await handleMcp(message as Parameters<typeof handleMcp>[0], ctx));
+    }
+    ctx.jobs.stopAll();
+  },
+});