@gtanczyk/diorama 0.0.0-stage → 0.2.0

package/app/src/actions/kids.ts

@@ -0,0 +1,215 @@
+// A workshop with children: photos of their drawings and recordings of them telling the story, kept with a
+// world that does not exist yet (drawings/, told/), heard at once, and turned into the idea the world is built from.
+import fs from 'node:fs';
+import path from 'node:path';
+import { z } from 'zod';
+import { previewDrawing } from '../scene/preview.ts';
+import { drawingFiles, keptName, Kind, normaliseDrawing, oneHero, PICTURE, readDrawing, readDrawings, readDrawn, readKinds } from '../world/drawings.ts';
+import { WorldFiles } from '../world/files.ts';
+import { Slug } from '../world/schema.ts';
+import { hear, hearAll, Idea, ideaFromTold, narrate, SOUND, toldFiles } from '../world/told.ts';
+import { ActionError, defineAction, runAction } from './registry.ts';
+
+/** One change at a time per world: photos arrive together, and each takes the next number and joins drawings.json. */
+const queues = new Map<string, Promise<unknown>>();
+function serial<T>(slug: string, run: () => Promise<T>): Promise<T> {
+  const next = (queues.get(slug) ?? Promise.resolve()).catch(() => {}).then(run);
+  queues.set(slug, next);
+  return next;
+}
+
+const stamp = (file: string) => Math.round(fs.statSync(file).mtimeMs);
+
+function readIdea(files: WorldFiles): Idea | null {
+  const parsed = Idea.safeParse(fs.existsSync(files.path('told', 'idea.json')) ? JSON.parse(fs.readFileSync(files.path('told', 'idea.json'), 'utf8')) : null);
+  return parsed.success ? parsed.data : null;
+}
+
+function status(files: WorldFiles) {
+  const drawn = readDrawn(files);
+  return {
+    drawings: drawingFiles(files).map((file) => {
+      const preview = files.path('drawings', 'preview', file);
+      return { file: `drawings/${file}`, v: stamp(files.path('drawings', file)), drawn: drawn.find((d) => d.file === file) ?? null, preview: fs.existsSync(preview) ? { file: `drawings/preview/${file}`, v: stamp(preview) } : null };
+    }),
+    told: toldFiles(files).map((clip) => {
+      const text = files.path('told', clip.replace(/\.[^.]+$/, '.txt'));
+      return { clip, file: `told/${clip}`, text: fs.existsSync(text) ? fs.readFileSync(text, 'utf8').trim() : null };
+    }),
+    idea: readIdea(files),
+    reading: fs.existsSync(files.path('idea.mp3')) ? { file: 'idea.mp3', v: stamp(files.path('idea.mp3')) } : null,
+    built: fs.existsSync(files.path('world.json')),
+  };
+}
+
+defineAction({
+  name: 'kids_status',
+  reads: true,
+  description: 'A workshop world\'s material: the photos of children\'s drawings (and what each became), the recordings of them telling the story with what was heard, and the idea written from them.',
+  input: z.object({ slug: Slug }),
+  async run({ slug }, ctx) {
+    return status(new WorldFiles(slug, ctx.root));
+  },
+});
+
+defineAction({
+  name: 'kids_add',
+  description: 'Keep a photo of a child\'s drawing (kind drawing) or a recording of children telling their story (kind told) with a world, even before it is built. A recording is heard at once (Muse Voice Transcribe) and its text returned.',
+  input: z.object({
+    slug: Slug,
+    kind: z.enum(['drawing', 'told']),
+    name: z.string().min(1).max(120).describe('the file name, for its extension'),
+    data: z.string().min(10).describe('the file, base64'),
+  }),
+  async run({ slug, kind, name, data }, ctx) {
+    const files = new WorldFiles(slug, ctx.root);
+    const bytes = Buffer.from(data.replace(/^data:[^,]*,/, ''), 'base64');
+    if (kind === 'drawing') {
+      if (!PICTURE.test(name)) throw new ActionError(400, `${name}: a drawing is a photo (png, jpg, webp, heic)`);
+      const image = await normaliseDrawing(bytes);
+      const file = await serial(slug, async () => {
+        const file = `${String(drawingFiles(files).length + 1).padStart(2, '0')}-${keptName(name)}.png`;
+        files.write(files.path('drawings', file), image);
+        return file;
+      });
+      // Read at once (what it is, its key, how it looks), so its preview can start while the next one is drawn.
+      const drawn = await readDrawing(files.memo, image, file, [...new Set(readDrawn(files).map((d) => d.key))]);
+      await serial(slug, async () => {
+        const kind = readKinds(files)[file];
+        const all = [...readDrawn(files).filter((d) => d.file !== file), { ...drawn, ...(kind && { kind }) }];
+        // An existing game's hero is already fixed. Do not promote its first newly drawn NPC to hero.
+        files.write(files.path('drawings.json'), fs.existsSync(files.path('world.json')) && kind !== 'hero' ? all : oneHero(all, kind === 'hero' ? drawn.key : undefined));
+      });
+      return { file: `drawings/${file}`, drawn };
+    }
+    if (!SOUND.test(name)) throw new ActionError(400, `${name}: a recording is webm, m4a, mp3, wav or ogg`);
+    const clip = await serial(slug, async () => {
+      const clip = `${String(toldFiles(files).length + 1).padStart(2, '0')}-${keptName(name)}${path.extname(name).toLowerCase()}`;
+      files.write(files.path('told', clip), bytes);
+      return clip;
+    });
+    return { clip, text: await hear(files, clip) };
+  },
+});
+
+// A chat picture is already on disk: the agent must not have to read or transmit its base64 itself.
+const pictureAdds = new Map<string, Promise<unknown>>();
+defineAction({
+  name: 'kids_from_picture',
+  description: 'Save and read a drawing already attached to the Studio conversation. Pass its studio/pictures/NN.jpg path from the message. Returns the drawings/ path for kids_status and world_extend; repeated calls for the same picture reuse it.',
+  input: z.object({ slug: Slug, picture: z.string().regex(/^studio\/pictures\/[0-9]{2,4}\.jpg$/) }),
+  async run({ slug, picture }, ctx) {
+    const key = `${ctx.root}/${slug}/${picture}`;
+    const pending = pictureAdds.get(key);
+    if (pending) return pending;
+    const task = (async () => {
+      const files = new WorldFiles(slug, ctx.root);
+      const source = files.path(picture);
+      if (!fs.existsSync(source)) throw new ActionError(404, `${slug} has no ${picture}`);
+      const index = files.path('drawings', 'picture-sources.json');
+      const read = (): Record<string, string> => fs.existsSync(index) ? JSON.parse(fs.readFileSync(index, 'utf8')) as Record<string, string> : {};
+      const old = read()[picture];
+      if (old && fs.existsSync(files.path(old))) {
+        const drawn = readDrawn(files).find(d => `drawings/${d.file}` === old);
+        if (drawn) return { file: old, drawn, reused: true };
+      }
+      const normalised = await normaliseDrawing(fs.readFileSync(source));
+      const match = readDrawn(files).find(d => fs.existsSync(files.path('drawings', d.file)) && fs.readFileSync(files.path('drawings', d.file)).equals(normalised));
+      if (match) {
+        const file = `drawings/${match.file}`;
+        files.write(index, { ...read(), [picture]: file });
+        return { file, drawn: match, reused: true };
+      }
+      const added = await runAction('kids_add', { slug, kind: 'drawing', name: path.basename(picture), data: fs.readFileSync(source).toString('base64') }, ctx) as { file: string; drawn: unknown };
+      files.write(index, { ...read(), [picture]: added.file });
+      return { ...added, reused: false };
+    })();
+    pictureAdds.set(key, task);
+    try { return await task; } finally { pictureAdds.delete(key); }
+  },
+});
+
+defineAction({
+  name: 'kids_remove',
+  description: 'Take a drawing or a recording out of a workshop world (the file as kids_status lists it, drawings/… or told/…).',
+  input: z.object({ slug: Slug, file: z.string().regex(/^(drawings|told)\/[a-z0-9-]+\.[a-z0-9]+$/) }),
+  async run({ slug, file }, ctx) {
+    return serial(slug, async () => remove(new WorldFiles(slug, ctx.root), file));
+  },
+});
+
+function remove(files: WorldFiles, file: string) {
+  const full = files.path(file);
+    if (!fs.existsSync(full)) throw new ActionError(404, `${files.slug} has no ${file}`);
+  fs.rmSync(full);
+  fs.rmSync(full.replace(/\.[^.]+$/, '.txt'), { force: true });
+  if (file.startsWith('drawings/')) {
+    const name = file.slice('drawings/'.length);
+    fs.rmSync(files.path('drawings', 'preview', name), { force: true });
+    if (fs.existsSync(files.path('drawings.json'))) files.write(files.path('drawings.json'), oneHero(readDrawn(files).filter((d) => d.file !== name)));
+  }
+  return status(files);
+}
+
+defineAction({
+  name: 'kids_idea',
+  description: 'Write the idea of a workshop world from everything the children said and drew: Spark joins what they told and drew into one short story in their language, says which drawn character is the hero, and a storyteller voice reads it aloud (idea.mp3). '
+    + 'The idea can be edited before world_build.',
+  input: z.object({ slug: Slug, note: z.string().max(2000).optional().describe('what the grown-up adds or wants'), read: z.boolean().optional().describe('read it aloud (idea.mp3); default true'), language: z.string().min(2).max(30).optional().describe('the workshop\'s language ("Polish"), whatever was spoken') }),
+  async run({ slug, note, read, language }, ctx) {
+    const files = new WorldFiles(slug, ctx.root);
+    const said = await hearAll(files);
+    if (said.length === 0 && drawingFiles(files).length === 0) throw new ActionError(400, 'record the children telling their story or add a drawing first');
+    const drawn = await readDrawings(files.memo, files);
+    const idea = await ideaFromTold(files.memo, said.map((s) => s.text), drawn, note ?? '', language);
+    // The story says who leads, unless the grown-up picked the hero.
+    if (idea.hero && !Object.values(readKinds(files)).includes('hero')) oneHero(drawn, idea.hero);
+    await serial(slug, async () => {
+      if (drawn.length) files.write(files.path('drawings.json'), drawn);
+      files.write(files.path('told', 'idea.json'), idea);
+    });
+    // Read aloud straight away; a story without a voice still stands.
+    if (read !== false) await narrate(files, idea).catch((e: Error) => console.log(`no reading of the story: ${e.message}`));
+    return status(files);
+  },
+});
+
+defineAction({
+  name: 'kids_kind',
+  description: 'Say what a drawing is in the game (hero, npc, item, prop or place), over what Spark saw; kind null gives the choice back to Spark. A changed character is painted again by kids_preview; write the idea again (kids_idea) afterwards.',
+  input: z.object({ slug: Slug, file: z.string().regex(/^drawings\/[a-z0-9-]+\.png$/), kind: Kind.nullable() }),
+  async run({ slug, file, kind }, ctx) {
+    return serial(slug, () => setKind(new WorldFiles(slug, ctx.root), file, kind));
+  },
+});
+
+async function setKind(files: WorldFiles, file: string, kind: Kind | null) {
+  const name = file.slice('drawings/'.length);
+  if (!fs.existsSync(files.path(file))) throw new ActionError(404, `${files.slug} has no ${file}`);
+  const kinds = readKinds(files);
+  if (kind) kinds[name] = kind;
+  else delete kinds[name];
+  files.write(files.path('drawings', 'kinds.json'), kinds);
+  // Shown at once; the next kids_idea writes the story with it.
+  const drawn = readDrawn(files);
+  const entry = drawn.find((d) => d.file === name);
+  if (entry && kind) {
+    entry.kind = kind;
+    files.write(files.path('drawings.json'), kind === 'hero' || kind === 'npc' || kind === 'player' ? oneHero(drawn, kind === 'hero' ? entry.key : drawn.find((d) => d.kind === 'hero' && d !== entry)?.key) : drawn);
+  }
+  return status(files);
+}
+
+defineAction({
+  name: 'kids_preview',
+  description: 'Bring one drawing to life at once (about half a minute): a character as the figure the game will use, a place or a thing as a quick painting (drawings/preview/<file>). For the children to see while the rest is drawn.',
+  input: z.object({ slug: Slug, file: z.string().regex(/^drawings\/[a-z0-9-]+\.png$/) }),
+  async run({ slug, file }, ctx) {
+    const files = new WorldFiles(slug, ctx.root);
+    const name = file.slice('drawings/'.length);
+    const drawn = readDrawn(files).find((d) => d.file === name);
+    if (!drawn) throw new ActionError(404, `${file} is not read yet: add it with kids_add`);
+    files.write(files.path('drawings', 'preview', name), await previewDrawing(files.memo, drawn, fs.readFileSync(files.path(file))));
+    return status(files);
+  },
+});

package/app/src/actions/language.ts

@@ -0,0 +1,52 @@
+// The language a world is told in: world_language translates every word the player reads or hears and has the
+// voices cast and read again in it.
+import fs from 'node:fs';
+import { z } from 'zod';
+import { credits } from '../sound/cost.ts';
+import { elevenKey } from '../sound/sources.ts';
+import { Story } from '../story/schema.ts';
+import { agentBlocked } from '../studio/locks.ts';
+import { WorldFiles } from '../world/files.ts';
+import { languagePlan, planLine } from '../world/language.ts';
+import { Slug } from '../world/schema.ts';
+import { ActionError, defineAction } from './registry.ts';
+
+defineAction({
+  name: 'world_language',
+  description: 'Tell the world in another language ("English", "Polish", "German"): the title, pitch, character and item names and every line of the story '
+    + '(intro, remarks, dialogue, answers, item names, endings) are translated in one cheap Spark pass, with names kept the same everywhere; the game\'s '
+    + 'own words (hints, goodbye, buttons) follow world.json\'s language by themselves. Keys, puzzles, paintings, masks and sounds stay. Then the voices are '
+    + 'cast again as native speakers of the new language (no accent of the old one), every spoken line is read again and the talking faces follow. '
+    + 'voices false translates only (cheap; the game shows text until run_step voices). Without go it only answers what it redoes and about what '
+    + 'it costs: tell the person, then call again with go true once they agree. Runs as a job; history_undo takes it back. Then play_reload.',
+  input: z.object({
+    slug: Slug,
+    language: z.string().trim().min(2).max(30).describe('The new language as its English name: "English", "Polish".'),
+    voices: z.boolean().optional().describe('false: translate the text only and leave the voices for later. Default true.'),
+    go: z.boolean().optional().describe('true runs it; leave out to get the estimate first.'),
+  }),
+  job: true,
+  async run({ slug, language, voices = true, go }, ctx) {
+    const files = new WorldFiles(slug, ctx.root);
+    if (!fs.existsSync(files.path('world.json'))) throw new ActionError(404, `no world ${slug} yet`);
+    const name = language[0].toUpperCase() + language.slice(1);
+    const world = files.world();
+    const story = fs.existsSync(files.path('story.json')) ? Story.parse(files.readJson(files.path('story.json'))) : null;
+    const plan = languagePlan(files, world, story, name);
+    if (!go) {
+      const left = await credits(elevenKey());
+      return {
+        estimate: plan,
+        line: voices ? planLine(plan) : `${plan.texts} texts translated from ${plan.from} to ${plan.to} in one Spark pass; voices left for later`,
+        ...(left && voices && { elevenLabsCreditsLeft: left.left }),
+        next: 'tell the person what it redoes and costs; call again with go: true when they agree',
+      };
+    }
+    const blocked = agentBlocked(files.dir, ctx.by);
+    if (blocked) throw new ActionError(423, blocked);
+    const busy = ctx.jobs.busy(slug);
+    if (busy) throw new ActionError(409, `${busy.label} is ${busy.state}; wait before changing the language`);
+    const job = ctx.jobs.start(slug, ['language', slug, name, ...(voices ? [] : ['--text-only'])], `language ${name}`, ctx.by);
+    return { job, line: voices ? planLine(plan) : 'text only' };
+  },
+});

package/app/src/actions/models.ts

@@ -0,0 +1,64 @@
+// Studio Setup's model pick: which model answers Spark's asks and which paints, with what each needs.
+import { z } from 'zod';
+import { readModelChoice, writeModelChoice } from '../meta/choice.ts';
+import { imageVia } from '../meta/muse.ts';
+import { sparkVia } from '../meta/spark.ts';
+import { envValue } from '../secrets.ts';
+import { defineAction } from './registry.ts';
+
+const google = () => !!envValue('GEMINI_API_KEY');
+const openrouter = () => !!envValue('OPENROUTER_API_KEY');
+
+/** What Setup offers; `ready` says whether this machine has the key it needs. */
+function options() {
+  const anthropic = !!envValue('DIORAMA_ANTHROPIC_API_KEY');
+  const gemini = google() || openrouter();
+  return {
+    spark: [
+      { id: 'meta', label: 'Muse Spark (Meta)', ready: !!envValue('META_API_KEY') },
+      { id: 'claude', label: 'Claude Sonnet 5.5', ready: anthropic },
+      { id: 'openrouter:anthropic/claude-sonnet-5.5', label: 'Claude Sonnet 5.5 via OpenRouter', ready: openrouter() },
+    ],
+    image: [
+      { id: 'meta', label: 'Muse Image (Meta)', ready: !!envValue('META_API_KEY') },
+      { id: 'gemini-flash', label: 'Gemini 3.1 Flash Image', ready: gemini },
+      { id: 'gemini-lite', label: 'Gemini 3.1 Flash Lite Image', ready: gemini },
+      { id: 'nano-banana', label: 'Nano Banana 2.1', ready: gemini },
+      { id: 'gemini-pro', label: 'Gemini 3 Pro Image', ready: gemini },
+    ],
+  };
+}
+
+function state() {
+  const picked = readModelChoice();
+  return {
+    spark: process.env.DIORAMA_SPARK_VIA || picked.spark || 'meta',
+    image: process.env.DIORAMA_IMAGE_VIA || picked.image || 'meta',
+    /** Set where the Studio was started: Setup shows it and cannot change it. */
+    env: { spark: process.env.DIORAMA_SPARK_VIA ?? null, image: process.env.DIORAMA_IMAGE_VIA ?? null },
+    resolved: { spark: sparkVia(), image: imageVia() },
+    options: options(),
+  };
+}
+
+defineAction({
+  name: 'models_get',
+  reads: true,
+  description: 'Which models make worlds now: Spark\'s asks (Muse Spark or Claude) and pictures (Muse Image or a Gemini image model), and what Setup offers.',
+  input: z.object({}),
+  async run() {
+    return state();
+  },
+});
+
+defineAction({
+  name: 'models_set',
+  description: 'Pick the model for Spark\'s asks (meta, claude, openrouter:<model>) and for pictures (meta, gemini-flash, gemini-lite, nano-banana, gemini-pro). '
+    + 'Other models\' answers keep their own memo keys; meta goes back to the Meta answers. The hackathon demo stays on meta.',
+  input: z.object({ spark: z.string().min(1).optional(), image: z.string().min(1).optional() }),
+  async run({ spark, image }) {
+    const was = readModelChoice();
+    writeModelChoice({ spark: spark ?? was.spark, image: image ?? was.image });
+    return state();
+  },
+});

package/app/src/actions/motion.ts

@@ -0,0 +1,119 @@
+// place_motion: what moves in a place as the game plays it (water, fog, flames, an NPC breathing), for an agent
+// that cannot watch the game: a few frames from the real play page, and the pixels that changed between them.
+import fs from 'node:fs';
+import type { AddressInfo } from 'node:net';
+import sharp from 'sharp';
+import { z } from 'zod';
+import { chromium } from '../commands/explore.ts';
+import { serveWorld } from '../commands/play.ts';
+import { svgLabel } from '../image/overlay.ts';
+import { Geometry } from '../play/geometry.ts';
+import { WorldFiles } from '../world/files.ts';
+import { Slug } from '../world/schema.ts';
+import { ActionError, defineAction } from './registry.ts';
+
+const FRAMES = 6;
+const EVERY_MS = 400;
+/** A pixel moved when its brightness changed by more than this between two frames (0-255): above noise, below a slow ripple. */
+const MOVED = 6;
+
+type Shot = { screenshot(o?: object): Promise<Buffer>; goto(url: string): Promise<unknown>; waitForFunction(js: string, arg?: unknown, o?: object): Promise<unknown>; waitForTimeout(ms: number): Promise<void>; evaluate<T>(js: string): Promise<T> };
+
+defineAction({
+  name: 'place_motion',
+  reads: true,
+  description: 'Watch a place in the real game for a few seconds and see what moves, since you cannot play it: '
+    + `${FRAMES} frames ${EVERY_MS} ms apart from the play page. You get a picture of the place with every pixel that moved tinted magenta, `
+    + 'a strip of three frames, and in words how much of the picture moved and, per thing, how much of its box moved. '
+    + 'Use it after a change to motion (water flow, fog, fire, weather, an animated thing) to check it shows, before telling the person it works. Free: no model calls.',
+  input: z.object({ slug: Slug, place: z.string().min(1) }),
+  async run({ slug, place }, ctx) {
+    const files = new WorldFiles(slug, ctx.root);
+    if (!fs.existsSync(files.path('places', place, 'geometry.json'))) throw new ActionError(404, `${slug}/${place} is not playable yet: build it to the derive step first`);
+    const g = Geometry.parse(files.readJson(files.path('places', place, 'geometry.json')));
+    const server = await serveWorld(files, 0);
+    const browser = await (await chromium()).launch({ args: ['--autoplay-policy=no-user-gesture-required'] });
+    const frames: Buffer[] = [];
+    let shows: Record<string, unknown> = {};
+    try {
+      const page = (await browser.newPage({ viewport: { width: 1200, height: 896 } })) as unknown as Shot;
+      await page.goto(`http://localhost:${(server.address() as AddressInfo).port}/?place=${encodeURIComponent(place)}&capture=1&cursor=0&sound=0&parallax=1`);
+      await page.waitForFunction('window.diorama && window.diorama.place', undefined, { timeout: 30000 });
+      // parallax=1: the game as played. Capture mode leaves parallax off by default, and without it the page loads no
+      // backdrop, so no water or mist layer either: every place then read "mist: none" however its fog was painted.
+      // The place fades in and the hero lands: let it settle first.
+      await page.waitForTimeout(1500);
+      // What the game loaded to move here: a 0% with no layer means the layer never reached the game.
+      shows = await page.evaluate<Record<string, unknown>>('(() => { const p = window.diorama.place; return { water: p.water ? `flowing ${p.water.flow.join(",") === "0,0" ? "still (swells)" : p.water.flow.join(",")}` : "none", mist: p.mist ? "drifting" : "none", clouds: p.clouds ? "drifting" : "none", waterfalls: (p.falls || []).length ? `${p.falls.length} pouring` : "none", animals: (p.critters || []).length ? `${p.critters.length} swimming or flying` : "none", weather: (() => { const w = window.diorama.data && window.diorama.data.places[p.key] && window.diorama.data.places[p.key].weather; return w ? `${w.fall}${w.fall !== "none" ? ` ${w.strength}` : ""}, fog ${w.fog}` : "none"; })() }; })()');
+      // Frames EVERY_MS of game time apart, not wall time: a headless browser draws ~12 frames a second, and capture
+      // mode steps 1/60 s per frame, so 2 s of waiting showed 0.4 s of the game and slow swells or drifting mist
+      // looked still.
+      for (let i = 0; i < FRAMES; i++) {
+        if (i) {
+          const t = await page.evaluate<number>('window.diorama.time');
+          await page.waitForFunction(`window.diorama.time >= ${t + EVERY_MS / 1000}`, undefined, { timeout: 30000 });
+        }
+        frames.push(await page.screenshot({ type: 'png' }));
+      }
+    } finally {
+      await browser.close();
+      server.close();
+    }
+
+    const raw = await Promise.all(frames.map((f) => sharp(f).greyscale().raw().toBuffer({ resolveWithObject: true })));
+    const { width: W, height: H } = raw[0].info;
+    const moved = new Uint8Array(W * H);
+    for (let k = 1; k < raw.length; k++) {
+      const [a, b] = [raw[k - 1].data, raw[k].data];
+      for (let i = 0; i < moved.length; i++) if (Math.abs(a[i] - b[i]) > MOVED) moved[i] = 1;
+    }
+    const share = (x0: number, y0: number, x1: number, y1: number) => {
+      // Boxes are in the painting's frame (1200 x 896), the screenshot may differ a little.
+      const [sx, sy] = [W / 1200, H / 896];
+      let n = 0;
+      let on = 0;
+      for (let y = Math.max(0, Math.floor(y0 * sy)); y < Math.min(H, Math.ceil(y1 * sy)); y++) {
+        for (let x = Math.max(0, Math.floor(x0 * sx)); x < Math.min(W, Math.ceil(x1 * sx)); x++) (n++, (on += moved[y * W + x]));
+      }
+      return n ? Math.round((on / n) * 100) : 0;
+    };
+    const total = share(0, 0, 1200, 896);
+    const things = Object.fromEntries(g.things.filter((t) => t.role !== 'npc').map((t) => [t.key, `${share(...t.box)}% of its box moved`]));
+
+    // The first frame, dimmed, with what moved in magenta; then three frames side by side.
+    const base = await sharp(frames[0]).removeAlpha().raw().toBuffer();
+    const tint = Buffer.alloc(W * H * 3);
+    for (let i = 0; i < W * H; i++) {
+      const m = moved[i];
+      tint[i * 3] = m ? 255 : base[i * 3] * 0.55;
+      tint[i * 3 + 1] = m ? 40 : base[i * 3 + 1] * 0.55;
+      tint[i * 3 + 2] = m ? 220 : base[i * 3 + 2] * 0.55;
+    }
+    const label = Buffer.from(`<svg xmlns="http://www.w3.org/2000/svg" width="${W}" height="${H}">${svgLabel(12, 28, `moved in ${(FRAMES - 1) * EVERY_MS / 1000} s: ${total}% of the picture`, [255, 255, 255])}</svg>`);
+    const whatMoved = await sharp(tint, { raw: { width: W, height: H, channels: 3 } }).composite([{ input: label }]).jpeg({ quality: 75 }).toBuffer();
+    const tile = 400;
+    const tileH = Math.round((tile * H) / W);
+    const picks = [0, Math.floor(FRAMES / 2), FRAMES - 1];
+    const strip = await sharp({ create: { width: tile * 3, height: tileH, channels: 3, background: '#111' } })
+      .composite(await Promise.all(picks.map(async (k, i) => ({ input: await sharp(frames[k]).resize(tile, tileH).png().toBuffer(), left: i * tile, top: 0 }))))
+      .jpeg({ quality: 72 }).toBuffer();
+    fs.writeFileSync(files.path('places', place, 'view-motion.jpg'), whatMoved);
+    // What the game loaded to move by itself: only then is a still frame a fault.
+    const expected = [
+      ...(shows.water && shows.water !== 'none' ? [`water (${shows.water})`] : []),
+      ...(shows.mist && shows.mist !== 'none' ? ['drifting mist'] : []),
+      ...(typeof shows.weather === 'string' && shows.weather !== 'none' && !/^none, fog 0$/.test(shows.weather) ? [`weather (${shows.weather})`] : []),
+    ];
+    return {
+      place,
+      inTheGame: shows,
+      moved: `${total}% of the picture changed over ${(FRAMES - 1) * EVERY_MS / 1000} s`,
+      ...(total < 1 && { note: expected.length
+        ? `almost nothing moves, though the game loaded ${expected.join(', ')}: that motion is not showing`
+        : 'nothing here is set to move by itself (no water, mist, weather or fog): a still frame while the hero stands is expected, not a fault. The place still moves with the camera and the hero as it is played.' }),
+      things,
+      picture: `places/${place}/view-motion.jpg`,
+      images: [{ mimeType: 'image/jpeg', data: whatMoved.toString('base64') }, { mimeType: 'image/jpeg', data: strip.toString('base64') }],
+    };
+  },
+});

package/app/src/actions/pictures.ts

@@ -0,0 +1,49 @@
+// Pictures the person hands the agent with a message (pasted, uploaded, a webcam photo): kept with the world
+// under studio/pictures/, and shown to the agent as an image, so it sees what the person means.
+import fs from 'node:fs';
+import path from 'node:path';
+import sharp from 'sharp';
+import { z } from 'zod';
+import { Slug } from '../world/schema.ts';
+import { ActionError, defineAction } from './registry.ts';
+
+const PICTURE = /\.(png|jpe?g|webp|gif|heic|heif|avif)$/i;
+const FILE = /^studio\/pictures\/[0-9]{2,4}\.jpg$/;
+
+defineAction({
+  name: 'picture_add',
+  description: 'Keep a picture the person hands you with a message (pasted, uploaded or a webcam photo) with a world, as studio/pictures/NN.jpg. For the Studio\'s chat; the agent reads one with picture_view.',
+  input: z.object({
+    slug: Slug,
+    name: z.string().min(1).max(120).describe('the file name, for its extension'),
+    data: z.string().min(10).describe('the file, base64 (a data: URL is fine)'),
+  }),
+  async run({ slug, name, data }, ctx) {
+    if (!PICTURE.test(name)) throw new ActionError(400, `${name}: a picture is png, jpg, webp, gif, heic or avif`);
+    const dir = path.join(ctx.root, slug, 'studio', 'pictures');
+    let jpeg: Buffer;
+    try {
+      jpeg = await sharp(Buffer.from(data.replace(/^data:[^,]*,/, ''), 'base64')).rotate()
+        .resize(1600, 1600, { fit: 'inside', withoutEnlargement: true }).flatten({ background: '#ffffff' }).jpeg({ quality: 85 }).toBuffer();
+    } catch {
+      throw new ActionError(400, `${name} is not a picture this machine can read`);
+    }
+    fs.mkdirSync(dir, { recursive: true });
+    const taken = fs.readdirSync(dir).map((f) => Number.parseInt(f, 10)).filter(Number.isFinite);
+    const file = `${String(Math.max(0, ...taken) + 1).padStart(2, '0')}.jpg`;
+    fs.writeFileSync(path.join(dir, file), jpeg);
+    return { file: `studio/pictures/${file}` };
+  },
+});
+
+defineAction({
+  name: 'picture_view',
+  reads: true,
+  description: 'Look at a picture the person handed you with their message: a message that starts with "[Pictures: studio/pictures/01.jpg, …]" comes with them. Call this for each one before you answer.',
+  input: z.object({ slug: Slug, file: z.string().regex(FILE).describe('as the message names it, studio/pictures/NN.jpg') }),
+  async run({ slug, file }, ctx) {
+    const at = path.join(ctx.root, slug, file);
+    if (!fs.existsSync(at)) throw new ActionError(404, `${slug} has no ${file}`);
+    return { picture: file, images: [{ mimeType: 'image/jpeg', data: fs.readFileSync(at).toString('base64') }] };
+  },
+});

package/app/src/actions/reactions.ts

@@ -0,0 +1,36 @@
+import fs from 'node:fs';
+import path from 'node:path';
+import { z } from 'zod';
+import { Slug } from '../world/schema.ts';
+import { agentBlocked } from '../studio/locks.ts';
+import { ActionError, defineAction } from './registry.ts';
+
+defineAction({
+  name: 'reactions_set',
+  description: 'Choose or adjust subtle scene reactions: footsteps on each surface, crossing a local floor area, and physical use/take interactions. A hint in the author\'s words guides the automatic vision-language planner; use "choose automatically" to reset the preference. Keeps the hint across rebuilds, runs reactions for this place, then play_reload. It never repaints the scene.',
+  input: z.object({ slug: Slug, place: Slug, hint: z.string().min(3).max(2000) }),
+  job: true,
+  async run({ slug, place, hint }, ctx) {
+    const dir = path.join(ctx.root, slug), blocked = agentBlocked(dir, ctx.by);
+    if (blocked) throw new ActionError(423, blocked);
+    const at = path.join(dir, 'places', place);
+    if (!fs.existsSync(path.join(at, 'geometry.json'))) throw new ActionError(404, `${place} has no geometry yet`);
+    return ctx.jobs.start(slug, ['reactions', slug, place], `reactions ${place}`, ctx.by, () => {
+      fs.writeFileSync(path.join(at, 'reactions-wish.json'), JSON.stringify({ hint }, null, 2) + '\n');
+    });
+  },
+});
+defineAction({
+  name: 'reactions_status', reads: true,
+  description: 'Read the chosen scene reactions and author preference. Area cells are summarized; triggers list step/enter and exact story interactions. Missing means the reactions step has not run.',
+  input: z.object({ slug: Slug, place: Slug }),
+  async run({ slug, place }, ctx) {
+    const at = path.join(ctx.root, slug, 'places', place), file = path.join(at, 'reactions.json');
+    const wish = path.join(at, 'reactions-wish.json');
+    const hint = fs.existsSync(wish) ? JSON.parse(fs.readFileSync(wish, 'utf8')).hint : undefined;
+    if (!fs.existsSync(file)) return { made: false, hint };
+    const { Reactions } = await import('../play/reactions.ts');
+    const rules = Reactions.parse(JSON.parse(fs.readFileSync(file, 'utf8')));
+    return { made: true, hint, ...rules, areas: rules.areas.map(({ cells, ...a }) => ({ ...a, cells: cells.reduce((n, row) => n + [...row].filter((c) => c === '#').length, 0) })) };
+  },
+});

package/app/src/actions/registry.ts

@@ -0,0 +1,101 @@
+// One definition per action: name, what it does, its input (zod) and run(). From that single entry come the
+// studio's POST /api/actions/<name>, the Muse Code tool of the same name (diorama mcp) and
+// `diorama action <name> '<json>'`, so nothing the GUI can do is out of reach of a prompt.
+import { z } from 'zod';
+import type { Events } from '../studio/events.ts';
+import type { Jobs } from '../studio/jobs.ts';
+
+export type ActionContext = {
+  /** Who asked: the person in the GUI, the agent through MCP, or someone at a terminal. */
+  by: 'user' | 'agent' | 'cli';
+  /** Where worlds live (worlds/ under the working directory). */
+  root: string;
+  jobs: Jobs;
+  events: Events;
+  /**
+   * For the agent: calls `listener` with what the person writes to it while it works, until the returned
+   * function is called. A long wait (job_wait) ends early on it, so the agent hears the person at once.
+   */
+  onPersonSays?: (listener: (text: string) => void) => () => void;
+};
+
+export type Action<I extends z.ZodTypeAny = z.ZodTypeAny> = {
+  name: string;
+  description: string;
+  input: I;
+  /** Starts work that outlives the call (a job), rather than answering right away. */
+  job?: boolean;
+  /** Only reads: the activity log leaves it out, since the page itself polls these. */
+  reads?: boolean;
+  run(input: z.infer<I>, ctx: ActionContext): Promise<unknown>;
+};
+
+export const actions = new Map<string, Action>();
+
+export function defineAction<I extends z.ZodTypeAny>(action: Action<I>) {
+  if (!/^[a-z][a-z0-9_]*$/.test(action.name)) throw new Error(`action names are snake_case: ${action.name}`);
+  actions.set(action.name, action as Action);
+}
+
+/** Validates, runs and announces an action; every caller goes through here, so the feed sees all of them. */
+export async function runAction(name: string, raw: unknown, ctx: ActionContext): Promise<unknown> {
+  const action = actions.get(name);
+  if (!action) throw new ActionError(404, `no action ${name}; try one of ${[...actions.keys()].join(', ')}`);
+  const parsed = action.input.safeParse(raw ?? {});
+  if (!parsed.success) throw new ActionError(400, `${name}: ${parsed.error.issues.map((i) => `${i.path.join('.') || 'input'} ${i.message}`).join('; ')}`);
+  const world = (parsed.data as { slug?: string }).slug;
+  // A file sent as base64 (a drawing, a recording) is shown by its size, not spilled into the feed.
+  const shown = Object.fromEntries(Object.entries(parsed.data as object).map(([k, v]) => [k, typeof v === 'string' && v.length > 4000 ? `[${Math.round(v.length * 0.75 / 1024)} KB]` : v]));
+  ctx.events.publish('action.start', { action: name, by: ctx.by, world, input: shown });
+  try {
+    const result = await action.run(parsed.data, ctx);
+    ctx.events.publish('action.end', { action: name, by: ctx.by, world, ok: true });
+    return result;
+  } catch (error) {
+    ctx.events.publish('action.end', { action: name, by: ctx.by, world, ok: false, error: error instanceof Error ? error.message : String(error) });
+    throw error;
+  }
+}
+
+export class ActionError extends Error {
+  constructor(readonly status: number, message: string) {
+    super(message);
+  }
+}
+
+/** The JSON schema of an action's input, for MCP tool listings and the GUI. */
+export function describeActions() {
+  return [...actions.values()].map((a) => ({ name: a.name, description: a.description, job: !!a.job, reads: !!a.reads, input: zodToJson(a.input) }));
+}
+
+/** Enough of JSON Schema for flat action inputs: objects of strings, numbers, booleans, enums and arrays. */
+export function zodToJson(schema: z.ZodTypeAny): Record<string, unknown> {
+  const def = schema._def as { typeName: string; description?: string };
+  const withDesc = (out: Record<string, unknown>) => (schema.description ? { ...out, description: schema.description } : out);
+  switch (def.typeName) {
+    case 'ZodObject': {
+      const shape = (schema as z.ZodObject<z.ZodRawShape>).shape;
+      const required = Object.entries(shape).filter(([, v]) => !(v as z.ZodTypeAny).isOptional()).map(([k]) => k);
+      return withDesc({ type: 'object', properties: Object.fromEntries(Object.entries(shape).map(([k, v]) => [k, zodToJson(v as z.ZodTypeAny)])), required, additionalProperties: false });
+    }
+    case 'ZodLiteral': return withDesc({ const: (schema as z.ZodLiteral<unknown>).value });
+    case 'ZodDiscriminatedUnion': return withDesc({ oneOf: (schema as z.ZodDiscriminatedUnion<string, z.ZodDiscriminatedUnionOption<string>[]>).options.map(zodToJson) });
+    case 'ZodString': return withDesc({ type: 'string' });
+    case 'ZodNumber': return withDesc({ type: 'number' });
+    case 'ZodBoolean': return withDesc({ type: 'boolean' });
+    case 'ZodEnum': return withDesc({ type: 'string', enum: (schema as z.ZodEnum<[string]>).options });
+    case 'ZodTuple': {
+      const items = (schema as z.ZodTuple).items.map((i: z.ZodTypeAny) => zodToJson(i));
+      return withDesc({ type: 'array', prefixItems: items, items: items[0] ?? {}, minItems: items.length, maxItems: items.length });
+    }
+    case 'ZodRecord': return withDesc({ type: 'object', additionalProperties: zodToJson((schema as z.ZodRecord).valueSchema) });
+    case 'ZodEffects': return { ...zodToJson((schema as z.ZodEffects<z.ZodTypeAny>).innerType()), ...(schema.description && { description: schema.description }) };
+    case 'ZodArray': return withDesc({ type: 'array', items: zodToJson((schema as z.ZodArray<z.ZodTypeAny>).element) });
+    case 'ZodOptional':
+    case 'ZodDefault': {
+      const inner = def.typeName === 'ZodOptional' ? (schema as z.ZodOptional<z.ZodTypeAny>).unwrap() : (schema as z.ZodDefault<z.ZodTypeAny>).removeDefault();
+      return { ...zodToJson(inner), ...(schema.description && { description: schema.description }) };
+    }
+    default: return withDesc({});
+  }
+}