@gtanczyk/diorama 0.1.0 → 0.3.0

package/README.md

@@ -113,6 +113,7 @@ Model answers are memoised by a hash of their inputs, so running a step again wi
 - [Characters, style and icons](docs/characters.md)
 - [NPC story stages, optional walks and travel](docs/npc-motion.md)
 - [Sound and voices](docs/sound.md)
+- [Releasing and the changelog](docs/releasing.md) ([CHANGELOG.md](CHANGELOG.md))
 
 ## Built at the Meta AI Global Hackathon
 

package/app/src/actions/budget.ts

@@ -4,7 +4,7 @@
 import fs from 'node:fs';
 import path from 'node:path';
 import { z } from 'zod';
-import type { ModelName } from '../meta/memo.ts';
+import { callVia, MUSE_VOICE, type ModelName } from '../meta/memo.ts';
 import { imageVia } from '../meta/muse.ts';
 import { sparkTier, sparkVia } from '../meta/spark.ts';
 import { configDir } from '../secrets.ts';
@@ -54,10 +54,18 @@ function tally(dir: string, since: string, credits?: string) {
   for (const line of fs.readFileSync(log, 'utf8').split('\n')) {
     if (!line) continue;
     try {
-      const c = JSON.parse(line) as { at: string; model: ModelName; hit: boolean; usd?: number; via?: string };
+      const c = JSON.parse(line) as { at: string; model: ModelName; hit: boolean; hash?: string; usd?: number; via?: string };
       if (c.hit) continue;
+      const via = callVia(dir, c);
+      // Muse Voice transcribing a take: Meta's, not an ElevenLabs call, so the ElevenLabs price is not put on it.
+      if (via === MUSE_VOICE) {
+        usd.all += c.usd ?? 0;
+        if (c.at >= since) usd.recent += c.usd ?? 0;
+        if (credits && c.at >= credits) usd.sinceCredits += c.usd ?? 0;
+        continue;
+      }
       // Answered by another model (Claude, Gemini through OpenRouter): not on Meta's bill, counted apart.
-      if (c.via && c.via !== 'contributor') {
+      if (via && via !== 'contributor') {
         usd.elsewhereCalls++;
         usd.elsewhereUsd += c.usd ?? 0;
         continue;

package/app/src/actions/cast.ts

@@ -89,6 +89,8 @@ defineAction({
         looksNow: fixes[c.key]?.looks ?? c.looks,
         fix: fixes[c.key] ?? null,
         figure: pic(`cast/${c.key}.png`),
+        // What the cut-out keeps beyond the body (cast_holds) and the share of its height the body takes.
+        cut: fs.existsSync(files.path('cast', c.key, 'cut.json')) ? files.readJson(files.path('cast', c.key, 'cut.json')) : null,
         face: faceStatus(files, c.key),
         painting: pic(`cast/${c.key}-painting.png`),
         rig: animationCurrent(files,c.key,'rig') && fs.existsSync(files.path('cast', c.key, 'rig.json')),
@@ -152,6 +154,23 @@ defineAction({
   },
 });
 
+defineAction({
+  name: 'cast_holds',
+  description: 'Keep what a character holds or carries on their cut-out (a giant slice of toast over the head, a lantern, a net), however big: SAM finds each '
+    + 'thing by its name and it is cut out with them and walks with their hands. Without holds, what their description says they hold is used (read by Spark); '
+    + 'holds names the things yourself in a few words each ("slice of toast"), [] keeps nothing extra. Kept across rebuilds. Cuts the figure again from its '
+    + 'painting and makes its 2D walk again: no Muse calls, a few SAM calls. Use it when the person says the figure lost what it holds. '
+    + 'A 3D model and its sprites never hold anything (held things tear its rig): when the held thing is who they are, say so. Then play_reload. Runs as a job.',
+  input: z.object({ slug: Slug, key: z.string().min(1), holds: z.array(z.string().min(2).max(60)).max(4).optional() }),
+  job: true,
+  async run({ slug, key, holds }, ctx) {
+    const prepare = prepareFix(ctx, slug, key, (f) => {
+      if (holds) f.holds = holds;
+    });
+    return ctx.jobs.start(slug, ['rig', slug, key], `cut out ${key} with what they hold`, ctx.by, prepare);
+  },
+});
+
 defineAction({
   name: 'cast_animate',
   description: 'Give a character a 3D model now, also one the build skips (an NPC with no lines, a creature or an animated object): '

package/app/src/actions/fix.ts

@@ -22,19 +22,11 @@ async function fixPlace(ctx: ActionContext, slug: string, place: string, from: '
   const dir = path.join(ctx.root, slug);
   const blocked = agentBlocked(dir, ctx.by, place);
   if (blocked) throw new ActionError(423, blocked);
-  // The agent's repair limit: after three fixes in a row that left the place unplayable, it is the person's turn.
-  if (ctx.by === 'agent') {
-    const since = new Date(Date.now() - 30 * 60_000).toISOString();
-    const tries = ctx.jobs.list(slug).filter((j) => j.by === 'agent' && j.args.includes('--place') && j.args[j.args.indexOf('--place') + 1] === place && (j.queuedAt ?? '') >= since);
-    const lastOk = tries.findLastIndex((j) => j.state === 'done');
-    if (tries.slice(lastOk + 1).filter((j) => j.state === 'failed').length >= 3) {
-      throw new ActionError(429, `three fixes of ${place} in a row left it unplayable: stop here and tell the person what you tried and what you would do next`);
-    }
-  }
+  const note = ctx.by === 'agent' ? repairGuard(ctx, slug, place, label, from) : undefined;
   const placeDir = path.join(dir, 'places', place);
   const needs = { layout: null, paint: 'blockin.png', masks: 'painting.png', derive: 'masks.json' }[from];
   if (needs && !fs.existsSync(path.join(placeDir, needs))) throw new ActionError(404, `${slug}/${place} has no ${needs} yet: build it first`);
-  return ctx.jobs.start(slug, ['build', slug, '--from', from, '--to', 'verify', '--place', place], `${label} · ${place}`, ctx.by, () => {
+  const job = ctx.jobs.start(slug, ['build', slug, '--from', from, '--to', 'verify', '--place', place], `${label} · ${place}`, ctx.by, () => {
     // The job snapshots first, then changes its source. Queued fixes must not change a running build.
     fs.mkdirSync(placeDir, { recursive: true });
     const fixes = readCorrections(placeDir);
@@ -42,6 +34,32 @@ async function fixPlace(ctx: ActionContext, slug: string, place: string, from: '
     writeCorrections(placeDir, Corrections.parse(fixes));
     fs.rmSync(path.join(placeDir, 'verify.json'), { force: true });
   }, addedThing ? { place, key: addedThing } : undefined);
+  return note ? { ...job, note } : job;
+}
+
+/** Fixes that repaint or relayout cost a Muse call; cheap ones only rerun SAM and the walk grid. */
+const COSTLY = new Set(['layout', 'paint']);
+
+/**
+ * The agent's repair guardrail. A streak of fixes that left the place unplayable does not stop the agent: it steers it.
+ * After three, a fix that already failed in the streak is refused (it would fail the same way) and a new path goes
+ * ahead with a short reminder; after six, costly repaints and relayouts wait for the person, while cheap fixes stay open.
+ */
+function repairGuard(ctx: ActionContext, slug: string, place: string, label: string, from: string): string | undefined {
+  const since = new Date(Date.now() - 30 * 60_000).toISOString();
+  const tries = ctx.jobs.list(slug).filter((j) => j.by === 'agent' && j.args.includes('--place') && j.args[j.args.indexOf('--place') + 1] === place && (j.queuedAt ?? '') >= since);
+  const lastOk = tries.findLastIndex((j) => j.state === 'done');
+  const failed = tries.slice(lastOk + 1).filter((j) => j.state === 'failed');
+  if (failed.length < 3) return undefined;
+  const tried = [...new Set(failed.map((j) => j.label.replace(` · ${place}`, '')))];
+  const paths = 'other paths: mask_brush (paint or erase a thing\'s mask by hand), floor_cells (add or take back 60 px floor cells), stand_set or hero_start_set, thing_remove, corrections_clear of a fix that made it worse, or ask_person';
+  if (tried.includes(label)) {
+    throw new ActionError(429, `${place}: "${label}" already failed in this streak (${failed.length} fixes in a row left it unplayable: ${tried.join(', ')}). Do not repeat it; look with place_view (walk, masks) and take a different path. ${paths}.`);
+  }
+  if (failed.length >= 6 && COSTLY.has(from)) {
+    throw new ActionError(429, `${place}: ${failed.length} fixes in a row left it unplayable (${tried.join(', ')}). A repaint or relayout now costs credits on a guess: ask_person first, or keep to the cheap fixes. ${paths}.`);
+  }
+  return `${failed.length} fixes of ${place} in a row left it unplayable (${tried.join(', ')}); this one is a new path. Check place_status after it ends; if it fails too, tell the person briefly what you tried.`;
 }
 
 const thingOf = (c: Corrections, key: string) => (c.things[key] ??= { strokes: [] });

package/app/src/actions/index.ts

@@ -24,6 +24,7 @@ import './pictures.ts';
 import './start.ts';
 import './sound.ts';
 import './step-work.ts';
+import './language.ts';
 
 import './video.ts';
 import './idea-review.ts';

package/app/src/actions/language.ts

@@ -0,0 +1,52 @@
+// The language a world is told in: world_language translates every word the player reads or hears and has the
+// voices cast and read again in it.
+import fs from 'node:fs';
+import { z } from 'zod';
+import { credits } from '../sound/cost.ts';
+import { elevenKey } from '../sound/sources.ts';
+import { Story } from '../story/schema.ts';
+import { agentBlocked } from '../studio/locks.ts';
+import { WorldFiles } from '../world/files.ts';
+import { languagePlan, planLine } from '../world/language.ts';
+import { Slug } from '../world/schema.ts';
+import { ActionError, defineAction } from './registry.ts';
+
+defineAction({
+  name: 'world_language',
+  description: 'Tell the world in another language ("English", "Polish", "German"): the title, pitch, character and item names and every line of the story '
+    + '(intro, remarks, dialogue, answers, item names, endings) are translated in one cheap Spark pass, with names kept the same everywhere; the game\'s '
+    + 'own words (hints, goodbye, buttons) follow world.json\'s language by themselves. Keys, puzzles, paintings, masks and sounds stay. Then the voices are '
+    + 'cast again as native speakers of the new language (no accent of the old one), every spoken line is read again and the talking faces follow. '
+    + 'voices false translates only (cheap; the game shows text until run_step voices). Without go it only answers what it redoes and about what '
+    + 'it costs: tell the person, then call again with go true once they agree. Runs as a job; history_undo takes it back. Then play_reload.',
+  input: z.object({
+    slug: Slug,
+    language: z.string().trim().min(2).max(30).describe('The new language as its English name: "English", "Polish".'),
+    voices: z.boolean().optional().describe('false: translate the text only and leave the voices for later. Default true.'),
+    go: z.boolean().optional().describe('true runs it; leave out to get the estimate first.'),
+  }),
+  job: true,
+  async run({ slug, language, voices = true, go }, ctx) {
+    const files = new WorldFiles(slug, ctx.root);
+    if (!fs.existsSync(files.path('world.json'))) throw new ActionError(404, `no world ${slug} yet`);
+    const name = language[0].toUpperCase() + language.slice(1);
+    const world = files.world();
+    const story = fs.existsSync(files.path('story.json')) ? Story.parse(files.readJson(files.path('story.json'))) : null;
+    const plan = languagePlan(files, world, story, name);
+    if (!go) {
+      const left = await credits(elevenKey());
+      return {
+        estimate: plan,
+        line: voices ? planLine(plan) : `${plan.texts} texts translated from ${plan.from} to ${plan.to} in one Spark pass; voices left for later`,
+        ...(left && voices && { elevenLabsCreditsLeft: left.left }),
+        next: 'tell the person what it redoes and costs; call again with go: true when they agree',
+      };
+    }
+    const blocked = agentBlocked(files.dir, ctx.by);
+    if (blocked) throw new ActionError(423, blocked);
+    const busy = ctx.jobs.busy(slug);
+    if (busy) throw new ActionError(409, `${busy.label} is ${busy.state}; wait before changing the language`);
+    const job = ctx.jobs.start(slug, ['language', slug, name, ...(voices ? [] : ['--text-only'])], `language ${name}`, ctx.by);
+    return { job, line: voices ? planLine(plan) : 'text only' };
+  },
+});

package/app/src/actions/world.ts

@@ -10,6 +10,7 @@ import { placeTrouble } from '../studio/trouble.ts';
 import { withThingEdits, WorldFiles } from '../world/files.ts';
 import { liveVoiceFailures } from '../sound/data.ts';
 import { Slug, World } from '../world/schema.ts';
+import { callVia } from '../meta/memo.ts';
 import { storyNeeds } from '../story/check.ts';
 import { ActionError, defineAction } from './registry.ts';
 
@@ -57,9 +58,9 @@ export function callCounts(dir: string, byVia = false): Record<string, number> {
   for (const line of fs.readFileSync(log, 'utf8').split('\n')) {
     if (!line) continue;
     try {
-      const call = JSON.parse(line) as { model: string; hit: boolean; via?: string };
-      // byVia: "model|via", so Studio names who answered (Claude standing in for Spark) as the live counts do.
-      const key = byVia ? `${call.model}|${typeof call.via === 'string' ? call.via : ''}` : call.model;
+      const call = JSON.parse(line) as { model: string; hit: boolean; hash?: string; via?: string };
+      // byVia: "model|via", so Studio names who answered (Claude standing in for Spark, Muse Voice transcribing) as the live counts do.
+      const key = byVia && !call.hit ? `${call.model}|${callVia(dir, call) ?? ''}` : call.model;
       if (!call.hit) counts[key] = (counts[key] ?? 0) + 1;
     } catch {}
   }

package/app/src/commands/backdrop.ts

@@ -9,7 +9,7 @@ import { Geometry } from '../play/geometry.ts';
 import { defineCommand } from '../registry.ts';
 import { BACKDROP_GROW, backdropFor, smearedBackdrop, type Card } from '../scene/backdrop.ts';
 import { carryBetween, carryFrom, paintingFor, ran, remakes, tilesLine, withChange } from '../scene/carry.ts';
-import { cloudSheet, CRITTER, critterSheet, Depth, FALLING, fallSheet, farFlats, floatSheet, cloneFill, mistSheet, smokeSheet, waterSheet } from '../scene/layers.ts';
+import { cloudSheet, CRITTER, critterSheet, Depth, FALLING, fallSheet, STREAM, streamSheet, farFlats, floatSheet, cloneFill, mistSheet, smokeSheet, waterSheet } from '../scene/layers.ts';
 import { maskFromPng as waterMaskOf } from '../image/mask.ts';
 import { WorldFiles } from '../world/files.ts';
 
@@ -80,8 +80,22 @@ defineCommand({
         png = await sharp(cloneFill(raw, FRAME.width, FRAME.height, dilate(critterMask, 3)), { raw: { width: FRAME.width, height: FRAME.height, channels: 3 } }).png().toBuffer();
         files.write(at('backdrop.png'), png);
       }
+      // Small things painted falling in a stream (code rain, snow, petals) fall on the page (layers/stream-<i>.png):
+      // cut out of the backdrop here, each column falling at its own pace. Other things' masks stay put (a doorway
+      // inside the rain keeps what is seen through it).
+      const looks = (k: string) => place.things.find((t) => t.key === k)?.looks ?? '';
+      const streamKeys = Object.keys(notes).filter((k) => !keys.includes(k) && !critterKeys.includes(k) && STREAM.test(`${k} ${notes[k].phrase} ${looks(k)}`) && fs.existsSync(at('masks', `${k}.png`)));
+      const streams: Array<{ png: Buffer; region: Buffer; box: [number, number, number, number] }> = [];
+      for (const k of streamKeys) {
+        const others = await Promise.all(Object.keys(notes).filter((o) => o !== k && fs.existsSync(at('masks', `${o}.png`))).map(async (o) => maskFromPng(fs.readFileSync(at('masks', `${o}.png`)))));
+        const made = await streamSheet(png, await maskFromPng(fs.readFileSync(at('masks', `${k}.png`))), dilate(unionAll(FRAME.width, FRAME.height, others), 2));
+        if (!made) continue;
+        png = made.backdrop;
+        streams.push(made);
+      }
+      if (streams.length) files.write(at('backdrop.png'), png);
       // Painted waterfalls pour (layers/fall-<i>.png); they stay out of the flat water below them.
-      const fallKeys = Object.keys(notes).filter((k) => !keys.includes(k) && FALLING.test(`${k} ${notes[k].phrase}`) && fs.existsSync(at('masks', `${k}.png`)));
+      const fallKeys = Object.keys(notes).filter((k) => !keys.includes(k) && !streamKeys.includes(k) && FALLING.test(`${k} ${notes[k].phrase}`) && fs.existsSync(at('masks', `${k}.png`)));
       const fallMasks = await Promise.all(fallKeys.map(async (k) => maskFromPng(fs.readFileSync(at('masks', `${k}.png`)))));
       // The far view as flats one behind another (layers/), so it has depth of its own.
       fs.rmSync(at('layers'), { recursive: true, force: true });
@@ -133,6 +147,11 @@ defineCommand({
       const falls = (await Promise.all(fallMasks.map((m) => fallSheet(png, m)))).filter((f) => !!f);
       falls.forEach((f, i) => files.write(at('layers', `fall-${i}.png`), f.png));
       if (falls.length) console.log(`  ${place.key}: ${falls.length} waterfall(s) pouring`);
+      streams.forEach((f, i) => {
+        files.write(at('layers', `stream-${i}.png`), f.png);
+        files.write(at('layers', `stream-${i}-region.png`), f.region);
+      });
+      if (streams.length) console.log(`  ${place.key}: ${streams.length} stream(s) of small things falling (${streamKeys.join(', ')})`);
       const critters = critterKeys.length ? await critterSheet(withCritters, critterMask, wetMask) : [];
       critters.forEach((c, i) => files.write(at('layers', `critter-${i}.png`), c.png));
       if (critters.length) console.log(`  ${place.key}: ${critters.length} painted animal(s) ${critters.some((c) => c.medium === 'air') ? 'flying and ' : ''}swimming (${critterKeys.join(', ')})`);
@@ -162,7 +181,7 @@ defineCommand({
         files.write(at('layers', 'mist-mask.png'), mist.mask);
         console.log(`  ${place.key}: painted ${mist.phrase} drifting ${mist.drift === null ? 'with the wind' : mist.drift < 0 ? 'left' : 'right'}${mist.speed !== 1 ? ` x${mist.speed}` : ''}`);
       }
-      if (far || water || mist || smoke.length || falls.length || critters.length || clouds) {
+      if (far || water || mist || smoke.length || falls.length || streams.length || critters.length || clouds) {
         if (far) {
           files.write(at('layers', 'sky.png'), far.sky);
           far.flats.forEach((f, i) => files.write(at('layers', `${i}.png`), f.png));
@@ -176,6 +195,7 @@ defineCommand({
           ...(smoke.length && { smoke: smoke.map((s, i) => ({ file: `layers/smoke-${i}.png`, box: s.box })) }),
           ...(floats.length && { floats: floats.map((s, i) => ({ file: `layers/float-${i}.png`, box: s.box })) }),
           ...(falls.length && { falls: falls.map((f, i) => ({ file: `layers/fall-${i}.png`, box: f.box })) }),
+          ...(streams.length && { streams: streams.map((f, i) => ({ file: `layers/stream-${i}.png`, region: `layers/stream-${i}-region.png`, box: f.box })) }),
           ...(critters.length && { critters: critters.map((c, i) => ({ file: `layers/critter-${i}.png`, box: c.box, medium: c.medium, facing: c.facing })) }),
           ...(clouds && { clouds: { box: clouds.box, clip: 'layers/cloud-clip.png', ...(clouds.keep && { keep: 'layers/cloud-keep.png' }), bands: clouds.bands.map((b, i) => ({ file: `layers/cloud-${i}.png`, box: b.box, speed: b.speed })) } }),
         });

package/app/src/commands/cast.ts

@@ -6,8 +6,8 @@ import { museEdit } from '../meta/muse.ts';
 import { askSparkJson } from '../meta/spark.ts';
 import { z } from 'zod';
 import { defineCommand, takeOption } from '../registry.ts';
-import { castBodies, cutHero, paintFigure, readBodies, type Who } from '../scene/cast.ts';
-import { samFind } from '../meta/sam.ts';
+import { castBodies, cutOut, findFigure, paintFigure, readBodies, type Who } from '../scene/cast.ts';
+import { heldMask, heldThings } from '../scene/held.ts';
 import { photoStanding } from '../scene/photo.ts';
 import { rigFigure } from '../scene/rig.ts';
 import { checkSheet } from '../scene/rig-check.ts';
@@ -64,14 +64,15 @@ defineCommand({
         return null;
       });
       if (!made) return;
-      const { painting, figure } = made;
+      const { painting, figure, share, holds } = made;
       bindExistingAnimation(files, who.key);
       files.write(files.path('cast', `${who.key}-painting.png`), painting);
       files.write(files.path('cast', `${who.key}.png`), figure);
+      writeCut(files, who.key, share, holds);
       markStyled(files, 'cast', who.key, style);
       // The same painting cut into parts, so the runtime can walk the figure.
       if (bodies[who.key]?.upright === false) return console.log(`${who.key}: ${files.path('cast', `${who.key}.png`)} · not upright: drawn as painted`);
-      await rigCast(files, who.key, painting);
+      await rigCast(files, who.key, painting, holds);
     }));
     for (const m of missing) console.log(`set aside, shown as a stand-in: ${m}`);
     console.log(`calls sent: muse ${files.memo.sent.muse}, sam ${files.memo.sent.sam}`);
@@ -83,9 +84,9 @@ defineCommand({
  * rig.json with the joints and how the walk measured. A walk that tears is kept but marked, and the game slides
  * the painted figure as one piece instead. Parts of an older cut-out rig are removed.
  */
-export async function rigCast(files: WorldFiles, key: string, painting: Buffer) {
+export async function rigCast(files: WorldFiles, key: string, painting: Buffer, holds: string[] = []) {
   try {
-    let { rig, parts, issues } = await rigFigure(files.memo, key, painting);
+    let { rig, parts, issues } = await rigFigure(files.memo, key, painting, 'warp', holds);
     rig.check = rig.sheet ? await checkSheet(rig, parts[rig.sheet.file]) : undefined;
     if (rig.check && !rig.check.ok) {
       // Try a separate neutral source for the warp; never overwrite the creator's painting.
@@ -96,7 +97,7 @@ export async function rigCast(files: WorldFiles, key: string, painting: Buffer)
         const verdict = schema.parse(await askSparkJson(files.memo, { label: `rig-neutral-check:${key}`, schema, images: [painting,neutral], personal: true,
           prompt: 'Compare the original and proposed neutral standing character. Check identity, clothing, colours and proportions match; legs are visible and not already walking; every object held in the original remains complete and held in the same hand. If none is held, heldPreserved is true. Reply JSON {"matches":true,"neutral":true,"heldPreserved":true,"reason":"short explanation"}.' }));
         if (!verdict.matches || !verdict.neutral || !verdict.heldPreserved) throw new Error(verdict.reason || 'neutral pose did not preserve the original');
-        const candidate = await rigFigure(files.memo, key, neutral);
+        const candidate = await rigFigure(files.memo, key, neutral, 'warp', holds);
         candidate.rig.check = candidate.rig.sheet ? await checkSheet(candidate.rig, candidate.parts[candidate.rig.sheet.file]) : undefined;
         if (!candidate.rig.check?.ok) throw new Error(candidate.rig.check?.why ?? 'neutral walk did not pass');
         rig = candidate.rig; parts = candidate.parts; issues = [...issues, 'walk prepared from a verified neutral pose'];
@@ -121,22 +122,40 @@ export async function rigCast(files: WorldFiles, key: string, painting: Buffer)
 
 export const rigAttemptFile = (files: WorldFiles, key: string) => files.path('cast', key, 'rig-attempt.json');
 
+/**
+ * cast/<key>/cut.json: what the cut-out keeps beyond the body (`holds`, SAM phrases) and the share of its height the
+ * body takes (`share`), so the game draws the body at its own height and a slice of toast held over it above.
+ */
+export const cutFile = (files: WorldFiles, key: string) => files.path('cast', key, 'cut.json');
+export function writeCut(files: WorldFiles, key: string, share: number, holds: string[]) {
+  files.write(cutFile(files, key), { share, holds });
+}
+
 defineCommand({
   name: 'rig',
-  usage: 'rig <slug> [key...]     make the walk again from the cast paintings already painted (no Muse; SAM answers are kept), and cut the figure out again so what it holds stays on',
+  usage: 'rig <slug> [key...]     make the walk again from the cast paintings already painted (no Muse; SAM answers are kept), and cut the figure out again so what it holds stays on (what the description says they hold, or cast/fixes.json holds)',
   async run(args) {
     const [slug, ...only] = args;
     if (!slug) throw new Error('usage: diorama rig <slug> [key...]');
     const files = new WorldFiles(slug);
+    const world = files.world();
     const bodies = readBodies(files, fs.existsSync);
+    const looksOf = new Map<string, string>([[world.hero.key, world.hero.looks], ...world.places.flatMap((p) => p.things.filter((t) => t.role === 'npc').map((t) => [t.key, t.looks] as [string, string])), ...players(files).map((p) => [p.key, p.looks] as [string, string])]);
     const keys = fs.readdirSync(files.path('cast')).filter((f) => f.endsWith('-painting.png')).map((f) => f.slice(0, -'-painting.png'.length)).filter((k) => !only.length || only.includes(k));
     for (const key of keys) {
-      if (bodies[key]?.upright === false) continue;
       const painting = fs.readFileSync(files.path('cast', `${key}-painting.png`));
-      const person = (await samFind(files.memo, { label: `figure:${key}`, image: painting, prompt: 'person' })).sort((a, b) => b.mask.bits.reduce((s, x) => s + x, 0) - a.mask.bits.reduce((s, x) => s + x, 0))[0];
-      if (person) files.write(files.path('cast', `${key}.png`), await cutHero(painting, person.mask));
-      await rigCast(files, key, painting);
+      const who = fixedLooks(files, { key, looks: looksOf.get(key) ?? '' });
+      const holds = await heldThings(files.memo, who);
+      const person = await findFigure(files.memo, who, painting);
+      if (person) {
+        const { figure, share } = await cutOut(painting, person.mask, await heldMask(files.memo, key, painting, person.mask, holds));
+        files.write(files.path('cast', `${key}.png`), figure);
+        writeCut(files, key, share, holds);
+        console.log(`${key}: cut out again${holds.length ? `, holding ${holds.join(', ')}` : ''}${share < 1 ? ` (the body is ${Math.round(share * 100)}% of its height)` : ''}`);
+      }
+      if (bodies[key]?.upright === false) continue;
+      await rigCast(files, key, painting, holds);
     }
-    console.log(`calls sent: sam ${files.memo.sent.sam}`);
+    console.log(`calls sent: spark ${files.memo.sent.spark}, sam ${files.memo.sent.sam}`);
   },
 });

package/app/src/commands/export.ts

@@ -7,6 +7,19 @@ import { defineCommand } from '../registry.ts';
 import { WorldFiles } from '../world/files.ts';
 import { bundle, WEB, worldData } from './play.ts';
 
+/** A language name ("Polish", "polski") or code ("pl", "pt-BR") as a BCP 47 code, when Intl knows it. */
+function langCode(language?: string): string | undefined {
+  if (!language) return undefined;
+  const t = language.trim();
+  if (/^[a-z]{2,3}(-[A-Za-z0-9]{2,8})*$/i.test(t)) return t;
+  const names = new Intl.DisplayNames(['en'], { type: 'language' });
+  for (const code of ['en', 'pl', 'de', 'fr', 'es', 'it', 'pt', 'nl', 'cs', 'sk', 'uk', 'ru', 'sv', 'no', 'da', 'fi', 'hu', 'ro', 'el', 'tr', 'ja', 'ko', 'zh', 'ar', 'he', 'hi']) {
+    const own = new Intl.DisplayNames([code], { type: 'language' }).of(code);
+    if ([names.of(code), own].some((n) => n?.toLowerCase() === t.toLowerCase())) return code;
+  }
+  return undefined;
+}
+
 const esc = (s: string) => s.replace(/&/g, '&amp;').replace(/"/g, '&quot;').replace(/</g, '&lt;').replace(/>/g, '&gt;');
 
 /** The link preview (X, Facebook, chats): the world's title and pitch, and a 1200x630 picture: the world's own og.jpg when it has one, else its first painted place. */
@@ -77,6 +90,7 @@ export async function exportWorld(slug: string, out: string, home?: string, url?
         entry.layers.mist.mask = await copy(entry.layers.mist.mask);
       }
       for (const f of entry.layers.flats) f.file = await art(f.file); // transparent around the flat
+      for (const f of entry.layers.streams ?? []) [f.file, f.region] = [await art(f.file), await copy(f.region)];
       for (const f of [...(entry.layers.smoke ?? []), ...(entry.layers.floats ?? []), ...(entry.layers.falls ?? []), ...(entry.layers.critters ?? []), ...(entry.layers.clouds?.bands ?? [])]) f.file = await art(f.file);
       if (entry.layers.clouds) {
         entry.layers.clouds.clip = await copy(entry.layers.clouds.clip);
@@ -115,6 +129,9 @@ export async function exportWorld(slug: string, out: string, home?: string, url?
   page = page.replace('<body>', '<body data-shipped>');
   if (home) page = page.replace('<div id="soundless"', `<a id="home" href="${esc(home)}">← Diorama</a>\n<div id="soundless"`);
   page = page.replace('<title>Diorama</title>', await sharePreview(files, out, url));
+  // The page's language follows the world's, so browsers and screen readers read a Polish world as Polish.
+  const lang = langCode(files.world().language);
+  if (lang) page = page.replace('<html lang="en">', `<html lang="${lang}">`);
   fs.writeFileSync(path.join(out, 'index.html'), page);
 }
 

package/app/src/commands/index.ts

@@ -24,6 +24,7 @@ import './cast.ts';
 import './model.ts';
 import './sprites.ts';
 import './style.ts';
+import './language.ts';
 import './play.ts';
 import './export.ts';
 import './publish.ts';

package/app/src/commands/language.ts

@@ -0,0 +1,87 @@
+// diorama language <slug> <Language>: the world told in another language. Title, pitch, names and every line of
+// the story are translated in one Spark pass; the paintings, masks, sounds and puzzles stay as they are. Then the
+// voices are cast again for the new language (no accent of the old one) and every spoken line is read again,
+// and the talking faces follow the new takes.
+import fs from 'node:fs';
+import { defineCommand, registry } from '../registry.ts';
+import { castFixesFile, readCastFixes } from '../scene/cast-fixes.ts';
+import { SoundPlan } from '../sound/plan.ts';
+import { readStoryFixes, StoryFixes, storyFixesFile, textAt } from '../story/fixes.ts';
+import { Story } from '../story/schema.ts';
+import { WorldFiles } from '../world/files.ts';
+import { applyWorldTexts, languagePlan, planLine, relanguage, translateTexts, worldTexts } from '../world/language.ts';
+import { placeSources, soundPlanFrom } from './sound.ts';
+
+const run = (name: string, args: string[]) => registry.get(name)!.run(args);
+
+defineCommand({
+  name: 'language',
+  usage: 'language <slug> <Language>  tell the world in another language ("English", "Polish"): title, pitch, names and the story translated in one Spark pass, then the voices cast again for that language and every spoken line read again [--text-only: leave the voices for later] [--estimate: only say what it redoes and costs]',
+  async run(args) {
+    const estimate = args.includes('--estimate');
+    const textOnly = args.includes('--text-only');
+    const [slug, language] = args.filter((a) => !a.startsWith('--'));
+    if (!slug || !language) throw new Error('usage: diorama language <slug> <Language> [--text-only] [--estimate]');
+    const files = new WorldFiles(slug);
+    const world = files.world();
+    const storyFile = files.path('story.json');
+    const story = fs.existsSync(storyFile) ? Story.parse(files.readJson(storyFile)) : null;
+    const plan = languagePlan(files, world, story, language);
+    console.log(`language ${slug}: ${planLine(plan)}`);
+    if (estimate) return;
+
+    if (!plan.same) {
+      const texts = worldTexts(world, story);
+      const kids = world.audience === 'kids' || fs.existsSync(files.path('drawings'));
+      console.log(`translating ${Object.keys(texts).length} texts from ${plan.from} to ${plan.to}`);
+      const translated = await translateTexts(files.memo, texts, plan.from, plan.to, kids);
+      const now = applyWorldTexts(world, story, translated, plan.to);
+      // sound.json is planned from the world and story: the words changed, not what makes a sound, so the plan stays.
+      const soundFile = files.path('sound.json');
+      const sound = fs.existsSync(soundFile) ? SoundPlan.safeParse(files.readJson(soundFile)) : undefined;
+      const sources = sound?.success ? placeSources(files, world) : undefined;
+      const soundWasCurrent = sound?.success && sources && sound.data.from === soundPlanFrom(world, story, sources);
+
+      files.write(files.path('world.json'), { ...JSON.parse(fs.readFileSync(files.path('world.json'), 'utf8')), ...now.world, ...(now.world.language ? {} : { language: undefined }) });
+      if (now.story) {
+        files.write(storyFile, now.story);
+        // The person's reworded lines are kept across rebuilds: they are reworded in the new language too.
+        const fixes = readStoryFixes(files);
+        if (Object.keys(fixes.texts).length) {
+          for (const path of Object.keys(fixes.texts)) {
+            const text = textAt(now.story, path);
+            if (text) fixes.texts[path] = { ...fixes.texts[path], text };
+          }
+          files.write(storyFixesFile(files), StoryFixes.parse(fixes));
+        }
+      }
+      // A voice the person described for the old language ("Native Polish speaker") is asked for in the new one.
+      const castFixes = readCastFixes(files);
+      let castChanged = false;
+      for (const fix of Object.values(castFixes)) {
+        if (fix.voice?.design && fix.voice.design !== relanguage(fix.voice.design, plan.from, plan.to)) {
+          fix.voice.design = relanguage(fix.voice.design, plan.from, plan.to);
+          castChanged = true;
+        }
+      }
+      if (castChanged) files.write(castFixesFile(files), castFixes);
+      if (soundWasCurrent && sound?.success && sources) files.write(soundFile, { ...sound.data, from: soundPlanFrom(files.world(), now.story, sources) });
+      for (const id of ['world.title', 'world.pitch']) console.log(`  ${id}: ${translated[id]}`);
+      for (const [id, text] of Object.entries(translated).filter(([id]) => id.startsWith('names.'))) console.log(`  ${id.slice(6)}: ${texts[id]} -> ${text}`);
+      console.log(`world.json and story.json are in ${plan.to} now`);
+    }
+    if (textOnly) return void console.log('voices left for later: run_step voices, then faces');
+    if (!story) return;
+    console.log('\n# voices');
+    await run('voices', [slug]);
+    if (fs.existsSync(files.path('cast'))) {
+      console.log('\n# faces');
+      try {
+        await run('build', [slug, '--from', 'faces', '--to', 'faces']);
+      } catch (error) {
+        // Talking mouths are polish: the voices are in, the game plays without them.
+        console.log(`faces not made again (${error instanceof Error ? error.message : error}): run_step faces later`);
+      }
+    }
+  },
+});

package/app/src/commands/play.ts

@@ -65,6 +65,7 @@ export function worldData(files: WorldFiles) {
       ...(flats.smoke?.length && { smoke: flats.smoke.map((f) => ({ ...f, file: rel(f.file) })) }),
       ...(flats.floats?.length && { floats: flats.floats.map((f) => ({ ...f, file: rel(f.file) })) }),
       ...(flats.falls?.length && { falls: flats.falls.map((f) => ({ ...f, file: rel(f.file) })) }),
+      ...(flats.streams?.length && { streams: flats.streams.map((f) => ({ ...f, file: rel(f.file), region: rel(f.region) })) }),
       ...(flats.critters?.length && { critters: flats.critters.map((f) => ({ ...f, file: rel(f.file) })) }),
       ...(flats.clouds && { clouds: { ...flats.clouds, clip: rel(flats.clouds.clip), ...(flats.clouds.keep && { keep: rel(flats.clouds.keep) }), bands: flats.clouds.bands.map((b) => ({ ...b, file: rel(b.file) })) } }),
     };
@@ -118,7 +119,10 @@ export function worldData(files: WorldFiles) {
       const walks = body?.upright !== false;
       // A creature has no legs for a 2D rig, but its own 3D model (no skeleton, model.json creature) hops and squashes.
       const creatureModel = model?.info.creature ? model : undefined;
-      return [key, { image: `cast/${f}`, ...(face && { face }), ...(body && { height: body.height }), ...(!walks && { upright: false }), ...(walks && rig && rig.check?.ok !== false ? { rig, parts: `cast/${key}` } : {}), ...(walks ? model && { model } : creatureModel && { model: creatureModel }), ...(walks && sheets && { sprites: sheets }) }];
+      // cast/<key>/cut.json (diorama cast, rig): the share of the cut-out's height the body takes when it holds something above it.
+      const cutAt = files.path('cast', key, 'cut.json');
+      const share = fs.existsSync(cutAt) ? Number((files.readJson(cutAt) as { share?: unknown }).share) : 1;
+      return [key, { image: `cast/${f}`, ...(share > 0 && share < 1 && { share }), ...(face && { face }), ...(body && { height: body.height }), ...(!walks && { upright: false }), ...(walks && rig && rig.check?.ok !== false ? { rig, parts: `cast/${key}` } : {}), ...(walks ? model && { model } : creatureModel && { model: creatureModel }), ...(walks && sheets && { sprites: sheets }) }];
     }));
   const story = fs.existsSync(files.path('story.json')) ? Story.parse(files.readJson(files.path('story.json'))) : null;
   // changes/<i>.png per place: how story.changes[i] looks once it holds.

package/app/src/commands/sound.ts

@@ -12,6 +12,20 @@ import { readSoundFixes } from '../sound/fixes.ts';
 import { elevenKey, musicgenPython } from '../sound/sources.ts';
 import { Story } from '../story/schema.ts';
 import { WorldFiles } from '../world/files.ts';
+import type { World } from '../world/schema.ts';
+
+/** What each place's sounds are planned from: its painting brief and floors. */
+export function placeSources(files: WorldFiles, world: World): Record<string, PlaceSources> {
+  const sources: Record<string, PlaceSources> = {};
+  for (const place of world.places) {
+    const read = <T>(file: string, schema: { parse: (v: unknown) => T }) => (fs.existsSync(files.place(place.key, file)) ? schema.parse(files.readJson(files.place(place.key, file))) : undefined);
+    sources[place.key] = { brief: read('brief.json', Brief), layout: read('layout.json', Layout) };
+  }
+  return sources;
+}
+
+/** sound.json's `from`: the plan is made again when this changes. */
+export const soundPlanFrom = (world: World, story: Story | null, sources: Record<string, PlaceSources>) => inputHash(JSON.stringify(world), JSON.stringify(story), JSON.stringify(sources));
 
 defineCommand({
   name: 'sound',
@@ -31,16 +45,12 @@ defineCommand({
     const files = new WorldFiles(slug);
     const world = files.world();
     const story = fs.existsSync(files.path('story.json')) ? Story.parse(files.readJson(files.path('story.json'))) : null;
-    const sources: Record<string, PlaceSources> = {};
-    for (const place of world.places) {
-      const read = <T>(file: string, schema: { parse: (v: unknown) => T }) => (fs.existsSync(files.place(place.key, file)) ? schema.parse(files.readJson(files.place(place.key, file))) : undefined);
-      sources[place.key] = { brief: read('brief.json', Brief), layout: read('layout.json', Layout) };
-    }
+    const sources = placeSources(files, world);
 
     // sound.json is the plan people may edit. It is made again with --replan, or when what it was made from
     // (world, story, briefs, floors) has changed since, so its shots and loops keep matching the story's things.
     const planFile = files.path('sound.json');
-    const from = inputHash(JSON.stringify(world), JSON.stringify(story), JSON.stringify(sources));
+    const from = soundPlanFrom(world, story, sources);
     const old = fs.existsSync(planFile) ? SoundPlan.parse(files.readJson(planFile)) : undefined;
     let plan: SoundPlan;
     // A plan made before silent things lost their loops drops them here too, before anything is paid for.

package/app/src/meta/memo.ts

@@ -23,6 +23,20 @@ export function inputHash(...parts: (string | Buffer)[]): string {
 /** What a call cost, when the asker knows: dollars and which backend answered (e.g. "claude" for a dev-mode Spark). */
 export type Paid = { data: Buffer; usd?: number; via?: string };
 
+/** The `via` of a Muse Voice Transcribe call: it shares the voice memo with ElevenLabs speech, but Meta answers it. */
+export const MUSE_VOICE = 'muse-voice';
+
+/**
+ * Who answered a logged call. Transcripts logged before they carried `via` are told apart from speech by the answer
+ * kept in the world's memo (a transcript is .txt or .json, speech .mp3), so old worlds stop counting them as ElevenLabs.
+ */
+export function callVia(dir: string, call: { model: string; hash?: string; via?: unknown }): string | undefined {
+  if (typeof call.via === 'string') return call.via;
+  if (call.model !== 'voice' || !call.hash) return undefined;
+  const at = (ext: string) => fs.existsSync(path.join(dir, 'memo', 'voice', `${call.hash}.${ext}`));
+  return at('txt') || at('json') ? MUSE_VOICE : undefined;
+}
+
 export class Memo {
   /** Calls actually sent to each model in this process (cache hits are not counted). */
   readonly sent: Record<ModelName, number> = { spark: 0, muse: 0, sam: 0, sound: 0, music: 0, voice: 0, space: 0 };

package/app/src/meta/voice.ts

@@ -1,6 +1,6 @@
 // Muse Voice Transcribe realtime transport. File uploads use voice-file.ts through speech/transcribe.ts.
 import { apiKey } from './http.ts';
-import { inputHash, type Memo } from './memo.ts';
+import { inputHash, MUSE_VOICE, type Memo } from './memo.ts';
 
 export const VOICE_TRANSCRIBE = 'muse-voice-transcribe-1.0';
 const URL_ASR = 'wss://api.meta.ai/v1/asr/realtime';
@@ -29,7 +29,7 @@ export function transcribe(memo: Memo, label: string, pcm24k: Buffer, pace = 1):
       }
     });
     queue = run.catch(() => {});
-    return run;
+    return run.then((data) => ({ data, via: MUSE_VOICE }));
   }).then((b) => b.toString('utf8'));
 }