@gtanczyk/diorama 0.1.0 → 0.2.0

package/app/src/actions/budget.ts

@@ -4,7 +4,7 @@
 import fs from 'node:fs';
 import path from 'node:path';
 import { z } from 'zod';
-import type { ModelName } from '../meta/memo.ts';
+import { callVia, MUSE_VOICE, type ModelName } from '../meta/memo.ts';
 import { imageVia } from '../meta/muse.ts';
 import { sparkTier, sparkVia } from '../meta/spark.ts';
 import { configDir } from '../secrets.ts';
@@ -54,10 +54,18 @@ function tally(dir: string, since: string, credits?: string) {
   for (const line of fs.readFileSync(log, 'utf8').split('\n')) {
     if (!line) continue;
     try {
-      const c = JSON.parse(line) as { at: string; model: ModelName; hit: boolean; usd?: number; via?: string };
+      const c = JSON.parse(line) as { at: string; model: ModelName; hit: boolean; hash?: string; usd?: number; via?: string };
       if (c.hit) continue;
+      const via = callVia(dir, c);
+      // Muse Voice transcribing a take: Meta's, not an ElevenLabs call, so the ElevenLabs price is not put on it.
+      if (via === MUSE_VOICE) {
+        usd.all += c.usd ?? 0;
+        if (c.at >= since) usd.recent += c.usd ?? 0;
+        if (credits && c.at >= credits) usd.sinceCredits += c.usd ?? 0;
+        continue;
+      }
       // Answered by another model (Claude, Gemini through OpenRouter): not on Meta's bill, counted apart.
-      if (c.via && c.via !== 'contributor') {
+      if (via && via !== 'contributor') {
         usd.elsewhereCalls++;
         usd.elsewhereUsd += c.usd ?? 0;
         continue;

package/app/src/actions/cast.ts

@@ -89,6 +89,8 @@ defineAction({
         looksNow: fixes[c.key]?.looks ?? c.looks,
         fix: fixes[c.key] ?? null,
         figure: pic(`cast/${c.key}.png`),
+        // What the cut-out keeps beyond the body (cast_holds) and the share of its height the body takes.
+        cut: fs.existsSync(files.path('cast', c.key, 'cut.json')) ? files.readJson(files.path('cast', c.key, 'cut.json')) : null,
         face: faceStatus(files, c.key),
         painting: pic(`cast/${c.key}-painting.png`),
         rig: animationCurrent(files,c.key,'rig') && fs.existsSync(files.path('cast', c.key, 'rig.json')),
@@ -152,6 +154,23 @@ defineAction({
   },
 });
 
+defineAction({
+  name: 'cast_holds',
+  description: 'Keep what a character holds or carries on their cut-out (a giant slice of toast over the head, a lantern, a net), however big: SAM finds each '
+    + 'thing by its name and it is cut out with them and walks with their hands. Without holds, what their description says they hold is used (read by Spark); '
+    + 'holds names the things yourself in a few words each ("slice of toast"), [] keeps nothing extra. Kept across rebuilds. Cuts the figure again from its '
+    + 'painting and makes its 2D walk again: no Muse calls, a few SAM calls. Use it when the person says the figure lost what it holds. '
+    + 'A 3D model and its sprites never hold anything (held things tear its rig): when the held thing is who they are, say so. Then play_reload. Runs as a job.',
+  input: z.object({ slug: Slug, key: z.string().min(1), holds: z.array(z.string().min(2).max(60)).max(4).optional() }),
+  job: true,
+  async run({ slug, key, holds }, ctx) {
+    const prepare = prepareFix(ctx, slug, key, (f) => {
+      if (holds) f.holds = holds;
+    });
+    return ctx.jobs.start(slug, ['rig', slug, key], `cut out ${key} with what they hold`, ctx.by, prepare);
+  },
+});
+
 defineAction({
   name: 'cast_animate',
   description: 'Give a character a 3D model now, also one the build skips (an NPC with no lines, a creature or an animated object): '

package/app/src/actions/index.ts

@@ -24,6 +24,7 @@ import './pictures.ts';
 import './start.ts';
 import './sound.ts';
 import './step-work.ts';
+import './language.ts';
 
 import './video.ts';
 import './idea-review.ts';

package/app/src/actions/language.ts

@@ -0,0 +1,52 @@
+// The language a world is told in: world_language translates every word the player reads or hears and has the
+// voices cast and read again in it.
+import fs from 'node:fs';
+import { z } from 'zod';
+import { credits } from '../sound/cost.ts';
+import { elevenKey } from '../sound/sources.ts';
+import { Story } from '../story/schema.ts';
+import { agentBlocked } from '../studio/locks.ts';
+import { WorldFiles } from '../world/files.ts';
+import { languagePlan, planLine } from '../world/language.ts';
+import { Slug } from '../world/schema.ts';
+import { ActionError, defineAction } from './registry.ts';
+
+defineAction({
+  name: 'world_language',
+  description: 'Tell the world in another language ("English", "Polish", "German"): the title, pitch, character and item names and every line of the story '
+    + '(intro, remarks, dialogue, answers, item names, endings) are translated in one cheap Spark pass, with names kept the same everywhere; the game\'s '
+    + 'own words (hints, goodbye, buttons) follow world.json\'s language by themselves. Keys, puzzles, paintings, masks and sounds stay. Then the voices are '
+    + 'cast again as native speakers of the new language (no accent of the old one), every spoken line is read again and the talking faces follow. '
+    + 'voices false translates only (cheap; the game shows text until run_step voices). Without go it only answers what it redoes and about what '
+    + 'it costs: tell the person, then call again with go true once they agree. Runs as a job; history_undo takes it back. Then play_reload.',
+  input: z.object({
+    slug: Slug,
+    language: z.string().trim().min(2).max(30).describe('The new language as its English name: "English", "Polish".'),
+    voices: z.boolean().optional().describe('false: translate the text only and leave the voices for later. Default true.'),
+    go: z.boolean().optional().describe('true runs it; leave out to get the estimate first.'),
+  }),
+  job: true,
+  async run({ slug, language, voices = true, go }, ctx) {
+    const files = new WorldFiles(slug, ctx.root);
+    if (!fs.existsSync(files.path('world.json'))) throw new ActionError(404, `no world ${slug} yet`);
+    const name = language[0].toUpperCase() + language.slice(1);
+    const world = files.world();
+    const story = fs.existsSync(files.path('story.json')) ? Story.parse(files.readJson(files.path('story.json'))) : null;
+    const plan = languagePlan(files, world, story, name);
+    if (!go) {
+      const left = await credits(elevenKey());
+      return {
+        estimate: plan,
+        line: voices ? planLine(plan) : `${plan.texts} texts translated from ${plan.from} to ${plan.to} in one Spark pass; voices left for later`,
+        ...(left && voices && { elevenLabsCreditsLeft: left.left }),
+        next: 'tell the person what it redoes and costs; call again with go: true when they agree',
+      };
+    }
+    const blocked = agentBlocked(files.dir, ctx.by);
+    if (blocked) throw new ActionError(423, blocked);
+    const busy = ctx.jobs.busy(slug);
+    if (busy) throw new ActionError(409, `${busy.label} is ${busy.state}; wait before changing the language`);
+    const job = ctx.jobs.start(slug, ['language', slug, name, ...(voices ? [] : ['--text-only'])], `language ${name}`, ctx.by);
+    return { job, line: voices ? planLine(plan) : 'text only' };
+  },
+});

package/app/src/actions/world.ts

@@ -10,6 +10,7 @@ import { placeTrouble } from '../studio/trouble.ts';
 import { withThingEdits, WorldFiles } from '../world/files.ts';
 import { liveVoiceFailures } from '../sound/data.ts';
 import { Slug, World } from '../world/schema.ts';
+import { callVia } from '../meta/memo.ts';
 import { storyNeeds } from '../story/check.ts';
 import { ActionError, defineAction } from './registry.ts';
 
@@ -57,9 +58,9 @@ export function callCounts(dir: string, byVia = false): Record<string, number> {
   for (const line of fs.readFileSync(log, 'utf8').split('\n')) {
     if (!line) continue;
     try {
-      const call = JSON.parse(line) as { model: string; hit: boolean; via?: string };
-      // byVia: "model|via", so Studio names who answered (Claude standing in for Spark) as the live counts do.
-      const key = byVia ? `${call.model}|${typeof call.via === 'string' ? call.via : ''}` : call.model;
+      const call = JSON.parse(line) as { model: string; hit: boolean; hash?: string; via?: string };
+      // byVia: "model|via", so Studio names who answered (Claude standing in for Spark, Muse Voice transcribing) as the live counts do.
+      const key = byVia && !call.hit ? `${call.model}|${callVia(dir, call) ?? ''}` : call.model;
       if (!call.hit) counts[key] = (counts[key] ?? 0) + 1;
     } catch {}
   }

package/app/src/commands/cast.ts

@@ -6,8 +6,8 @@ import { museEdit } from '../meta/muse.ts';
 import { askSparkJson } from '../meta/spark.ts';
 import { z } from 'zod';
 import { defineCommand, takeOption } from '../registry.ts';
-import { castBodies, cutHero, paintFigure, readBodies, type Who } from '../scene/cast.ts';
-import { samFind } from '../meta/sam.ts';
+import { castBodies, cutOut, findFigure, paintFigure, readBodies, type Who } from '../scene/cast.ts';
+import { heldMask, heldThings } from '../scene/held.ts';
 import { photoStanding } from '../scene/photo.ts';
 import { rigFigure } from '../scene/rig.ts';
 import { checkSheet } from '../scene/rig-check.ts';
@@ -64,14 +64,15 @@ defineCommand({
         return null;
       });
       if (!made) return;
-      const { painting, figure } = made;
+      const { painting, figure, share, holds } = made;
       bindExistingAnimation(files, who.key);
       files.write(files.path('cast', `${who.key}-painting.png`), painting);
       files.write(files.path('cast', `${who.key}.png`), figure);
+      writeCut(files, who.key, share, holds);
       markStyled(files, 'cast', who.key, style);
       // The same painting cut into parts, so the runtime can walk the figure.
       if (bodies[who.key]?.upright === false) return console.log(`${who.key}: ${files.path('cast', `${who.key}.png`)} · not upright: drawn as painted`);
-      await rigCast(files, who.key, painting);
+      await rigCast(files, who.key, painting, holds);
     }));
     for (const m of missing) console.log(`set aside, shown as a stand-in: ${m}`);
     console.log(`calls sent: muse ${files.memo.sent.muse}, sam ${files.memo.sent.sam}`);
@@ -83,9 +84,9 @@ defineCommand({
  * rig.json with the joints and how the walk measured. A walk that tears is kept but marked, and the game slides
  * the painted figure as one piece instead. Parts of an older cut-out rig are removed.
  */
-export async function rigCast(files: WorldFiles, key: string, painting: Buffer) {
+export async function rigCast(files: WorldFiles, key: string, painting: Buffer, holds: string[] = []) {
   try {
-    let { rig, parts, issues } = await rigFigure(files.memo, key, painting);
+    let { rig, parts, issues } = await rigFigure(files.memo, key, painting, 'warp', holds);
     rig.check = rig.sheet ? await checkSheet(rig, parts[rig.sheet.file]) : undefined;
     if (rig.check && !rig.check.ok) {
       // Try a separate neutral source for the warp; never overwrite the creator's painting.
@@ -96,7 +97,7 @@ export async function rigCast(files: WorldFiles, key: string, painting: Buffer)
         const verdict = schema.parse(await askSparkJson(files.memo, { label: `rig-neutral-check:${key}`, schema, images: [painting,neutral], personal: true,
           prompt: 'Compare the original and proposed neutral standing character. Check identity, clothing, colours and proportions match; legs are visible and not already walking; every object held in the original remains complete and held in the same hand. If none is held, heldPreserved is true. Reply JSON {"matches":true,"neutral":true,"heldPreserved":true,"reason":"short explanation"}.' }));
         if (!verdict.matches || !verdict.neutral || !verdict.heldPreserved) throw new Error(verdict.reason || 'neutral pose did not preserve the original');
-        const candidate = await rigFigure(files.memo, key, neutral);
+        const candidate = await rigFigure(files.memo, key, neutral, 'warp', holds);
         candidate.rig.check = candidate.rig.sheet ? await checkSheet(candidate.rig, candidate.parts[candidate.rig.sheet.file]) : undefined;
         if (!candidate.rig.check?.ok) throw new Error(candidate.rig.check?.why ?? 'neutral walk did not pass');
         rig = candidate.rig; parts = candidate.parts; issues = [...issues, 'walk prepared from a verified neutral pose'];
@@ -121,22 +122,40 @@ export async function rigCast(files: WorldFiles, key: string, painting: Buffer)
 
 export const rigAttemptFile = (files: WorldFiles, key: string) => files.path('cast', key, 'rig-attempt.json');
 
+/**
+ * cast/<key>/cut.json: what the cut-out keeps beyond the body (`holds`, SAM phrases) and the share of its height the
+ * body takes (`share`), so the game draws the body at its own height and a slice of toast held over it above.
+ */
+export const cutFile = (files: WorldFiles, key: string) => files.path('cast', key, 'cut.json');
+export function writeCut(files: WorldFiles, key: string, share: number, holds: string[]) {
+  files.write(cutFile(files, key), { share, holds });
+}
+
 defineCommand({
   name: 'rig',
-  usage: 'rig <slug> [key...]     make the walk again from the cast paintings already painted (no Muse; SAM answers are kept), and cut the figure out again so what it holds stays on',
+  usage: 'rig <slug> [key...]     make the walk again from the cast paintings already painted (no Muse; SAM answers are kept), and cut the figure out again so what it holds stays on (what the description says they hold, or cast/fixes.json holds)',
   async run(args) {
     const [slug, ...only] = args;
     if (!slug) throw new Error('usage: diorama rig <slug> [key...]');
     const files = new WorldFiles(slug);
+    const world = files.world();
     const bodies = readBodies(files, fs.existsSync);
+    const looksOf = new Map<string, string>([[world.hero.key, world.hero.looks], ...world.places.flatMap((p) => p.things.filter((t) => t.role === 'npc').map((t) => [t.key, t.looks] as [string, string])), ...players(files).map((p) => [p.key, p.looks] as [string, string])]);
     const keys = fs.readdirSync(files.path('cast')).filter((f) => f.endsWith('-painting.png')).map((f) => f.slice(0, -'-painting.png'.length)).filter((k) => !only.length || only.includes(k));
     for (const key of keys) {
-      if (bodies[key]?.upright === false) continue;
       const painting = fs.readFileSync(files.path('cast', `${key}-painting.png`));
-      const person = (await samFind(files.memo, { label: `figure:${key}`, image: painting, prompt: 'person' })).sort((a, b) => b.mask.bits.reduce((s, x) => s + x, 0) - a.mask.bits.reduce((s, x) => s + x, 0))[0];
-      if (person) files.write(files.path('cast', `${key}.png`), await cutHero(painting, person.mask));
-      await rigCast(files, key, painting);
+      const who = fixedLooks(files, { key, looks: looksOf.get(key) ?? '' });
+      const holds = await heldThings(files.memo, who);
+      const person = await findFigure(files.memo, who, painting);
+      if (person) {
+        const { figure, share } = await cutOut(painting, person.mask, await heldMask(files.memo, key, painting, person.mask, holds));
+        files.write(files.path('cast', `${key}.png`), figure);
+        writeCut(files, key, share, holds);
+        console.log(`${key}: cut out again${holds.length ? `, holding ${holds.join(', ')}` : ''}${share < 1 ? ` (the body is ${Math.round(share * 100)}% of its height)` : ''}`);
+      }
+      if (bodies[key]?.upright === false) continue;
+      await rigCast(files, key, painting, holds);
     }
-    console.log(`calls sent: sam ${files.memo.sent.sam}`);
+    console.log(`calls sent: spark ${files.memo.sent.spark}, sam ${files.memo.sent.sam}`);
   },
 });

package/app/src/commands/export.ts

@@ -7,6 +7,19 @@ import { defineCommand } from '../registry.ts';
 import { WorldFiles } from '../world/files.ts';
 import { bundle, WEB, worldData } from './play.ts';
 
+/** A language name ("Polish", "polski") or code ("pl", "pt-BR") as a BCP 47 code, when Intl knows it. */
+function langCode(language?: string): string | undefined {
+  if (!language) return undefined;
+  const t = language.trim();
+  if (/^[a-z]{2,3}(-[A-Za-z0-9]{2,8})*$/i.test(t)) return t;
+  const names = new Intl.DisplayNames(['en'], { type: 'language' });
+  for (const code of ['en', 'pl', 'de', 'fr', 'es', 'it', 'pt', 'nl', 'cs', 'sk', 'uk', 'ru', 'sv', 'no', 'da', 'fi', 'hu', 'ro', 'el', 'tr', 'ja', 'ko', 'zh', 'ar', 'he', 'hi']) {
+    const own = new Intl.DisplayNames([code], { type: 'language' }).of(code);
+    if ([names.of(code), own].some((n) => n?.toLowerCase() === t.toLowerCase())) return code;
+  }
+  return undefined;
+}
+
 const esc = (s: string) => s.replace(/&/g, '&amp;').replace(/"/g, '&quot;').replace(/</g, '&lt;').replace(/>/g, '&gt;');
 
 /** The link preview (X, Facebook, chats): the world's title and pitch, and a 1200x630 picture: the world's own og.jpg when it has one, else its first painted place. */
@@ -115,6 +128,9 @@ export async function exportWorld(slug: string, out: string, home?: string, url?
   page = page.replace('<body>', '<body data-shipped>');
   if (home) page = page.replace('<div id="soundless"', `<a id="home" href="${esc(home)}">← Diorama</a>\n<div id="soundless"`);
   page = page.replace('<title>Diorama</title>', await sharePreview(files, out, url));
+  // The page's language follows the world's, so browsers and screen readers read a Polish world as Polish.
+  const lang = langCode(files.world().language);
+  if (lang) page = page.replace('<html lang="en">', `<html lang="${lang}">`);
   fs.writeFileSync(path.join(out, 'index.html'), page);
 }
 

package/app/src/commands/index.ts

@@ -24,6 +24,7 @@ import './cast.ts';
 import './model.ts';
 import './sprites.ts';
 import './style.ts';
+import './language.ts';
 import './play.ts';
 import './export.ts';
 import './publish.ts';

package/app/src/commands/language.ts

@@ -0,0 +1,87 @@
+// diorama language <slug> <Language>: the world told in another language. Title, pitch, names and every line of
+// the story are translated in one Spark pass; the paintings, masks, sounds and puzzles stay as they are. Then the
+// voices are cast again for the new language (no accent of the old one) and every spoken line is read again,
+// and the talking faces follow the new takes.
+import fs from 'node:fs';
+import { defineCommand, registry } from '../registry.ts';
+import { castFixesFile, readCastFixes } from '../scene/cast-fixes.ts';
+import { SoundPlan } from '../sound/plan.ts';
+import { readStoryFixes, StoryFixes, storyFixesFile, textAt } from '../story/fixes.ts';
+import { Story } from '../story/schema.ts';
+import { WorldFiles } from '../world/files.ts';
+import { applyWorldTexts, languagePlan, planLine, relanguage, translateTexts, worldTexts } from '../world/language.ts';
+import { placeSources, soundPlanFrom } from './sound.ts';
+
+const run = (name: string, args: string[]) => registry.get(name)!.run(args);
+
+defineCommand({
+  name: 'language',
+  usage: 'language <slug> <Language>  tell the world in another language ("English", "Polish"): title, pitch, names and the story translated in one Spark pass, then the voices cast again for that language and every spoken line read again [--text-only: leave the voices for later] [--estimate: only say what it redoes and costs]',
+  async run(args) {
+    const estimate = args.includes('--estimate');
+    const textOnly = args.includes('--text-only');
+    const [slug, language] = args.filter((a) => !a.startsWith('--'));
+    if (!slug || !language) throw new Error('usage: diorama language <slug> <Language> [--text-only] [--estimate]');
+    const files = new WorldFiles(slug);
+    const world = files.world();
+    const storyFile = files.path('story.json');
+    const story = fs.existsSync(storyFile) ? Story.parse(files.readJson(storyFile)) : null;
+    const plan = languagePlan(files, world, story, language);
+    console.log(`language ${slug}: ${planLine(plan)}`);
+    if (estimate) return;
+
+    if (!plan.same) {
+      const texts = worldTexts(world, story);
+      const kids = world.audience === 'kids' || fs.existsSync(files.path('drawings'));
+      console.log(`translating ${Object.keys(texts).length} texts from ${plan.from} to ${plan.to}`);
+      const translated = await translateTexts(files.memo, texts, plan.from, plan.to, kids);
+      const now = applyWorldTexts(world, story, translated, plan.to);
+      // sound.json is planned from the world and story: the words changed, not what makes a sound, so the plan stays.
+      const soundFile = files.path('sound.json');
+      const sound = fs.existsSync(soundFile) ? SoundPlan.safeParse(files.readJson(soundFile)) : undefined;
+      const sources = sound?.success ? placeSources(files, world) : undefined;
+      const soundWasCurrent = sound?.success && sources && sound.data.from === soundPlanFrom(world, story, sources);
+
+      files.write(files.path('world.json'), { ...JSON.parse(fs.readFileSync(files.path('world.json'), 'utf8')), ...now.world, ...(now.world.language ? {} : { language: undefined }) });
+      if (now.story) {
+        files.write(storyFile, now.story);
+        // The person's reworded lines are kept across rebuilds: they are reworded in the new language too.
+        const fixes = readStoryFixes(files);
+        if (Object.keys(fixes.texts).length) {
+          for (const path of Object.keys(fixes.texts)) {
+            const text = textAt(now.story, path);
+            if (text) fixes.texts[path] = { ...fixes.texts[path], text };
+          }
+          files.write(storyFixesFile(files), StoryFixes.parse(fixes));
+        }
+      }
+      // A voice the person described for the old language ("Native Polish speaker") is asked for in the new one.
+      const castFixes = readCastFixes(files);
+      let castChanged = false;
+      for (const fix of Object.values(castFixes)) {
+        if (fix.voice?.design && fix.voice.design !== relanguage(fix.voice.design, plan.from, plan.to)) {
+          fix.voice.design = relanguage(fix.voice.design, plan.from, plan.to);
+          castChanged = true;
+        }
+      }
+      if (castChanged) files.write(castFixesFile(files), castFixes);
+      if (soundWasCurrent && sound?.success && sources) files.write(soundFile, { ...sound.data, from: soundPlanFrom(files.world(), now.story, sources) });
+      for (const id of ['world.title', 'world.pitch']) console.log(`  ${id}: ${translated[id]}`);
+      for (const [id, text] of Object.entries(translated).filter(([id]) => id.startsWith('names.'))) console.log(`  ${id.slice(6)}: ${texts[id]} -> ${text}`);
+      console.log(`world.json and story.json are in ${plan.to} now`);
+    }
+    if (textOnly) return void console.log('voices left for later: run_step voices, then faces');
+    if (!story) return;
+    console.log('\n# voices');
+    await run('voices', [slug]);
+    if (fs.existsSync(files.path('cast'))) {
+      console.log('\n# faces');
+      try {
+        await run('build', [slug, '--from', 'faces', '--to', 'faces']);
+      } catch (error) {
+        // Talking mouths are polish: the voices are in, the game plays without them.
+        console.log(`faces not made again (${error instanceof Error ? error.message : error}): run_step faces later`);
+      }
+    }
+  },
+});

package/app/src/commands/play.ts

@@ -118,7 +118,10 @@ export function worldData(files: WorldFiles) {
       const walks = body?.upright !== false;
       // A creature has no legs for a 2D rig, but its own 3D model (no skeleton, model.json creature) hops and squashes.
       const creatureModel = model?.info.creature ? model : undefined;
-      return [key, { image: `cast/${f}`, ...(face && { face }), ...(body && { height: body.height }), ...(!walks && { upright: false }), ...(walks && rig && rig.check?.ok !== false ? { rig, parts: `cast/${key}` } : {}), ...(walks ? model && { model } : creatureModel && { model: creatureModel }), ...(walks && sheets && { sprites: sheets }) }];
+      // cast/<key>/cut.json (diorama cast, rig): the share of the cut-out's height the body takes when it holds something above it.
+      const cutAt = files.path('cast', key, 'cut.json');
+      const share = fs.existsSync(cutAt) ? Number((files.readJson(cutAt) as { share?: unknown }).share) : 1;
+      return [key, { image: `cast/${f}`, ...(share > 0 && share < 1 && { share }), ...(face && { face }), ...(body && { height: body.height }), ...(!walks && { upright: false }), ...(walks && rig && rig.check?.ok !== false ? { rig, parts: `cast/${key}` } : {}), ...(walks ? model && { model } : creatureModel && { model: creatureModel }), ...(walks && sheets && { sprites: sheets }) }];
     }));
   const story = fs.existsSync(files.path('story.json')) ? Story.parse(files.readJson(files.path('story.json'))) : null;
   // changes/<i>.png per place: how story.changes[i] looks once it holds.

package/app/src/commands/sound.ts

@@ -12,6 +12,20 @@ import { readSoundFixes } from '../sound/fixes.ts';
 import { elevenKey, musicgenPython } from '../sound/sources.ts';
 import { Story } from '../story/schema.ts';
 import { WorldFiles } from '../world/files.ts';
+import type { World } from '../world/schema.ts';
+
+/** What each place's sounds are planned from: its painting brief and floors. */
+export function placeSources(files: WorldFiles, world: World): Record<string, PlaceSources> {
+  const sources: Record<string, PlaceSources> = {};
+  for (const place of world.places) {
+    const read = <T>(file: string, schema: { parse: (v: unknown) => T }) => (fs.existsSync(files.place(place.key, file)) ? schema.parse(files.readJson(files.place(place.key, file))) : undefined);
+    sources[place.key] = { brief: read('brief.json', Brief), layout: read('layout.json', Layout) };
+  }
+  return sources;
+}
+
+/** sound.json's `from`: the plan is made again when this changes. */
+export const soundPlanFrom = (world: World, story: Story | null, sources: Record<string, PlaceSources>) => inputHash(JSON.stringify(world), JSON.stringify(story), JSON.stringify(sources));
 
 defineCommand({
   name: 'sound',
@@ -31,16 +45,12 @@ defineCommand({
     const files = new WorldFiles(slug);
     const world = files.world();
     const story = fs.existsSync(files.path('story.json')) ? Story.parse(files.readJson(files.path('story.json'))) : null;
-    const sources: Record<string, PlaceSources> = {};
-    for (const place of world.places) {
-      const read = <T>(file: string, schema: { parse: (v: unknown) => T }) => (fs.existsSync(files.place(place.key, file)) ? schema.parse(files.readJson(files.place(place.key, file))) : undefined);
-      sources[place.key] = { brief: read('brief.json', Brief), layout: read('layout.json', Layout) };
-    }
+    const sources = placeSources(files, world);
 
     // sound.json is the plan people may edit. It is made again with --replan, or when what it was made from
     // (world, story, briefs, floors) has changed since, so its shots and loops keep matching the story's things.
     const planFile = files.path('sound.json');
-    const from = inputHash(JSON.stringify(world), JSON.stringify(story), JSON.stringify(sources));
+    const from = soundPlanFrom(world, story, sources);
     const old = fs.existsSync(planFile) ? SoundPlan.parse(files.readJson(planFile)) : undefined;
     let plan: SoundPlan;
     // A plan made before silent things lost their loops drops them here too, before anything is paid for.

package/app/src/meta/memo.ts

@@ -23,6 +23,20 @@ export function inputHash(...parts: (string | Buffer)[]): string {
 /** What a call cost, when the asker knows: dollars and which backend answered (e.g. "claude" for a dev-mode Spark). */
 export type Paid = { data: Buffer; usd?: number; via?: string };
 
+/** The `via` of a Muse Voice Transcribe call: it shares the voice memo with ElevenLabs speech, but Meta answers it. */
+export const MUSE_VOICE = 'muse-voice';
+
+/**
+ * Who answered a logged call. Transcripts logged before they carried `via` are told apart from speech by the answer
+ * kept in the world's memo (a transcript is .txt or .json, speech .mp3), so old worlds stop counting them as ElevenLabs.
+ */
+export function callVia(dir: string, call: { model: string; hash?: string; via?: unknown }): string | undefined {
+  if (typeof call.via === 'string') return call.via;
+  if (call.model !== 'voice' || !call.hash) return undefined;
+  const at = (ext: string) => fs.existsSync(path.join(dir, 'memo', 'voice', `${call.hash}.${ext}`));
+  return at('txt') || at('json') ? MUSE_VOICE : undefined;
+}
+
 export class Memo {
   /** Calls actually sent to each model in this process (cache hits are not counted). */
   readonly sent: Record<ModelName, number> = { spark: 0, muse: 0, sam: 0, sound: 0, music: 0, voice: 0, space: 0 };

package/app/src/meta/voice.ts

@@ -1,6 +1,6 @@
 // Muse Voice Transcribe realtime transport. File uploads use voice-file.ts through speech/transcribe.ts.
 import { apiKey } from './http.ts';
-import { inputHash, type Memo } from './memo.ts';
+import { inputHash, MUSE_VOICE, type Memo } from './memo.ts';
 
 export const VOICE_TRANSCRIBE = 'muse-voice-transcribe-1.0';
 const URL_ASR = 'wss://api.meta.ai/v1/asr/realtime';
@@ -29,7 +29,7 @@ export function transcribe(memo: Memo, label: string, pcm24k: Buffer, pace = 1):
       }
     });
     queue = run.catch(() => {});
-    return run;
+    return run.then((data) => ({ data, via: MUSE_VOICE }));
   }).then((b) => b.toString('utf8'));
 }
 

package/app/src/scene/cast-fixes.ts

@@ -6,6 +6,8 @@ import type { WorldFiles } from '../world/files.ts';
 
 export const CastFixes = z.record(z.string(), z.object({
   looks: z.string().min(10).optional(),
+  /** What they hold, as short SAM phrases ("slice of toast"), kept on the cut-out whatever its size (cast_holds); [] keeps nothing. */
+  holds: z.array(z.string().min(2)).max(4).optional(),
   take: z.number().int().min(0).optional(),
   /** The person asked for a 3D model (cast_animate) though the story would not build one: the model step builds it every time. */
   animate: z.boolean().optional(),
@@ -25,7 +27,7 @@ export function readCastFixes(files: WorldFiles): CastFixes {
 }
 
 /** A character with the person's description in place of the world's. */
-export function fixedLooks<T extends { key: string; looks: string }>(files: WorldFiles, who: T): T {
-  const looks = readCastFixes(files)[who.key]?.looks;
-  return looks ? { ...who, looks } : who;
+export function fixedLooks<T extends { key: string; looks: string }>(files: WorldFiles, who: T): T & { holds?: string[] } {
+  const { looks, holds } = readCastFixes(files)[who.key] ?? {};
+  return { ...who, ...(looks && { looks }), ...(holds && { holds }) };
 }

package/app/src/scene/cast.ts

@@ -3,7 +3,7 @@ import { STYLE_REFERENCE_PROMPT } from '../world/style-reference.ts';
 import sharp from 'sharp';
 import type { Mask } from '../image/mask.ts';
 import { maskBox, union } from '../image/raster.ts';
-import { bodyShare, heldMask, heldThings } from './held.ts';
+import { bodyShare, groundGaps, heldMask, heldThings } from './held.ts';
 import { companions, silhouette } from './warp.ts';
 import type { Memo } from '../meta/memo.ts';
 import { museEdit, musePaint } from '../meta/muse.ts';
@@ -40,7 +40,10 @@ export async function cutOut(painting: Buffer, mask: Mask, held?: Mask): Promise
   const { data, info } = await sharp(painting).resize(mask.width, mask.height, { fit: 'fill' }).ensureAlpha().raw().toBuffer({ resolveWithObject: true });
   // SAM's person leaves out what they hold (a lantern, a net of fish) and leaves holes where it hangs over them.
   // ... and what floats beside them as part of who they are (a heart, the bubble they are in).
-  const whole = companions(data, silhouette(data, held ? union(mask, held) : mask));
+  const body = held ? union(mask, held) : mask;
+  const whole = companions(data, silhouette(data, body));
+  // Held up in both hands, the thing closes a window of plain ground with the arms and head: it stays see-through.
+  if (held) for (const gap of groundGaps(data, whole, body)) gap.bits.forEach((b, i) => b && (whole.bits[i] = 0));
   for (let i = 0; i < whole.bits.length; i++) if (!whole.bits[i]) data[i * info.channels + 3] = 0;
   const [x0, y0, x1, y1] = maskBox(whole)!;
   const figure = await sharp(data, { raw: { width: info.width, height: info.height, channels: 4 } })
@@ -112,13 +115,8 @@ export async function paintFigure(memo: Memo, world: World, who: Who, take = 0,
         : (from?.styleReference ? await edit({ label: `figure:${who.key}`, prompt: figurePrompt(world, who), images: [], take: t }) : await musePaint(memo, { label: `figure:${who.key}`, prompt: figurePrompt(world, who), take: t }));
     const png = await sharp(painting).png().toBuffer();
     // Not every character is a person (a flock of pigeons, a cat): SAM is asked by their own description next.
-    let found: Awaited<ReturnType<typeof samFind>> = [];
-    for (const prompt of ['person', who.looks.split(/[,.;]/)[0].trim().split(/\s+/).slice(0, 6).join(' '), 'animal']) {
-      found = await samFind(memo, { label: `figure:${who.key}`, image: png, prompt });
-      if (found.length) break;
-    }
-    if (!found.length) continue;
-    const biggest = found.reduce((a, b) => (area(b.mask) > area(a.mask) ? b : a));
+    const biggest = await findFigure(memo, who, png);
+    if (!biggest) continue;
     const made = { painting: png, ...(await cutOut(png, biggest.mask, await heldMask(memo, who.key, png, biggest.mask, holds))), holds };
     const why = (await boxed(made.figure)) ?? (upright ? await notStanding(memo, who.key, made.figure, t) : undefined);
     if (!why) return made;
@@ -132,6 +130,15 @@ export async function paintFigure(memo: Memo, world: World, who: Who, take = 0,
   throw new Error(`${who.key}: SAM found no one in the painting, two takes; try another take (cast_redo anotherTake)`);
 }
 
+/** The biggest one SAM finds of the character: as a person, else by its own description (a flock of pigeons, a cat), else as an animal. */
+export async function findFigure(memo: Memo, who: Who, png: Buffer) {
+  for (const prompt of ['person', who.looks.split(/[,.;]/)[0].trim().split(/\s+/).slice(0, 6).join(' '), 'animal']) {
+    const found = await samFind(memo, { label: `figure:${who.key}`, image: png, prompt });
+    if (found.length) return found.reduce((a, b) => (area(b.mask) > area(a.mask) ? b : a));
+  }
+  return undefined;
+}
+
 /**
  * Why a cut-out of a person on two legs is not one standing whole, or undefined when it is. A standing figure is
  * about twice as tall as wide (robots and big-headed drawn children down to about 1.4), so only a squatter one is