@gtanczyk/diorama 0.0.0-stage → 0.2.0

package/app/src/secrets.ts

@@ -0,0 +1,219 @@
+// Keys typed into `diorama setup` are kept encrypted on disk, never as plain text and never in a file the
+// repo or a world could pick up. A random 32-byte master key lives in the OS vault (macOS Keychain via
+// `security`, Linux Secret Service via `secret-tool`); the secrets themselves are AES-256-GCM sealed in
+// ~/.config/diorama/credentials.bin (0600). Environment variables always win over stored keys.
+import { execFile, spawn } from 'node:child_process';
+import { createCipheriv, createDecipheriv, randomBytes } from 'node:crypto';
+import fs from 'node:fs';
+import os from 'node:os';
+import path from 'node:path';
+
+/** The keys diorama knows how to keep, with the other names each one is accepted under. */
+export const SECRETS = {
+  META_API_KEY: { label: 'Meta Model API key', aliases: ['META_MUSE_API_KEY', 'META_MODEL_API_KEY'], required: true },
+  ELEVENLABS_API_KEY: { label: 'ElevenLabs key', aliases: ['ELEVEN_API_KEY'], required: false },
+  HF_TOKEN: { label: 'Hugging Face token', aliases: ['HUGGING_FACE_API_KEY'], required: false },
+  // For DIORAMA_SPARK_VIA=claude. Its own name first, so setting it never switches Claude Code itself to API billing.
+  DIORAMA_ANTHROPIC_API_KEY: { label: 'Anthropic API key (Spark via Claude)', aliases: ['ANTHROPIC_API_KEY', 'CLAUDE_API_KEY'], required: false },
+  OPENROUTER_API_KEY: { label: 'OpenRouter key (Gemini pictures, other Spark models)', aliases: [], required: false },
+  GEMINI_API_KEY: { label: 'Google Gemini key (Gemini pictures straight from Google)', aliases: ['GOOGLE_API_KEY'], required: false },
+} as const;
+export type SecretName = keyof typeof SECRETS;
+export const SECRET_NAMES = Object.keys(SECRETS) as SecretName[];
+
+/** Holds the master key. */
+export interface Vault {
+  readonly name: string;
+  get(): Promise<Buffer | null>;
+  set(key: Buffer): Promise<void>;
+}
+
+const SERVICE = 'diorama';
+const ACCOUNT = 'credentials-key';
+
+function run(cmd: string, args: string[], input?: string): Promise<{ code: number; out: string }> {
+  return new Promise((resolve) => {
+    const child = spawn(cmd, args, { stdio: ['pipe', 'pipe', 'ignore'] });
+    let out = '';
+    child.stdout.on('data', (d) => (out += d));
+    child.on('error', () => resolve({ code: -1, out: '' }));
+    child.on('close', (code) => resolve({ code: code ?? -1, out }));
+    child.stdin.end(input ?? '');
+  });
+}
+
+function onPath(cmd: string): Promise<boolean> {
+  return new Promise((resolve) => execFile('/bin/sh', ['-c', `command -v ${cmd}`], (error) => resolve(!error)));
+}
+
+/** macOS Keychain. The key goes in through `security -i` on stdin, so it never shows up in a process list. */
+const keychain: Vault = {
+  name: 'macOS Keychain',
+  async get() {
+    const { code, out } = await run('security', ['find-generic-password', '-s', SERVICE, '-a', ACCOUNT, '-w']);
+    return code === 0 && out.trim() ? Buffer.from(out.trim(), 'hex') : null;
+  },
+  async set(key) {
+    const { code } = await run('security', ['-i'], `add-generic-password -U -s ${SERVICE} -a ${ACCOUNT} -l "diorama credentials" -w ${key.toString('hex')}\n`);
+    if (code !== 0) throw new Error('could not write the diorama key to the macOS Keychain');
+  },
+};
+
+/** Linux Secret Service (GNOME Keyring, KWallet); `secret-tool store` reads the value from stdin. */
+const secretService: Vault = {
+  name: 'Secret Service',
+  async get() {
+    const { code, out } = await run('secret-tool', ['lookup', 'service', SERVICE, 'account', ACCOUNT]);
+    return code === 0 && out.trim() ? Buffer.from(out.trim(), 'hex') : null;
+  },
+  async set(key) {
+    const { code } = await run('secret-tool', ['store', '--label=diorama credentials', 'service', SERVICE, 'account', ACCOUNT], key.toString('hex'));
+    if (code !== 0) throw new Error('could not write the diorama key to the Secret Service (is a keyring unlocked?)');
+  },
+};
+
+/** Opt-in only: the master key in a 0600 file next to the secrets. Anyone who can read your home can read both. */
+export function fileVault(dir: string): Vault {
+  const file = path.join(dir, 'credentials.key');
+  return {
+    name: 'key file (DIORAMA_ALLOW_FILE_KEYCHAIN)',
+    async get() {
+      return fs.existsSync(file) ? Buffer.from(fs.readFileSync(file, 'utf8').trim(), 'hex') : null;
+    },
+    async set(key) {
+      writeAtomic(file, key.toString('hex'));
+    },
+  };
+}
+
+export function configDir(): string {
+  if (process.env.DIORAMA_CONFIG_DIR) return path.resolve(process.env.DIORAMA_CONFIG_DIR);
+  return path.join(process.env.XDG_CONFIG_HOME || path.join(os.homedir(), '.config'), 'diorama');
+}
+
+/** The vault this machine offers, or null when there is none and the file fallback was not asked for. */
+export async function findVault(dir = configDir()): Promise<Vault | null> {
+  if (process.platform === 'darwin' && (await onPath('security'))) return keychain;
+  if (process.platform === 'linux' && (await onPath('secret-tool'))) return secretService;
+  if (process.env.DIORAMA_ALLOW_FILE_KEYCHAIN === 'true') return fileVault(dir);
+  return null;
+}
+
+export const NO_VAULT = 'No system keychain found (macOS Keychain, or secret-tool on Linux). Set the key as an environment variable instead, '
+  + 'or run with DIORAMA_ALLOW_FILE_KEYCHAIN=true to keep it in a key file under ~/.config/diorama (weaker: anyone who can read your home can read it).';
+
+function writeAtomic(file: string, data: string | Buffer) {
+  fs.mkdirSync(path.dirname(file), { recursive: true, mode: 0o700 });
+  const tmp = `${file}.${process.pid}.${randomBytes(4).toString('hex')}.tmp`;
+  fs.writeFileSync(tmp, data, { mode: 0o600 });
+  fs.renameSync(tmp, file);
+}
+
+const MAGIC = Buffer.from('DIO1');
+
+export function seal(key: Buffer, plain: Record<string, string>): Buffer {
+  const iv = randomBytes(12);
+  const cipher = createCipheriv('aes-256-gcm', key, iv);
+  cipher.setAAD(MAGIC);
+  const body = Buffer.concat([cipher.update(JSON.stringify(plain), 'utf8'), cipher.final()]);
+  return Buffer.concat([MAGIC, iv, cipher.getAuthTag(), body]);
+}
+
+export function open(key: Buffer, sealed: Buffer): Record<string, string> {
+  if (!sealed.subarray(0, 4).equals(MAGIC)) throw new Error('credentials.bin is not a diorama credentials file');
+  const decipher = createDecipheriv('aes-256-gcm', key, sealed.subarray(4, 16));
+  decipher.setAAD(MAGIC);
+  decipher.setAuthTag(sealed.subarray(16, 32));
+  return JSON.parse(Buffer.concat([decipher.update(sealed.subarray(32)), decipher.final()]).toString('utf8'));
+}
+
+/** The encrypted secrets file. Writes take a lock file, so two `diorama setup` runs cannot lose a key. */
+export class SecretStore {
+  readonly file: string;
+
+  constructor(readonly vault: Vault, readonly dir = configDir()) {
+    this.file = path.join(dir, 'credentials.bin');
+  }
+
+  async read(): Promise<Record<string, string>> {
+    if (!fs.existsSync(this.file)) return {};
+    const key = await this.vault.get();
+    if (!key) throw new Error(`credentials.bin exists but its key is missing from the ${this.vault.name}; run diorama setup to store your keys again`);
+    return open(key, fs.readFileSync(this.file));
+  }
+
+  async set(name: SecretName, value: string | null) {
+    await this.locked(async () => {
+      let key = await this.vault.get();
+      if (!key || key.length !== 32) {
+        key = randomBytes(32);
+        await this.vault.set(key);
+      }
+      const all = fs.existsSync(this.file) ? open(key, fs.readFileSync(this.file)) : {};
+      if (value === null) delete all[name];
+      else all[name] = value;
+      writeAtomic(this.file, seal(key, all));
+    });
+  }
+
+  private async locked(work: () => Promise<void>) {
+    fs.mkdirSync(this.dir, { recursive: true, mode: 0o700 });
+    const lock = path.join(this.dir, 'credentials.lock');
+    for (let tries = 0; ; tries++) {
+      try {
+        fs.closeSync(fs.openSync(lock, 'wx', 0o600));
+        break;
+      } catch (error) {
+        const stale = fs.existsSync(lock) && Date.now() - fs.statSync(lock).mtimeMs > 30_000;
+        if (stale) fs.rmSync(lock, { force: true });
+        else if (tries > 50) throw new Error(`credentials are locked by another diorama process (${lock})`);
+        else await new Promise((r) => setTimeout(r, 100));
+      }
+    }
+    try {
+      await work();
+    } finally {
+      fs.rmSync(lock, { force: true });
+    }
+  }
+}
+
+/** Where a key comes from right now: the environment (under any of its names), the store, or nowhere. */
+export function envValue(name: SecretName): string | undefined {
+  return [name, ...SECRETS[name].aliases].map((n) => process.env[n]).find(Boolean);
+}
+
+/** At startup: stored keys fill in whatever the environment does not set. Quiet when nothing is stored. */
+export async function loadSecrets(dir = configDir()): Promise<Partial<Record<SecretName, 'env' | 'store'>>> {
+  const sources: Partial<Record<SecretName, 'env' | 'store'>> = {};
+  for (const name of SECRET_NAMES) if (envValue(name)) sources[name] = 'env';
+  if (fs.existsSync(path.join(dir, 'credentials.bin')) && SECRET_NAMES.some((n) => !sources[n])) {
+    const vault = await findVault(dir);
+    if (vault) {
+      const stored = await new SecretStore(vault, dir).read();
+      for (const name of SECRET_NAMES) {
+        if (sources[name] || !stored[name]) continue;
+        process.env[name] = stored[name];
+        sources[name] = 'store';
+      }
+    }
+  }
+  return sources;
+}
+
+/** Every known key value in this process, long enough to be worth hiding. */
+function knownValues(): string[] {
+  return SECRET_NAMES.flatMap((n) => [n, ...SECRETS[n].aliases]).map((n) => process.env[n]).filter((v): v is string => !!v && v.length >= 8);
+}
+
+/** Replaces any key value in text, for logs, recordings and anything sent to a browser or agent. */
+export function redact(text: string): string {
+  for (const value of knownValues()) text = text.split(value).join('[redacted]');
+  return text.replace(/\b(Bearer\s+)[A-Za-z0-9._~+/=-]{16,}/g, '$1[redacted]');
+}
+
+/** The environment without any provider key, for a process that must not hold them (the agent). */
+export function withoutSecrets(env: NodeJS.ProcessEnv): NodeJS.ProcessEnv {
+  const drop = new Set<string>(SECRET_NAMES.flatMap((n) => [n, ...SECRETS[n].aliases]));
+  return Object.fromEntries(Object.entries(env).filter(([k]) => !drop.has(k)));
+}

package/app/src/setup/checks.ts

@@ -0,0 +1,110 @@
+// What a machine needs before diorama can build: shown by `diorama setup`, the Studio's Setup screen and
+// the setup_status action. A check never returns a key, only where it comes from.
+import { execFile } from 'node:child_process';
+import fs from 'node:fs';
+import path from 'node:path';
+import { blenderState } from '../model/blender.ts';
+import { envValue, findVault, loadSecrets, NO_VAULT, type SecretName } from '../secrets.ts';
+
+export type Check = {
+  id: string;
+  label: string;
+  /** ok, missing (needed to build), optional (missing but not needed), fail (present but broken). */
+  state: 'ok' | 'missing' | 'optional' | 'fail';
+  detail: string;
+  /** Commands that fix it, typed in a terminal. */
+  fix?: string[];
+};
+
+function sh(cmd: string, args: string[], timeout = 8000): Promise<string | null> {
+  return new Promise((resolve) => execFile(cmd, args, { timeout }, (error, out) => resolve(error ? null : String(out).trim())));
+}
+
+async function keyCheck(name: SecretName, label: string, required: boolean, what: string, source?: 'env' | 'store'): Promise<Check> {
+  const value = envValue(name);
+  if (!value) return { id: name, label, state: required ? 'missing' : 'optional', detail: what, fix: [`diorama setup --key ${name}`] };
+  return { id: name, label, state: 'ok', detail: source === 'store' ? 'in your system keychain' : 'from the environment' };
+}
+
+export async function runChecks(opts: { live?: boolean } = {}): Promise<Check[]> {
+  const sources = await loadSecrets().catch(() => ({}) as Partial<Record<SecretName, 'env' | 'store'>>);
+  const checks: Check[] = [];
+
+  const major = Number(process.versions.node.split('.')[0]);
+  checks.push({ id: 'node', label: 'Node.js', state: major >= 22 ? 'ok' : 'fail', detail: `${process.versions.node}${major >= 22 ? '' : ': diorama needs 22 or newer'}` });
+
+  const vault = await findVault();
+  checks.push(vault
+    ? { id: 'vault', label: 'Key storage', state: 'ok', detail: vault.name }
+    : { id: 'vault', label: 'Key storage', state: 'optional', detail: NO_VAULT });
+
+  const meta = await keyCheck('META_API_KEY', 'Meta Model API key', true, 'Plans, paints and cuts every scene (Muse Spark, Muse Image, SAM 3.1). Needed to build.', sources.META_API_KEY);
+  if (meta.state === 'ok' && opts.live) {
+    const { askSparkJson } = await import('../meta/spark.ts');
+    const { Memo } = await import('../meta/memo.ts');
+    const started = Date.now();
+    try {
+      // A fresh label each time, so the memo cannot answer for the API.
+      await askSparkJson(new Memo(path.resolve('.diorama/setup/memo')), { label: 'setup', prompt: `Reply as JSON {"ok": true}. (${Date.now()})` });
+      meta.detail += ` · live check ${Date.now() - started} ms`;
+    } catch (error) {
+      Object.assign(meta, { state: 'fail', detail: `the API refused it: ${error instanceof Error ? error.message.slice(0, 200) : error}` });
+    }
+  }
+  checks.push(meta);
+
+  const muse = await sh('muse', ['--version']);
+  // Other agents the Studio can switch to (genaicode drives them the same way, with the Studio's MCP tools).
+  const others = await Promise.all([['claude', 'Claude Code'], ['codex', 'Codex']].map(async ([cmd, label]) => ({ label, version: await sh(cmd, ['--version']) })));
+  const found = others.filter((o) => o.version);
+  const install = ['curl -fsSL https://dev.meta.ai/install.sh | bash', 'muse login', 'muse --version'];
+  checks.push(muse
+    ? { id: 'muse', label: 'Muse Code', state: 'ok', detail: muse.split('\n')[0] }
+    : found.length
+      // Another agent stands in, so the Studio works; Muse Code stays the recommended one.
+      ? { id: 'muse', label: 'Muse Code', state: 'optional', detail: `Not found on PATH, so the Studio's agent is ${found[0].label} for now. Muse Code is the default once installed. macOS and Linux; on Windows use WSL2.`, fix: install }
+      : { id: 'muse', label: 'Muse Code', state: 'missing', detail: 'The agent that builds and fixes worlds with you. Not found on PATH. macOS and Linux; on Windows use WSL2.', fix: install });
+  checks.push({
+    id: 'agents',
+    label: 'Other agents',
+    state: found.length ? 'ok' : 'optional',
+    detail: `${others.map((o) => (o.version ? `${o.label} ${/\d+\.\d+[\w.-]*/.exec(o.version)?.[0] ?? 'found'}` : `${o.label} not found`)).join(' · ')}. Switch in the agent panel; DIORAMA_AGENT=muse, claude or codex sets the default.`,
+  });
+
+  let chromium: string | null = null;
+  try {
+    const { chromium: pw } = await import('playwright');
+    chromium = pw.executablePath();
+  } catch {}
+  checks.push(chromium && fs.existsSync(chromium)
+    ? { id: 'chromium', label: 'Chromium for Explore and Playthrough', state: 'ok', detail: 'installed' }
+    : { id: 'chromium', label: 'Chromium for Explore and Playthrough', state: 'missing', detail: 'Explore and the playthrough bot drive a browser.', fix: ['diorama browsers'] });
+
+  checks.push(await keyCheck('ELEVENLABS_API_KEY', 'ElevenLabs key', false, 'Optional: sound effects, music and voices. Without it the world plays silent.', sources.ELEVENLABS_API_KEY));
+  checks.push(await keyCheck('HF_TOKEN', 'Hugging Face token', false, 'Optional: 3D hero figures (TRELLIS, auto-rig) on ZeroGPU Spaces.', sources.HF_TOKEN));
+  checks.push(await keyCheck('DIORAMA_ANTHROPIC_API_KEY', 'Anthropic API key', false, 'Optional: with DIORAMA_SPARK_VIA=claude, Spark\'s text and vision asks go to Claude (Sonnet 5.5) instead of Meta.', sources.DIORAMA_ANTHROPIC_API_KEY));
+  checks.push(await keyCheck('OPENROUTER_API_KEY', 'OpenRouter key', false, 'Optional: DIORAMA_IMAGE_VIA=gemini-flash, gemini-lite, nano-banana or gemini-pro paints with Gemini instead of Muse Image; DIORAMA_SPARK_VIA=openrouter:<model> for any chat model.', sources.OPENROUTER_API_KEY));
+  checks.push(await keyCheck('GEMINI_API_KEY', 'Google Gemini key', false, 'Optional: with DIORAMA_IMAGE_VIA=gemini-flash (or gemini-lite, nano-banana, gemini-pro) Gemini paints straight from Google instead of through OpenRouter.', sources.GEMINI_API_KEY));
+
+  const blender = await blenderState();
+  const blenderWhat = 'With Blender 3.6+ installed, 3D figures get a calm rest pose (arms down, feet under the hips) before they walk, so they stop walking straddle-legged with arms out.';
+  checks.push(blender.installed
+    ? { id: 'blender', label: 'Blender for natural 3D walks', state: blender.enabled ? 'ok' : 'optional',
+      detail: blender.enabled ? `Blender ${blender.installed.version} (${blender.installed.bin}): on. Turn it off in Setup.` : `Blender ${blender.installed.version} found but turned off in Setup.` }
+    : { id: 'blender', label: 'Blender for natural 3D walks', state: 'optional', detail: `Optional: ${blenderWhat} Without it they walk from the pose TRELLIS built. Or set DIORAMA_BLENDER to its path.`,
+      fix: [process.platform === 'darwin' ? 'brew install --cask blender' : 'https://www.blender.org/download/'] });
+
+  const py = process.env.DIORAMA_PYTHON;
+  checks.push(!py
+    ? { id: 'python', label: 'Python for local music', state: 'optional', detail: 'DIORAMA_PYTHON not set: music comes from ElevenLabs. MusicGen needs torch, transformers, scipy.' }
+    : fs.existsSync(py)
+      ? { id: 'python', label: 'Python for local music', state: 'ok', detail: py }
+      : { id: 'python', label: 'Python for local music', state: 'fail', detail: `DIORAMA_PYTHON points at ${py}, which does not exist` });
+
+  const remote = await sh('git', ['remote', 'get-url', 'origin']);
+  checks.push(remote
+    ? { id: 'publish', label: 'Publishing', state: 'ok', detail: `git remote ${remote.replace(/\/\/[^@/]+@/, '//')}; diorama publish pushes to gh-pages` }
+    : { id: 'publish', label: 'Publishing', state: 'optional', detail: 'No git remote: diorama export --zip still makes an itch.io upload.' });
+
+  return checks;
+}

package/app/src/sound/cost.ts

@@ -0,0 +1,74 @@
+// What sound costs. ElevenLabs bills every effect, line and design, so while a world is being developed the
+// pipeline uses final quality unless an author explicitly requests a cheaper draft.
+import os from 'node:os';
+import path from 'node:path';
+import { readSoundQuality } from './settings.ts';
+import { trackOperation } from '../studio/operation.ts';
+
+/** CLI flags > DIORAMA_SOUND > Studio Setup > final. */
+export type Quality = 'draft' | 'final';
+
+export function soundQuality(args: string[] = []): Quality {
+  if (args.includes('--draft') && args.includes('--final')) throw new Error('choose --draft or --final, not both');
+  if (args.includes('--final')) process.env.DIORAMA_SOUND = 'final';
+  if (args.includes('--draft')) process.env.DIORAMA_SOUND = 'draft';
+  const quality = process.env.DIORAMA_SOUND ?? readSoundQuality();
+  if (quality !== 'final' && quality !== 'draft') throw new Error('DIORAMA_SOUND must be final or draft');
+  return quality;
+}
+
+/** Speech: Flash v2.5 costs half of Multilingual v2 per character. */
+export const FINAL_TTS = 'eleven_multilingual_v2';
+export const ttsModel = () => (soundQuality() === 'final' ? FINAL_TTS : 'eleven_flash_v2_5');
+/** Takes per line before it is flagged: each take is paid. */
+export const lineTakes = () => (soundQuality() === 'final' ? 3 : 2);
+
+/**
+ * Sounds, speech and voice designs every world on this machine shares: the same request is never paid twice,
+ * and generic sounds (footsteps per surface, clicks, pick-ups) are reused across worlds in draft quality.
+ */
+export function sharedDir(...parts: string[]): string {
+  return path.join(process.env.DIORAMA_CACHE || path.join(os.homedir(), '.cache', 'diorama'), ...parts);
+}
+
+let outOf: string | undefined;
+
+/** Called with every ElevenLabs refusal: an empty account stops all later calls at once instead of each failing. */
+export function noteRefusal(status: number, body: string) {
+  if (/quota_exceeded|insufficient_credits|credits remaining/i.test(body) || (status === 402)) outOf = 'ElevenLabs has no credits left (top up, or the world stays silent)';
+}
+
+export function checkCredits() {
+  if (outOf) throw new Error(outOf);
+}
+
+export type Credits = { used: number; limit: number; left: number };
+
+/** The account's credits, or undefined when the key cannot read them. */
+export async function credits(key: string | undefined): Promise<Credits | undefined> {
+  if (!key) return undefined;
+  try {
+    return await trackOperation('elevenlabs', 'check subscription', async () => {
+      const res = await fetch('https://api.elevenlabs.io/v1/user/subscription', { headers: { 'xi-api-key': key } });
+      if (!res.ok) return undefined;
+      const s = (await res.json()) as { character_count: number; character_limit: number };
+      return { used: s.character_count, limit: s.character_limit, left: s.character_limit - s.character_count };
+    });
+  } catch {
+    return undefined;
+  }
+}
+
+/** Credits a step may still spend: DIORAMA_ELEVEN_FLOOR (default 2000) are always left for other work. */
+export async function creditGuard(key: string | undefined, step: string): Promise<{ start?: Credits; report: () => Promise<string> }> {
+  const start = await credits(key);
+  const floor = Number(process.env.DIORAMA_ELEVEN_FLOOR ?? 2000);
+  if (start && start.left <= floor) throw new Error(`${step}: ElevenLabs has ${start.left} credits left (floor ${floor}); top up or set DIORAMA_ELEVEN_FLOOR lower. The world plays silent until then.`);
+  return {
+    start,
+    report: async () => {
+      const end = start && (await credits(key));
+      return start && end ? `credits: ${end.used - start.used} used, ${end.left} left (${soundQuality()} quality)` : `${soundQuality()} quality`;
+    },
+  };
+}

package/app/src/sound/data.ts

@@ -0,0 +1,89 @@
+// The play page's SoundData from sound.json (the plan) and sound/sounds.json (what was made).
+import fs from 'node:fs';
+import type { SoundData } from '../play/sound.ts';
+import { Layout } from '../scene/layout.ts';
+import type { WorldFiles } from '../world/files.ts';
+import { SoundManifest } from './make.ts';
+import { SoundPlan, soundIds } from './plan.ts';
+import { jobsOf } from './make.ts';
+import { readSoundFixes } from './fixes.ts';
+import { storyLines, Voices } from './voices.ts';
+import { Story } from '../story/schema.ts';
+import { lipSyncData } from './lipsync.ts';
+
+/**
+ * The spoken lines the last voices run could not make that the story still has. A line rewritten since (story_text)
+ * is history, not a missing recording: story_status unvoiced lists the new wording until voices runs again.
+ */
+export function liveVoiceFailures(files: WorldFiles): Voices['failures'] {
+  const voicesFile = files.path('voices.json');
+  if (!fs.existsSync(voicesFile)) return [];
+  const voices = Voices.safeParse(files.readJson(voicesFile));
+  if (!voices.success) return [];
+  try {
+    const world = files.world();
+    const story = Story.parse(files.readJson(files.path('story.json')));
+    const live = new Set(storyLines(world, story, true).map((l) => JSON.stringify([l.speaker, l.text])));
+    return voices.data.failures.filter((f) => live.has(JSON.stringify([f.speaker, f.text])));
+  } catch {
+    return voices.data.failures;
+  }
+}
+
+export function soundData(files: WorldFiles): SoundData | undefined {
+  const sound = effects(files);
+  const voicesFile = files.path('voices.json');
+  if (!fs.existsSync(voicesFile)) return sound;
+  const recordings = Voices.parse(files.readJson(voicesFile)).lines
+    .filter((l) => !l.flags.some((f) => f.startsWith('heard')) && fs.existsSync(files.path(l.file)));
+  const lines: NonNullable<SoundData['lines']> = {};
+  const bySpeaker: NonNullable<SoundData['bySpeaker']> = {};
+  const sync = lipSyncData(files);
+  for (const l of recordings) {
+    const entry = { file: l.file, seconds: l.seconds, speaker: l.speaker, ...(sync[l.file] && { lipsync: sync[l.file] }) };
+    (bySpeaker[l.speaker] ??= {})[l.text] = entry;
+    if (recordings.filter((r) => r.text === l.text).length === 1) lines[l.text] = entry;
+  }
+  return { ...(sound ?? { files: {}, places: {}, shots: {} }), lines, bySpeaker };
+}
+
+function effects(files: WorldFiles): SoundData | undefined {
+  const planFile = files.path('sound.json');
+  const madeFile = files.path('sound', 'sounds.json');
+  if (!fs.existsSync(planFile) || !fs.existsSync(madeFile)) return undefined;
+  const plan = SoundPlan.parse(files.readJson(planFile));
+  const made = SoundManifest.parse(files.readJson(madeFile)).sounds;
+  const wanted = new Map(jobsOf(plan, readSoundFixes(files)).map((j) => [j.id, j.prompt]));
+  // A take made from an older wording of its prompt still plays until a new one is made: the plan changing (a
+  // sound fix, a re-plan) once left a finished world with no music and no beds at all, silent instead of close.
+  // Only for music, beds and loops (mood under the game); a one-shot or a step made for other words may be the
+  // wrong sound for its moment, so those wait for their new take.
+  const have = (id: string) => (made[id] && (made[id].prompt === wanted.get(id) || (wanted.has(id) && ['music', 'bed', 'loop'].includes(made[id].kind)))
+    && !made[id].flags.some((f) => /does not loop well|still hissy|only \d+ footstep/.test(f))
+    && fs.existsSync(files.path(made[id].file)) ? id : undefined);
+  const data: SoundData = {
+    files: Object.fromEntries(Object.entries(made).filter(([id]) => have(id)).map(([id, s]) => [id, { file: s.file, kind: s.kind, level: s.level, ...(s.slices && { slices: s.slices }) }])),
+    music: have(soundIds.music()),
+    places: {},
+    shots: {},
+  };
+  for (const place of files.world().places) {
+    const p = plan.places[place.key];
+    if (!p) continue;
+    const layoutFile = files.place(place.key, 'layout.json');
+    const floors = fs.existsSync(layoutFile) ? Layout.parse(files.readJson(layoutFile)).floors : [];
+    data.places[place.key] = {
+      bed: have(soundIds.bed(place.key)),
+      loops: p.loops.flatMap((l) => {
+        const id = have(soundIds.loop(place.key, l.thing));
+        return id ? [{ id, thing: l.thing, level: l.level, when: l.when }] : [];
+      }),
+      zones: floors.map((f) => (p.floors[f.key] ? have(soundIds.step(p.floors[f.key])) ?? null : null)),
+    };
+  }
+  for (const shot of plan.shots) {
+    const id = have(soundIds.shot(shot.on));
+    if (id) data.shots[shot.on] = id;
+  }
+  return data;
+}

package/app/src/sound/fixes.ts

@@ -0,0 +1,23 @@
+// The person's corrections to single sounds (sound_redo): a hint added to Spark's prompt, or simply another take.
+// Kept in sound/fixes.json, so a rebuild or a new sound plan keeps them; the plan itself stays Spark's.
+import fs from 'node:fs';
+import { z } from 'zod';
+import type { WorldFiles } from '../world/files.ts';
+
+export const SoundFixes = z.record(z.string(), z.object({
+  /** Words added to the sound's prompt ("louder, a clear clock tick"). */
+  hint: z.string().optional(),
+  /** Takes asked for again: each new round asks ElevenLabs for new takes instead of the cached ones. */
+  take: z.number().int().min(0).optional(),
+}));
+export type SoundFixes = z.infer<typeof SoundFixes>;
+
+export const soundFixesFile = (files: WorldFiles) => files.path('sound', 'fixes.json');
+
+export function readSoundFixes(files: WorldFiles): SoundFixes {
+  try {
+    return SoundFixes.parse(JSON.parse(fs.readFileSync(soundFixesFile(files), 'utf8')));
+  } catch {
+    return {};
+  }
+}

package/app/src/sound/issues.ts

@@ -0,0 +1,21 @@
+/** Shared with Studio; old manifests are classified too, without regenerating any audio. */
+export type AudioMisses = {
+  sounds: Array<{ id: string; reason: string }>;
+  lines: Array<{ speaker: string; text: string; reason: string }>;
+};
+type Kind = 'validation' | 'design' | 'rejected' | 'unavailable';
+export type AudioIssue = { kind: Kind; speaker?: string; id?: string; count: number; reason: string };
+
+export function audioIssues(misses: AudioMisses): AudioIssue[] {
+  const groups = new Map<string, AudioIssue>();
+  for (const line of misses.lines) {
+    const kind: Kind = /validation pending|transcribe .*?(backlog|busy|overload|socket|timed out|closed)|transcription unavailable/i.test(line.reason) ? 'validation'
+      : /waiting for a designed voice|Voice Design pending/i.test(line.reason) ? 'design'
+      : /speech rejected/i.test(line.reason) ? 'rejected' : 'unavailable';
+    const key = `${kind}:${line.speaker}`;
+    const group = groups.get(key);
+    if (group) group.count++;
+    else groups.set(key, { kind, speaker: line.speaker, count: 1, reason: line.reason });
+  }
+  return [...groups.values(), ...misses.sounds.map((s): AudioIssue => ({ kind: /rejected/i.test(s.reason) ? 'rejected' : 'unavailable', id: s.id, count: 1, reason: s.reason }))];
+}

package/app/src/sound/language.ts

@@ -0,0 +1,11 @@
+/** World language names and voice-library language labels use different formats. */
+export function languageKey(language = 'English'): string {
+  const name = language.trim().toLowerCase();
+  const names: Record<string, string> = {
+    polish: 'pl', polski: 'pl', english: 'en', german: 'de', deutsch: 'de', french: 'fr', français: 'fr',
+    spanish: 'es', español: 'es', italian: 'it', portuguese: 'pt', dutch: 'nl', ukrainian: 'uk',
+    russian: 'ru', czech: 'cs', slovak: 'sk', swedish: 'sv', norwegian: 'no', danish: 'da', finnish: 'fi',
+    japanese: 'ja', korean: 'ko', chinese: 'zh', arabic: 'ar', hindi: 'hi', turkish: 'tr',
+  };
+  return names[name] ?? (/^[a-z]{2}(-[a-z]{2})?$/.test(name) ? name.slice(0, 2) : name);
+}

package/app/src/sound/lipsync-make.ts

@@ -0,0 +1,98 @@
+// Offline, cached analysis of the final recordings. No TTS request and no changes to paid audio.
+import fs from 'node:fs';
+import os from 'node:os';
+import path from 'node:path';
+import { execFile } from 'node:child_process';
+import { promisify } from 'node:util';
+import { z } from 'zod';
+import type { WorldFiles } from '../world/files.ts';
+import { sha256 } from '../scene/faces-data.ts';
+import { MOUTHS, type LipSync } from '../play/lipsync.ts';
+import { decode, envelope, RATE } from './pcm.ts';
+import { lipSyncData } from './lipsync.ts';
+
+const exec = promisify(execFile);
+const VERSION = '1.14.0';
+const Cue = z.object({ start: z.number().nonnegative(), end: z.number().positive(), value: z.enum(MOUTHS as ['X', 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H']) }).refine(c => c.end > c.start);
+export type SyncEntry = LipSync & { sha256: string };
+export type SyncManifest = { version: 1; recognizer: string; byFile: Record<string, SyncEntry>; failures?: Array<{ speaker: string; text: string; reason: string }> };
+
+/** A pinned official release, kept locally. An explicit executable is useful on managed machines. */
+export async function rhubarbExecutable(explicit = process.env.DIORAMA_RHUBARB): Promise<string> {
+  if (explicit) { await exec(explicit, ['--version']); return explicit; }
+  try { await exec('rhubarb', ['--version']); return 'rhubarb'; } catch { /* Prepare the local dependency. */ }
+  const platform = process.platform === 'darwin' ? 'macOS' : process.platform === 'linux' ? 'Linux' : process.platform === 'win32' ? 'Windows' : undefined;
+  if (!platform) throw new Error('Install Rhubarb Lip Sync and set DIORAMA_RHUBARB to its executable');
+  const name = `Rhubarb-Lip-Sync-${VERSION}-${platform}`;
+  const root = process.env.DIORAMA_TOOLS_DIR ?? path.join(process.env.DIORAMA_CACHE ?? path.join(os.homedir(), '.cache', 'diorama'), 'tools');
+  const executable = path.join(root, name, `rhubarb${process.platform === 'win32' ? '.exe' : ''}`);
+  if (fs.existsSync(executable)) { await exec(executable, ['--version']); return executable; }
+  fs.mkdirSync(root, { recursive: true });
+  const temporary = fs.mkdtempSync(path.join(root, '.rhubarb-'));
+  try {
+    console.log(`faces: preparing Rhubarb ${VERSION} (${platform})`);
+    const response = await fetch(`https://github.com/DanielSWolf/rhubarb-lip-sync/releases/download/v${VERSION}/${name}.zip`, { signal: AbortSignal.timeout(120_000) });
+    if (!response.ok) throw new Error(`Rhubarb download: HTTP ${response.status}`);
+    const archive = path.join(temporary, 'rhubarb.zip');
+    fs.writeFileSync(archive, Buffer.from(await response.arrayBuffer()));
+    if (process.platform === 'win32') await exec('tar', ['-xf', archive, '-C', temporary]);
+    else await exec('unzip', ['-q', archive, '-d', temporary]);
+    const unpacked = path.join(temporary, name);
+    const binary = path.join(unpacked, path.basename(executable));
+    if (process.platform !== 'win32') fs.chmodSync(binary, 0o755);
+    await exec(binary, ['--version']);
+    // A concurrent run may have installed the same release while this one downloaded it.
+    if (!fs.existsSync(executable)) fs.renameSync(unpacked, path.dirname(executable));
+    return executable;
+  } finally { fs.rmSync(temporary, { recursive: true, force: true }); }
+}
+
+export async function analyseRecording(audio: string, executable: string, language?: string): Promise<LipSync> {
+  const temporary = fs.mkdtempSync(path.join(os.tmpdir(), 'diorama-lipsync-'));
+  try {
+    const wav = path.join(temporary, 'line.wav'), json = path.join(temporary, 'cues.json');
+    await exec('ffmpeg', ['-v', 'error', '-y', '-i', audio, '-ac', '1', '-ar', '16000', wav]);
+    await exec(executable, ['-r', /^(en|english)\b/i.test(language ?? '') ? 'pocketSphinx' : 'phonetic', '-f', 'json', '--quiet', '-o', json, wav], { timeout: 120_000 });
+    const cues = z.array(Cue).parse(JSON.parse(fs.readFileSync(json, 'utf8')).mouthCues);
+    if (!cues.length || cues.some((c, i) => i > 0 && c.start < cues[i - 1].end)) throw new Error('Rhubarb returned an empty or overlapping timeline');
+    const pcm = decode(fs.readFileSync(audio));
+    if (cues.at(-1)!.end > pcm.left.length / RATE + 0.05) throw new Error('Rhubarb timeline exceeds the recording');
+    const rms = envelope(pcm, 40);
+    const sorted = [...rms].sort((a, b) => a - b), scale = sorted[Math.floor(sorted.length * 0.95)] || 1;
+    return { cues, step: 0.04, envelope: rms.map(v => Math.round(Math.min(1, v / scale) * 100) / 100) };
+  } finally { fs.rmSync(temporary, { recursive: true, force: true }); }
+}
+
+export async function prepareLipSync(files: WorldFiles, only: string[] = [], explicit?: string): Promise<SyncManifest> {
+  const recordings = files.readJson(files.path('voices.json')) as { lines: Array<{ speaker: string; text: string; file: string }> };
+  const previous = lipSyncData(files);
+  const byFile: SyncManifest['byFile'] = {};
+  const failures: NonNullable<SyncManifest['failures']> = [];
+  let executable: Promise<string> | undefined;
+  const language = files.world().language;
+  for (const line of recordings.lines) {
+    if (line.speaker === 'narrator' || !fs.existsSync(files.path(line.file))) continue;
+    const sha = sha256(fs.readFileSync(files.path(line.file)));
+    if (previous[line.file]) { byFile[line.file] = { ...previous[line.file], sha256: sha }; continue; }
+    if (only.length && !only.includes(line.speaker)) continue;
+    try {
+      executable ??= rhubarbExecutable(explicit);
+      byFile[line.file] = { ...await analyseRecording(files.path(line.file), await executable, language), sha256: sha };
+      console.log(`${line.speaker}: lip-sync ${byFile[line.file].cues.length} cues — ${line.text.slice(0, 70)}`);
+    } catch (e) { failures.push({ speaker: line.speaker, text: line.text, reason: String(e instanceof Error ? e.message : e) }); }
+  }
+  const manifest: SyncManifest = { version: 1, recognizer: `rhubarb-${VERSION}`, byFile, failures };
+  const target = files.path('sound', 'lipsync.json');
+  files.write(`${target}.tmp`, manifest);
+  fs.renameSync(`${target}.tmp`, target);
+  return manifest;
+}
+
+/** Existing speaking characters keep their timing when a voice is retried; preparing art is a separate operation. */
+export async function refreshExistingLipSync(files: WorldFiles, only: string[] = []) {
+  if (!fs.existsSync(files.path('sound', 'lipsync.json'))) return;
+  try {
+    const result = await prepareLipSync(files, only);
+    if (result.failures?.length) console.log(`warning: ${result.failures.length} lines need lip-sync preparation; recordings are retained`);
+  } catch (e) { console.log(`warning: lip-sync preparation failed; recordings are retained: ${e instanceof Error ? e.message : e}`); }
+}