@gtanczyk/diorama 0.0.0-stage → 0.2.0

package/app/src/commands/voices.ts

@@ -0,0 +1,145 @@
+import { refreshExistingLipSync } from '../sound/lipsync-make.ts';
+// diorama voices <slug>: every line of the story spoken. Spark casts the voices (kept in voices.json; voice_set in the Studio recasts one),
+// ElevenLabs reads, Muse Voice Transcribe checks each take says its line.
+import fs from 'node:fs';
+import { inputHash } from '../meta/memo.ts';
+import { defineCommand, emit } from '../registry.ts';
+import { creditGuard, soundQuality } from '../sound/cost.ts';
+import { elevenKey } from '../sound/sources.ts';
+import { castVoices, DESIGNED, PENDING_DESIGN, freeVoiceSlot, voiceCandidates, DesignedVoices, designDescriptions, designVoice, needsDesign, stockVoices, storyLines, voiceLine, Voices, VOICE_PIPELINE, type Cast } from '../sound/voices.ts';
+import { fixedLooks, readCastFixes } from '../scene/cast-fixes.ts';
+import { Story } from '../story/schema.ts';
+import { WorldFiles } from '../world/files.ts';
+import { resumePendingVoices } from '../sound/voices.ts';
+
+defineCommand({
+  name: 'voices',
+  usage: 'voices <slug>            cast a voice per character (stock, or designed by ElevenLabs Voice Design for children, teens and the old) and speak every story line, checked by Muse Voice Transcribe; conversations, intro and endings [--all: every line; always for a world for kids, whose players cannot read] [--recast] [--draft: Flash and 2 takes; default final uses Multilingual v2 and up to 3 takes]',
+  async run(args) {
+    // Keep this generation at one quality even if Setup changes while it runs.
+    process.env.DIORAMA_SOUND = soundQuality(args);
+    const recast = args.includes('--recast');
+    const [slug] = args.filter((a) => !a.startsWith('--'));
+    if (!slug) throw new Error('usage: diorama voices <slug>');
+    const files = new WorldFiles(slug);
+    const world = files.world();
+    if (!fs.existsSync(files.path('story.json'))) return void console.log('no story.json: nothing to say');
+    if (!elevenKey()) return void console.log('no ELEVENLABS_API_KEY: lines stay unspoken');
+    const story = Story.parse(files.readJson(files.path('story.json')));
+    const guard = await creditGuard(elevenKey(), 'voices');
+    const lines = storyLines(world, story, args.includes('--all') || world.audience === 'kids');
+    const outFile = files.path('voices.json');
+    const old = fs.existsSync(outFile) ? Voices.parse(files.readJson(outFile)) : undefined;
+    let cast: Cast;
+    // A cast is kept only when it was made from profiles (who Spark saw in each figure) and every child, teen and old
+    // speaker has a designed voice; older casts are redone.
+    const fixes = readCastFixes(files);
+    const figure = (key: string) => (fs.existsSync(files.path('cast', `${key}.png`)) ? fs.readFileSync(files.path('cast', `${key}.png`)) : undefined);
+    const people = [world.hero, ...world.places.flatMap((p) => p.things).filter((t) => t.role === 'npc')];
+    const speakers = people.map((p) => ({ ...fixedLooks(files, { key: p.key, looks: p.looks }), figure: figure(p.key) }));
+    const castingFrom = inputHash(JSON.stringify(world), JSON.stringify(lines), ...speakers.flatMap((p) => [p.key, p.looks ?? '', p.figure ?? Buffer.alloc(0)]));
+    const current = (k: string) => { const c = old?.cast[k]; return old?.pipeline === VOICE_PIPELINE && old.castingFrom === castingFrom && c?.profile?.language === (world.language ?? 'English') && (!needsDesign(c.profile) || !!c.design || c.voice === PENDING_DESIGN || !!fixes[k]?.voice); };
+    if (old && !recast && lines.every((l) => current(l.speaker))) cast = await resumePendingVoices(files.memo, old.cast, lines);
+    else {
+      cast = await castVoices(files.memo, world, lines, await stockVoices(), speakers);
+    }
+    // The voices the person picked (cast/fixes.json) replace Spark's casting.
+    for (const [key, fix] of Object.entries(fixes)) {
+      if (!fix.voice || !cast[key]) continue;
+      const { design: wish, ...voice } = fix.voice;
+      const entry = { ...cast[key], ...voice };
+      if (voice.voice === DESIGNED) {
+        // A designed voice: from the person's description, or from the profile; designed again only when that changes.
+        const description = wish ?? (cast[key].profile ? designDescriptions(cast[key].profile!)[0] : 'A clear warm voice.');
+        const says = lines.filter((l) => l.speaker === key).map((l) => l.text);
+        try {
+          entry.design = cast[key].design?.request === description ? cast[key].design : await designVoice(files.memo, cast[key].profile ?? { gender: 'any', age: 'middle_aged', sound: '', language: world.language ?? 'English' }, says, description);
+          entry.pitch = 0;
+        } catch (error) {
+          // The voice Spark cast stays: the lines are still read. Studio shows the note on the character's card.
+          console.log(`  ${key}: the designed voice could not be made (${error instanceof Error ? error.message : error}); the cast voice stays`);
+          cast = { ...cast, [key]: { ...cast[key], warnings: [...(cast[key].warnings ?? []), 'The designed voice could not be made this time; the cast voice reads the lines. Try the design again later.'] } };
+          continue;
+        }
+      } else delete entry.design;
+      cast = { ...cast, [key]: entry };
+    }
+    for (const [who, v] of Object.entries(cast)) console.log(`  ${who.padEnd(16)} ${v.profile ? `${v.profile.age} ${v.profile.gender}`.padEnd(18) : ''} ${v.design ? 'designed' : v.voice}${v.pitch ? ` +${v.pitch} st` : ''}  ${v.why}`);
+
+    // Lines no longer in the story go; the rest are rewritten (from the cache when nothing about them changed).
+    // Keep existing files until replacements pass validation; a failed run must not delete paid recordings.
+    // No free voice slot on the account: a designed voice cannot be created, so the nearest stock voice stands in.
+    if (Object.values(cast).some((c) => c.design) && !(await freeVoiceSlot())) {
+      const stock = await stockVoices();
+      for (const [k, c] of Object.entries(cast)) {
+        if (!c.design || !c.profile) continue;
+        const v = voiceCandidates(stock, c.profile)[0] ?? stock[0];
+        if (!v) throw new Error(`no stock voice available for ${k}`);
+        console.log(`  ${k}: no free voice slot on the ElevenLabs account for a designed voice; ${v.name} stands in`);
+        cast = { ...cast, [k]: { ...c, voice: v.voice_id, design: undefined, pitch: 0, warnings: ['No free Voice Design slot; a stock actor stands in. Audition this voice.'] } };
+      }
+    }
+    const done: Voices['lines'] = new Array(lines.length);
+    const failures: Voices['failures'] = [];
+    let completed = 0;
+    const progress = () => emit('work.progress', { kind: 'voices', completed, total: lines.length, failed: failures.length });
+    progress();
+    // Designed voices live on the account only while their character's lines are read, one character at a time,
+    // so a world needs a single free voice slot.
+    const designed = new DesignedVoices(`diorama ${slug}`);
+    const index = lines.map((_, i) => i);
+    const groups = [index.filter((i) => !cast[lines[i].speaker].design), ...Object.keys(cast).filter((k) => cast[k].design).map((k) => index.filter((i) => lines[i].speaker === k))];
+    try {
+      for (const group of groups) {
+        let next = 0;
+        await Promise.all(Array.from({ length: 4 }, async () => {
+          while (next < group.length) await read(group[next++]);
+        }));
+        // A line the transcriber dropped (its stream failed, not the take) is listened to again once, after the
+        // group: the take is paid and cached, so this asks nothing of ElevenLabs. 42 lines of one world failed so.
+        const dropped = group.filter((i) => failures.some((f) => f.speaker === lines[i].speaker && f.text === lines[i].text && /^transcribe /.test(f.reason)));
+        if (dropped.length) console.log(`  listening again to ${dropped.length} line(s) the transcriber dropped`);
+        for (const i of dropped) {
+          failures.splice(failures.findIndex((f) => f.speaker === lines[i].speaker && f.text === lines[i].text), 1);
+          completed--;
+          await read(i);
+        }
+        await designed.close();
+      }
+    } finally {
+      await designed.close();
+    }
+    async function read(i: number) {
+      // A child, teen or old speaker whose designed voice is not made yet waits: no stock adult reads for them, and
+      // no request goes out with a voice id that does not exist.
+      const voice = cast[lines[i].speaker];
+      if (voice.voice === PENDING_DESIGN) {
+        failures.push({ speaker: lines[i].speaker, text: lines[i].text, reason: `waiting for a designed voice: ${voice.pending ?? 'Voice Design pending'}` });
+        completed++;
+        progress();
+        return;
+      }
+      try {
+        done[i] = await voiceLine(files.memo, lines[i], cast[lines[i].speaker], (id, mp3) => {
+          files.write(files.path('sound', 'voice', `${id}.mp3`), mp3);
+          return `sound/voice/${id}.mp3`;
+        }, designed, { retryRejected: old?.failures.some((f) => f.speaker === lines[i].speaker && f.text === lines[i].text && f.reason.startsWith('speech rejected')) });
+        const l = done[i];
+        console.log(`  ${!l.flags.some((f) => f.startsWith('heard')) ? 'ok  ' : 'FLAG'} ${l.speaker.padEnd(14)} ${l.seconds.toFixed(1).padStart(5)} s  ${l.text.slice(0, 60)}${l.flags.length ? `  (${l.flags.join('; ')}; heard "${l.heard.slice(0, 60)}")` : ''}`);
+      } catch (error) {
+        failures.push({ speaker: lines[i].speaker, text: lines[i].text, reason: error instanceof Error ? error.message : String(error) });
+        console.log(`  FAIL ${lines[i].speaker}: ${lines[i].text.slice(0, 50)}: ${error instanceof Error ? error.message : error}`);
+      } finally {
+        completed++;
+        progress();
+      }
+    }
+    const spoken = done.filter(Boolean);
+    files.write(outFile, { pipeline: VOICE_PIPELINE, castingFrom, cast, lines: spoken, failures });
+    await refreshExistingLipSync(files);
+    const s = files.memo.sent;
+    console.log(`voices ${slug}: ${spoken.length}/${lines.length} lines, ${spoken.filter((l) => l.flags.some((f) => f.startsWith('heard'))).length} flagged · sent: spark ${s.spark}, voice ${s.voice} · ${await guard.report()} · ${outFile}`);
+    // A line without a voice still shows as text: the build goes on and Studio shows it with a retry.
+    if (failures.length) console.log(`warning: ${failures.length} spoken lines stay text only (rejected audio is not used); see voices.json failures`);
+  },
+});

package/app/src/commands/weather.ts

@@ -0,0 +1,52 @@
+// diorama weather <slug> [place...]: weather.json per place, what falls there and where water drips, for the
+// page to animate (rain or snow over the frame, rings on water and puddles, splashes, drips off the eaves).
+import fs from 'node:fs';
+import sharp from 'sharp';
+import { maskFromPng, maskToPng } from '../image/mask.ts';
+import { Geometry } from '../play/geometry.ts';
+import { decodeDepth } from '../play/shadow.ts';
+import { WeatherWish, type Weather } from '../play/weather.ts';
+import { defineCommand } from '../registry.ts';
+import { dryOf, readWeather } from '../scene/weather.ts';
+import { WorldFiles } from '../world/files.ts';
+
+defineCommand({
+  name: 'weather',
+  usage: 'weather <slug> [place...]  read what falls on each place (rain, snow) and which edges drip, for the page to animate',
+  async run(args) {
+    const [slug, ...only] = args;
+    if (!slug) throw new Error('usage: diorama weather <slug> [place...]');
+    const files = new WorldFiles(slug);
+    const world = files.world();
+    const places = world.places.filter((p) => only.length === 0 || only.includes(p.key));
+    await Promise.all(places.map(async (place) => {
+      const at = (...parts: string[]) => files.place(place.key, ...parts);
+      fs.rmSync(at('weather.json'), { force: true });
+      fs.rmSync(at('weather'), { recursive: true, force: true });
+      if (!fs.existsSync(at('geometry.json')) || !fs.existsSync(at('painting.png'))) {
+        console.log(`  ${place.key}: no geometry.json or painting.png, no weather`);
+        return;
+      }
+      const g = Geometry.parse(files.readJson(at('geometry.json')));
+      const painting = fs.readFileSync(at('painting.png'));
+      let depth;
+      if (g.depth && fs.existsSync(at(g.depth))) {
+        const { data, info } = await sharp(fs.readFileSync(at(g.depth))).ensureAlpha().raw().toBuffer({ resolveWithObject: true });
+        depth = decodeDepth(data, info.width, info.height);
+      }
+      const maskFiles = [
+        ...(fs.existsSync(at('masks', 'floors')) ? fs.readdirSync(at('masks', 'floors')).filter((f) => f.endsWith('.png')).map((f) => at('masks', 'floors', f)) : []),
+        ...[...new Set(g.occluders.map((o) => o.mask).filter((m): m is string => !!m))].map((m) => at(m)),
+      ].filter((f) => fs.existsSync(f));
+      const dry = dryOf(g.frame.width, g.frame.height, await Promise.all(maskFiles.map(async (f) => maskFromPng(fs.readFileSync(f)))));
+      const wish = WeatherWish.parse(fs.existsSync(at('weather-wish.json')) ? files.readJson(at('weather-wish.json')) : {});
+      const { weather, ripples } = await readWeather(files.memo, place, painting, g, depth, dry, wish);
+      if (ripples) files.write(at('weather', 'ripples.png'), await maskToPng(ripples));
+      const out: Weather = { ...weather, ...(ripples && { ripples: 'weather/ripples.png' }) };
+      files.write(at('weather.json'), out);
+      const drips = weather.drips.map((d) => `${d.phrase} (${d.from.length})`).join(', ');
+      console.log(`${place.key}: ${weather.fall === 'none' ? 'nothing falls' : `${weather.fall} ${weather.strength}, wind ${weather.wind}`}${weather.fog ? `, fog ${weather.fog}` : ''}${ripples ? ', rings on water and puddles' : ''}${drips ? `, drips from ${drips}` : ''}`);
+    }));
+    console.log(`calls sent: spark ${files.memo.sent.spark}, sam ${files.memo.sent.sam}`);
+  },
+});

package/app/src/commands/world.ts

@@ -0,0 +1,104 @@
+// diorama world <slug> "<idea>": the first step of every build.
+import fs from 'node:fs';
+import { defineCommand, takeOption } from '../registry.ts';
+import { describePerson, describePhoto, framePhoto, normalisePhoto } from '../scene/photo.ts';
+import { designWorld, type Extra } from '../world/design.ts';
+import { drawingFiles, drawingsBrief, drawnIssues, fitPlaces, keepDrawings, KIDS_STYLE, oneHero, readDrawings, readKinds } from '../world/drawings.ts';
+import { WorldFiles } from '../world/files.ts';
+import { hearAll, Idea, ideaFromTold, keepTold } from '../world/told.ts';
+import { CreativeChoices } from '../world/idea-review.ts';
+
+defineCommand({
+  name: 'world',
+  usage: 'world <slug> "<idea>"   design a world (world.json) with Muse Spark [--places N] [--photo room.jpg: a real room is the first place] [--hero-photo me.jpg: a real person is the hero] '
+    + '[--drawings DIR: photos of children\'s drawings become the hero, NPCs, items and places] [--told DIR: recordings of them telling the story; without an idea, the idea is written from them] [--kids: a game for 5-7 year olds; on with drawings or recordings] [--language Polish: the game\'s text and voices; with recordings, the language spoken]',
+  async run(args) {
+    const places = Number(takeOption(args, 'places', '4'));
+    const photoFile = takeOption(args, 'photo', '');
+    const heroFile = takeOption(args, 'hero-photo', '');
+    const drawingsFrom = takeOption(args, 'drawings', '');
+    const toldFrom = takeOption(args, 'told', '');
+    let language: string | undefined = takeOption(args, 'language', '') || undefined;
+    const kidsFlag = args.indexOf('--kids');
+    if (kidsFlag >= 0) args.splice(kidsFlag, 1);
+    const [slug, given] = args;
+    if (!slug) throw new Error('usage: diorama world <slug> "<idea>" [--places N]');
+    const files = new WorldFiles(slug);
+    await designInitialWorld(files, { places, photoFile, heroFile, drawingsFrom, toldFrom, language, kidsFlag: kidsFlag >= 0, given });
+  },
+});
+
+/** The same first stage is used by the optional preview, without starting later build steps. */
+export async function designInitialWorld(files: WorldFiles, { places, photoFile = '', heroFile = '', drawingsFrom = '', toldFrom = '', language, kidsFlag = false, given }: { places: number; photoFile?: string; heroFile?: string; drawingsFrom?: string; toldFrom?: string; language?: string; kidsFlag?: boolean; given?: string }) {
+  const briefFile = files.path('creative-brief.json');
+  const choices = fs.existsSync(briefFile) ? CreativeChoices.parse((files.readJson(briefFile) as { choices: unknown }).choices) : undefined;
+  const chosenStyle = choices?.style;
+  // Drawings and recordings already kept with the world (the Studio uploads them there) count too.
+  if (drawingsFrom) console.log(`drawings kept: ${(await keepDrawings(files, drawingsFrom)).join(', ')}`);
+  if (toldFrom) console.log(`recordings kept: ${keepTold(files, toldFrom).join(', ')}`);
+  const said = await hearAll(files);
+  for (const s of said) console.log(`heard ${s.clip}: ${s.text}`);
+  // A drawn place over the places asked for goes into a place as a prop: places are what makes a build long.
+  const drawn = fitPlaces(await readDrawings(files.memo, files), places);
+  for (const d of drawn) console.log(`drawing ${d.file}: ${d.kind} ${d.key}, ${d.looks}`);
+  const kids = kidsFlag || drawn.length > 0 || said.length > 0;
+
+  let idea = given;
+  // The story says who leads, unless the grown-up already picked a drawing as the hero.
+  const heroOf = (told: Idea) => {
+    if (told.hero && !drawn.some((d) => readKinds(files)[d.file] === 'hero')) oneHero(drawn, told.hero);
+  };
+  if (idea) {
+    // An idea written from what the children said (the Studio's kids_idea), maybe edited since: its language and hero stand.
+    const told = Idea.safeParse(fs.existsSync(files.path('told', 'idea.json')) ? JSON.parse(fs.readFileSync(files.path('told', 'idea.json'), 'utf8')) : null);
+    if (told.success) (language ??= told.data.language, heroOf(told.data));
+  } else if (said.length || drawn.length) {
+    // What the children said and drew is the idea.
+    const told = await ideaFromTold(files.memo, said.map((s) => s.text), drawn);
+    idea = told.idea;
+    language ??= told.language;
+    heroOf(told);
+    files.write(files.path('told', 'idea.json'), told);
+    console.log(`idea from what they said (${told.language}): ${told.title}. ${told.idea}`);
+  }
+  if (!idea) throw new Error('usage: diorama world <slug> "<idea>" (or --told DIR with recordings of the story)');
+  files.write(files.path('idea.txt'), `${idea}\n`);
+  let room;
+  if (photoFile) {
+    const { image, crop } = await framePhoto(fs.readFileSync(photoFile));
+    room = await describePhoto(files.memo, image, idea);
+    files.write(files.place(room.key, 'photo.png'), image);
+    files.write(files.place(room.key, 'photo.json'), crop);
+    console.log(`photo: ${room.key}, ${room.things.map((t) => t.key).join(', ')} · crop ${JSON.stringify(crop.crop)}`);
+  }
+  let hero;
+  if (heroFile) {
+    const photo = await normalisePhoto(fs.readFileSync(heroFile));
+    hero = await describePerson(files.memo, photo);
+    files.write(files.path('cast', `${hero.key}-photo.png`), photo);
+    console.log(`hero from photo: ${hero.key}, ${hero.looks}`);
+  }
+  const other = language && !/^english$/i.test(language) ? language : undefined;
+  // A clarification can choose a different lead. Do not let the default first drawing override it.
+  const characters = drawn.filter(d => d.kind === 'hero' || d.kind === 'npc').map(d => d.key);
+  const chooseHero = !!(choices?.answers.length || choices?.note) && characters.length > 1 && !drawn.some(d => readKinds(files)[d.file] === 'hero');
+  const asks = [
+    drawn.length ? drawingsBrief(drawn, chosenStyle ?? KIDS_STYLE, chooseHero) : '',
+    chosenStyle ? `The creator explicitly chose this visual style: ${chosenStyle}. Preserve recognizable shapes and identities from all source drawings.` : '',
+    other ? `Write the title and pitch in ${other}, the language of the players; everything else (style, shows, mood, looks, keys) stays in English for the painters.` : '',
+    // The game shows a name over each speaker: in the players' language, as they call them.
+    other || kids ? `Add "names": {"<key>": "name"} for the hero and every npc: a short name ${other ? `in ${other} ` : ''}the players would call them by (the name the children gave them, if they gave one; else a short friendly name a child would give).${drawn.length ? ` Name every drawn item and place too, as a child would say it${other ? ` in ${other}` : ''} ("the blue hat", "the mushroom forest").` : ''}` : '',
+  ].filter(Boolean);
+  const extra: Extra | undefined = asks.length ? { brief: asks.join('\n\n'), issues: (w) => {
+    if (!chooseHero) return drawnIssues(w, drawn);
+    if (!characters.includes(w.hero.key)) return [`Choose the hero from the drawn characters: ${characters.join(', ')}`];
+    return drawnIssues(w, oneHero(drawn.map(d => ({ ...d })), w.hero.key));
+  } } : undefined;
+  const designed = await designWorld(files.memo, hero ? `${idea}\nThe hero is ${hero.looks}` : idea, places, room, extra);
+  if (chooseHero) oneHero(drawn, designed.hero.key);
+  const world = { ...designed, ...(drawn.length && { style: KIDS_STYLE }), ...(chosenStyle && { style: chosenStyle }), ...(hero && { hero }), ...(other && { language: other }), ...(kids && { audience: 'kids' as const }) };
+  files.write(files.path('world.json'), world);
+  if (drawingFiles(files).length) files.write(files.path('drawings.json'), drawn);
+  console.log(`${world.title}: ${world.places.map((p) => p.key).join(' · ')}`);
+  console.log(`spark calls sent: ${files.memo.sent.spark}`);
+}