@gtanczyk/diorama 0.0.0-stage → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +202 -0
- package/README.md +157 -2
- package/app/bin/diorama.mjs +13 -0
- package/app/src/actions/budget.ts +152 -0
- package/app/src/actions/build.ts +158 -0
- package/app/src/actions/cast.ts +248 -0
- package/app/src/actions/depth.ts +95 -0
- package/app/src/actions/extend.ts +36 -0
- package/app/src/actions/faces.ts +160 -0
- package/app/src/actions/feedback.ts +87 -0
- package/app/src/actions/fix.ts +382 -0
- package/app/src/actions/history.ts +93 -0
- package/app/src/actions/idea-preview.ts +65 -0
- package/app/src/actions/idea-review.ts +102 -0
- package/app/src/actions/idea-sketch.ts +172 -0
- package/app/src/actions/index.ts +31 -0
- package/app/src/actions/items.ts +94 -0
- package/app/src/actions/kids.ts +215 -0
- package/app/src/actions/models.ts +64 -0
- package/app/src/actions/motion.ts +119 -0
- package/app/src/actions/pictures.ts +49 -0
- package/app/src/actions/reactions.ts +36 -0
- package/app/src/actions/registry.ts +101 -0
- package/app/src/actions/share.ts +32 -0
- package/app/src/actions/sound.ts +73 -0
- package/app/src/actions/start.ts +154 -0
- package/app/src/actions/step-work.ts +150 -0
- package/app/src/actions/story.ts +101 -0
- package/app/src/actions/studio.ts +133 -0
- package/app/src/actions/style.ts +72 -0
- package/app/src/actions/video.ts +46 -0
- package/app/src/actions/view.ts +109 -0
- package/app/src/actions/weather.ts +38 -0
- package/app/src/actions/world.ts +289 -0
- package/app/src/commands/action.ts +39 -0
- package/app/src/commands/backdrop.ts +188 -0
- package/app/src/commands/browsers.ts +21 -0
- package/app/src/commands/build.ts +503 -0
- package/app/src/commands/cast.ts +142 -0
- package/app/src/commands/clip.ts +431 -0
- package/app/src/commands/derive.ts +167 -0
- package/app/src/commands/doctor.ts +74 -0
- package/app/src/commands/explore.ts +123 -0
- package/app/src/commands/export.ts +142 -0
- package/app/src/commands/extend.ts +75 -0
- package/app/src/commands/faces.ts +80 -0
- package/app/src/commands/icons.ts +69 -0
- package/app/src/commands/index.ts +36 -0
- package/app/src/commands/layout.ts +31 -0
- package/app/src/commands/masks.ts +117 -0
- package/app/src/commands/mcp.ts +52 -0
- package/app/src/commands/model.ts +284 -0
- package/app/src/commands/paint.ts +182 -0
- package/app/src/commands/plates.ts +97 -0
- package/app/src/commands/play.ts +192 -0
- package/app/src/commands/playthrough.ts +111 -0
- package/app/src/commands/publish.ts +78 -0
- package/app/src/commands/reactions.ts +40 -0
- package/app/src/commands/reel.ts +46 -0
- package/app/src/commands/setup.ts +85 -0
- package/app/src/commands/sound.ts +90 -0
- package/app/src/commands/sprites.ts +232 -0
- package/app/src/commands/story.ts +37 -0
- package/app/src/commands/studio.ts +65 -0
- package/app/src/commands/style.ts +87 -0
- package/app/src/commands/verify.ts +45 -0
- package/app/src/commands/voice-retry.ts +14 -0
- package/app/src/commands/voices.ts +145 -0
- package/app/src/commands/weather.ts +52 -0
- package/app/src/commands/world.ts +104 -0
- package/app/src/film/clip.ts +386 -0
- package/app/src/film/reel.ts +126 -0
- package/app/src/film/screencast.ts +123 -0
- package/app/src/image/mask.ts +29 -0
- package/app/src/image/overlay.ts +38 -0
- package/app/src/image/raster.ts +224 -0
- package/app/src/main.ts +28 -0
- package/app/src/meta/choice.ts +27 -0
- package/app/src/meta/http.ts +99 -0
- package/app/src/meta/memo.ts +91 -0
- package/app/src/meta/muse.ts +70 -0
- package/app/src/meta/openrouter.ts +93 -0
- package/app/src/meta/prices.ts +25 -0
- package/app/src/meta/sam.ts +83 -0
- package/app/src/meta/schema.ts +63 -0
- package/app/src/meta/spark.ts +220 -0
- package/app/src/meta/step.ts +14 -0
- package/app/src/meta/voice-file.ts +78 -0
- package/app/src/meta/voice.ts +103 -0
- package/app/src/model/animation.ts +38 -0
- package/app/src/model/blender/relax.py +284 -0
- package/app/src/model/blender.ts +115 -0
- package/app/src/model/build.ts +50 -0
- package/app/src/model/face.ts +21 -0
- package/app/src/model/fit.ts +181 -0
- package/app/src/model/glb.ts +241 -0
- package/app/src/model/plan.ts +151 -0
- package/app/src/model/quality.ts +48 -0
- package/app/src/model/space.ts +168 -0
- package/app/src/model/sprites.ts +293 -0
- package/app/src/model/views.ts +163 -0
- package/app/src/play/creature.ts +43 -0
- package/app/src/play/depth-occlusion.ts +150 -0
- package/app/src/play/exit-outline.ts +48 -0
- package/app/src/play/face3d.ts +34 -0
- package/app/src/play/focus.ts +78 -0
- package/app/src/play/geometry.ts +272 -0
- package/app/src/play/gestures.ts +23 -0
- package/app/src/play/glow.ts +187 -0
- package/app/src/play/lipsync.ts +69 -0
- package/app/src/play/living.ts +83 -0
- package/app/src/play/outline-path.ts +130 -0
- package/app/src/play/parallax.ts +200 -0
- package/app/src/play/path.ts +262 -0
- package/app/src/play/preview.ts +71 -0
- package/app/src/play/reactions.ts +92 -0
- package/app/src/play/rig.ts +172 -0
- package/app/src/play/shadow.ts +253 -0
- package/app/src/play/sound.ts +51 -0
- package/app/src/play/weather.ts +58 -0
- package/app/src/qa/explore.ts +136 -0
- package/app/src/qa/probe.ts +139 -0
- package/app/src/qa/route.ts +118 -0
- package/app/src/registry.ts +23 -0
- package/app/src/scene/animation-source.ts +36 -0
- package/app/src/scene/assets.ts +37 -0
- package/app/src/scene/autofix.ts +133 -0
- package/app/src/scene/backdrop.ts +326 -0
- package/app/src/scene/blockin.ts +189 -0
- package/app/src/scene/brief.ts +74 -0
- package/app/src/scene/carry.ts +177 -0
- package/app/src/scene/cast-fixes.ts +31 -0
- package/app/src/scene/cast.ts +192 -0
- package/app/src/scene/corrections.ts +164 -0
- package/app/src/scene/critic.ts +169 -0
- package/app/src/scene/depthmap.ts +149 -0
- package/app/src/scene/derive.ts +475 -0
- package/app/src/scene/diagnose.ts +132 -0
- package/app/src/scene/face-calibration.ts +65 -0
- package/app/src/scene/face-registration.ts +21 -0
- package/app/src/scene/face-work.ts +71 -0
- package/app/src/scene/faces-data.ts +48 -0
- package/app/src/scene/faces-settings.ts +20 -0
- package/app/src/scene/faces.ts +435 -0
- package/app/src/scene/glows.ts +153 -0
- package/app/src/scene/held.ts +55 -0
- package/app/src/scene/icons.ts +185 -0
- package/app/src/scene/layers.ts +822 -0
- package/app/src/scene/layout.ts +216 -0
- package/app/src/scene/locate.ts +118 -0
- package/app/src/scene/maskcheck.ts +43 -0
- package/app/src/scene/masks.ts +392 -0
- package/app/src/scene/occluders.ts +123 -0
- package/app/src/scene/occlusion-check.ts +138 -0
- package/app/src/scene/paint-layout.ts +135 -0
- package/app/src/scene/paint.ts +81 -0
- package/app/src/scene/photo.ts +123 -0
- package/app/src/scene/plates.ts +256 -0
- package/app/src/scene/playbook.ts +121 -0
- package/app/src/scene/preview.ts +23 -0
- package/app/src/scene/reactions.ts +118 -0
- package/app/src/scene/restyle.ts +53 -0
- package/app/src/scene/rig-check.ts +68 -0
- package/app/src/scene/rig.ts +375 -0
- package/app/src/scene/scale.ts +148 -0
- package/app/src/scene/shadows.ts +207 -0
- package/app/src/scene/stairs.ts +183 -0
- package/app/src/scene/status.ts +32 -0
- package/app/src/scene/verify.ts +156 -0
- package/app/src/scene/walk.ts +326 -0
- package/app/src/scene/warp.ts +417 -0
- package/app/src/scene/weather.ts +118 -0
- package/app/src/scene/words.ts +65 -0
- package/app/src/secrets.ts +219 -0
- package/app/src/setup/checks.ts +110 -0
- package/app/src/sound/cost.ts +74 -0
- package/app/src/sound/data.ts +89 -0
- package/app/src/sound/fixes.ts +23 -0
- package/app/src/sound/issues.ts +21 -0
- package/app/src/sound/language.ts +11 -0
- package/app/src/sound/lipsync-make.ts +98 -0
- package/app/src/sound/lipsync.ts +24 -0
- package/app/src/sound/make.ts +229 -0
- package/app/src/sound/musicgen.py +20 -0
- package/app/src/sound/pcm.ts +280 -0
- package/app/src/sound/plan.ts +154 -0
- package/app/src/sound/retry.ts +43 -0
- package/app/src/sound/settings.ts +28 -0
- package/app/src/sound/sources.ts +81 -0
- package/app/src/sound/spoken.ts +70 -0
- package/app/src/sound/voices.ts +579 -0
- package/app/src/speech/transcribe.ts +38 -0
- package/app/src/speech/types.ts +18 -0
- package/app/src/story/check.ts +230 -0
- package/app/src/story/design.ts +108 -0
- package/app/src/story/engine.ts +233 -0
- package/app/src/story/fixes.ts +54 -0
- package/app/src/story/npc-guidance.ts +7 -0
- package/app/src/story/phrases.ts +4 -0
- package/app/src/story/schema.ts +92 -0
- package/app/src/studio/agent.ts +554 -0
- package/app/src/studio/discovery.ts +28 -0
- package/app/src/studio/events.ts +48 -0
- package/app/src/studio/history.ts +124 -0
- package/app/src/studio/jobs.ts +291 -0
- package/app/src/studio/locks.ts +33 -0
- package/app/src/studio/mcp.ts +86 -0
- package/app/src/studio/operation.ts +18 -0
- package/app/src/studio/repair.ts +31 -0
- package/app/src/studio/server.ts +421 -0
- package/app/src/studio/trouble.ts +48 -0
- package/app/src/world/design.ts +72 -0
- package/app/src/world/drawings.ts +198 -0
- package/app/src/world/extend-stage.ts +109 -0
- package/app/src/world/extend.ts +123 -0
- package/app/src/world/extension-progress.ts +10 -0
- package/app/src/world/files.ts +62 -0
- package/app/src/world/idea-review.ts +32 -0
- package/app/src/world/photo-limits.ts +2 -0
- package/app/src/world/pool.ts +41 -0
- package/app/src/world/schedule.ts +64 -0
- package/app/src/world/schema.ts +74 -0
- package/app/src/world/sketch.ts +81 -0
- package/app/src/world/style-plan.ts +90 -0
- package/app/src/world/style-reference.ts +16 -0
- package/app/src/world/styled.ts +51 -0
- package/app/src/world/told.ts +105 -0
- package/app/src/world/video-limits.ts +3 -0
- package/app/src/world/video.ts +98 -0
- package/app/studio/activity.tsx +124 -0
- package/app/studio/agent.tsx +384 -0
- package/app/studio/api.ts +111 -0
- package/app/studio/atelier.tsx +234 -0
- package/app/studio/audio-missing.tsx +25 -0
- package/app/studio/audio-setup.tsx +33 -0
- package/app/studio/blender-setup.tsx +30 -0
- package/app/studio/build.tsx +677 -0
- package/app/studio/camera.tsx +59 -0
- package/app/studio/cast-work.tsx +207 -0
- package/app/studio/cast.tsx +178 -0
- package/app/studio/concept-edit.tsx +47 -0
- package/app/studio/creator-idea.tsx +46 -0
- package/app/studio/custom-look.tsx +36 -0
- package/app/studio/faces-setup.tsx +28 -0
- package/app/studio/favicon.svg +11 -0
- package/app/studio/figure.tsx +314 -0
- package/app/studio/fonts/OFL.txt +93 -0
- package/app/studio/fonts/fonts.css +7 -0
- package/app/studio/fonts/fraunces-italic-latin-ext.woff2 +0 -0
- package/app/studio/fonts/fraunces-italic-latin.woff2 +0 -0
- package/app/studio/fonts/fraunces-normal-latin-ext.woff2 +0 -0
- package/app/studio/fonts/fraunces-normal-latin.woff2 +0 -0
- package/app/studio/fonts/instrument-sans-normal-latin-ext.woff2 +0 -0
- package/app/studio/fonts/instrument-sans-normal-latin.woff2 +0 -0
- package/app/studio/icons.tsx +42 -0
- package/app/studio/idea-card.tsx +42 -0
- package/app/studio/idea-review.tsx +181 -0
- package/app/studio/idea-storyboard.tsx +134 -0
- package/app/studio/index.html +15 -0
- package/app/studio/job-words.ts +33 -0
- package/app/studio/lightbox.tsx +49 -0
- package/app/studio/main.tsx +1328 -0
- package/app/studio/nav.ts +14 -0
- package/app/studio/place.tsx +264 -0
- package/app/studio/review-sketch.tsx +47 -0
- package/app/studio/say-it.tsx +49 -0
- package/app/studio/step-work.tsx +134 -0
- package/app/studio/story.tsx +394 -0
- package/app/studio/style-menu.tsx +81 -0
- package/app/studio/style.css +1390 -0
- package/app/studio/sure.tsx +54 -0
- package/app/studio/tip.tsx +74 -0
- package/app/studio/tsconfig.json +5 -0
- package/app/studio/video.tsx +73 -0
- package/app/studio/world.tsx +188 -0
- package/app/tsconfig.json +13 -0
- package/app/web/audio.ts +315 -0
- package/app/web/face.ts +131 -0
- package/app/web/face3d.ts +35 -0
- package/app/web/glows.ts +387 -0
- package/app/web/hero3d.ts +728 -0
- package/app/web/index.html +118 -0
- package/app/web/living.ts +404 -0
- package/app/web/main.ts +3751 -0
- package/app/web/pad.ts +174 -0
- package/app/web/reactions.ts +79 -0
- package/app/web/sheet.ts +36 -0
- package/app/web/synth.ts +140 -0
- package/app/web/tsconfig.json +5 -0
- package/app/web/ui.ts +263 -0
- package/app/web/weather.ts +306 -0
- package/app/web/words.ts +12 -0
- package/docs/characters.md +58 -0
- package/docs/npc-motion.md +69 -0
- package/docs/scenes.md +75 -0
- package/docs/setup.md +60 -0
- package/docs/sound.md +23 -0
- package/docs/studio.md +62 -0
- package/package.json +70 -4
|
@@ -0,0 +1,154 @@
|
|
|
1
|
+
// sound.json: what a world sounds like. Spark plans it from the world, each place's brief and floors,
|
|
2
|
+
// and the story; code turns each line into a stable sound id, so a re-plan only regenerates what changed.
|
|
3
|
+
import { z } from 'zod';
|
|
4
|
+
import type { Memo } from '../meta/memo.ts';
|
|
5
|
+
import { askSparkJson } from '../meta/spark.ts';
|
|
6
|
+
import type { Brief } from '../scene/brief.ts';
|
|
7
|
+
import type { Layout } from '../scene/layout.ts';
|
|
8
|
+
import type { Story } from '../story/schema.ts';
|
|
9
|
+
import { Fact } from '../story/schema.ts';
|
|
10
|
+
import type { World } from '../world/schema.ts';
|
|
11
|
+
|
|
12
|
+
const Prompt = z.string().min(8);
|
|
13
|
+
/** 1 quiet, 2 normal, 3 loud: how it sits in the mix next to the place bed. */
|
|
14
|
+
const Level = z.union([z.literal(1), z.literal(2), z.literal(3)]).default(2);
|
|
15
|
+
|
|
16
|
+
export const SoundPlan = z.object({
|
|
17
|
+
/** Hash of what the plan was made from (world, story, briefs, floors); when that changes the plan is made again. */
|
|
18
|
+
from: z.string().optional(),
|
|
19
|
+
/** A music bed for the whole world, looped under everything. */
|
|
20
|
+
music: z.object({ prompt: Prompt }),
|
|
21
|
+
places: z.record(z.string(), z.object({
|
|
22
|
+
/** The constant air of the place: wind, sea, room tone. */
|
|
23
|
+
bed: Prompt,
|
|
24
|
+
/** A sound that comes from one thing and pans and fades with where the hero stands. */
|
|
25
|
+
loops: z.array(z.object({ thing: z.string(), prompt: Prompt, level: Level, when: z.array(Fact).default([]) })).max(6).default([]),
|
|
26
|
+
/** Floor key (layout.json) to the surface the hero's feet hit there. */
|
|
27
|
+
floors: z.record(z.string(), z.string()),
|
|
28
|
+
})),
|
|
29
|
+
/** Surface name to what a few footsteps on it sound like. */
|
|
30
|
+
steps: z.record(z.string(), Prompt),
|
|
31
|
+
/** Short one-shots: picking up an item, an exit, using a thing, the ending. */
|
|
32
|
+
shots: z.array(z.object({ on: z.string(), prompt: Prompt, seconds: z.number().min(0.5).max(5).default(1.5), level: Level })).max(16).default([]),
|
|
33
|
+
});
|
|
34
|
+
export type SoundPlan = z.infer<typeof SoundPlan>;
|
|
35
|
+
|
|
36
|
+
/** What fires a one-shot; the runtime matches these. */
|
|
37
|
+
export const SHOT_TRIGGERS = ['take', 'exit:<thing key>', 'use:<thing key>', 'talk', 'ending', 'click'] as const;
|
|
38
|
+
|
|
39
|
+
export const slugOf = (text: string) => text.toLowerCase().replace(/[^a-z0-9]+/g, '-').replace(/^-|-$/g, '').slice(0, 40) || 'x';
|
|
40
|
+
|
|
41
|
+
/** The id a planned sound is stored and played under. Ids depend on what the sound is for, never on its prompt. */
|
|
42
|
+
export const soundIds = {
|
|
43
|
+
music: () => 'music',
|
|
44
|
+
bed: (place: string) => `${place}-bed`,
|
|
45
|
+
loop: (place: string, thing: string) => `${place}-${thing}-loop`,
|
|
46
|
+
step: (surface: string) => `step-${slugOf(surface)}`,
|
|
47
|
+
shot: (on: string) => `shot-${slugOf(on)}`,
|
|
48
|
+
};
|
|
49
|
+
|
|
50
|
+
export type PlaceSources = { brief?: Brief; layout?: Layout };
|
|
51
|
+
|
|
52
|
+
export function planIssues(plan: SoundPlan, world: World, sources: Record<string, PlaceSources>): string[] {
|
|
53
|
+
const issues: string[] = [];
|
|
54
|
+
for (const place of world.places) {
|
|
55
|
+
const p = plan.places[place.key];
|
|
56
|
+
if (!p) {
|
|
57
|
+
issues.push(`no sounds for place ${place.key}`);
|
|
58
|
+
continue;
|
|
59
|
+
}
|
|
60
|
+
const things = new Set(place.things.map((t) => t.key));
|
|
61
|
+
for (const loop of p.loops) if (!things.has(loop.thing)) issues.push(`${place.key}: loop on "${loop.thing}", which is not one of its things`);
|
|
62
|
+
for (const floor of sources[place.key]?.layout?.floors ?? []) {
|
|
63
|
+
const surface = p.floors[floor.key];
|
|
64
|
+
if (!surface) issues.push(`${place.key}: floor ${floor.key} has no surface`);
|
|
65
|
+
else if (!plan.steps[surface]) issues.push(`${place.key}: surface "${surface}" of ${floor.key} has no footsteps in steps`);
|
|
66
|
+
}
|
|
67
|
+
// Every thing the brief animates should be heard too, unless it is silent by nature (a light, cloth).
|
|
68
|
+
// A moving floor (water under a quay) has nothing to pan to; the bed carries it.
|
|
69
|
+
// A boat the brief sets bobbing on the water ("water") is heard in the water's bed, not on its own.
|
|
70
|
+
for (const alive of sources[place.key]?.brief?.alive ?? []) {
|
|
71
|
+
if (things.has(alive.key) && ['water', 'fire', 'smoke'].includes(alive.as) && !(alive.as === 'water' && AFLOAT.test(alive.key)) && !p.loops.some((l) => l.thing === alive.key)) issues.push(`${place.key}: ${alive.key} is ${alive.as} but makes no sound`);
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
return issues;
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
/** Things that ride the water rather than being it. */
|
|
78
|
+
const AFLOAT = /(^|-)(boats?|rowboat|dinghy|skiff|canoe|kayak|raft|ship|barge|ferry|yacht|sailboats?|buoy|[łl][óo]d[źz]|[łl]odzie|[łl][óo]dka|statek|tratwa)(-|$)/;
|
|
79
|
+
|
|
80
|
+
/** Drops loops on keys that are not things of their place (a floor, say): nothing in the page to pan them to. */
|
|
81
|
+
export function withoutStrayLoops(plan: SoundPlan, world: World): SoundPlan {
|
|
82
|
+
const places = Object.fromEntries(Object.entries(plan.places).map(([key, p]) => {
|
|
83
|
+
const things = new Set(world.places.find((w) => w.key === key)?.things.map((t) => t.key) ?? []);
|
|
84
|
+
return [key, { ...p, loops: p.loops.filter((l) => things.has(l.thing)) }];
|
|
85
|
+
}));
|
|
86
|
+
return { ...plan, places };
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
/**
|
|
90
|
+
* Things silent by nature (a lamp, a sign) or heard only when used (a ticket machine): a loop on them is three takes
|
|
91
|
+
* of hiss or one click in silence, all turned down. Dropped before anything is paid for, unless the brief animates
|
|
92
|
+
* the thing as fire, water or smoke (an oil lantern's flame).
|
|
93
|
+
*/
|
|
94
|
+
const SILENT = /(^|-)(lamp|lamps|lantern|lanterns|light|lights|streetlight|sign|signs|poster|posters|timetable|schedule|map|ticket|tickets|ticket-machine|validator|kasownik|latarnia|latarka|lampa|znak|plakat|rozklad|bilet|biletomat)(-|$)/;
|
|
95
|
+
export function withoutSilentLoops(plan: SoundPlan, sources: Record<string, PlaceSources> = {}): SoundPlan {
|
|
96
|
+
const places = Object.fromEntries(Object.entries(plan.places).map(([key, p]) => {
|
|
97
|
+
const alive = new Set((sources[key]?.brief?.alive ?? []).filter((a) => ['water', 'fire', 'smoke'].includes(a.as)).map((a) => a.key));
|
|
98
|
+
return [key, { ...p, loops: p.loops.filter((l) => alive.has(l.thing) || !SILENT.test(l.thing)) }];
|
|
99
|
+
}));
|
|
100
|
+
return { ...plan, places };
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
function storyLines(story: Story | null): string[] {
|
|
104
|
+
if (!story) return [];
|
|
105
|
+
const lines = story.interactions.filter((i) => i.verb !== 'look').map((i) => `- ${i.place}: ${i.verb} ${i.thing}${i.with ? ` with ${i.with}` : ''} -> "${i.say}"`);
|
|
106
|
+
for (const c of story.changes) lines.push(`- ${c.place}: once ${c.when.join(' and ')}, ${c.thing} becomes "${c.looks}"`);
|
|
107
|
+
return lines;
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
export async function writeSoundPlan(memo: Memo, world: World, story: Story | null, sources: Record<string, PlaceSources>): Promise<SoundPlan> {
|
|
111
|
+
const places = world.places.map((place) => {
|
|
112
|
+
const brief = sources[place.key]?.brief;
|
|
113
|
+
const floors = sources[place.key]?.layout?.floors ?? [];
|
|
114
|
+
return `## ${place.key}
|
|
115
|
+
Shows: ${place.shows}
|
|
116
|
+
Mood: ${place.mood}
|
|
117
|
+
Things: ${place.things.map((t) => `${t.key} (${t.role}: ${brief?.looks[t.key] ?? t.looks})`).join('; ')}
|
|
118
|
+
Moving things: ${brief?.alive.map((a) => `${a.key} is ${a.as}`).join(', ') || 'none'}
|
|
119
|
+
Floors: ${floors.map((f) => `${f.key} (${brief?.looks[f.key] ?? f.is})`).join('; ') || 'none'}
|
|
120
|
+
Exits: ${place.exits.map((e) => `${e.via} to ${e.to}`).join(', ') || 'none'}`;
|
|
121
|
+
}).join('\n\n');
|
|
122
|
+
const prompt = `You are the sound designer of a 2D painted point-and-click adventure. Each sound you describe is generated by a text-to-sound model, so every prompt names concrete sources, materials, distance and character, never music theory or feelings alone.
|
|
123
|
+
|
|
124
|
+
Game: ${world.title}. ${world.pitch}
|
|
125
|
+
Look: ${world.style}
|
|
126
|
+
|
|
127
|
+
${places}
|
|
128
|
+
|
|
129
|
+
What the hero does in the story:
|
|
130
|
+
${storyLines(story).join('\n') || '- (no story yet)'}
|
|
131
|
+
|
|
132
|
+
Plan:
|
|
133
|
+
- music: one instrumental music bed for the whole game, under everything: instruments, tempo, mood. No vocals.
|
|
134
|
+
- per place: "bed", the quiet ambience of the place (water, distant life, birds far off, a room's warm hum) that loops for minutes without a single event standing out. Not wind, rustling leaves, "air tone" or breathing: those come out as hiss that tires the ear under the whole game. No footsteps, no voices, no music in it.
|
|
135
|
+
- per place: "loops", at most 2 sounds that come from one of its things (thing key from its Things) and go on without stopping: running water, fire, a running engine or fan, a ticking clock, animals. Most places need 0 or 1. Only for things that make that sound all by themselves; every loop costs money to make. Never for lights, lamps, lanterns, signs, posters or timetables (they are silent), not for trees, bushes or grass in the wind, not for someone breathing, and not for things that sound only when used (a ticket machine, a door, a bell, a till): give those a "use:<thing>" shot instead. "when" lists story facts that must hold for it to play, e.g. ["lamp-lit"] for a fire that starts in the story (use [] when always). level 1 quiet, 2 normal, 3 loud.
|
|
136
|
+
- per place: "floors", each floor key to a surface name such as "wood", "wet-wood", "stone", "gravel", "grass", "sand", "metal", "carpet". Reuse surface names across places.
|
|
137
|
+
- "steps": each surface name used to a prompt for a few separate footsteps of one person walking on it, e.g. "slow footsteps of boots on wet wooden planks, 6 separate steps, dry close recording".
|
|
138
|
+
- "shots": short one-shot effects. "on" is one of: ${SHOT_TRIGGERS.join(', ')}. Give one for "take" (picking up an item), "click" (a soft UI click), "talk" (a short sound as a conversation starts), "ending", one "exit:<thing>" per exit thing (exits of the same kind, two wooden doors or two garden gates, share one sound: give them word for word the same prompt, seconds and level, so it is made once), and "use:<thing>" for things the story changes when used. "seconds" 0.5 to 4.
|
|
139
|
+
|
|
140
|
+
Reply with JSON only:
|
|
141
|
+
{"music": {"prompt": "..."}, "places": {"<place key>": {"bed": "...", "loops": [{"thing": "<thing key>", "prompt": "...", "level": 2, "when": []}], "floors": {"<floor key>": "<surface>"}}}, "steps": {"<surface>": "..."}, "shots": [{"on": "take", "prompt": "...", "seconds": 1.5, "level": 2}]}`;
|
|
142
|
+
let feedback = '';
|
|
143
|
+
for (let attempt = 1; ; attempt++) {
|
|
144
|
+
const answer = await askSparkJson(memo, { label: `sound-plan${attempt > 1 ? `-${attempt}` : ''}`, prompt: prompt + feedback });
|
|
145
|
+
const parsed = SoundPlan.safeParse(answer);
|
|
146
|
+
const issues = parsed.success ? planIssues(parsed.data, world, sources) : parsed.error.issues.map((i) => `${i.path.join('.')}: ${i.message}`);
|
|
147
|
+
if (!issues.length && parsed.success) return withoutSilentLoops(parsed.data, sources);
|
|
148
|
+
if (attempt === 2) {
|
|
149
|
+
if (!parsed.success) throw new Error(`sound plan: ${issues.join('; ')}`);
|
|
150
|
+
return withoutSilentLoops(withoutStrayLoops(parsed.data, world), sources); // usable; the step prints what is still missing
|
|
151
|
+
}
|
|
152
|
+
feedback = `\n\nYour last plan had these problems, fix them:\n${issues.map((i) => `- ${i}`).join('\n')}`;
|
|
153
|
+
}
|
|
154
|
+
}
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
import { refreshExistingLipSync } from './lipsync-make.ts';
|
|
2
|
+
import type { WorldFiles } from '../world/files.ts';
|
|
3
|
+
import { Story } from '../story/schema.ts';
|
|
4
|
+
import { creditGuard } from './cost.ts';
|
|
5
|
+
import { elevenKey } from './sources.ts';
|
|
6
|
+
import { DesignedVoices, freeVoiceSlot, PENDING_DESIGN, storyLines, voiceLine, Voices } from './voices.ts';
|
|
7
|
+
|
|
8
|
+
/** Resolve at queue time and again when the job starts, so removed dialogue cannot be regenerated. */
|
|
9
|
+
export function voiceRetryTarget(files: WorldFiles, speaker: string, text: string) {
|
|
10
|
+
const saved = files.readJson(files.path('voices.json')) as Voices;
|
|
11
|
+
const old = Voices.parse(saved);
|
|
12
|
+
const line = storyLines(files.world(), Story.parse(files.readJson(files.path('story.json'))), true).find((l) => l.speaker === speaker && l.text === text);
|
|
13
|
+
if (!line) throw new Error('this speaker no longer says that line');
|
|
14
|
+
const voice = old.cast[speaker];
|
|
15
|
+
if (!voice || voice.voice === PENDING_DESIGN) throw new Error('the voice is not ready; run voices to finish voice design first');
|
|
16
|
+
return { old, saved, line, voice };
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
/** Retry only this line, preserving the cast, every other recording and every other failure. */
|
|
20
|
+
export async function retryVoiceLine(files: WorldFiles, speaker: string, text: string) {
|
|
21
|
+
const { old, saved, line, voice } = voiceRetryTarget(files, speaker, text);
|
|
22
|
+
if (!elevenKey()) throw new Error('no ELEVENLABS_API_KEY');
|
|
23
|
+
const guard = await creditGuard(elevenKey(), 'voice retry');
|
|
24
|
+
if (voice.design && !(await freeVoiceSlot())) throw new Error('No free ElevenLabs voice slot; free one and retry. The selected voice is unchanged.');
|
|
25
|
+
const same = (l: { speaker: string; text: string }) => l.speaker === speaker && l.text === text;
|
|
26
|
+
const failed = old.failures.find(same);
|
|
27
|
+
const designed = new DesignedVoices(`diorama ${files.slug}`);
|
|
28
|
+
try {
|
|
29
|
+
const result = await voiceLine(files.memo, line, voice, (id, mp3) => {
|
|
30
|
+
const file = `sound/voice/${id}.mp3`;
|
|
31
|
+
files.write(files.path(file), mp3);
|
|
32
|
+
return file;
|
|
33
|
+
}, designed, { retryRejected: failed?.reason.startsWith('speech rejected') });
|
|
34
|
+
files.write(files.path('voices.json'), { ...saved, lines: [...saved.lines.filter((l) => !same(l)), result], failures: old.failures.filter((l) => !same(l)) });
|
|
35
|
+
} catch (error) {
|
|
36
|
+
const reason = error instanceof Error ? error.message : String(error);
|
|
37
|
+
// Do not discard a previous recording if a requested replacement fails.
|
|
38
|
+
files.write(files.path('voices.json'), { ...saved, failures: [...old.failures.filter((l) => !same(l)), { speaker, text, reason }] });
|
|
39
|
+
throw error;
|
|
40
|
+
} finally { if (voice.design) await designed.close(); }
|
|
41
|
+
await refreshExistingLipSync(files, [speaker]);
|
|
42
|
+
console.log(`voice retry ${speaker}: ${text} · ${await guard.report()}`);
|
|
43
|
+
}
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
// Machine-wide audio quality, shared by Studio and CLI jobs.
|
|
2
|
+
import fs from 'node:fs';
|
|
3
|
+
import path from 'node:path';
|
|
4
|
+
import { randomUUID } from 'node:crypto';
|
|
5
|
+
import { configDir } from '../secrets.ts';
|
|
6
|
+
import type { Quality } from './cost.ts';
|
|
7
|
+
|
|
8
|
+
const file = () => path.join(configDir(), 'sound.json');
|
|
9
|
+
|
|
10
|
+
export function readSoundQuality(): Quality {
|
|
11
|
+
let raw: string;
|
|
12
|
+
try { raw = fs.readFileSync(file(), 'utf8'); }
|
|
13
|
+
catch (e) { if ((e as NodeJS.ErrnoException).code === 'ENOENT') return 'final'; throw e; }
|
|
14
|
+
const quality = JSON.parse(raw).quality;
|
|
15
|
+
if (quality !== 'final' && quality !== 'draft') throw new Error('Saved audio quality must be final or draft');
|
|
16
|
+
return quality;
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
export function writeSoundQuality(quality: Quality) {
|
|
20
|
+
if (quality !== 'final' && quality !== 'draft') throw new Error('Audio quality must be final or draft');
|
|
21
|
+
const target = file();
|
|
22
|
+
fs.mkdirSync(path.dirname(target), { recursive: true, mode: 0o700 });
|
|
23
|
+
const tmp = `${target}.${randomUUID()}.tmp`;
|
|
24
|
+
try {
|
|
25
|
+
fs.writeFileSync(tmp, `${JSON.stringify({ quality }, null, 2)}\n`, { mode: 0o600 });
|
|
26
|
+
fs.renameSync(tmp, target);
|
|
27
|
+
} finally { fs.rmSync(tmp, { force: true }); }
|
|
28
|
+
}
|
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
// Where sounds come from. The Meta Model API has no audio generation, so effects come from ElevenLabs
|
|
2
|
+
// and music from Meta's open-weights MusicGen run locally (or ElevenLabs music when Python is not set up).
|
|
3
|
+
// Every take is kept in the memo by a hash of its request, so a rerun never pays twice.
|
|
4
|
+
import { execFileSync } from 'node:child_process';
|
|
5
|
+
import fs from 'node:fs';
|
|
6
|
+
import os from 'node:os';
|
|
7
|
+
import path from 'node:path';
|
|
8
|
+
import { fileURLToPath } from 'node:url';
|
|
9
|
+
import { inputHash, type Memo } from '../meta/memo.ts';
|
|
10
|
+
import { checkCredits, noteRefusal, ttsModel } from './cost.ts';
|
|
11
|
+
|
|
12
|
+
const ELEVEN = 'https://api.elevenlabs.io/v1';
|
|
13
|
+
|
|
14
|
+
export function elevenKey(): string | undefined {
|
|
15
|
+
return process.env.ELEVENLABS_API_KEY || process.env.ELEVEN_API_KEY;
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
async function elevenPost(route: string, body: unknown): Promise<Buffer> {
|
|
19
|
+
const key = elevenKey();
|
|
20
|
+
if (!key) throw new Error('no ElevenLabs key: set ELEVENLABS_API_KEY');
|
|
21
|
+
checkCredits();
|
|
22
|
+
for (let n = 1; ; n++) {
|
|
23
|
+
const res = await fetch(ELEVEN + route, { method: 'POST', headers: { 'xi-api-key': key, 'Content-Type': 'application/json' }, body: JSON.stringify(body) });
|
|
24
|
+
if (res.ok) return Buffer.from(await res.arrayBuffer());
|
|
25
|
+
const text = (await res.text()).slice(0, 300);
|
|
26
|
+
noteRefusal(res.status, text);
|
|
27
|
+
if (![429, 500, 502, 503].includes(res.status) || n === 4) throw new Error(`ElevenLabs ${route}: ${res.status} ${text}`);
|
|
28
|
+
await new Promise((r) => setTimeout(r, 1000 * 2 ** n));
|
|
29
|
+
}
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
export type EffectAsk = { label: string; prompt: string; seconds: number; loop: boolean; take?: number };
|
|
33
|
+
|
|
34
|
+
/** An effect or ambience from ElevenLabs text-to-sound, as mp3. `take` > 1 asks again for a different take. */
|
|
35
|
+
const effectBody = (ask: EffectAsk) => ({ text: ask.prompt, duration_seconds: ask.seconds, loop: ask.loop, prompt_influence: 0.45, model_id: 'eleven_text_to_sound_v2' });
|
|
36
|
+
const effectHash = (ask: EffectAsk) => inputHash('eleven-sfx', JSON.stringify(effectBody(ask)), String(ask.take ?? 1));
|
|
37
|
+
|
|
38
|
+
export function soundEffect(memo: Memo, ask: EffectAsk): Promise<Buffer> {
|
|
39
|
+
return memo.remember('sound', ask.label, effectHash(ask), 'mp3', () => elevenPost('/sound-generation', effectBody(ask)));
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
/** A take the quality checks turned down is forgotten, so the next run asks for a new one instead of replaying it. */
|
|
43
|
+
export function forgetEffect(memo: Memo, ask: EffectAsk) {
|
|
44
|
+
memo.forget('sound', effectHash(ask), 'mp3');
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
/** Music from ElevenLabs, for when MusicGen is not available. */
|
|
48
|
+
export function elevenMusic(memo: Memo, label: string, prompt: string, seconds: number): Promise<Buffer> {
|
|
49
|
+
const body = { prompt, music_length_ms: Math.round(seconds * 1000), model_id: 'music_v1', force_instrumental: true };
|
|
50
|
+
return memo.remember('sound', label, inputHash('eleven-music', JSON.stringify(body)), 'mp3', () => elevenPost('/music', body));
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
const MUSICGEN_PY = fileURLToPath(new URL('./musicgen.py', import.meta.url));
|
|
54
|
+
export const MUSICGEN_MODEL = 'facebook/musicgen-small';
|
|
55
|
+
|
|
56
|
+
/** The Python that has torch and transformers: DIORAMA_PYTHON, or none. */
|
|
57
|
+
export function musicgenPython(): string | undefined {
|
|
58
|
+
const py = process.env.DIORAMA_PYTHON;
|
|
59
|
+
if (!py) return undefined;
|
|
60
|
+
try {
|
|
61
|
+
execFileSync(py, ['-c', 'import torch, transformers'], { stdio: 'ignore' });
|
|
62
|
+
return py;
|
|
63
|
+
} catch {
|
|
64
|
+
return undefined;
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
/** Music from Meta's MusicGen (open weights, runs on CPU in a few minutes), as wav. */
|
|
69
|
+
export function musicgen(memo: Memo, label: string, prompt: string, seconds: number, python: string): Promise<Buffer> {
|
|
70
|
+
return memo.remember('music', label, inputHash(MUSICGEN_MODEL, prompt, String(seconds)), 'wav', async () => {
|
|
71
|
+
const out = path.join(fs.mkdtempSync(path.join(os.tmpdir(), 'musicgen-')), 'out.wav');
|
|
72
|
+
execFileSync(python, ['-I', MUSICGEN_PY, MUSICGEN_MODEL, prompt, String(seconds), out], { stdio: ['ignore', 'inherit', 'inherit'] });
|
|
73
|
+
return fs.readFileSync(out);
|
|
74
|
+
});
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
/** Speech in any language ElevenLabs reads (Flash v2.5: cheap and quick), as mp3. */
|
|
78
|
+
export function elevenSpeech(memo: Memo, label: string, text: string, voice: string): Promise<Buffer> {
|
|
79
|
+
const body = { text, model_id: ttsModel() };
|
|
80
|
+
return memo.remember('voice', label, inputHash('eleven-tts', voice, JSON.stringify(body)), 'mp3', () => elevenPost(`/text-to-speech/${voice}?output_format=mp3_44100_128`, body));
|
|
81
|
+
}
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
// What a line sounds like read aloud: "10+10=20" on screen is "dziesięć plus dziesięć równa się dwadzieścia" spoken.
|
|
2
|
+
// The voice actor reads the spoken form, and the transcript is checked against it, so a line with digits and signs
|
|
3
|
+
// is neither left to the actor's guess nor failed because the transcriber wrote words where the line has digits.
|
|
4
|
+
import { languageKey } from './language.ts';
|
|
5
|
+
|
|
6
|
+
type Words = { number: (n: number) => string; ops: Record<string, string> };
|
|
7
|
+
|
|
8
|
+
const PL_UNITS = ['zero', 'jeden', 'dwa', 'trzy', 'cztery', 'pięć', 'sześć', 'siedem', 'osiem', 'dziewięć'];
|
|
9
|
+
const PL_TEENS = ['dziesięć', 'jedenaście', 'dwanaście', 'trzynaście', 'czternaście', 'piętnaście', 'szesnaście', 'siedemnaście', 'osiemnaście', 'dziewiętnaście'];
|
|
10
|
+
const PL_TENS = ['', '', 'dwadzieścia', 'trzydzieści', 'czterdzieści', 'pięćdziesiąt', 'sześćdziesiąt', 'siedemdziesiąt', 'osiemdziesiąt', 'dziewięćdziesiąt'];
|
|
11
|
+
const PL_HUNDREDS = ['', 'sto', 'dwieście', 'trzysta', 'czterysta', 'pięćset', 'sześćset', 'siedemset', 'osiemset', 'dziewięćset'];
|
|
12
|
+
|
|
13
|
+
function plBelow1000(n: number): string[] {
|
|
14
|
+
const h = Math.floor(n / 100), r = n % 100, t = Math.floor(r / 10), u = r % 10;
|
|
15
|
+
return [PL_HUNDREDS[h], r >= 10 && r < 20 ? PL_TEENS[r - 10] : PL_TENS[t], r >= 10 && r < 20 ? '' : u ? PL_UNITS[u] : ''].filter(Boolean);
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
function plNumber(n: number): string {
|
|
19
|
+
if (n === 0) return PL_UNITS[0];
|
|
20
|
+
const thousands = Math.floor(n / 1000), rest = n % 1000;
|
|
21
|
+
const k = thousands === 0 ? [] : thousands === 1 ? ['tysiąc']
|
|
22
|
+
: [...plBelow1000(thousands), [2, 3, 4].includes(thousands % 10) && ![12, 13, 14].includes(thousands % 100) ? 'tysiące' : 'tysięcy'];
|
|
23
|
+
return [...k, ...plBelow1000(rest)].join(' ');
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
const EN_UNITS = ['zero', 'one', 'two', 'three', 'four', 'five', 'six', 'seven', 'eight', 'nine', 'ten', 'eleven', 'twelve', 'thirteen', 'fourteen', 'fifteen', 'sixteen', 'seventeen', 'eighteen', 'nineteen'];
|
|
27
|
+
const EN_TENS = ['', '', 'twenty', 'thirty', 'forty', 'fifty', 'sixty', 'seventy', 'eighty', 'ninety'];
|
|
28
|
+
|
|
29
|
+
function enBelow1000(n: number): string[] {
|
|
30
|
+
const h = Math.floor(n / 100), r = n % 100;
|
|
31
|
+
const tail = r === 0 ? [] : r < 20 ? [EN_UNITS[r]] : [EN_TENS[Math.floor(r / 10)] + (r % 10 ? `-${EN_UNITS[r % 10]}` : '')];
|
|
32
|
+
return [...(h ? [EN_UNITS[h], 'hundred'] : []), ...tail];
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
function enNumber(n: number): string {
|
|
36
|
+
if (n === 0) return 'zero';
|
|
37
|
+
const thousands = Math.floor(n / 1000), rest = n % 1000;
|
|
38
|
+
return [...(thousands ? [...enBelow1000(thousands), 'thousand'] : []), ...enBelow1000(rest)].join(' ');
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
const LANGUAGES: Record<string, Words> = {
|
|
42
|
+
pl: { number: plNumber, ops: { '+': 'plus', '-': 'minus', '−': 'minus', '×': 'razy', '*': 'razy', '÷': 'podzielić przez', '=': 'równa się' } },
|
|
43
|
+
en: { number: enNumber, ops: { '+': 'plus', '-': 'minus', '−': 'minus', '×': 'times', '*': 'times', '÷': 'divided by', '=': 'equals' } },
|
|
44
|
+
};
|
|
45
|
+
|
|
46
|
+
/** Whole numbers up to 999 999 read as words; others (a year said its own way, decimals) are left to the actor. */
|
|
47
|
+
const NUMBER = /(?<![\d.,:])\d{1,6}(?![\d]|[.,:]\d)/g;
|
|
48
|
+
|
|
49
|
+
/** The line as it is said: whole numbers as words, and a sign between two numbers ("10+10", "3 × 4 = 12") as its word. */
|
|
50
|
+
export function spokenText(text: string, language?: string): string {
|
|
51
|
+
const words = LANGUAGES[languageKey(language)];
|
|
52
|
+
if (!words) return text;
|
|
53
|
+
const signs = Object.keys(words.ops).map((s) => s.replace(/[-*+]/g, '\\$&')).join('');
|
|
54
|
+
// Only between numbers: a dash between words is punctuation, not "minus".
|
|
55
|
+
const between = new RegExp(`(\\d)\\s*([${signs}])\\s*(?=\\d)`, 'g');
|
|
56
|
+
// "22 000" is one number: a space (or a no-break one) between thousands goes first.
|
|
57
|
+
let out = text.replace(/(?<![\d.,])(\d{1,3})[ \u00a0](\d{3})(?![\d])/g, '$1$2');
|
|
58
|
+
for (let i = 0; i < 2; i++) out = out.replace(between, (_, a: string, op: string) => `${a} ${words.ops[op]} `);
|
|
59
|
+
return out.replace(NUMBER, (d) => words.number(Number(d))).replace(/\s+/g, ' ');
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
/** The number words of a language, in the order said: "dziesięć plus dziesięć równa się dwadzieścia" has dziesięć, dziesięć, dwadzieścia. */
|
|
63
|
+
export function numberWords(spoken: string, language?: string): string[] {
|
|
64
|
+
const words = LANGUAGES[languageKey(language)];
|
|
65
|
+
if (!words) return [];
|
|
66
|
+
const vocabulary = new Set(language && languageKey(language) === 'pl'
|
|
67
|
+
? [...PL_UNITS, ...PL_TEENS, ...PL_TENS, ...PL_HUNDREDS, 'tysiąc', 'tysiące', 'tysięcy'].filter(Boolean)
|
|
68
|
+
: [...EN_UNITS, ...EN_TENS, 'hundred', 'thousand'].filter(Boolean));
|
|
69
|
+
return spoken.normalize('NFC').toLowerCase().split(/[^\p{L}]+/u).flatMap((w) => w.split('-')).filter((w) => vocabulary.has(w));
|
|
70
|
+
}
|