@gtanczyk/diorama 0.4.0 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/app/src/actions/fix.ts +14 -0
- package/app/src/actions/studio.ts +25 -0
- package/app/src/cli.ts +53 -0
- package/app/src/commands/browsers.ts +24 -8
- package/app/src/commands/derive.ts +5 -2
- package/app/src/commands/explore.ts +2 -1
- package/app/src/commands/model.ts +8 -0
- package/app/src/commands/paint.ts +2 -2
- package/app/src/commands/setup.ts +25 -1
- package/app/src/commands/studio.ts +1 -0
- package/app/src/commands/voices.ts +4 -2
- package/app/src/main.ts +5 -4
- package/app/src/model/views.ts +41 -2
- package/app/src/scene/backdrop.ts +34 -12
- package/app/src/scene/corrections.ts +6 -0
- package/app/src/scene/critic.ts +13 -3
- package/app/src/scene/derive.ts +22 -3
- package/app/src/secrets.ts +2 -1
- package/app/src/setup/checks.ts +5 -4
- package/app/src/studio/agent.ts +29 -2
- package/app/src/studio/mcp.ts +2 -1
- package/app/src/studio/server.ts +3 -2
- package/app/studio/agent.tsx +4 -2
- package/app/studio/main.tsx +5 -2
- package/app/web/main.ts +7 -1
- package/docs/setup.md +1 -1
- package/docs/studio.md +1 -1
- package/package.json +2 -2
package/app/src/actions/fix.ts
CHANGED
|
@@ -171,6 +171,20 @@ defineAction({
|
|
|
171
171
|
},
|
|
172
172
|
});
|
|
173
173
|
|
|
174
|
+
defineAction({
|
|
175
|
+
name: 'hero_size_set',
|
|
176
|
+
description: 'Say how tall the hero stands in a place, against a thing painted there: times that thing\'s painted height ("about as tall as the candle jar": 1). '
|
|
177
|
+
+ 'Use it when the hero is far too big or small for the place, or differs from place to place in one world (a desk seen whole and a close-up of a mug), '
|
|
178
|
+
+ 'or when only a strip of floor is walkable because the hero would be taller than the picture at the front. Sets the place\'s scale. Kept across rebuilds. No repaint: reruns derive to verify for that place.',
|
|
179
|
+
input: z.object({ slug: Slug, place: z.string().min(1), like: z.string().min(1), times: z.number().positive().max(20) }),
|
|
180
|
+
job: true,
|
|
181
|
+
async run({ slug, place, like, times }, ctx) {
|
|
182
|
+
return fixPlace(ctx, slug, place, 'derive', `hero size ${times} x ${like}`, (c) => {
|
|
183
|
+
c.heroSize = { like, times };
|
|
184
|
+
});
|
|
185
|
+
},
|
|
186
|
+
});
|
|
187
|
+
|
|
174
188
|
defineAction({
|
|
175
189
|
name: 'hero_start_set',
|
|
176
190
|
description: 'Set where the hero begins in a place (painting pixels, on walkable floor; the nearest roomy cell of the main floor is used): '
|
|
@@ -8,6 +8,8 @@ import { soundQuality } from '../sound/cost.ts';
|
|
|
8
8
|
import { writeSoundQuality } from '../sound/settings.ts';
|
|
9
9
|
import { blenderEnv, blenderState, writeBlenderSettings } from '../model/blender.ts';
|
|
10
10
|
import { facesState, writeFacesSettings } from '../scene/faces-settings.ts';
|
|
11
|
+
import { installChromium } from '../commands/browsers.ts';
|
|
12
|
+
import { cli } from '../cli.ts';
|
|
11
13
|
|
|
12
14
|
const audioState = () => ({ quality: soundQuality(), env: process.env.DIORAMA_SOUND ?? null });
|
|
13
15
|
|
|
@@ -87,6 +89,29 @@ defineAction({
|
|
|
87
89
|
},
|
|
88
90
|
});
|
|
89
91
|
|
|
92
|
+
defineAction({
|
|
93
|
+
name: 'setup_cli',
|
|
94
|
+
reads: true,
|
|
95
|
+
description: 'The command that runs diorama in a terminal on this machine (diorama, npx @gtanczyk/diorama or npm run diorama --), for hints to the person.',
|
|
96
|
+
input: z.object({}),
|
|
97
|
+
async run() {
|
|
98
|
+
return { command: cli(), setup: cli('setup') };
|
|
99
|
+
},
|
|
100
|
+
});
|
|
101
|
+
|
|
102
|
+
let downloading: Promise<unknown> | null = null;
|
|
103
|
+
|
|
104
|
+
defineAction({
|
|
105
|
+
name: 'setup_browsers',
|
|
106
|
+
description: 'Download Chromium for Explore, the playthrough bot and sprites (Playwright, ~150 MB, once), then check the machine again. Ask the person first.',
|
|
107
|
+
input: z.object({}),
|
|
108
|
+
async run(_input, ctx) {
|
|
109
|
+
downloading ??= installChromium([], (line) => /\d%/.test(line) && !/100%/.test(line) ? undefined : ctx.events.publish('log', { stream: 'out', line: `chromium: ${line}` })).finally(() => (downloading = null));
|
|
110
|
+
await downloading;
|
|
111
|
+
return runChecks();
|
|
112
|
+
},
|
|
113
|
+
});
|
|
114
|
+
|
|
90
115
|
defineAction({
|
|
91
116
|
name: 'ui_focus',
|
|
92
117
|
description: 'Point the Studio view at a world, a screen or a place, so the person sees what the agent is talking about (when they follow the agent).',
|
package/app/src/cli.ts
ADDED
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
// How to run diorama again from this shell, so every hint (setup fixes, errors, the Studio's Setup cards, the
|
|
2
|
+
// agent's prompt) shows a command that works: `npx @gtanczyk/diorama` when it was started through npx, `diorama`
|
|
3
|
+
// when the global install is on PATH, `npm run diorama --` from a checkout.
|
|
4
|
+
import fs from 'node:fs';
|
|
5
|
+
import path from 'node:path';
|
|
6
|
+
import { fileURLToPath } from 'node:url';
|
|
7
|
+
|
|
8
|
+
const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '../..');
|
|
9
|
+
const BIN = path.join(ROOT, 'app', 'bin', 'diorama.mjs');
|
|
10
|
+
const PACKAGE = '@gtanczyk/diorama';
|
|
11
|
+
|
|
12
|
+
const real = (p: string) => {
|
|
13
|
+
try {
|
|
14
|
+
return fs.realpathSync(p);
|
|
15
|
+
} catch {
|
|
16
|
+
return null;
|
|
17
|
+
}
|
|
18
|
+
};
|
|
19
|
+
|
|
20
|
+
/** A `diorama` on PATH that is this very package (npx and npm scripts put their own node_modules/.bin first: not those). */
|
|
21
|
+
function globalOnPath(): boolean {
|
|
22
|
+
const mine = real(BIN);
|
|
23
|
+
for (const dir of (process.env.PATH ?? '').split(path.delimiter)) {
|
|
24
|
+
if (!dir || dir.split(/[\\/]/).slice(-2).join('/') === 'node_modules/.bin') continue;
|
|
25
|
+
if (real(path.join(dir, 'diorama')) === mine) return true;
|
|
26
|
+
// Windows: npm's global shim sits next to the package's node_modules folder, not as a link.
|
|
27
|
+
if (process.platform === 'win32' && fs.existsSync(path.join(dir, 'diorama.cmd')) && real(path.join(dir, 'node_modules', '@gtanczyk', 'diorama')) === real(ROOT)) return true;
|
|
28
|
+
}
|
|
29
|
+
return false;
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
function detect(): string {
|
|
33
|
+
if (process.env.DIORAMA_CLI) return process.env.DIORAMA_CLI;
|
|
34
|
+
const parts = ROOT.split(/[\\/]/);
|
|
35
|
+
if (parts.includes('_npx')) return `npx ${PACKAGE}`;
|
|
36
|
+
if (globalOnPath()) return 'diorama';
|
|
37
|
+
if (!parts.includes('node_modules') && fs.existsSync(path.join(ROOT, '.git'))) return 'npm run diorama --';
|
|
38
|
+
return `npx ${PACKAGE}`;
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
let prefix: string | null = null;
|
|
42
|
+
|
|
43
|
+
/** The command to type for `diorama <rest>` here, e.g. cli('browsers') → `npx @gtanczyk/diorama browsers`. */
|
|
44
|
+
export function cli(rest = ''): string {
|
|
45
|
+
prefix ??= detect();
|
|
46
|
+
return rest ? `${prefix} ${rest}` : prefix;
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
/** Rewrites the `diorama <command>` a message names (a usage line, an error) into the command that works here. */
|
|
50
|
+
export function cliText(text: string): string {
|
|
51
|
+
if (cli() === 'diorama') return text;
|
|
52
|
+
return text.replace(/(^|usage: |`|run )diorama (?=[a-z])/g, (_, lead: string) => `${lead}${cli()} `);
|
|
53
|
+
}
|
|
@@ -1,21 +1,37 @@
|
|
|
1
1
|
// diorama browsers: downloads Chromium for the Playwright that ships with diorama, so explore, playthrough and
|
|
2
2
|
// sprites work the same from a checkout, a global install or npx (a bare `npx playwright` may fetch another version).
|
|
3
|
+
// `diorama setup` and the Studio's Setup page offer the same download, so nobody has to find this command.
|
|
3
4
|
import { spawn } from 'node:child_process';
|
|
4
5
|
import { createRequire } from 'node:module';
|
|
5
6
|
import path from 'node:path';
|
|
6
7
|
import { defineCommand } from '../registry.ts';
|
|
7
8
|
|
|
9
|
+
/** Runs Playwright's installer for Chromium; `onLine` gets its output (the terminal gets it as is when absent). */
|
|
10
|
+
export async function installChromium(args: string[] = [], onLine?: (line: string) => void): Promise<void> {
|
|
11
|
+
const cli = path.join(path.dirname(createRequire(import.meta.url).resolve('playwright/package.json')), 'cli.js');
|
|
12
|
+
const code = await new Promise<number>((resolve) => {
|
|
13
|
+
// Loaded through -e so argv[1] is not the npx cache path, which makes Playwright warn about missing project dependencies.
|
|
14
|
+
const child = spawn(process.execPath, ['-e', `require(${JSON.stringify(cli)})`, 'diorama-browsers', 'install', 'chromium', ...args], { stdio: onLine ? ['ignore', 'pipe', 'pipe'] : 'inherit' });
|
|
15
|
+
if (onLine) {
|
|
16
|
+
for (const stream of [child.stdout, child.stderr]) {
|
|
17
|
+
let rest = '';
|
|
18
|
+
stream?.setEncoding('utf8').on('data', (chunk: string) => {
|
|
19
|
+
const lines = (rest + chunk).split(/\r?\n|\r/);
|
|
20
|
+
rest = lines.pop() ?? '';
|
|
21
|
+
for (const line of lines) if (line.trim()) onLine(line);
|
|
22
|
+
});
|
|
23
|
+
}
|
|
24
|
+
}
|
|
25
|
+
child.on('error', () => resolve(1));
|
|
26
|
+
child.on('close', (c) => resolve(c ?? 1));
|
|
27
|
+
});
|
|
28
|
+
if (code !== 0) throw new Error(`playwright install chromium exited with ${code}`);
|
|
29
|
+
}
|
|
30
|
+
|
|
8
31
|
defineCommand({
|
|
9
32
|
name: 'browsers',
|
|
10
33
|
usage: 'browsers download Chromium for explore, playthrough and sprites (Playwright, ~150 MB, once)',
|
|
11
34
|
async run(args) {
|
|
12
|
-
|
|
13
|
-
const code = await new Promise<number>((resolve) => {
|
|
14
|
-
// Loaded through -e so argv[1] is not the npx cache path, which makes Playwright warn about missing project dependencies.
|
|
15
|
-
const child = spawn(process.execPath, ['-e', `require(${JSON.stringify(cli)})`, 'diorama-browsers', 'install', 'chromium', ...args], { stdio: 'inherit' });
|
|
16
|
-
child.on('error', () => resolve(1));
|
|
17
|
-
child.on('close', (c) => resolve(c ?? 1));
|
|
18
|
-
});
|
|
19
|
-
if (code !== 0) throw new Error(`playwright install chromium exited with ${code}`);
|
|
35
|
+
await installChromium(args);
|
|
20
36
|
},
|
|
21
37
|
});
|
|
@@ -16,6 +16,7 @@ import { closeFaces, placeDepth } from '../scene/occlusion-check.ts';
|
|
|
16
16
|
import { gridOf } from '../scene/verify.ts';
|
|
17
17
|
import { unionAll } from '../image/raster.ts';
|
|
18
18
|
import { WorldFiles } from '../world/files.ts';
|
|
19
|
+
import { readBodies } from '../scene/cast.ts';
|
|
19
20
|
|
|
20
21
|
type MasksJson = { things: Record<string, Omit<ThingIn, 'key' | 'mask'>> };
|
|
21
22
|
|
|
@@ -54,7 +55,8 @@ async function walkPicture(painting: Buffer, g: Geometry, footprint: ThingIn['ma
|
|
|
54
55
|
// The hero's size at three depths, as little figures spread over the floor (far left, middle, near right):
|
|
55
56
|
// stacked boxes in one column read as a path, not as a person.
|
|
56
57
|
const rows = g.walk.map((row, r) => (row.includes('#') ? r : -1)).filter((r) => r >= 0);
|
|
57
|
-
|
|
58
|
+
// No walkable row at all (NO_FLOOR): no figures to draw; verify reports the missing floor.
|
|
59
|
+
for (const [q, across] of rows.length ? ([[0.1, 0.2], [0.5, 0.5], [0.95, 0.8]] as const) : []) {
|
|
58
60
|
const r = rows[Math.floor(q * (rows.length - 1))];
|
|
59
61
|
const cols = [...g.walk[r]].flatMap((ch, c) => (ch === '#' ? [c] : []));
|
|
60
62
|
// The walkable cell nearest to a quarter, the middle and three quarters of the frame, so they stand apart.
|
|
@@ -94,6 +96,7 @@ defineCommand({
|
|
|
94
96
|
const files = new WorldFiles(slug);
|
|
95
97
|
const world = files.world();
|
|
96
98
|
const places = world.places.filter((p) => only.length === 0 || only.includes(p.key));
|
|
99
|
+
const heroMetres = readBodies(files, fs.existsSync)[world.hero.key]?.height;
|
|
97
100
|
await Promise.all(places.map(async (place) => {
|
|
98
101
|
const at = (...parts: string[]) => files.place(place.key, ...parts);
|
|
99
102
|
// NPCs are drawn by the runtime, not painted: moving one only moves its box (and where the hero talks to it).
|
|
@@ -121,7 +124,7 @@ defineCommand({
|
|
|
121
124
|
console.log(` ${place.key}: no depth model for walls (${e.message})`);
|
|
122
125
|
return undefined;
|
|
123
126
|
});
|
|
124
|
-
const out = await derive(files.memo, { place, layout: asPainted, things, floors, painting, plates, carry, lastScale, previousScale, ...(start && { start }), ...(upright && { upright }), ...(said && { said }) }, (line) => console.log(` ${place.key}: ${line}`));
|
|
127
|
+
const out = await derive(files.memo, { place, layout: asPainted, things, floors, painting, plates, carry, lastScale, previousScale, ...(start && { start }), ...(upright && { upright }), ...(said && { said }), ...(heroMetres && { heroMetres }), ...(readCorrections(at()).heroSize && { heroSize: readCorrections(at()).heroSize }) }, (line) => console.log(` ${place.key}: ${line}`));
|
|
125
128
|
// Shadows only with a light: without one the runtime has nothing to darken by.
|
|
126
129
|
const lit = out.shadows.light && maskArea(out.shadows.mask) > 0;
|
|
127
130
|
if (lit) files.write(at('masks', 'shadows.png'), await maskToPng(out.shadows.mask));
|
|
@@ -9,6 +9,7 @@ import { CROP_JS, PROBE_JS, type ProbeHit } from '../qa/probe.ts';
|
|
|
9
9
|
import { defineCommand, takeOption } from '../registry.ts';
|
|
10
10
|
import { WorldFiles } from '../world/files.ts';
|
|
11
11
|
import { serveWorld } from './play.ts';
|
|
12
|
+
import { cli } from '../cli.ts';
|
|
12
13
|
|
|
13
14
|
export type Browser = { newPage(o?: object): Promise<Page>; close(): Promise<void> };
|
|
14
15
|
export type Page = {
|
|
@@ -26,7 +27,7 @@ export async function chromium(): Promise<{ launch(o?: object): Promise<Browser>
|
|
|
26
27
|
try {
|
|
27
28
|
return (createRequire(import.meta.url)('playwright') as { chromium: never }).chromium;
|
|
28
29
|
} catch {
|
|
29
|
-
throw new Error(
|
|
30
|
+
throw new Error(`this command needs Playwright with Chromium: ${cli('browsers')}`);
|
|
30
31
|
}
|
|
31
32
|
}
|
|
32
33
|
}
|
|
@@ -149,6 +149,14 @@ async function makeModel(files: WorldFiles, key: string, take: number, given: st
|
|
|
149
149
|
try {
|
|
150
150
|
views = await paintViews(files.memo, key, who.looks ?? '', painting, take, creature, (name, png) => files.write(out(`${name}.png`), png));
|
|
151
151
|
} catch (error) {
|
|
152
|
+
// A creature whose sides never showed one body: no model, the painted figure hops as it is (an older model too
|
|
153
|
+
// was made from such sides, so it goes).
|
|
154
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
155
|
+
if (creature && message.startsWith('its 4 sides do not show one creature')) {
|
|
156
|
+
if (fs.existsSync(out('model.json'))) fs.renameSync(out('model.json'), out('model-rejected.json'));
|
|
157
|
+
console.log(`${key}: no 3D model: ${message}; the painted figure hops as it is`);
|
|
158
|
+
return message;
|
|
159
|
+
}
|
|
152
160
|
// Refused even in calm words: the model made before stays (a changed prompt reran it), else the 2D rig walks.
|
|
153
161
|
if (!refusedByFilter(error)) throw error;
|
|
154
162
|
if (fs.existsSync(out('model.json')) && fs.existsSync(out('model.glb'))) {
|
|
@@ -4,7 +4,7 @@ import { createHash } from 'node:crypto';
|
|
|
4
4
|
import fs from 'node:fs';
|
|
5
5
|
import { writeBrief } from '../scene/brief.ts';
|
|
6
6
|
import { seedRan } from '../scene/carry.ts';
|
|
7
|
-
import { addsPerson, critique, paintedPeople, repaint } from '../scene/critic.ts';
|
|
7
|
+
import { addsPerson, castLooks, critique, paintedPeople, repaint } from '../scene/critic.ts';
|
|
8
8
|
import { readCorrections, writeCorrections } from '../scene/corrections.ts';
|
|
9
9
|
import { Layout } from '../scene/layout.ts';
|
|
10
10
|
import { paintPlace } from '../scene/paint.ts';
|
|
@@ -98,7 +98,7 @@ defineCommand({
|
|
|
98
98
|
const again = await critique(files.memo, worded(place, words), brief, first);
|
|
99
99
|
// People are looked for on the painting as the game shows it, with every fix in, not only the first one.
|
|
100
100
|
const shown = fs.existsSync(files.place(place.key, 'painting.png')) ? fs.readFileSync(files.place(place.key, 'painting.png')) : first;
|
|
101
|
-
for (const p of shown === first || place.kind === 'drive' ? [] : await paintedPeople(files.memo, place.key, shown)) {
|
|
101
|
+
for (const p of shown === first || place.kind === 'drive' ? [] : await paintedPeople(files.memo, place.key, shown, castLooks(place))) {
|
|
102
102
|
again.issues.push({ key: 'person', problem: 'unwanted', region: p.box, fix: `Paint out the ${p.what}: continue what is behind them (planks, water, wall), add nothing new` });
|
|
103
103
|
}
|
|
104
104
|
const had = new Set(found.issues.map((i) => JSON.stringify([i.region, i.fix])));
|
|
@@ -2,6 +2,8 @@
|
|
|
2
2
|
import { defineCommand, takeOption } from '../registry.ts';
|
|
3
3
|
import { findVault, NO_VAULT, SECRET_NAMES, SECRETS, SecretStore, type SecretName } from '../secrets.ts';
|
|
4
4
|
import { runChecks, type Check } from '../setup/checks.ts';
|
|
5
|
+
import { installChromium } from './browsers.ts';
|
|
6
|
+
import { cli } from '../cli.ts';
|
|
5
7
|
|
|
6
8
|
const MARK: Record<Check['state'], string> = { ok: ' ok ', missing: ' MISSING ', optional: ' - ', fail: ' FAIL ' };
|
|
7
9
|
|
|
@@ -15,7 +17,7 @@ export function printChecks(checks: Check[]) {
|
|
|
15
17
|
/** Reads a line without echoing it. Only from a terminal: a pasted key never lands in shell history or argv. */
|
|
16
18
|
function askHidden(question: string): Promise<string> {
|
|
17
19
|
const stdin = process.stdin;
|
|
18
|
-
if (!stdin.isTTY) throw new Error('
|
|
20
|
+
if (!stdin.isTTY) throw new Error(`${cli('setup')} reads keys from a terminal only; or set them as environment variables`);
|
|
19
21
|
process.stdout.write(question);
|
|
20
22
|
return new Promise((resolve, reject) => {
|
|
21
23
|
let value = '';
|
|
@@ -42,6 +44,17 @@ function askHidden(question: string): Promise<string> {
|
|
|
42
44
|
});
|
|
43
45
|
}
|
|
44
46
|
|
|
47
|
+
/** A yes/no question in the terminal; Enter means yes. */
|
|
48
|
+
async function askYes(question: string): Promise<boolean> {
|
|
49
|
+
const { createInterface } = await import('node:readline/promises');
|
|
50
|
+
const rl = createInterface({ input: process.stdin, output: process.stdout });
|
|
51
|
+
try {
|
|
52
|
+
return !/^n/i.test((await rl.question(question)).trim());
|
|
53
|
+
} finally {
|
|
54
|
+
rl.close();
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
|
|
45
58
|
async function storeKey(name: SecretName, remove: boolean) {
|
|
46
59
|
const vault = await findVault();
|
|
47
60
|
if (!vault) throw new Error(NO_VAULT);
|
|
@@ -79,6 +92,17 @@ defineCommand({
|
|
|
79
92
|
console.log('');
|
|
80
93
|
printChecks(checks);
|
|
81
94
|
}
|
|
95
|
+
// Chromium is one download: offer it here instead of sending the person off to find another command.
|
|
96
|
+
if (process.stdin.isTTY && checks.some((c) => c.id === 'chromium' && c.state === 'missing') && (await askYes('\nDownload Chromium for Explore and Playthrough now (~150 MB, once)? [Y/n] '))) {
|
|
97
|
+
try {
|
|
98
|
+
await installChromium();
|
|
99
|
+
} catch (error) {
|
|
100
|
+
console.log(`Chromium download failed (${error instanceof Error ? error.message : error}); try again with ${cli('browsers')}`);
|
|
101
|
+
}
|
|
102
|
+
checks = await runChecks({ live });
|
|
103
|
+
console.log('');
|
|
104
|
+
printChecks(checks);
|
|
105
|
+
}
|
|
82
106
|
const blocking = checks.filter((c) => c.state === 'missing' || c.state === 'fail');
|
|
83
107
|
console.log(blocking.length ? `\n${blocking.length} to fix before a full build: ${blocking.map((c) => c.label).join(', ')}` : '\nready to build');
|
|
84
108
|
},
|
|
@@ -58,6 +58,7 @@ function describe(e: StudioEvent, verbose: boolean): string | null {
|
|
|
58
58
|
case 'agent.message': return `agent: ${String(e.text).slice(0, 200)}`;
|
|
59
59
|
case 'agent.turn.end': return e.status === 'completed' ? null : `agent stopped: ${e.error ?? e.status}`;
|
|
60
60
|
case 'agent.error': return `agent error: ${e.text}`;
|
|
61
|
+
case 'agent.limit': return `agent ${e.text}`;
|
|
61
62
|
case 'agent.agent': return `agent is now ${e.label}`;
|
|
62
63
|
case 'log': return verbose || e.stream === 'err' ? ` ${e.line}` : null;
|
|
63
64
|
default: return null;
|
|
@@ -14,7 +14,7 @@ import { resumePendingVoices } from '../sound/voices.ts';
|
|
|
14
14
|
|
|
15
15
|
defineCommand({
|
|
16
16
|
name: 'voices',
|
|
17
|
-
usage: 'voices <slug> cast a voice per character (stock, or designed by ElevenLabs Voice Design for children, teens and the old) and speak every story line, checked by Muse Voice Transcribe;
|
|
17
|
+
usage: 'voices <slug> cast a voice per character (stock, or designed by ElevenLabs Voice Design for children, teens and the old) and speak every story line, checked by Muse Voice Transcribe; every line the hero and the others say, intro and endings [--essential: only conversations, intro and endings, not the hero\'s looks, uses and locked doors] [--recast] [--draft: Flash and 2 takes; default final uses Multilingual v2 and up to 3 takes]',
|
|
18
18
|
async run(args) {
|
|
19
19
|
// Keep this generation at one quality even if Setup changes while it runs.
|
|
20
20
|
process.env.DIORAMA_SOUND = soundQuality(args);
|
|
@@ -27,7 +27,9 @@ defineCommand({
|
|
|
27
27
|
if (!elevenKey()) return void console.log('no ELEVENLABS_API_KEY: lines stay unspoken');
|
|
28
28
|
const story = Story.parse(files.readJson(files.path('story.json')));
|
|
29
29
|
const guard = await creditGuard(elevenKey(), 'voices');
|
|
30
|
-
|
|
30
|
+
// Every line by default: a hero who answers out loud in conversations and is silent looking at things reads as broken.
|
|
31
|
+
// --essential keeps to conversations, intro and endings (a world for kids always gets every line).
|
|
32
|
+
const lines = storyLines(world, story, !args.includes('--essential') || world.audience === 'kids');
|
|
31
33
|
const outFile = files.path('voices.json');
|
|
32
34
|
const old = fs.existsSync(outFile) ? Voices.parse(files.readJson(outFile)) : undefined;
|
|
33
35
|
let cast: Cast;
|
package/app/src/main.ts
CHANGED
|
@@ -1,11 +1,12 @@
|
|
|
1
1
|
import './commands/index.ts';
|
|
2
2
|
import { registry } from './registry.ts';
|
|
3
3
|
import { loadSecrets, redact } from './secrets.ts';
|
|
4
|
+
import { cli, cliText } from './cli.ts';
|
|
4
5
|
|
|
5
6
|
function help() {
|
|
6
|
-
console.log(
|
|
7
|
+
console.log(`diorama: a story prompt becomes painted scenes you can walk in\n\nusage: ${cli('<command> [args]')}\n`);
|
|
7
8
|
for (const cmd of registry.values()) console.log(` ${cmd.usage}`);
|
|
8
|
-
console.log(
|
|
9
|
+
console.log(`\nfirst time: ${cli('setup')} (keys, what is missing), then ${cli('studio')}. Worlds are written to ./worlds here.`);
|
|
9
10
|
}
|
|
10
11
|
|
|
11
12
|
const [name, ...args] = process.argv.slice(2);
|
|
@@ -15,7 +16,7 @@ if (!cmd) {
|
|
|
15
16
|
process.exit(!name || name === 'help' ? 0 : 2);
|
|
16
17
|
}
|
|
17
18
|
if (args.includes('--help') || args.includes('-h')) {
|
|
18
|
-
console.log(`usage:
|
|
19
|
+
console.log(`usage: ${cli(cmd.usage)}`);
|
|
19
20
|
process.exit(0);
|
|
20
21
|
}
|
|
21
22
|
try {
|
|
@@ -23,6 +24,6 @@ try {
|
|
|
23
24
|
if (name !== 'setup') await loadSecrets();
|
|
24
25
|
await cmd.run(args);
|
|
25
26
|
} catch (error) {
|
|
26
|
-
console.error(redact(`diorama ${name}: ${error instanceof Error ? error.message : error}`));
|
|
27
|
+
console.error(redact(`diorama ${name}: ${cliText(error instanceof Error ? error.message : String(error))}`));
|
|
27
28
|
process.exit(1);
|
|
28
29
|
}
|
package/app/src/model/views.ts
CHANGED
|
@@ -158,6 +158,45 @@ async function paintCreatureViews(memo: Memo, key: string, looks: string, painti
|
|
|
158
158
|
label: `${key}-view-${facing}`, size: SIZE, take, images: [front, back],
|
|
159
159
|
prompt: `The same creature as in both pictures (front and back views), same colours and painted style, now seen exactly from the side in profile, facing to the ${facing} of the picture (${looks}), ${REST}. Same size and position in the frame as the pictures. ${PLAIN}`,
|
|
160
160
|
}).then((png) => (onView?.(facing, png), png));
|
|
161
|
-
const
|
|
162
|
-
|
|
161
|
+
const views: Record<ViewName, Buffer> = { front, back, left: front, right: front };
|
|
162
|
+
[views.left, views.right] = await Promise.all([side('left'), side('right')]);
|
|
163
|
+
// Muse can leave out or add a part between views (a mug drawn standing on a bottle loses the bottle in front and
|
|
164
|
+
// keeps it from the side): TRELLIS cannot make one body of that and the model comes out torn. Views that do not
|
|
165
|
+
// show the same creature as the front are painted again; still not one creature after that, there is no model.
|
|
166
|
+
for (let round = 0; round < CREATURE_ROUNDS; round++) {
|
|
167
|
+
const wrong = await checkCreature(memo, key, views, round);
|
|
168
|
+
const redo = Object.keys(wrong) as ViewName[];
|
|
169
|
+
if (!redo.length) return views;
|
|
170
|
+
console.log(` ${key}: ${redo.map((v) => `${v} view: ${wrong[v]}`).join('; ')}; painting ${redo.length === 1 ? 'it' : 'them'} again`);
|
|
171
|
+
const t = take + 1 + round;
|
|
172
|
+
if (wrong.back) (views.back = await edit({
|
|
173
|
+
label: `${key}-view-back`, size: SIZE, take: t, images: [front],
|
|
174
|
+
prompt: `The same creature as in the picture (${looks}), with exactly the same parts and nothing more or less, same colours and painted style, in exactly the same pose, seen straight from behind: we see its back, its face is not visible. Same size and position in the frame as the picture. ${PLAIN}`,
|
|
175
|
+
}), onView?.('back', views.back));
|
|
176
|
+
await Promise.all((['left', 'right'] as const).filter((f) => wrong[f]).map(async (f) => {
|
|
177
|
+
views[f] = await edit({
|
|
178
|
+
label: `${key}-view-${f}`, size: SIZE, take: t, images: [front, views.back],
|
|
179
|
+
prompt: `The same creature as in both pictures (front and back views), with exactly the same parts as the front and nothing more or less, same colours and painted style, now seen exactly from the side in profile, facing to the ${f} of the picture (${looks}), ${REST}. Same size and position in the frame as the pictures. ${PLAIN}`,
|
|
180
|
+
});
|
|
181
|
+
onView?.(f, views[f]);
|
|
182
|
+
}));
|
|
183
|
+
}
|
|
184
|
+
const still = await checkCreature(memo, key, views, CREATURE_ROUNDS);
|
|
185
|
+
if (Object.keys(still).length) throw new Error(`its 4 sides do not show one creature (${Object.entries(still).map(([v, why]) => `${v}: ${why}`).join('; ')})`);
|
|
186
|
+
return views;
|
|
187
|
+
}
|
|
188
|
+
|
|
189
|
+
/** Rounds of repainting creature views that do not match the front. */
|
|
190
|
+
const CREATURE_ROUNDS = 2;
|
|
191
|
+
|
|
192
|
+
/** Which of the back and side views show a different creature from the front: a part left out or added (what it stands on, legs, a handle), another shape. */
|
|
193
|
+
export async function checkCreature(memo: Memo, key: string, views: Record<ViewName, Buffer>, round = 0): Promise<Partial<Record<ViewName, string>>> {
|
|
194
|
+
const names: ViewName[] = ['front', 'back', 'left', 'right'];
|
|
195
|
+
const a = await askSparkJson<Record<string, { ok?: unknown; why?: unknown }>>(memo, {
|
|
196
|
+
label: `views-creature-check:${key}#${round}`, images: await Promise.all(names.map((v) => small(views[v]))),
|
|
197
|
+
prompt: 'Four pictures of one creature or living object, meant to be the same body seen from the front, from behind, from its left side and from its right side, so a 3D model can be built from them. '
|
|
198
|
+
+ 'Compare pictures 2, 3 and 4 with picture 1 (the front). One is wrong when it shows a part the front does not, or lacks one the front has (something it stands on or in, a base, a bottle, a stand, legs, arms, a tail, a handle, a hat), or a clearly different body shape or size. Colours, light and the turned view itself are fine. '
|
|
199
|
+
+ `Reply with JSON only: {${names.slice(1).map((v) => `"${v}": {"ok": true or false, "why": "a few words"}`).join(', ')}}`,
|
|
200
|
+
});
|
|
201
|
+
return Object.fromEntries(names.slice(1).flatMap((v) => (a[v]?.ok === false ? [[v, String(a[v]?.why ?? 'not the same creature as the front')]] : [])));
|
|
163
202
|
}
|
|
@@ -19,8 +19,10 @@ import { pasteInside } from './plates.ts';
|
|
|
19
19
|
export const BACKDROP_GROW = 14;
|
|
20
20
|
|
|
21
21
|
/**
|
|
22
|
-
* Fills the masked pixels from their surroundings: a pyramid of averages of the known pixels,
|
|
23
|
-
* read
|
|
22
|
+
* Fills the masked pixels from their surroundings: a pyramid of averages of the known pixels (pull), then each
|
|
23
|
+
* level's empty cells read from the next coarser level, bilinearly (push), down to the hole's pixels. Smooth,
|
|
24
|
+
* never sharp. Read back nearest-cell, a big hole came out in square blocks of the coarse levels (64 px and more),
|
|
25
|
+
* which a tone correction carried into the backdrop as square patches round a lamp's light.
|
|
24
26
|
* With `known`, only those pixels feed the fill.
|
|
25
27
|
*/
|
|
26
28
|
export function smearFill(rgb: Uint8Array, W: number, H: number, hole: Mask, known?: Mask): Uint8Array {
|
|
@@ -46,18 +48,38 @@ export function smearFill(rgb: Uint8Array, W: number, H: number, hole: Mask, kno
|
|
|
46
48
|
}
|
|
47
49
|
levels.push(next);
|
|
48
50
|
}
|
|
51
|
+
// Each level as colours: its own average where it has known pixels, else read from the level above it.
|
|
52
|
+
const colours = levels.map((l) => new Float32Array(l.w * l.h * 3));
|
|
53
|
+
const filled = levels.map((l) => Uint8Array.from(l.weight, (v) => (v > 0 ? 1 : 0)));
|
|
54
|
+
const sample = (k: number, fx: number, fy: number, out: Float32Array, at: number) => {
|
|
55
|
+
const l = levels[k];
|
|
56
|
+
const col = colours[k];
|
|
57
|
+
const x = Math.max(0, Math.min(l.w - 1, fx));
|
|
58
|
+
const y = Math.max(0, Math.min(l.h - 1, fy));
|
|
59
|
+
const [x0, y0] = [Math.floor(x), Math.floor(y)];
|
|
60
|
+
const [x1, y1] = [Math.min(l.w - 1, x0 + 1), Math.min(l.h - 1, y0 + 1)];
|
|
61
|
+
const [ax, ay] = [x - x0, y - y0];
|
|
62
|
+
for (let c = 0; c < 3; c++) {
|
|
63
|
+
const top = col[(y0 * l.w + x0) * 3 + c] * (1 - ax) + col[(y0 * l.w + x1) * 3 + c] * ax;
|
|
64
|
+
const bottom = col[(y1 * l.w + x0) * 3 + c] * (1 - ax) + col[(y1 * l.w + x1) * 3 + c] * ax;
|
|
65
|
+
out[at + c] = top * (1 - ay) + bottom * ay;
|
|
66
|
+
}
|
|
67
|
+
};
|
|
68
|
+
for (let k = levels.length - 1; k >= 1; k--) {
|
|
69
|
+
const l = levels[k];
|
|
70
|
+
for (let j = 0; j < l.w * l.h; j++) {
|
|
71
|
+
if (filled[k][j]) for (let c = 0; c < 3; c++) colours[k][j * 3 + c] = l.sum[j * 3 + c] / l.weight[j];
|
|
72
|
+
else if (k < levels.length - 1) sample(k + 1, ((j % l.w) + 0.5) / 2 - 0.5, (Math.floor(j / l.w) + 0.5) / 2 - 0.5, colours[k], j * 3);
|
|
73
|
+
}
|
|
74
|
+
}
|
|
49
75
|
const out = rgb.slice();
|
|
76
|
+
if (levels.length === 1) return out;
|
|
77
|
+
const px = new Float32Array(3);
|
|
50
78
|
for (let i = 0; i < W * H; i++) {
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
if (l.weight[j] > 0) {
|
|
56
|
-
for (let c = 0; c < 3; c++) out[i * 3 + c] = l.sum[j * 3 + c] / l.weight[j];
|
|
57
|
-
break;
|
|
58
|
-
}
|
|
59
|
-
[x, y] = [x >> 1, y >> 1];
|
|
60
|
-
}
|
|
79
|
+
// A known pixel in the hole (with `known`) keeps its own colour.
|
|
80
|
+
if (!hole.bits[i] || base.weight[i] > 0) continue;
|
|
81
|
+
sample(1, ((i % W) + 0.5) / 2 - 0.5, (Math.floor(i / W) + 0.5) / 2 - 0.5, px, 0);
|
|
82
|
+
for (let c = 0; c < 3; c++) out[i * 3 + c] = Math.round(px[c]);
|
|
61
83
|
}
|
|
62
84
|
return out;
|
|
63
85
|
}
|
|
@@ -67,6 +67,12 @@ export const Corrections = z.object({
|
|
|
67
67
|
take: z.number().int().min(0).optional(),
|
|
68
68
|
/** Where the hero begins in this place (hero_start_set), instead of the front middle of the biggest floor. */
|
|
69
69
|
start: XY.optional(),
|
|
70
|
+
/**
|
|
71
|
+
* How tall the hero stands here, against a thing painted in this place (hero_size_set): `times` that thing's painted
|
|
72
|
+
* height. The place's scale is set from it, over Spark's guess of a person. One world's places can come out ten
|
|
73
|
+
* times apart (a desk seen whole, people-sized, and a close-up of a mug, props-sized), and the hero with them.
|
|
74
|
+
*/
|
|
75
|
+
heroSize: z.object({ like: z.string().min(1), times: z.number().positive().max(20) }).optional(),
|
|
70
76
|
/** This place's own painting style over world.json's (style_set with scope place). */
|
|
71
77
|
style: z.string().min(10).optional(),
|
|
72
78
|
});
|
package/app/src/scene/critic.ts
CHANGED
|
@@ -43,7 +43,7 @@ Reply with JSON only: {"issues": [{"key": "...", "problem": "missing | wrong | f
|
|
|
43
43
|
// The critic weighs the whole picture and lets a figure pass when the place's words name one ("a child floats
|
|
44
44
|
// candle boats"); people are asked about on their own, as every one of them is a double or a stranger.
|
|
45
45
|
// A drive's hero is painted at the wheel on purpose: no one is looked for there.
|
|
46
|
-
const people = place.kind === 'drive' ? [] : await paintedPeople(memo, place.key, painting);
|
|
46
|
+
const people = place.kind === 'drive' ? [] : await paintedPeople(memo, place.key, painting, castLooks(place));
|
|
47
47
|
const near = (a: number[], b: number[]) => Math.abs(a[0] - b[0]) + Math.abs(a[1] - b[1]) < 120;
|
|
48
48
|
const extra = people.filter((p) => !found.issues.some((i) => i.problem === 'unwanted' && near(i.region, p.box))).map((p) => ({
|
|
49
49
|
key: 'person', problem: 'unwanted' as const, region: pad(p.box, 24),
|
|
@@ -52,6 +52,13 @@ Reply with JSON only: {"issues": [{"key": "...", "problem": "missing | wrong | f
|
|
|
52
52
|
return { issues: [...found.issues, ...extra].slice(0, 6) };
|
|
53
53
|
}
|
|
54
54
|
|
|
55
|
+
/**
|
|
56
|
+
* How the place's NPCs look. A drawn world's NPC is often a thing (a steaming mug, a candle in a jar) that the place's
|
|
57
|
+
* words also name as part of the scene, so the painter paints it in and the critic, looking for people, lets it pass:
|
|
58
|
+
* the game then draws its figure beside the painted one, a double. They are looked for by their looks too.
|
|
59
|
+
*/
|
|
60
|
+
export const castLooks = (place: Place) => place.things.filter((t) => t.role === 'npc').map((t) => t.looks ?? t.key.replace(/-/g, ' '));
|
|
61
|
+
|
|
55
62
|
const PERSON = /\b(child|children|kid|boy|girl|man|woman|men|women|person|people|figure|keeper|seller|vendor|fisherman|sailor|someone|crowd|silhouette)s?\b/i;
|
|
56
63
|
|
|
57
64
|
/**
|
|
@@ -63,9 +70,12 @@ export const addsPerson = (i: { problem: string; fix: string }) => i.problem !==
|
|
|
63
70
|
const People = z.object({ people: z.array(z.object({ what: z.string(), box: z.tuple([z.number(), z.number(), z.number(), z.number()]) })).max(6) });
|
|
64
71
|
|
|
65
72
|
/** Every person or figure painted in, with a box: none belongs in a background, the game adds its people itself. */
|
|
66
|
-
export async function paintedPeople(memo: Memo, key: string, painting: Buffer): Promise<Array<{ what: string; box: [number, number, number, number] }>> {
|
|
73
|
+
export async function paintedPeople(memo: Memo, key: string, painting: Buffer, cast: string[] = []): Promise<Array<{ what: string; box: [number, number, number, number] }>> {
|
|
74
|
+
const others = cast.length
|
|
75
|
+
? `\nAlso list each of these characters if it is painted in, however it is drawn (as an object, an animal or a creature, standing on or inside something else): ${cast.join('; ')}. The game draws them over the picture itself, so a painted one would be a double. Box only the character, not what it stands on.`
|
|
76
|
+
: '';
|
|
67
77
|
const prompt = `This is a painted background (${FRAME.width}x${FRAME.height} px) for a 2D adventure. It must show no people at all.
|
|
68
|
-
List every person or human figure painted in it: adults, children, sitting, crouching or far away, a silhouette in a window or a doorway. Not statues, scarecrows, posters or pictures of people
|
|
78
|
+
List every person or human figure painted in it: adults, children, sitting, crouching or far away, a silhouette in a window or a doorway. Not statues, scarecrows, posters or pictures of people.${others}
|
|
69
79
|
For each give what it is in a few words ("a crouching child in a grey hoodie") and its box [x0, y0, x1, y1] in pixels, tight around the figure.
|
|
70
80
|
Reply with JSON only: {"people": [{"what": "...", "box": [0, 0, 0, 0]}]}`;
|
|
71
81
|
try {
|
package/app/src/scene/derive.ts
CHANGED
|
@@ -33,10 +33,15 @@ export type DeriveIn = { place: Place; layout: Layout; things: ThingIn[]; floors
|
|
|
33
33
|
* Floor someone said is floor: brush strokes that add to a floor (the person's, the agent's, the free bridge's)
|
|
34
34
|
* and floor cells the person added. It is walked on whatever thing's mask lies over it.
|
|
35
35
|
*/
|
|
36
|
-
said?: Mask
|
|
36
|
+
said?: Mask;
|
|
37
|
+
/** How tall the hero stands (cast/figures.json), in metres; the scale is for a grown-up. Absent: a grown-up. */
|
|
38
|
+
heroMetres?: number;
|
|
39
|
+
/** The hero as tall as `times` this thing's painted height (corrections heroSize): sets the scale. */
|
|
40
|
+
heroSize?: { like: string; times: number } };
|
|
37
41
|
export type Derived = { geometry: Geometry; grid: Grid; blobs: Blob[]; footprint: Mask; probes: Sample[]; shadows: Shadows };
|
|
38
42
|
|
|
39
43
|
const { width: W, height: H } = FRAME;
|
|
44
|
+
const round3 = (v: number) => Math.round(v * 1000) / 1000;
|
|
40
45
|
/** The tallest the hero may be: three quarters of the picture. */
|
|
41
46
|
export const NEAR_MAX_PX = Math.round(0.75 * H);
|
|
42
47
|
/** How far (px) a layout light's point may lie outside the thing it belongs to: a glow drawn just off the lamp. */
|
|
@@ -362,14 +367,28 @@ export async function derive(memo: Memo, input: DeriveIn, log: (line: string) =>
|
|
|
362
367
|
scale = { horizonY: Math.min(layout.horizon, topY - 40), pxPerY: 0.5, source: 'layout' };
|
|
363
368
|
flag('scale: guessed from the layout horizon');
|
|
364
369
|
}
|
|
370
|
+
// The person said how tall the hero is here, against a painted thing: the scale follows it (the horizon stays).
|
|
371
|
+
const heroMetres = input.heroMetres ?? PERSON_M;
|
|
372
|
+
const like = input.heroSize && things.find((t) => t.key === input.heroSize!.like);
|
|
373
|
+
const likeBox = like ? maskBox(like.mask) : undefined;
|
|
374
|
+
if (input.heroSize && !likeBox) flag(`hero size: no thing ${input.heroSize.like} with a mask here; the scale stays`);
|
|
375
|
+
if (input.heroSize && likeBox) {
|
|
376
|
+
const footY = likeBox[3];
|
|
377
|
+
const horizonY = scale.horizonY < footY - 20 ? scale.horizonY : Math.min(layout.horizon, footY - 40, topY - 40);
|
|
378
|
+
const personAt = (input.heroSize.times * (likeBox[3] - likeBox[1]) * PERSON_M) / heroMetres;
|
|
379
|
+
scale = { horizonY, pxPerY: round3(personAt / Math.max(1, footY - horizonY)), source: scale.source };
|
|
380
|
+
log(`scale from the hero's size: ${input.heroSize.times} x ${input.heroSize.like} (${Math.round((likeBox[3] - likeBox[1]) * input.heroSize.times)} px at y ${footY})`);
|
|
381
|
+
}
|
|
365
382
|
|
|
366
383
|
// The frame gate: no standing where a good part of the hero would be outside the picture.
|
|
367
384
|
const sized = scale;
|
|
385
|
+
// The gates are for the hero as tall as it is: a small one (a figurine among props) fits where a grown-up would not.
|
|
386
|
+
const heroShare = Math.min(1, heroMetres / PERSON_M);
|
|
368
387
|
let framed = 0;
|
|
369
388
|
grid.open.forEach((v, i) => {
|
|
370
389
|
if (!v) return;
|
|
371
390
|
const [x, y] = grid.centre(i);
|
|
372
|
-
const h = Math.max(8, sized.pxPerY * (y - sized.horizonY));
|
|
391
|
+
const h = Math.max(8, heroShare * sized.pxPerY * (y - sized.horizonY));
|
|
373
392
|
if (x < 0.1 * h || x > W - 0.1 * h || y < 0.7 * h) {
|
|
374
393
|
grid.open[i] = 0;
|
|
375
394
|
shut[i] = 1;
|
|
@@ -378,7 +397,7 @@ export async function derive(memo: Memo, input: DeriveIn, log: (line: string) =>
|
|
|
378
397
|
});
|
|
379
398
|
// The near gate: in a close-up interior the front rows sit right under the camera, where the hero would be
|
|
380
399
|
// taller than the picture (a bakery at 942 px). Those rows are foreground to look past, not floor to stand on.
|
|
381
|
-
const tooNear = [...grid.open.keys()].filter((i) => grid.open[i] && sized.pxPerY * (grid.centre(i)[1] - sized.horizonY) > NEAR_MAX_PX);
|
|
400
|
+
const tooNear = [...grid.open.keys()].filter((i) => grid.open[i] && heroShare * sized.pxPerY * (grid.centre(i)[1] - sized.horizonY) > NEAR_MAX_PX);
|
|
382
401
|
if (tooNear.length && grid.count() - tooNear.length >= 200) {
|
|
383
402
|
tooNear.forEach((i) => (grid.open[i] = 0, shut[i] = 1));
|
|
384
403
|
framed += tooNear.length;
|
package/app/src/secrets.ts
CHANGED
|
@@ -7,6 +7,7 @@ import { createCipheriv, createDecipheriv, randomBytes } from 'node:crypto';
|
|
|
7
7
|
import fs from 'node:fs';
|
|
8
8
|
import os from 'node:os';
|
|
9
9
|
import path from 'node:path';
|
|
10
|
+
import { cli } from './cli.ts';
|
|
10
11
|
|
|
11
12
|
/** The keys diorama knows how to keep, with the other names each one is accepted under. */
|
|
12
13
|
export const SECRETS = {
|
|
@@ -138,7 +139,7 @@ export class SecretStore {
|
|
|
138
139
|
async read(): Promise<Record<string, string>> {
|
|
139
140
|
if (!fs.existsSync(this.file)) return {};
|
|
140
141
|
const key = await this.vault.get();
|
|
141
|
-
if (!key) throw new Error(`credentials.bin exists but its key is missing from the ${this.vault.name}; run
|
|
142
|
+
if (!key) throw new Error(`credentials.bin exists but its key is missing from the ${this.vault.name}; run ${cli('setup')} to store your keys again`);
|
|
142
143
|
return open(key, fs.readFileSync(this.file));
|
|
143
144
|
}
|
|
144
145
|
|
package/app/src/setup/checks.ts
CHANGED
|
@@ -5,6 +5,7 @@ import fs from 'node:fs';
|
|
|
5
5
|
import path from 'node:path';
|
|
6
6
|
import { blenderState } from '../model/blender.ts';
|
|
7
7
|
import { envValue, findVault, loadSecrets, NO_VAULT, type SecretName } from '../secrets.ts';
|
|
8
|
+
import { cli } from '../cli.ts';
|
|
8
9
|
|
|
9
10
|
export type Check = {
|
|
10
11
|
id: string;
|
|
@@ -22,7 +23,7 @@ function sh(cmd: string, args: string[], timeout = 8000): Promise<string | null>
|
|
|
22
23
|
|
|
23
24
|
async function keyCheck(name: SecretName, label: string, required: boolean, what: string, source?: 'env' | 'store'): Promise<Check> {
|
|
24
25
|
const value = envValue(name);
|
|
25
|
-
if (!value) return { id: name, label, state: required ? 'missing' : 'optional', detail: what, fix: [`
|
|
26
|
+
if (!value) return { id: name, label, state: required ? 'missing' : 'optional', detail: what, fix: [cli(`setup --key ${name}`)] };
|
|
26
27
|
return { id: name, label, state: 'ok', detail: source === 'store' ? 'in your system keychain' : 'from the environment' };
|
|
27
28
|
}
|
|
28
29
|
|
|
@@ -78,7 +79,7 @@ export async function runChecks(opts: { live?: boolean } = {}): Promise<Check[]>
|
|
|
78
79
|
} catch {}
|
|
79
80
|
checks.push(chromium && fs.existsSync(chromium)
|
|
80
81
|
? { id: 'chromium', label: 'Chromium for Explore and Playthrough', state: 'ok', detail: 'installed' }
|
|
81
|
-
: { id: 'chromium', label: 'Chromium for Explore and Playthrough', state: 'missing', detail: 'Explore and the playthrough bot drive a browser.', fix: ['
|
|
82
|
+
: { id: 'chromium', label: 'Chromium for Explore and Playthrough', state: 'missing', detail: 'Explore and the playthrough bot drive a browser.', fix: [cli('browsers')] });
|
|
82
83
|
|
|
83
84
|
checks.push(await keyCheck('ELEVENLABS_API_KEY', 'ElevenLabs key', false, 'Optional: sound effects, music and voices. Without it the world plays silent.', sources.ELEVENLABS_API_KEY));
|
|
84
85
|
checks.push(await keyCheck('HF_TOKEN', 'Hugging Face token', false, 'Optional: 3D hero figures (TRELLIS, auto-rig) on ZeroGPU Spaces.', sources.HF_TOKEN));
|
|
@@ -103,8 +104,8 @@ export async function runChecks(opts: { live?: boolean } = {}): Promise<Check[]>
|
|
|
103
104
|
|
|
104
105
|
const remote = await sh('git', ['remote', 'get-url', 'origin']);
|
|
105
106
|
checks.push(remote
|
|
106
|
-
? { id: 'publish', label: 'Publishing', state: 'ok', detail: `git remote ${remote.replace(/\/\/[^@/]+@/, '//')};
|
|
107
|
-
: { id: 'publish', label: 'Publishing', state: 'optional', detail:
|
|
107
|
+
? { id: 'publish', label: 'Publishing', state: 'ok', detail: `git remote ${remote.replace(/\/\/[^@/]+@/, '//')}; ${cli('publish <slug>')} pushes to gh-pages` }
|
|
108
|
+
: { id: 'publish', label: 'Publishing', state: 'optional', detail: `No git remote: ${cli('export <slug> --zip')} still makes an itch.io upload.` });
|
|
108
109
|
|
|
109
110
|
return checks;
|
|
110
111
|
}
|
package/app/src/studio/agent.ts
CHANGED
|
@@ -5,11 +5,12 @@
|
|
|
5
5
|
import { randomBytes } from 'node:crypto';
|
|
6
6
|
import fs from 'node:fs';
|
|
7
7
|
import path from 'node:path';
|
|
8
|
-
import { antigravity, claude, codexLive, createAgentSession, detectAgents, museLive, type CodingAgent, withoutProviderCredentials, type AgentSession, type ApprovalDecision, type ApprovalRequest, type SessionState, type SessionTurn } from 'genaicode/agents';
|
|
8
|
+
import { antigravity, claude, codexLive, createAgentSession, detectAgents, museLive, type CodingAgent, withoutProviderCredentials, type AgentSession, type ApprovalDecision, type ApprovalRequest, type SessionState, type SessionTurn, type AgentBudget, type RateLimit } from 'genaicode/agents';
|
|
9
9
|
import { configDir, redact, withoutSecrets } from '../secrets.ts';
|
|
10
10
|
import type { Events } from './events.ts';
|
|
11
11
|
import { History } from './history.ts';
|
|
12
12
|
import { NPC_STORY_GUIDANCE } from '../story/npc-guidance.ts';
|
|
13
|
+
import { cli } from '../cli.ts';
|
|
13
14
|
|
|
14
15
|
export const AGENT_BRIEF = (slug: string) => `You are the coding agent inside Diorama Studio, working on the world "${slug}" with the person.
|
|
15
16
|
Use only the diorama MCP tools: they are the same actions as the Studio's buttons, and the person watches each call live.
|
|
@@ -26,7 +27,7 @@ When a job seems slow or the person asks what it is doing, call job_inspect for
|
|
|
26
27
|
To fix a place use the fix tools, which are kept across rebuilds: mask_fix (a better SAM phrase, the real box, or the kind of a thing), floor_cells, mask_brush, stand_set, repaint_region (costs a Muse edit), paint_words (how the place, its things and floors are described to the painter); corrections_get shows what is fixed already, corrections_clear takes fixes back. When asked to go through the person's notes, read feedback_pending, fix each one and close it with feedback_done in one sentence. Check budget before a rebuild of many places.
|
|
27
28
|
When the person wants the game in another language ("make it English", "po polsku"), call world_language: it translates the title, names and the whole story in one pass and casts and records the voices again as native speakers; first without go, tell them what it redoes and what the voices cost, then call it with go true once they agree (voices false when they only want the text now). Never translate it line by line with story_text, and never say the language cannot be changed.
|
|
28
29
|
When the person wants the game to look different (more realistic, watercolour, like a picture book, less cartoon), call style_set with the style in a few words (medium, palette, light; scope place or cast for only one place or the characters): first without go, tell them what it repaints and about what it costs, then call it with go true once they agree. It repaints each picture keeping where everything is, so masks and their fixes still fit; children's drawings stay the reference; history_undo takes it back. Never write world.json's style by hand.
|
|
29
|
-
Before a risky change, say what you will do; history_undo can take back a job. Respect locks. Keys are never yours: if a key is missing, ask the person to run
|
|
30
|
+
Before a risky change, say what you will do; history_undo can take back a job. Respect locks. Keys are never yours: if a key is missing, ask the person to run \`${cli('setup')}\` in a terminal.
|
|
30
31
|
Read files freely to find out why something failed, but do not edit world files or run commands that change them: make every change through the diorama MCP tools, so the person sees it live, can undo it, and it survives the next build (a hand edit is overwritten by the next build). When no single tool makes the change the person asks for, do not stop at "there is no tool": find the nearest way with the tools you have (several story_text calls, a story_redo with their wish, a run_step) and offer it with what it costs, using ask_person; mention the missing tool only as a side note, so it can be added later. Refuse only what would need a hand edit or would throw away their work. Start from world_status, place_status, place_view, job_wait, corrections_get and feedback_pending: place_view shows you a place as a picture (layer walk, masks or painting), which files and the shell cannot. Reach for the shell only when no Studio tool answers the question.
|
|
31
32
|
When a build fails for another reason (a step that broke, a service that did not answer), the Studio wakes you with the error and the job's last lines: job_inspect it, give a service error (HTTP 5xx, a timeout, a busy Space) one job_retry, which picks up at the failed step, and fix anything else with that step's own tool; never retry the same error twice, tell the person instead. A character the 3D service left in 2D (cast_status: the 3D service did not answer) still plays with its 2D rig; cast_animate tries its model again once the service is back. When a build leaves places not playable, repair them one by one: place_status lists each error with its playbook (where to look, the fix tools in order, the last resort), and place_view shows you the walkable floor with the 60 px cells floor_cells takes, the gaps to bridge and the spots off the floor. An unreached patch where nothing is floor (a wall, the sky) is taken back, not bridged: corrections_clear the fix that made it, or mask_brush erase. Try the tools in that order; each fix rebuilds only that place, so read place_status again after its job ends. After three fixes that did not change the error, do not repeat one that failed: look again with place_view and take a different path (mask_brush by hand, floor_cells, thing_remove, corrections_clear); the fix tools refuse an exact repeat, and after six they keep repaints and relayouts for the person. If the new path fails too, tell the person briefly what you tried and what you would do next. Ask before the last resort, except one repaint_region per place.
|
|
32
33
|
When layout failed and there is no picture, do not wait for place_view: inspect place_status errors, then call layout_fix with concrete layout guidance, or place_fix to resume. Both work before the place directory exists. Failed layout batches retain their diagnostics and retry with new feedback instead of replaying the same rejected batch. When paint fails with content_policy_violation, the image safety filter refused a word, not the picture: paint already lets Spark reword the place twice, so call paint_words with calm, plain words for the scene and the looks the brief had (leafless, not bare; a seat that rests, not hangs on chains; no children, blood or weapons), never an edit of world.json. A place that never got past a step is rebuilt from that step: place_fix (the Fix it button on its card) or job_retry does that by itself.
|
|
@@ -60,6 +61,22 @@ export function studioTool(request: ApprovalRequest): boolean {
|
|
|
60
61
|
*/
|
|
61
62
|
export const shellTool = (name: string) => /(^|[/_.])(shell|bash|sh|exec|exec_command|repl|js)$/i.test(name);
|
|
62
63
|
|
|
64
|
+
/** DIORAMA_AGENT_MAX_COST (dollars) and DIORAMA_AGENT_MAX_TOKENS cap each world's conversation; unset, nothing is capped. */
|
|
65
|
+
export function agentBudget(env: NodeJS.ProcessEnv = process.env): AgentBudget | undefined {
|
|
66
|
+
const cost = Number(env.DIORAMA_AGENT_MAX_COST);
|
|
67
|
+
const tokens = Number(env.DIORAMA_AGENT_MAX_TOKENS);
|
|
68
|
+
const budget: AgentBudget = { ...(cost > 0 && { maxCostUsd: cost }), ...(tokens > 0 && { maxTokens: Math.floor(tokens) }) };
|
|
69
|
+
return Object.keys(budget).length ? budget : undefined;
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
/** An agent's plan limit close to running out or used up, in words for the feed; null while it is fine. */
|
|
73
|
+
export function limitText(limit: RateLimit): string | null {
|
|
74
|
+
if (limit.status === 'ok') return null;
|
|
75
|
+
const used = limit.usedPercent === undefined ? '' : ` (${Math.round(limit.usedPercent)}% used)`;
|
|
76
|
+
const resets = limit.resetsAt === undefined ? '' : `, resets ${new Date(limit.resetsAt).toLocaleString()}`;
|
|
77
|
+
return `${limit.name ? `${limit.name} ` : ''}${limit.window} limit ${limit.status === 'exceeded' ? 'reached' : 'nearly used up'}${used}${resets}`;
|
|
78
|
+
}
|
|
79
|
+
|
|
63
80
|
/** Turns a session snapshot into what the page may see: secrets blanked. */
|
|
64
81
|
const safe = (state: SessionState): SessionState => JSON.parse(redact(JSON.stringify(state))) as SessionState;
|
|
65
82
|
|
|
@@ -108,6 +125,8 @@ class Talk {
|
|
|
108
125
|
/** Tools announced already: a running tool is announced when it starts and must not be again once it settles. */
|
|
109
126
|
private tools = new Set<string>();
|
|
110
127
|
private ended = new Set<number>();
|
|
128
|
+
/** Plan limits announced already, by limit and state, so a warning shows once. */
|
|
129
|
+
private limits = new Set<string>();
|
|
111
130
|
|
|
112
131
|
/**
|
|
113
132
|
* The Studio's tools as the Studio saw them called (arguments, outcome). Some agents report a tool call by
|
|
@@ -213,6 +232,13 @@ class Talk {
|
|
|
213
232
|
|
|
214
233
|
/** The feed (Activity, the terminal) gets the gist: what was said, each tool, refusals, how a turn ended. */
|
|
215
234
|
private announce(state: SessionState) {
|
|
235
|
+
for (const limit of state.rateLimits ?? []) {
|
|
236
|
+
const key = `${limit.name ?? ''}/${limit.window}/${limit.status}`;
|
|
237
|
+
const text = limitText(limit);
|
|
238
|
+
if (!text || this.limits.has(key)) continue;
|
|
239
|
+
this.limits.add(key);
|
|
240
|
+
this.publish('limit', { text });
|
|
241
|
+
}
|
|
216
242
|
for (const turn of state.turns) {
|
|
217
243
|
if (!this.seen.has(turn.id)) {
|
|
218
244
|
this.seen.set(turn.id, 0);
|
|
@@ -415,6 +441,7 @@ export class AgentHub {
|
|
|
415
441
|
agent,
|
|
416
442
|
cwd,
|
|
417
443
|
model: agentModel(name),
|
|
444
|
+
budget: agentBudget(),
|
|
418
445
|
task: {
|
|
419
446
|
env: withoutProviderCredentials(withoutSecrets(process.env)) as Record<string, string>,
|
|
420
447
|
// genaicode hands the server to Muse as -c overrides, the token through an environment variable, never argv.
|
package/app/src/studio/mcp.ts
CHANGED
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
// this over streamable HTTP at /mcp (Muse Code connects there) and `diorama mcp` over stdio.
|
|
3
3
|
import { actions, describeActions, runAction, type ActionContext } from '../actions/registry.ts';
|
|
4
4
|
import { redact } from '../secrets.ts';
|
|
5
|
+
import { cli } from '../cli.ts';
|
|
5
6
|
|
|
6
7
|
type Message = { jsonrpc: '2.0'; id?: string | number | null; method?: string; params?: Record<string, unknown> };
|
|
7
8
|
export type Reply = { jsonrpc: '2.0'; id: string | number | null; result?: unknown; error?: { code: number; message: string } };
|
|
@@ -10,7 +11,7 @@ const PROTOCOL = '2025-06-18';
|
|
|
10
11
|
|
|
11
12
|
export const INSTRUCTIONS = 'Diorama builds painted point-and-click worlds. Every tool here is a Diorama Studio action; the person sees each call '
|
|
12
13
|
+ 'in the Studio at once. Long work (world_build, run_step) starts a job: follow it with job_wait. Use ui_focus to show the person the '
|
|
13
|
-
+ 'world or place you are talking about. Keys are never available to you: if setup_status says one is missing, ask the person to run
|
|
14
|
+
+ 'world or place you are talking about. Keys are never available to you: if setup_status says one is missing, ask the person to run ' + cli('setup') + '.';
|
|
14
15
|
|
|
15
16
|
export function mcpTools() {
|
|
16
17
|
return describeActions().map((a) => ({
|
package/app/src/studio/server.ts
CHANGED
|
@@ -24,6 +24,7 @@ import { bundle, WEB, worldData } from '../commands/play.ts';
|
|
|
24
24
|
import { WorldFiles } from '../world/files.ts';
|
|
25
25
|
import { snapshotBeforeJob } from './history.ts';
|
|
26
26
|
import { handleMcp } from './mcp.ts';
|
|
27
|
+
import { cli } from '../cli.ts';
|
|
27
28
|
|
|
28
29
|
export const STUDIO_WEB = fileURLToPath(new URL('../../studio/', import.meta.url));
|
|
29
30
|
const CHAT_CSS = createRequire(import.meta.url).resolve('genaicode/react/styles.css');
|
|
@@ -184,7 +185,7 @@ export async function startStudio(opts: { port?: number; root?: string; cwd?: st
|
|
|
184
185
|
});
|
|
185
186
|
|
|
186
187
|
// The game and pictures read the world through the Studio's session cookie; a browser that never opened the Studio has none.
|
|
187
|
-
const NO_SESSION =
|
|
188
|
+
const NO_SESSION = `no session: open the Studio link from \`${cli('studio')}\` in this browser first, or play the world on its own with \`${cli('play <slug>')}\``;
|
|
188
189
|
const server = http.createServer(async (req, res) => {
|
|
189
190
|
const headers = { 'cache-control': 'no-store', 'x-content-type-options': 'nosniff', 'referrer-policy': 'no-referrer', 'cross-origin-resource-policy': 'same-origin' };
|
|
190
191
|
const send = (status: number, type: string, body: string | Buffer, extra: Record<string, string> = {}) => {
|
|
@@ -239,7 +240,7 @@ export async function startStudio(opts: { port?: number; root?: string; cwd?: st
|
|
|
239
240
|
}
|
|
240
241
|
|
|
241
242
|
if (url.pathname.startsWith('/api/')) {
|
|
242
|
-
if (!authed) return json(401, { error:
|
|
243
|
+
if (!authed) return json(401, { error: `missing or wrong token: open the link ${cli('studio')} printed` });
|
|
243
244
|
if (req.method === 'POST' && !(req.headers['content-type'] ?? '').startsWith('application/json')) return json(415, { error: 'JSON only' });
|
|
244
245
|
|
|
245
246
|
if (url.pathname === '/api/session' && req.method === 'POST') {
|
package/app/studio/agent.tsx
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
// The Muse Code panel: genaicode's own chat (genaicode/react), fed the session state the Studio server keeps
|
|
2
2
|
// for this world. The page holds no agent and no key: it sends what the person types and answers approvals.
|
|
3
|
-
import { AgentChat, type ChatNotice, type SessionState, type SessionTurn } from 'genaicode/react';
|
|
3
|
+
import { AgentChat, limitChips, type ChatNotice, type SessionState, type SessionTurn } from 'genaicode/react';
|
|
4
4
|
import type { ComponentChildren } from 'preact';
|
|
5
5
|
import { useEffect, useRef, useState } from 'preact/hooks';
|
|
6
6
|
import { call, fileUrl, token } from './api.ts';
|
|
@@ -19,7 +19,7 @@ export const SAYS: Record<string, string> = {
|
|
|
19
19
|
corrections_get: 'Read fixes', corrections_clear: 'Take fixes back', mask_fix: 'Fix a mask', thing_remove: 'Remove a thing', thing_add: 'Add a thing', mask_brush: 'Brush a mask',
|
|
20
20
|
floor_cells: 'Fix floor cells', stand_set: 'Move a stand spot', repaint_region: 'Repaint a region', water_flow: 'Set water flow', water_area: 'Fix where water moves', fog_drift: 'Set fog drift', weather_set: 'Set weather', reactions_set: 'Set reactions', reactions_status: 'Scene reactions',
|
|
21
21
|
feedback_pending: 'Read open notes', feedback_list: 'Read notes', feedback_done: 'Close a note', feedback_add: 'Leave a note', feedback_reopen: 'Reopen a note',
|
|
22
|
-
history_list: 'Read history', history_undo: 'Undo a job', history_redo: 'Redo', lock_set: 'Change a lock', ui_focus: 'Show a place', ask_person: 'Ask you', budget: 'Check budget', setup_status: 'Check Setup',
|
|
22
|
+
history_list: 'Read history', history_undo: 'Undo a job', history_redo: 'Redo', lock_set: 'Change a lock', ui_focus: 'Show a place', ask_person: 'Ask you', budget: 'Check budget', setup_status: 'Check Setup', setup_cli: 'Read the terminal command', setup_browsers: 'Download Chromium',
|
|
23
23
|
cast_status: 'Look at the cast', cast_redo: 'Repaint a character', voice_list: 'List voices', voice_set: 'Change a voice',
|
|
24
24
|
story_status: 'Read the story', story_text: 'Reword a line', story_redo: 'Rewrite the story', place_view: 'Look at the floor', play_reload: 'Reload the game', npc_move: 'Move an NPC',
|
|
25
25
|
};
|
|
@@ -309,6 +309,8 @@ export function AgentPanel({ slug, replay, speed = 1, jobs = [], notices, afterT
|
|
|
309
309
|
</select>
|
|
310
310
|
)}
|
|
311
311
|
<span class="grow" />
|
|
312
|
+
{/* The agent's plan limits and the conversation's budget, once near their end or reported. */}
|
|
313
|
+
{!replay && state && limitChips(state).map((chip) => <span key={chip.key} class="gc-chip gc-limit" data-level={chip.level} title={chip.title}>{chip.text}</span>)}
|
|
312
314
|
{replay
|
|
313
315
|
? <span class="live on">{replayDone ? 'replay done' : 'replay'}</span>
|
|
314
316
|
: <span class={`live ${live ? 'on' : ''}`}>{running ? 'working…' : live ? 'ready' : 'reconnecting'}</span>}
|
package/app/studio/main.tsx
CHANGED
|
@@ -224,7 +224,9 @@ function Setup() {
|
|
|
224
224
|
const [checks, setChecks] = useState<Check[] | null>(null);
|
|
225
225
|
const [busy, setBusy] = useState('');
|
|
226
226
|
const [error, setError] = useState('');
|
|
227
|
-
const
|
|
227
|
+
const [setupCommand, setSetupCommand] = useState('diorama setup');
|
|
228
|
+
useEffect(() => void call<{ setup: string }>('setup_cli').then((c) => setSetupCommand(c.setup)).catch(() => {}), []);
|
|
229
|
+
const run = async (name: 'setup_status' | 'setup_check' | 'setup_browsers') => {
|
|
228
230
|
setBusy(name);
|
|
229
231
|
setError('');
|
|
230
232
|
try {
|
|
@@ -250,7 +252,7 @@ function Setup() {
|
|
|
250
252
|
<h1>Setup</h1>
|
|
251
253
|
</div>
|
|
252
254
|
<p class="lead">{!checks ? 'Checking this machine…' : blocking.length ? `${blocking.map((c) => c.label).join(' and ')} still ${blocking.length > 1 ? 'need' : 'needs'} you. Everything else is ready or optional.` : `Everything diorama needs is here.${ok < checks.length ? ' The unticked ones are optional.' : ''}`}</p>
|
|
253
|
-
<p class="note">Keys are typed only in your terminal with <code>
|
|
255
|
+
<p class="note">Keys are typed only in your terminal with <code>{setupCommand}</code>, hidden, and kept in your system keychain. This page never sees them.</p>
|
|
254
256
|
{error && <p class="error">{error}</p>}
|
|
255
257
|
<div class="row">
|
|
256
258
|
<button type="button" class="primary" disabled={!!busy} title="Look at this computer again: keys, tools, the agent. Free" onClick={() => run('setup_status')}>{busy === 'setup_status' ? 'Checking…' : 'Check again'}</button>
|
|
@@ -273,6 +275,7 @@ function Setup() {
|
|
|
273
275
|
<div class="fix">
|
|
274
276
|
<pre>{c.fix.join('\n')}</pre>
|
|
275
277
|
<Copy text={c.fix.join('\n')} />
|
|
278
|
+
{c.id === 'chromium' && <button type="button" class="primary small" disabled={!!busy} title="Download Playwright's Chromium now (~150 MB, once), then check again. Free" onClick={() => run('setup_browsers')}>{busy === 'setup_browsers' ? 'Downloading…' : 'Download'}</button>}
|
|
276
279
|
</div>
|
|
277
280
|
)}
|
|
278
281
|
</div>
|
package/app/web/main.ts
CHANGED
|
@@ -2935,9 +2935,15 @@ class Game {
|
|
|
2935
2935
|
const img = c.getImageData(0, 0, w, h);
|
|
2936
2936
|
const d = img.data;
|
|
2937
2937
|
const ramp = (v: number, a: number, b: number) => Math.max(0, Math.min(1, (v - a) / (b - a)));
|
|
2938
|
+
// Only the flame's own pixels: near white-hot and strongly warm. Wood or paper lit warm by it (a letter by a
|
|
2939
|
+
// candle) passed a looser test, and the whole box flared over them as a bright square. Faded to nothing at
|
|
2940
|
+
// the box's edge, so what still passes never ends on a straight line.
|
|
2938
2941
|
for (let i = 0; i < d.length; i += 4) {
|
|
2939
2942
|
const luma = 0.3 * d[i] + 0.59 * d[i + 1] + 0.11 * d[i + 2];
|
|
2940
|
-
|
|
2943
|
+
const k = i / 4;
|
|
2944
|
+
const [u, v] = [((k % w) + 0.5) / w * 2 - 1, (Math.floor(k / w) + 0.5) / h * 2 - 1];
|
|
2945
|
+
const edge = Math.max(0, 1 - (u * u + v * v)) ** 0.5;
|
|
2946
|
+
d[i + 3] = Math.round(255 * ramp(luma, 170, 235) * ramp(d[i] - d[i + 2], 70, 130) * edge);
|
|
2941
2947
|
}
|
|
2942
2948
|
c.putImageData(img, 0, 0);
|
|
2943
2949
|
return { canvas, x0, y0 };
|
package/docs/setup.md
CHANGED
|
@@ -8,7 +8,7 @@ Without a checkout, from npm. Run it in an empty folder: worlds are written to `
|
|
|
8
8
|
|
|
9
9
|
```sh
|
|
10
10
|
mkdir my-worlds && cd my-worlds
|
|
11
|
-
npx -y @gtanczyk/diorama setup # what this machine still needs; asks for keys, hidden
|
|
11
|
+
npx -y @gtanczyk/diorama setup # what this machine still needs; asks for keys, hidden, and offers to download Chromium
|
|
12
12
|
npx -y @gtanczyk/diorama doctor # one live call to each model
|
|
13
13
|
npx -y @gtanczyk/diorama studio # the GUI
|
|
14
14
|
```
|
package/docs/studio.md
CHANGED
|
@@ -34,7 +34,7 @@ The agent can use `job_inspect` to see a job's timestamped recent activity, how
|
|
|
34
34
|
|
|
35
35
|
Every fix is kept in a file next to what it fixes and applied again by every later build, so a rebuild never loses it: `places/<place>/corrections.json` (rebuilding only that place from the step it feeds), `story-fixes.json`, `cast/fixes.json`. A place's thing list is edited the same way: Remove takes a prop off it (kept in the picture, or painted out with one Muse edit) and Add makes something painted a thing of its own (`removed` and `added` in `corrections.json`, which every step reads with world.json); exits, items and people belong to the story. A small repaint rebuilds only what it reaches (`app/src/scene/carry.ts`): each place step keeps the painting it last ran on (`places/<place>/.ran/`), compares the new one with it tile by tile, and asks again only about what the changed tiles reach (a thing's or floor's mask, a plate, the backdrop's cards, the light, the stairs); everything else is asked about the old painting, which the memo answers for nothing, and the place keeps its scale. Lighting the lanterns of a harbour takes about 100 s instead of 270 s, and a rebuild on the same painting asks nothing new. A change over a third of the painting rebuilds everything. A place whose steps have not kept their painting (built before this, or put back by an Undo) takes the painting on disk as theirs before a repaint, so that repaint carries too. **History** lists the jobs the world went through, the agent's included, each with who ran it, when and which places it changed, and puts the world back as it was before any of them (Redo takes that back right after); a locked world or place is left alone by the agent.
|
|
36
36
|
|
|
37
|
-
The conversation is genaicode's chat with Muse Code (default), Claude Code or Codex, picked in its header before the first message (Setup's **Default agent**, or a pick here, is remembered in `~/.config/diorama/studio.json` for new worlds; a world keeps the agent it last talked with, and `DIORAMA_AGENT=codex` overrides both; `DIORAMA_AGY_MODEL=gemini-4-argon-high`, `DIORAMA_CODEX_MODEL`, `DIORAMA_MUSE_MODEL`, `DIORAMA_AGENT_CLAUDE_MODEL` or `DIORAMA_AGENT_MODEL` picks the agent's model); after it, "Hand over…" passes the conversation to another agent and a line marks where it changed hands. After a Studio restart the earlier turns come back on screen from `studio/sessions/`. The conversation is also kept in words in `studio/conversation.md`: a new agent session (another agent, or after the Studio restarts) is told to read it first, so nothing agreed is lost. The agent works through the same actions as the page, so its steps show live and its changes survive rebuilds; when a build leaves places unplayable it is woken to repair them, three tries a place. Every conversation is recorded in `studio/sessions/`: open one from "Recorded sessions", or `/w/<slug>?replay=<file>&speed=2`, to play it back for a film.
|
|
37
|
+
The conversation is genaicode's chat with Muse Code (default), Claude Code or Codex, picked in its header before the first message (Setup's **Default agent**, or a pick here, is remembered in `~/.config/diorama/studio.json` for new worlds; a world keeps the agent it last talked with, and `DIORAMA_AGENT=codex` overrides both; `DIORAMA_AGY_MODEL=gemini-4-argon-high`, `DIORAMA_CODEX_MODEL`, `DIORAMA_MUSE_MODEL`, `DIORAMA_AGENT_CLAUDE_MODEL` or `DIORAMA_AGENT_MODEL` picks the agent's model); after it, "Hand over…" passes the conversation to another agent and a line marks where it changed hands. After a Studio restart the earlier turns come back on screen from `studio/sessions/`. The conversation is also kept in words in `studio/conversation.md`: a new agent session (another agent, or after the Studio restarts) is told to read it first, so nothing agreed is lost. The agent works through the same actions as the page, so its steps show live and its changes survive rebuilds; when a build leaves places unplayable it is woken to repair them, three tries a place. When the agent reports its plan limits (Claude Code's 5-hour and weekly limits, Codex's), the header shows how much is used and when it resets once a limit is near its end, and the terminal says so once. `DIORAMA_AGENT_MAX_COST` (dollars) and `DIORAMA_AGENT_MAX_TOKENS` cap what one world's conversation may spend: the header shows the budget, warns at 80%, and once it is used up a new message does not start. Every conversation is recorded in `studio/sessions/`: open one from "Recorded sessions", or `/w/<slug>?replay=<file>&speed=2`, to play it back for a film.
|
|
38
38
|
|
|
39
39
|
**Setup** shows what this machine still needs, which agents it found, and the model calls sent across all worlds, priced once you enter the per-call prices from your billing pages. Keys are only ever typed in the terminal (`diorama setup`).
|
|
40
40
|
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@gtanczyk/diorama",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.5.0",
|
|
4
4
|
"description": "Turn a story prompt into playable painted point-and-click scenes with Meta Muse and SAM 3.1",
|
|
5
5
|
"license": "Apache-2.0",
|
|
6
6
|
"repository": {
|
|
@@ -54,7 +54,7 @@
|
|
|
54
54
|
"@meta-sam/parser": "^0.1.0",
|
|
55
55
|
"@preact/signals": "^2.11.3",
|
|
56
56
|
"esbuild": "^0.25.0",
|
|
57
|
-
"genaicode": "^2.
|
|
57
|
+
"genaicode": "^2.29.0",
|
|
58
58
|
"onnxruntime-web": "^1.31.1",
|
|
59
59
|
"playwright": "^1.63.0",
|
|
60
60
|
"preact": "^10.29.8",
|