@gtanczyk/diorama 0.3.0 → 0.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/app/src/actions/story.ts +28 -1
- package/app/src/actions/studio.ts +25 -0
- package/app/src/actions/style.ts +3 -2
- package/app/src/cli.ts +53 -0
- package/app/src/commands/browsers.ts +24 -8
- package/app/src/commands/clip.ts +7 -0
- package/app/src/commands/derive.ts +9 -0
- package/app/src/commands/explore.ts +2 -1
- package/app/src/commands/language.ts +2 -1
- package/app/src/commands/layout.ts +9 -4
- package/app/src/commands/masks.ts +2 -1
- package/app/src/commands/paint.ts +15 -7
- package/app/src/commands/playthrough.ts +2 -2
- package/app/src/commands/setup.ts +25 -1
- package/app/src/commands/world.ts +9 -6
- package/app/src/main.ts +5 -4
- package/app/src/play/car.ts +2 -0
- package/app/src/qa/route.ts +12 -2
- package/app/src/scene/cast.ts +4 -2
- package/app/src/scene/critic.ts +3 -2
- package/app/src/scene/layout.ts +13 -5
- package/app/src/scene/paint-layout.ts +2 -1
- package/app/src/scene/paint.ts +15 -8
- package/app/src/scene/photo.ts +25 -2
- package/app/src/scene/status.ts +2 -1
- package/app/src/scene/vehicle.ts +22 -0
- package/app/src/secrets.ts +2 -1
- package/app/src/setup/checks.ts +5 -4
- package/app/src/sound/plan.ts +6 -1
- package/app/src/story/engine.ts +9 -4
- package/app/src/story/fixes.ts +11 -2
- package/app/src/studio/agent.ts +16 -3
- package/app/src/studio/mcp.ts +2 -1
- package/app/src/studio/server.ts +3 -2
- package/app/src/world/design.ts +1 -0
- package/app/src/world/drawings.ts +47 -14
- package/app/src/world/schema.ts +5 -0
- package/app/studio/agent.tsx +1 -1
- package/app/studio/main.tsx +5 -2
- package/app/web/main.ts +151 -12
- package/docs/scenes.md +9 -0
- package/docs/setup.md +1 -1
- package/docs/studio.md +3 -1
- package/package.json +1 -1
package/app/src/scene/layout.ts
CHANGED
|
@@ -154,6 +154,15 @@ export function closeUpIssues(layout: Layout): string[] {
|
|
|
154
154
|
: [];
|
|
155
155
|
}
|
|
156
156
|
|
|
157
|
+
/** A drive place: a side view of a road that loops sideways, the car standing on it, the signs beside it. */
|
|
158
|
+
const DRIVE_LAYOUT = `
|
|
159
|
+
This place is the hero's car: it stands at the kerb with its engine running, the hero at the wheel, and from here the player picks where to drive. So:
|
|
160
|
+
- the car is the main thing: in the middle of the frame, a little below the centre, seen from behind at three-quarters so the driver shows through the window, about half the frame wide, its wheels on the street
|
|
161
|
+
- one floor, the street and pavement around the car, reaching the bottom edge
|
|
162
|
+
- the street and the city behind it fill the rest; leave the top right quarter of the frame calm (sky, a dark wall), the game lays its destination board there
|
|
163
|
+
|
|
164
|
+
`;
|
|
165
|
+
|
|
157
166
|
function brief(world: World, place: Place): string {
|
|
158
167
|
const { width: W, height: H } = FRAME;
|
|
159
168
|
return `Lay out one place of a 2D point-and-click adventure as a block-in for a painter. The frame is ${W}x${H} px, x to the right, y down; the camera looks a little down at the place, like at a stage.
|
|
@@ -169,8 +178,7 @@ Give:
|
|
|
169
178
|
- floors: 1 to 4 walkable areas below the horizon as outlines, nearest first; the first reaches the bottom edge; every other floor touches one listed before it (a bridge, steps or a path across water or a drop is a floor too), so the hero can walk everywhere; leave room to walk around things
|
|
170
179
|
- things: every thing as a box standing on the floor: x (centre), footY (where it touches the floor), width, height in px, metres (real height), colour (main colour), shape (its silhouette: block = a box, stall, cupboard; low = bench, table, boat, cart, bed; post = lamp, pole, sign, column; tree; arch = door, gate, archway, window, tunnel; round = barrel, pot, rock, bush, well; steps = stairs or steps; flat = lies on the floor: rug, water, tracks); nearer things sit lower and are bigger; exits sit at the edge of a floor; every exit, item and person stands on a floor; leave a way round every big thing, and never stand one across the only way to a floor
|
|
171
180
|
- lights: up to 6 visible sources (lamps, fires, windows) with x, y, colour
|
|
172
|
-
|
|
173
|
-
Reply with JSON only:
|
|
181
|
+
${place.kind === 'drive' ? DRIVE_LAYOUT : ''}Reply with JSON only:
|
|
174
182
|
{"horizon": 0, "sky": "#rrggbb", "ground": "#rrggbb",
|
|
175
183
|
"floors": [{"key": "...", "is": "what the floor is", "colour": "#rrggbb", "outline": [[x, y], ...]}],
|
|
176
184
|
"things": [{"key": "...", "colour": "#rrggbb", "x": 0, "footY": 0, "width": 0, "height": 0, "metres": 1.0, "shape": "block"}],
|
|
@@ -185,9 +193,9 @@ export function repairSmallBoxes(layout: Layout): Layout {
|
|
|
185
193
|
})) };
|
|
186
194
|
}
|
|
187
195
|
|
|
188
|
-
/** With a photo, measure existing positions without resizing them. */
|
|
189
|
-
export async function layOut(memo: Memo, world: World, place: Place, photo?: Buffer, guidance?: string, concept?: Buffer): Promise<Layout> {
|
|
190
|
-
const ask = () => (photo ? `${brief(world, place)}\n\nThe place already exists: it is the attached photo, ${FRAME.width}x${FRAME.height} px. Do not place anything; measure where each thing, floor and light is in it.` : brief(world, place));
|
|
196
|
+
/** With a photo (or a grown-up's sketch of the place), measure existing positions without resizing them. */
|
|
197
|
+
export async function layOut(memo: Memo, world: World, place: Place, photo?: Buffer, guidance?: string, concept?: Buffer, picture: 'photo' | 'sketch' = 'photo'): Promise<Layout> {
|
|
198
|
+
const ask = () => (photo ? `${brief(world, place)}\n\nThe place already exists: it is the attached ${picture === 'sketch' ? "artist's sketch (ignore the sketchbook page, notes and colour swatches)" : 'photo'}, ${FRAME.width}x${FRAME.height} px. Do not place anything; measure where each thing, floor and light is in it.` : brief(world, place));
|
|
191
199
|
const base = ask() + (!photo && concept ? '\n\nThe attached image is the creator-approved concept for this place. Start from its camera and relative arrangement of the listed things, scaled to the output frame. Preserve the recognizable composition where possible, but adapt positions, sizes and floor connections to satisfy every walkability rule above. This is a concept, not a fixed photograph; do not copy inaccessible areas or invent extra things.' : '') + (guidance ? `\n\nLayout repair guidance: ${guidance}` : '');
|
|
192
200
|
// Keep paid replies. An exhausted batch becomes feedback for the next batch, never an identical replay.
|
|
193
201
|
const stateFile = path.join(memo.dir, 'layout-retries', `${inputHash(base, photo ?? '', ...(!photo && concept ? [concept] : []))}.json`);
|
|
@@ -51,7 +51,8 @@ const saysBlocked = (why: string) => BLOCKED.test(why);
|
|
|
51
51
|
/** Fail closed on incomplete observations: an omitted floor is not evidence that it is connected. */
|
|
52
52
|
export function paintedLayoutIssues(place: Place, layout: Layout, seen: PaintedLayout): string[] {
|
|
53
53
|
const issues: string[] = [];
|
|
54
|
-
|
|
54
|
+
// A drive's hero is painted at the wheel of the car: the car's card carries them.
|
|
55
|
+
if (seen.people && place.kind !== 'drive') issues.push('unwanted people painted into the background; the game adds its cast separately');
|
|
55
56
|
if (Math.abs(seen.horizon - layout.horizon) > 70) issues.push(`camera horizon moved from ${layout.horizon} to ${Math.round(seen.horizon)}`);
|
|
56
57
|
for (const t of place.things.filter((t) => t.role !== 'npc')) {
|
|
57
58
|
const planned = layout.things.find((b) => b.key === t.key);
|
package/app/src/scene/paint.ts
CHANGED
|
@@ -1,6 +1,8 @@
|
|
|
1
1
|
import { STYLE_REFERENCE_PROMPT } from '../world/style-reference.ts';
|
|
2
|
+
import { drawers } from '../world/drawings.ts';
|
|
2
3
|
// paint: blockin.png becomes painting.png through one Muse edit.
|
|
3
4
|
import sharp from 'sharp';
|
|
5
|
+
import { VEHICLE_PROMPT } from './vehicle.ts';
|
|
4
6
|
import type { Memo } from '../meta/memo.ts';
|
|
5
7
|
import { museEdit } from '../meta/muse.ts';
|
|
6
8
|
import type { Place, World } from '../world/schema.ts';
|
|
@@ -31,6 +33,9 @@ export function whereIs(layout: Layout, box: Layout['things'][number]): string {
|
|
|
31
33
|
return `${back < 0.3 ? 'at the back' : back < 0.65 ? 'halfway back' : 'at the front'}, ${across}`;
|
|
32
34
|
}
|
|
33
35
|
|
|
36
|
+
/** A drive place is seen from the side at car height, and its scenery loops sideways in the page. */
|
|
37
|
+
const DRIVE_PAINT = (world: World) => `The hero's car stands at the kerb with its engine running, lights on, a wisp of exhaust, seen from behind at three-quarters, with the hero at the wheel visible through the window or over the open top: ${world.hero.looks}. Keep the top right quarter of the picture calm, nothing important there.`;
|
|
38
|
+
|
|
34
39
|
export function paintPrompt(world: World, place: Place, layout: Layout, brief?: Brief): string {
|
|
35
40
|
const look = (key: string, fallback: string) => brief?.looks[key] ?? fallback;
|
|
36
41
|
const masses = place.things
|
|
@@ -51,7 +56,7 @@ ${[...masses, ...floors].join('\n')}
|
|
|
51
56
|
|
|
52
57
|
${layoutContract(place, layout)}
|
|
53
58
|
|
|
54
|
-
Seen a little from above, like a stage. Keep the floor areas clear to walk on. No people, ${noAnimals(brief)}, no text or letters, no border.${castNote(place)}${fallsIn(`${place.shows} ${place.mood}`) ? '\nThe rain or snow itself is animated over the picture later: paint no falling streaks, drops or flakes in the air, only what the weather leaves (wet sheen, puddles, snow lying on the ground).' : ''}`;
|
|
59
|
+
${place.kind === 'drive' ? DRIVE_PAINT(world) : 'Seen a little from above, like a stage. Keep the floor areas clear to walk on.'} No ${place.kind === 'drive' ? 'other ' : ''}people, ${noAnimals(brief)}, no text or letters, no border.${castNote(place)}${fallsIn(`${place.shows} ${place.mood}`) ? '\nThe rain or snow itself is animated over the picture later: paint no falling streaks, drops or flakes in the air, only what the weather leaves (wet sheen, puddles, snow lying on the ground).' : ''}`;
|
|
55
60
|
}
|
|
56
61
|
|
|
57
62
|
function whereFloor(layout: Layout, f: Layout['floors'][number]): string {
|
|
@@ -60,22 +65,24 @@ function whereFloor(layout: Layout, f: Layout['floors'][number]): string {
|
|
|
60
65
|
return whereIs(layout, mid);
|
|
61
66
|
}
|
|
62
67
|
|
|
63
|
-
/**
|
|
68
|
+
/** Drawings of the place or of things in it (children's, or an artist's sketches), as more images after the block-in. */
|
|
64
69
|
export type Drawing = { key: string; image: Buffer };
|
|
65
70
|
|
|
66
|
-
export function drawingsPrompt(place: Place, drawings: Drawing[]): string {
|
|
71
|
+
export function drawingsPrompt(place: Place, drawings: Drawing[], kids = true): string {
|
|
67
72
|
if (drawings.length === 0) return '';
|
|
73
|
+
const drawn = drawers({ audience: kids ? 'kids' : undefined });
|
|
68
74
|
const which = drawings.map((d, i) => `image ${i + 2} is ${d.key === place.key ? 'this place' : `the ${d.key}`}`);
|
|
69
75
|
return `
|
|
70
76
|
|
|
71
|
-
The other images are
|
|
77
|
+
The other images are ${drawn.many} on paper: ${which.join(', ')}. ${kids ? 'Paint what they drew so the children know it at once: their colours, shapes and ideas.' : 'Paint what was drawn so the artist knows it at once: the same shapes, colours, ideas and the same hand and medium.'} A drawing fills its paper, but its thing does not fill the place: paint each at its real size beside the others (a hat is smaller than a log, a person comes up to a door's top). The composition still comes only from the first image; leave out the paper, the table and any written names.`;
|
|
72
78
|
}
|
|
73
79
|
|
|
74
80
|
/** One Muse edit, scaled back to the layout frame (Muse answers larger than asked). */
|
|
75
|
-
export async function paintPlace(memo: Memo, world: World, place: Place, layout: Layout, blockin: Buffer, brief?: Brief, take = 0, drawings: Drawing[] = [], concept?: Buffer, styleReference?: Buffer): Promise<Buffer> {
|
|
76
|
-
const prompt = paintPrompt(world, place, layout, brief) + drawingsPrompt(place, drawings)
|
|
81
|
+
export async function paintPlace(memo: Memo, world: World, place: Place, layout: Layout, blockin: Buffer, brief?: Brief, take = 0, drawings: Drawing[] = [], concept?: Buffer, styleReference?: Buffer, vehicle?: Buffer): Promise<Buffer> {
|
|
82
|
+
const prompt = paintPrompt(world, place, layout, brief) + drawingsPrompt(place, drawings, world.audience === 'kids')
|
|
77
83
|
+ (styleReference ? `\nImage ${drawings.length + 2}: ${STYLE_REFERENCE_PROMPT}` : '')
|
|
78
|
-
+ (concept ?
|
|
79
|
-
|
|
84
|
+
+ (concept ? `\nImage ${drawings.length + 2 + (styleReference ? 1 : 0)} is the creator-approved concept for this place. Preserve its distinctive architecture, palette, materials and visual identity. The FIRST image remains the authoritative layout: adapt the approved appearance to those positions and clear walking areas. Do not add characters or extra objects from the concept.` : '')
|
|
85
|
+
+ (vehicle ? `\nThe LAST image ${VEHICLE_PROMPT}` : '');
|
|
86
|
+
const painted = await museEdit(memo, { label: `paint:${place.key}`, prompt, images: [blockin, ...drawings.map((d) => d.image), ...(styleReference ? [styleReference] : []), ...(concept ? [concept] : []), ...(vehicle ? [vehicle] : [])], take });
|
|
80
87
|
return sharp(painted).resize(FRAME.width, FRAME.height, { fit: 'fill' }).png().toBuffer();
|
|
81
88
|
}
|
package/app/src/scene/photo.ts
CHANGED
|
@@ -1,6 +1,8 @@
|
|
|
1
1
|
import { STYLE_REFERENCE_PROMPT } from '../world/style-reference.ts';
|
|
2
2
|
// A real photo as a place: crop it to the game frame, let Spark read the room into things,
|
|
3
3
|
// and let Muse repaint it in the world's style without moving anything.
|
|
4
|
+
import fs from 'node:fs';
|
|
5
|
+
import path from 'node:path';
|
|
4
6
|
import sharp from 'sharp';
|
|
5
7
|
import { z } from 'zod';
|
|
6
8
|
import type { Memo } from '../meta/memo.ts';
|
|
@@ -58,6 +60,27 @@ Reply with JSON only:
|
|
|
58
60
|
throw new Error('Spark could not read the photo into a place in 3 tries');
|
|
59
61
|
}
|
|
60
62
|
|
|
63
|
+
/** The picture a place was made from, which stands in for its block-in: a real photo, or a grown-up's sketch of the place. */
|
|
64
|
+
export function placePicture(dir: string): { file: string; kind: 'photo' | 'sketch' } | undefined {
|
|
65
|
+
for (const kind of ['photo', 'sketch'] as const) {
|
|
66
|
+
const file = path.join(dir, `${kind}.png`);
|
|
67
|
+
if (fs.existsSync(file)) return { file, kind };
|
|
68
|
+
}
|
|
69
|
+
return undefined;
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
export function sketchPaintPrompt(world: World, place: Place, brief?: Brief): string {
|
|
73
|
+
const looks = place.things.filter((t) => t.role !== 'npc').map((t) => `- ${t.key}: ${brief?.looks[t.key] ?? t.looks}`);
|
|
74
|
+
return `Finish this artist's sketchbook drawing as a background of a 2D point-and-click adventure game. Keep the camera and every object exactly where and as big as it is in the sketch: same buildings, same furniture, same floor, same perspective. Keep the artist's hand and medium; only finish it: clean confident lines, full colour, depth and light.
|
|
75
|
+
|
|
76
|
+
Style: ${world.style}
|
|
77
|
+
Mood: ${place.mood}
|
|
78
|
+
Objects to keep recognisable:
|
|
79
|
+
${looks.join('\n')}
|
|
80
|
+
|
|
81
|
+
Leave out the sketchbook page itself: its edges and spine, the handwritten notes, colour swatches and any writing; continue the scene out to every edge of the frame. Remove every person; paint what is behind them. Keep the floor clear to walk on. No text or letters, no border.`;
|
|
82
|
+
}
|
|
83
|
+
|
|
61
84
|
export function photoPaintPrompt(world: World, place: Place, brief?: Brief): string {
|
|
62
85
|
const looks = place.things.filter((t) => t.role !== 'npc').map((t) => `- ${t.key}: ${brief?.looks[t.key] ?? t.looks}`);
|
|
63
86
|
return `Repaint this photo as a finished background of a 2D point-and-click adventure game. Keep the camera and every object exactly where and as big as it is in the photo: same walls, same furniture, same door, same floor. Only the medium changes.
|
|
@@ -70,8 +93,8 @@ ${looks.join('\n')}
|
|
|
70
93
|
Remove every person; paint the furniture and floor behind them. Keep the floor clear to walk on. No text or letters, no border.`;
|
|
71
94
|
}
|
|
72
95
|
|
|
73
|
-
export async function paintPhoto(memo: Memo, world: World, place: Place, photo: Buffer, brief?: Brief, take = 0, concept?: Buffer, styleReference?: Buffer): Promise<Buffer> {
|
|
74
|
-
const prompt = photoPaintPrompt(world, place, brief) + (styleReference ? '\nThe LAST image: ' + STYLE_REFERENCE_PROMPT : '') + (concept ? '\nThe second image is the creator-approved visual concept: preserve its palette and treatment. The first photo is authoritative for the real room, furniture, perspective and layout.' : '');
|
|
96
|
+
export async function paintPhoto(memo: Memo, world: World, place: Place, photo: Buffer, brief?: Brief, take = 0, concept?: Buffer, styleReference?: Buffer, picture: 'photo' | 'sketch' = 'photo'): Promise<Buffer> {
|
|
97
|
+
const prompt = (picture === 'sketch' ? sketchPaintPrompt : photoPaintPrompt)(world, place, brief) + (styleReference ? '\nThe LAST image: ' + STYLE_REFERENCE_PROMPT : '') + (concept ? '\nThe second image is the creator-approved visual concept: preserve its palette and treatment. The first photo is authoritative for the real room, furniture, perspective and layout.' : '');
|
|
75
98
|
const painted = await museEdit(memo, { label: `paint:${place.key}`, prompt, images: [photo, ...(concept ? [concept] : []), ...(styleReference ? [styleReference] : [])], take });
|
|
76
99
|
return sharp(painted).resize(FRAME.width, FRAME.height, { fit: 'fill' }).png().toBuffer();
|
|
77
100
|
}
|
package/app/src/scene/status.ts
CHANGED
|
@@ -7,6 +7,7 @@ import { World, type Place } from '../world/schema.ts';
|
|
|
7
7
|
import { verifyPlace, type BackdropNotes, type MaskNotes, type PlateNotes, type Verdict } from './verify.ts';
|
|
8
8
|
import { Layout } from './layout.ts';
|
|
9
9
|
import { paintingLayoutHash } from './paint-layout.ts';
|
|
10
|
+
import { placePicture } from './photo.ts';
|
|
10
11
|
|
|
11
12
|
export function currentVerdict(worldDir: string, key: string, place?: Place): Verdict | null {
|
|
12
13
|
const dir = path.join(worldDir, 'places', key);
|
|
@@ -18,7 +19,7 @@ export function currentVerdict(worldDir: string, key: string, place?: Place): Ve
|
|
|
18
19
|
if (!place) throw new Error(`no place ${key}`);
|
|
19
20
|
const read = (file: string) => JSON.parse(fs.readFileSync(path.join(dir, file), 'utf8'));
|
|
20
21
|
const optional = <T>(file: string) => (fs.existsSync(path.join(dir, file)) ? read(file) as T : {} as T);
|
|
21
|
-
if (!
|
|
22
|
+
if (!placePicture(dir) && fs.existsSync(path.join(dir, 'painting-layout.json'))) {
|
|
22
23
|
const r = read('painting-layout.json') as { ok?: boolean; inputs?: string; attempts?: Array<{ issues: string[] }> };
|
|
23
24
|
if (!r.ok || r.inputs !== paintingLayoutHash(place, Layout.parse(read('layout.json')), fs.readFileSync(path.join(dir, 'blockin.png')), fs.readFileSync(path.join(dir, 'painting.png')))) {
|
|
24
25
|
return { ok: false, errors: [{ code: 'PAINT_LAYOUT_MISMATCH', detail: r.attempts?.at(-1)?.issues.join('; ') || 'Composition check is pending or stale: rebuild from paint.' }], flags: [] };
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
// The hero's car (a drive place, kind "drive") is parked in every place it drives to. Muse paints each place on its
|
|
2
|
+
// own, so without a reference each parks a different car; the car as painted in the drive place, cut out by its
|
|
3
|
+
// mask, goes to every such place's painting as how the car looks.
|
|
4
|
+
import fs from 'node:fs';
|
|
5
|
+
import sharp from 'sharp';
|
|
6
|
+
import type { WorldFiles } from '../world/files.ts';
|
|
7
|
+
import type { Place, World } from '../world/schema.ts';
|
|
8
|
+
|
|
9
|
+
/** The drive place's car, cut out on white, for a place it is parked in; undefined before it is painted and masked. */
|
|
10
|
+
export async function heroVehicle(files: WorldFiles, world: World, place: Place): Promise<Buffer | undefined> {
|
|
11
|
+
const drive = world.places.find((p) => p.kind === 'drive');
|
|
12
|
+
if (!drive || drive.key === place.key || !place.exits.some((e) => e.to === drive.key)) return undefined;
|
|
13
|
+
const car = drive.exits[0]?.via;
|
|
14
|
+
const [painting, mask] = [files.place(drive.key, 'painting.png'), files.place(drive.key, 'masks', `${car}.png`)];
|
|
15
|
+
if (!car || !fs.existsSync(painting) || !fs.existsSync(mask)) return undefined;
|
|
16
|
+
const { width, height } = await sharp(painting).metadata();
|
|
17
|
+
const alpha = await sharp(mask).resize(width, height, { fit: 'fill' }).greyscale().raw().toBuffer();
|
|
18
|
+
const cut = await sharp(painting).removeAlpha().joinChannel(alpha, { raw: { width: width!, height: height!, channels: 1 } }).png().toBuffer();
|
|
19
|
+
return sharp(cut).trim().flatten({ background: '#ffffff' }).png().toBuffer();
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
export const VEHICLE_PROMPT = 'shows the hero\'s car: wherever the car stands in this place, paint exactly this car (body shape, colour, trim, wheels), parked and seen from this place\'s camera; whoever sits in it is not part of it, and nothing else in that image belongs in the place.';
|
package/app/src/secrets.ts
CHANGED
|
@@ -7,6 +7,7 @@ import { createCipheriv, createDecipheriv, randomBytes } from 'node:crypto';
|
|
|
7
7
|
import fs from 'node:fs';
|
|
8
8
|
import os from 'node:os';
|
|
9
9
|
import path from 'node:path';
|
|
10
|
+
import { cli } from './cli.ts';
|
|
10
11
|
|
|
11
12
|
/** The keys diorama knows how to keep, with the other names each one is accepted under. */
|
|
12
13
|
export const SECRETS = {
|
|
@@ -138,7 +139,7 @@ export class SecretStore {
|
|
|
138
139
|
async read(): Promise<Record<string, string>> {
|
|
139
140
|
if (!fs.existsSync(this.file)) return {};
|
|
140
141
|
const key = await this.vault.get();
|
|
141
|
-
if (!key) throw new Error(`credentials.bin exists but its key is missing from the ${this.vault.name}; run
|
|
142
|
+
if (!key) throw new Error(`credentials.bin exists but its key is missing from the ${this.vault.name}; run ${cli('setup')} to store your keys again`);
|
|
142
143
|
return open(key, fs.readFileSync(this.file));
|
|
143
144
|
}
|
|
144
145
|
|
package/app/src/setup/checks.ts
CHANGED
|
@@ -5,6 +5,7 @@ import fs from 'node:fs';
|
|
|
5
5
|
import path from 'node:path';
|
|
6
6
|
import { blenderState } from '../model/blender.ts';
|
|
7
7
|
import { envValue, findVault, loadSecrets, NO_VAULT, type SecretName } from '../secrets.ts';
|
|
8
|
+
import { cli } from '../cli.ts';
|
|
8
9
|
|
|
9
10
|
export type Check = {
|
|
10
11
|
id: string;
|
|
@@ -22,7 +23,7 @@ function sh(cmd: string, args: string[], timeout = 8000): Promise<string | null>
|
|
|
22
23
|
|
|
23
24
|
async function keyCheck(name: SecretName, label: string, required: boolean, what: string, source?: 'env' | 'store'): Promise<Check> {
|
|
24
25
|
const value = envValue(name);
|
|
25
|
-
if (!value) return { id: name, label, state: required ? 'missing' : 'optional', detail: what, fix: [`
|
|
26
|
+
if (!value) return { id: name, label, state: required ? 'missing' : 'optional', detail: what, fix: [cli(`setup --key ${name}`)] };
|
|
26
27
|
return { id: name, label, state: 'ok', detail: source === 'store' ? 'in your system keychain' : 'from the environment' };
|
|
27
28
|
}
|
|
28
29
|
|
|
@@ -78,7 +79,7 @@ export async function runChecks(opts: { live?: boolean } = {}): Promise<Check[]>
|
|
|
78
79
|
} catch {}
|
|
79
80
|
checks.push(chromium && fs.existsSync(chromium)
|
|
80
81
|
? { id: 'chromium', label: 'Chromium for Explore and Playthrough', state: 'ok', detail: 'installed' }
|
|
81
|
-
: { id: 'chromium', label: 'Chromium for Explore and Playthrough', state: 'missing', detail: 'Explore and the playthrough bot drive a browser.', fix: ['
|
|
82
|
+
: { id: 'chromium', label: 'Chromium for Explore and Playthrough', state: 'missing', detail: 'Explore and the playthrough bot drive a browser.', fix: [cli('browsers')] });
|
|
82
83
|
|
|
83
84
|
checks.push(await keyCheck('ELEVENLABS_API_KEY', 'ElevenLabs key', false, 'Optional: sound effects, music and voices. Without it the world plays silent.', sources.ELEVENLABS_API_KEY));
|
|
84
85
|
checks.push(await keyCheck('HF_TOKEN', 'Hugging Face token', false, 'Optional: 3D hero figures (TRELLIS, auto-rig) on ZeroGPU Spaces.', sources.HF_TOKEN));
|
|
@@ -103,8 +104,8 @@ export async function runChecks(opts: { live?: boolean } = {}): Promise<Check[]>
|
|
|
103
104
|
|
|
104
105
|
const remote = await sh('git', ['remote', 'get-url', 'origin']);
|
|
105
106
|
checks.push(remote
|
|
106
|
-
? { id: 'publish', label: 'Publishing', state: 'ok', detail: `git remote ${remote.replace(/\/\/[^@/]+@/, '//')};
|
|
107
|
-
: { id: 'publish', label: 'Publishing', state: 'optional', detail:
|
|
107
|
+
? { id: 'publish', label: 'Publishing', state: 'ok', detail: `git remote ${remote.replace(/\/\/[^@/]+@/, '//')}; ${cli('publish <slug>')} pushes to gh-pages` }
|
|
108
|
+
: { id: 'publish', label: 'Publishing', state: 'optional', detail: `No git remote: ${cli('export <slug> --zip')} still makes an itch.io upload.` });
|
|
108
109
|
|
|
109
110
|
return checks;
|
|
110
111
|
}
|
package/app/src/sound/plan.ts
CHANGED
|
@@ -13,6 +13,9 @@ const Prompt = z.string().min(8);
|
|
|
13
13
|
/** 1 quiet, 2 normal, 3 loud: how it sits in the mix next to the place bed. */
|
|
14
14
|
const Level = z.union([z.literal(1), z.literal(2), z.literal(3)]).default(2);
|
|
15
15
|
|
|
16
|
+
/** One-shots kept from a plan; the same prompt is made once, so exits of one kind cost one sound. */
|
|
17
|
+
const MAX_SHOTS = 24;
|
|
18
|
+
|
|
16
19
|
export const SoundPlan = z.object({
|
|
17
20
|
/** Hash of what the plan was made from (world, story, briefs, floors); when that changes the plan is made again. */
|
|
18
21
|
from: z.string().optional(),
|
|
@@ -29,7 +32,9 @@ export const SoundPlan = z.object({
|
|
|
29
32
|
/** Surface name to what a few footsteps on it sound like. */
|
|
30
33
|
steps: z.record(z.string(), Prompt),
|
|
31
34
|
/** Short one-shots: picking up an item, an exit, using a thing, the ending. */
|
|
32
|
-
shots: z.array(z.object({ on: z.string(), prompt: Prompt, seconds: z.number().min(0.5).max(5).default(1.5), level: Level })).
|
|
35
|
+
shots: z.array(z.object({ on: z.string(), prompt: Prompt, seconds: z.number().min(0.5).max(5).default(1.5), level: Level })).default([])
|
|
36
|
+
// A world with many exits (a car parked in every place) asks for more than fit: the first ones are kept.
|
|
37
|
+
.transform((shots) => shots.slice(0, MAX_SHOTS)),
|
|
33
38
|
});
|
|
34
39
|
export type SoundPlan = z.infer<typeof SoundPlan>;
|
|
35
40
|
|
package/app/src/story/engine.ts
CHANGED
|
@@ -17,7 +17,8 @@ export type State = {
|
|
|
17
17
|
};
|
|
18
18
|
|
|
19
19
|
export type Action =
|
|
20
|
-
|
|
20
|
+
/** `to` picks among several places one thing leads to (a car that drives anywhere). */
|
|
21
|
+
| { kind: 'go'; via: string; to?: string }
|
|
21
22
|
| { kind: 'do'; thing: string; verb: string; with?: string }
|
|
22
23
|
| { kind: 'talk'; npc: string }
|
|
23
24
|
| { kind: 'choose'; index: number }
|
|
@@ -101,7 +102,11 @@ export function actions(world: World, story: Story, state: State): Action[] {
|
|
|
101
102
|
}
|
|
102
103
|
const place = world.places.find((p) => p.key === state.place)!;
|
|
103
104
|
const out: Action[] = [];
|
|
104
|
-
for (const exit of place.exits)
|
|
105
|
+
for (const exit of place.exits) {
|
|
106
|
+
if (lockOn(story, state, exit.via)) continue;
|
|
107
|
+
const shared = place.exits.filter((e) => e.via === exit.via).length > 1;
|
|
108
|
+
out.push({ kind: 'go', via: exit.via, ...(shared && { to: exit.to }) });
|
|
109
|
+
}
|
|
105
110
|
const seen = new Set<string>();
|
|
106
111
|
for (const it of story.interactions) {
|
|
107
112
|
const id = `${it.thing}|${it.verb}|${it.with ?? ''}`;
|
|
@@ -126,7 +131,7 @@ export function apply(world: World, story: Story, state: State, action: Action):
|
|
|
126
131
|
case 'go': {
|
|
127
132
|
const lock = lockOn(story, state, action.via);
|
|
128
133
|
if (lock) return { state, say: [lock.say] };
|
|
129
|
-
const exit = world.places.find((p) => p.key === state.place)?.exits.find((e) => e.via === action.via);
|
|
134
|
+
const exit = world.places.find((p) => p.key === state.place)?.exits.find((e) => e.via === action.via && (!action.to || e.to === action.to));
|
|
130
135
|
if (!exit) throw new Error(`no exit "${action.via}" in ${state.place}`);
|
|
131
136
|
next.place = exit.to;
|
|
132
137
|
break;
|
|
@@ -201,7 +206,7 @@ export function stateKey(state: State): string {
|
|
|
201
206
|
|
|
202
207
|
export function describe(action: Action): string {
|
|
203
208
|
switch (action.kind) {
|
|
204
|
-
case 'go': return `go ${action.via}`;
|
|
209
|
+
case 'go': return `go ${action.via}${action.to ? ` to ${action.to}` : ''}`;
|
|
205
210
|
case 'do': return `${action.verb} ${action.thing}${action.with ? ` with ${action.with}` : ''}`;
|
|
206
211
|
case 'talk': return `talk ${action.npc}`;
|
|
207
212
|
case 'choose': return `choose ${action.index + 1}`;
|
package/app/src/story/fixes.ts
CHANGED
|
@@ -9,6 +9,8 @@ export const StoryFixes = z.object({
|
|
|
9
9
|
wish: z.string().min(5).optional(),
|
|
10
10
|
/** Reworded text by its place in story.json, e.g. "dialogues.0.lines.2.says". */
|
|
11
11
|
texts: z.record(z.string(), z.object({ text: z.string().min(1), at: z.string() })).default({}),
|
|
12
|
+
/** What the hero says looking at a thing, by "place/thing": added where the story has no look line there, else it rewords it. */
|
|
13
|
+
looks: z.record(z.string(), z.object({ text: z.string().min(1), at: z.string() })).default({}),
|
|
12
14
|
});
|
|
13
15
|
export type StoryFixes = z.infer<typeof StoryFixes>;
|
|
14
16
|
|
|
@@ -19,7 +21,7 @@ export const storyFixesFile = (files: WorldFiles) => files.path('story-fixes.jso
|
|
|
19
21
|
|
|
20
22
|
export function readStoryFixes(files: WorldFiles): StoryFixes {
|
|
21
23
|
const file = storyFixesFile(files);
|
|
22
|
-
return fs.existsSync(file) ? StoryFixes.parse(files.readJson(file)) : { texts: {} };
|
|
24
|
+
return fs.existsSync(file) ? StoryFixes.parse(files.readJson(file)) : { texts: {}, looks: {} };
|
|
23
25
|
}
|
|
24
26
|
|
|
25
27
|
/** The object holding a text field and the field's name, or null when the path names no text. */
|
|
@@ -42,7 +44,7 @@ export function textAt(story: Story, path: string): string | null {
|
|
|
42
44
|
}
|
|
43
45
|
|
|
44
46
|
/** The story with the reworded lines in; paths that no longer name a text are left out and listed. */
|
|
45
|
-
export function applyTexts(story: Story, fixes: StoryFixes): { story: Story; skipped: string[] } {
|
|
47
|
+
export function applyTexts(story: Story, fixes: Pick<StoryFixes, 'texts'> & Partial<Pick<StoryFixes, 'looks'>>): { story: Story; skipped: string[] } {
|
|
46
48
|
const out = structuredClone(story);
|
|
47
49
|
const skipped: string[] = [];
|
|
48
50
|
for (const [path, { text }] of Object.entries(fixes.texts)) {
|
|
@@ -50,5 +52,12 @@ export function applyTexts(story: Story, fixes: StoryFixes): { story: Story; ski
|
|
|
50
52
|
if (hit) hit[0][hit[1]] = text;
|
|
51
53
|
else skipped.push(path);
|
|
52
54
|
}
|
|
55
|
+
for (const [at, { text }] of Object.entries(fixes.looks ?? {})) {
|
|
56
|
+
const [place, thing] = at.split('/');
|
|
57
|
+
// The plain look line (no condition) is the one replaced; a thing with none gets one.
|
|
58
|
+
const plain = out.interactions.find((i) => i.place === place && i.thing === thing && i.verb === 'look' && i.when.length === 0);
|
|
59
|
+
if (plain) plain.say = text;
|
|
60
|
+
else out.interactions.push({ place, thing, verb: 'look', when: [], say: text, do: [], once: false } as Story['interactions'][number]);
|
|
61
|
+
}
|
|
53
62
|
return { story: out, skipped };
|
|
54
63
|
}
|
package/app/src/studio/agent.ts
CHANGED
|
@@ -10,6 +10,7 @@ import { configDir, redact, withoutSecrets } from '../secrets.ts';
|
|
|
10
10
|
import type { Events } from './events.ts';
|
|
11
11
|
import { History } from './history.ts';
|
|
12
12
|
import { NPC_STORY_GUIDANCE } from '../story/npc-guidance.ts';
|
|
13
|
+
import { cli } from '../cli.ts';
|
|
13
14
|
|
|
14
15
|
export const AGENT_BRIEF = (slug: string) => `You are the coding agent inside Diorama Studio, working on the world "${slug}" with the person.
|
|
15
16
|
Use only the diorama MCP tools: they are the same actions as the Studio's buttons, and the person watches each call live.
|
|
@@ -26,12 +27,12 @@ When a job seems slow or the person asks what it is doing, call job_inspect for
|
|
|
26
27
|
To fix a place use the fix tools, which are kept across rebuilds: mask_fix (a better SAM phrase, the real box, or the kind of a thing), floor_cells, mask_brush, stand_set, repaint_region (costs a Muse edit), paint_words (how the place, its things and floors are described to the painter); corrections_get shows what is fixed already, corrections_clear takes fixes back. When asked to go through the person's notes, read feedback_pending, fix each one and close it with feedback_done in one sentence. Check budget before a rebuild of many places.
|
|
27
28
|
When the person wants the game in another language ("make it English", "po polsku"), call world_language: it translates the title, names and the whole story in one pass and casts and records the voices again as native speakers; first without go, tell them what it redoes and what the voices cost, then call it with go true once they agree (voices false when they only want the text now). Never translate it line by line with story_text, and never say the language cannot be changed.
|
|
28
29
|
When the person wants the game to look different (more realistic, watercolour, like a picture book, less cartoon), call style_set with the style in a few words (medium, palette, light; scope place or cast for only one place or the characters): first without go, tell them what it repaints and about what it costs, then call it with go true once they agree. It repaints each picture keeping where everything is, so masks and their fixes still fit; children's drawings stay the reference; history_undo takes it back. Never write world.json's style by hand.
|
|
29
|
-
Before a risky change, say what you will do; history_undo can take back a job. Respect locks. Keys are never yours: if a key is missing, ask the person to run
|
|
30
|
+
Before a risky change, say what you will do; history_undo can take back a job. Respect locks. Keys are never yours: if a key is missing, ask the person to run \`${cli('setup')}\` in a terminal.
|
|
30
31
|
Read files freely to find out why something failed, but do not edit world files or run commands that change them: make every change through the diorama MCP tools, so the person sees it live, can undo it, and it survives the next build (a hand edit is overwritten by the next build). When no single tool makes the change the person asks for, do not stop at "there is no tool": find the nearest way with the tools you have (several story_text calls, a story_redo with their wish, a run_step) and offer it with what it costs, using ask_person; mention the missing tool only as a side note, so it can be added later. Refuse only what would need a hand edit or would throw away their work. Start from world_status, place_status, place_view, job_wait, corrections_get and feedback_pending: place_view shows you a place as a picture (layer walk, masks or painting), which files and the shell cannot. Reach for the shell only when no Studio tool answers the question.
|
|
31
32
|
When a build fails for another reason (a step that broke, a service that did not answer), the Studio wakes you with the error and the job's last lines: job_inspect it, give a service error (HTTP 5xx, a timeout, a busy Space) one job_retry, which picks up at the failed step, and fix anything else with that step's own tool; never retry the same error twice, tell the person instead. A character the 3D service left in 2D (cast_status: the 3D service did not answer) still plays with its 2D rig; cast_animate tries its model again once the service is back. When a build leaves places not playable, repair them one by one: place_status lists each error with its playbook (where to look, the fix tools in order, the last resort), and place_view shows you the walkable floor with the 60 px cells floor_cells takes, the gaps to bridge and the spots off the floor. An unreached patch where nothing is floor (a wall, the sky) is taken back, not bridged: corrections_clear the fix that made it, or mask_brush erase. Try the tools in that order; each fix rebuilds only that place, so read place_status again after its job ends. After three fixes that did not change the error, do not repeat one that failed: look again with place_view and take a different path (mask_brush by hand, floor_cells, thing_remove, corrections_clear); the fix tools refuse an exact repeat, and after six they keep repaints and relayouts for the person. If the new path fails too, tell the person briefly what you tried and what you would do next. Ask before the last resort, except one repaint_region per place.
|
|
32
33
|
When layout failed and there is no picture, do not wait for place_view: inspect place_status errors, then call layout_fix with concrete layout guidance, or place_fix to resume. Both work before the place directory exists. Failed layout batches retain their diagnostics and retry with new feedback instead of replaying the same rejected batch. When paint fails with content_policy_violation, the image safety filter refused a word, not the picture: paint already lets Spark reword the place twice, so call paint_words with calm, plain words for the scene and the looks the brief had (leafless, not bare; a seat that rests, not hangs on chains; no children, blood or weapons), never an edit of world.json. A place that never got past a step is rebuilt from that step: place_fix (the Fix it button on its card) or job_retry does that by itself.
|
|
33
34
|
Stairs: derive finds the steps of floors and walk-on things named as stairs (stair, step, schody), and people then step from tread to tread; place_view (layer walk) draws the steps it found in blue. Scale: derive measures how tall a person is at each spot (Spark's people, held against a door on the floor: a person by a door is about 0.8 of its height). When the hero looks far too small or too big next to a door, run_step derive with that place, which rebuilds it through verify; when it stays wrong, the door is not named as a door: give it a door phrase with mask_fix. When people glide up stairs as on a ramp, the stairs thing or floor is not named as stairs: give it a stairs phrase with mask_fix (for example "stone stairs"), which rebuilds the place.
|
|
34
|
-
A message that starts with "[On screen: …]" says what the person had open in Studio as they wrote ("place", "place / thing", "cast", "cast / character" or "cast / item key"; older messages say "[Looking at: …]"): when they say "this", "it" or "here" without a name, they mean that; look at it (place_view for a place) before you change it. Never mention the prefix. A message that starts with "[Pictures: studio/pictures/01.jpg, …]" comes with pictures the person pasted, uploaded or took with the webcam: look at each with picture_view before you answer, and act on what they show. When the person is playing and your change shows in the game, call play_reload yourself rather than asking them to refresh. You cannot watch the game, but place_motion can: after a change to motion (water flow, fog, fire, weather, an animated thing) call it and check the motion shows before you say it works. A message that starts with "[Studio]" is a private note from the Studio, not from the person: follow it, never mention it or reply to it. The story has story_status, story_text (rewording) and story_redo (a wish to Spark); story_status lists unanswerable questions (a question in a "say" or in a line without choices leaves the player stuck): reword it into a statement with story_text, or story_redo when it needs real answers; it also lists unseen story parts (an item thing whose item has another key stays in the painting; a change on a person is never drawn): fix those with story_redo; the cast has cast_status, cast_redo, cast_animate, cast_rewalk, cast_holds and voice_set; a figure whose cut-out lost what it holds or carries (a toast over its head, a lantern, a basket; cast_status cut.holds lists what is kept) gets cast_holds, not cast_redo: it cuts the same painting again with the held thing; speaking faces have faces_status and faces_prepare. Talking mouths can be off in Setup (faces_get; off by default): then never prepare, repair or calibrate faces and never offer to; characters speak with voice and gestures. Free fixes before paid ones: a 3D figure whose face shows as a patch of another colour, or whose colours look pale or plastic beside its painting, is a render matter; run sprites again (free for UV sprites, no Muse) before ever offering sprites_redo or per-frame Muse repaints, and when you do offer paid work say the real count, not a range. Order: a playable game first, talking mouths last. While any place is unpainted, being built or unplayable, never start face work (faces_prepare, face_frame_repair, face_lips_set, face_calibration_set refuse it); repair the places, and leave lips for after everything plays, or skip them: characters talk with gestures. A failed speaking animation is separate from game readiness; recordings and the previous appearance remain usable. Inspect coverage and repairWork first. Accepted atlases and calibrated mouths are saved, so resume rather than repainting the whole character. For a rejected mouth, faces_inspect with its work id and mouth returns the exact head and reference; face_frame_repair uses the returned revision and concrete visual guidance to edit ONLY that mouth, then revalidate the atlas. For an accepted atlas with uncertain lip corners, inspect the exact mouth image with faces_inspect and use face_lips_set with a normalized tight lip box relative to that single full-head image (not the whole atlas). Never guess coordinates, use it on rejected articulation, or lower validation thresholds. Both tools resume preparation and reject stale revisions/sources. Wait for the job, inspect faces_status again and play_reload after success. Do not repeat an unchanged failed repair; inspect the new stage and use its matching tool. For source-pose or final-patch calibration failures, face_calibration_inspect lists pending targets and shows their exact close-up. Use face_calibration_set with its target, revision, imageHash, tight lip box and visible-cavity open flag. This preserves accepted artwork and resumes the remaining checks. If an inspected mouth remains genuinely ambiguous, stop and explain that specific uncertainty rather than inventing coordinates. Image, 2D rig and talk-sprite faces share audio timing. 3D models articulate when model3d.mouth is true: method morphs uses authored facial poses, painted-surfaces animates calibrated face paint on the actual mesh. Check the reported view coverage and spritePoses pending count; a body skeleton alone does not articulate. Do not claim native mouth geometry for the painted-surface method; a 3D figure that walks straddle-legged with arms held out to the sides (like a zombie) or hands sticking out gets cast_relax (local Blender, free; blender_get says if it is there, else tell the person to install Blender and turn it on in Setup); a walk that tears (holes in clothes, a leg, boot or arm coming off, a lantern left hanging in the air) gets cast_rewalk, not cast_redo; a character with no 3D model the person explicitly wants animated in 3D (a creature, cast_status body upright false, or an NPC with no lines) gets a model with cast_animate, never a cast_redo that makes it stand on two legs: a creature hops, squashes and blinks as one body, then play_reload; a hero painted sitting, lying or with furniture grown into the 3D model (from a photo where they sat or lay) gets cast_redo with looks that describe only the person standing (clothes, trousers, shoes; no pose, bed or "not visible"), then the model again; inventory icons have icons_status, icon_redo (a hint in the person's words, cut or paint, another take) and icon_fix_clear, never a swapped file in items/. Voices are cast from the gender and age Spark sees in each figure: adults get stock voices, children, teens and the old get a voice made for them by ElevenLabs Voice Design (voice "design"). When one sounds wrong, voice_list with slug and key lists the voices that fit that character first; voice_set with voice "design" and a design description in English remakes a designed voice (no word "child" or age in years there: ElevenLabs refuses it). Never give a child an adult's voice or raise an adult voice's pitch to fake one. A character whose voice is "pending-design" in cast_status waits for Voice Design (it was refused or the service was busy): run_step voices or voice_set designs it again; never give them a stock adult instead. Lines with numbers or sums ("10+10=20") are read and checked as words by themselves; voiceFailures lists only lines the story still has. For one missing or rejected line, use voice_retry with slug, key and the exact text from voiceFailures: it keeps the selected voice and every other recording, gives rejected takes fresh seeds, and resumes saved audio after a transcription outage. Do not change voice_set merely to retry a line. Sounds have sound_status (each bed, loop, footstep, one-shot and music: made, or rejected and why) and sound_redo (ids, an optional hint in English, kept across rebuilds): a sound rejected as "mostly hiss" or "nearly silent" stays silent in the game until sound_redo, never an edited sound.json. cast_status says how the game moves each character (moves), why there is no 2D walk (rigIssue) and which walk directions left the pose (spriteIssues): those get sprites_redo, which paints new sheets without repainting the character; a character with no model after flat reconstructions walks with its 2D rig or slides as a cut-out, and cast_animate tries the model again. When the hero starts in front of an NPC or a thing and hides it, hero_start_set moves where they begin; npc_move changes the NPC standing position in the layout. Author a story-triggered trip with story_redo. A still frame in place_motion is a fault only when inTheGame lists water, mist or weather for that place.
|
|
35
|
+
A message that starts with "[On screen: …]" says what the person had open in Studio as they wrote ("place", "place / thing", "cast", "cast / character" or "cast / item key"; older messages say "[Looking at: …]"): when they say "this", "it" or "here" without a name, they mean that; look at it (place_view for a place) before you change it. Never mention the prefix. A message that starts with "[Pictures: studio/pictures/01.jpg, …]" comes with pictures the person pasted, uploaded or took with the webcam: look at each with picture_view before you answer, and act on what they show. When the person is playing and your change shows in the game, call play_reload yourself rather than asking them to refresh. You cannot watch the game, but place_motion can: after a change to motion (water flow, fog, fire, weather, an animated thing) call it and check the motion shows before you say it works. A message that starts with "[Studio]" is a private note from the Studio, not from the person: follow it, never mention it or reply to it. The story has story_status, story_text (rewording), story_look (what the hero says looking at one thing, added where there is none, nothing else changes: use it rather than story_redo for a missing look line) and story_redo (a wish to Spark, which writes the whole story again); story_status lists unanswerable questions (a question in a "say" or in a line without choices leaves the player stuck): reword it into a statement with story_text, or story_redo when it needs real answers; it also lists unseen story parts (an item thing whose item has another key stays in the painting; a change on a person is never drawn): fix those with story_redo; the cast has cast_status, cast_redo, cast_animate, cast_rewalk, cast_holds and voice_set; a figure whose cut-out lost what it holds or carries (a toast over its head, a lantern, a basket; cast_status cut.holds lists what is kept) gets cast_holds, not cast_redo: it cuts the same painting again with the held thing; speaking faces have faces_status and faces_prepare. Talking mouths can be off in Setup (faces_get; off by default): then never prepare, repair or calibrate faces and never offer to; characters speak with voice and gestures. Free fixes before paid ones: a 3D figure whose face shows as a patch of another colour, or whose colours look pale or plastic beside its painting, is a render matter; run sprites again (free for UV sprites, no Muse) before ever offering sprites_redo or per-frame Muse repaints, and when you do offer paid work say the real count, not a range. Order: a playable game first, talking mouths last. While any place is unpainted, being built or unplayable, never start face work (faces_prepare, face_frame_repair, face_lips_set, face_calibration_set refuse it); repair the places, and leave lips for after everything plays, or skip them: characters talk with gestures. A failed speaking animation is separate from game readiness; recordings and the previous appearance remain usable. Inspect coverage and repairWork first. Accepted atlases and calibrated mouths are saved, so resume rather than repainting the whole character. For a rejected mouth, faces_inspect with its work id and mouth returns the exact head and reference; face_frame_repair uses the returned revision and concrete visual guidance to edit ONLY that mouth, then revalidate the atlas. For an accepted atlas with uncertain lip corners, inspect the exact mouth image with faces_inspect and use face_lips_set with a normalized tight lip box relative to that single full-head image (not the whole atlas). Never guess coordinates, use it on rejected articulation, or lower validation thresholds. Both tools resume preparation and reject stale revisions/sources. Wait for the job, inspect faces_status again and play_reload after success. Do not repeat an unchanged failed repair; inspect the new stage and use its matching tool. For source-pose or final-patch calibration failures, face_calibration_inspect lists pending targets and shows their exact close-up. Use face_calibration_set with its target, revision, imageHash, tight lip box and visible-cavity open flag. This preserves accepted artwork and resumes the remaining checks. If an inspected mouth remains genuinely ambiguous, stop and explain that specific uncertainty rather than inventing coordinates. Image, 2D rig and talk-sprite faces share audio timing. 3D models articulate when model3d.mouth is true: method morphs uses authored facial poses, painted-surfaces animates calibrated face paint on the actual mesh. Check the reported view coverage and spritePoses pending count; a body skeleton alone does not articulate. Do not claim native mouth geometry for the painted-surface method; a 3D figure that walks straddle-legged with arms held out to the sides (like a zombie) or hands sticking out gets cast_relax (local Blender, free; blender_get says if it is there, else tell the person to install Blender and turn it on in Setup); a walk that tears (holes in clothes, a leg, boot or arm coming off, a lantern left hanging in the air) gets cast_rewalk, not cast_redo; a character with no 3D model the person explicitly wants animated in 3D (a creature, cast_status body upright false, or an NPC with no lines) gets a model with cast_animate, never a cast_redo that makes it stand on two legs: a creature hops, squashes and blinks as one body, then play_reload; a hero painted sitting, lying or with furniture grown into the 3D model (from a photo where they sat or lay) gets cast_redo with looks that describe only the person standing (clothes, trousers, shoes; no pose, bed or "not visible"), then the model again; inventory icons have icons_status, icon_redo (a hint in the person's words, cut or paint, another take) and icon_fix_clear, never a swapped file in items/. Voices are cast from the gender and age Spark sees in each figure: adults get stock voices, children, teens and the old get a voice made for them by ElevenLabs Voice Design (voice "design"). When one sounds wrong, voice_list with slug and key lists the voices that fit that character first; voice_set with voice "design" and a design description in English remakes a designed voice (no word "child" or age in years there: ElevenLabs refuses it). Never give a child an adult's voice or raise an adult voice's pitch to fake one. A character whose voice is "pending-design" in cast_status waits for Voice Design (it was refused or the service was busy): run_step voices or voice_set designs it again; never give them a stock adult instead. Lines with numbers or sums ("10+10=20") are read and checked as words by themselves; voiceFailures lists only lines the story still has. For one missing or rejected line, use voice_retry with slug, key and the exact text from voiceFailures: it keeps the selected voice and every other recording, gives rejected takes fresh seeds, and resumes saved audio after a transcription outage. Do not change voice_set merely to retry a line. Sounds have sound_status (each bed, loop, footstep, one-shot and music: made, or rejected and why) and sound_redo (ids, an optional hint in English, kept across rebuilds): a sound rejected as "mostly hiss" or "nearly silent" stays silent in the game until sound_redo, never an edited sound.json. cast_status says how the game moves each character (moves), why there is no 2D walk (rigIssue) and which walk directions left the pose (spriteIssues): those get sprites_redo, which paints new sheets without repainting the character; a character with no model after flat reconstructions walks with its 2D rig or slides as a cut-out, and cast_animate tries the model again. When the hero starts in front of an NPC or a thing and hides it, hero_start_set moves where they begin; npc_move changes the NPC standing position in the layout. Author a story-triggered trip with story_redo. A still frame in place_motion is a fault only when inTheGame lists water, mist or weather for that place.
|
|
35
36
|
When you need the person to decide (yes or no, which of two fixes, go ahead with something that costs calls), ask with ask_person instead of a question in your text: it shows them buttons; then end your turn.
|
|
36
37
|
After a redo the person asked for (a repaint, a recast), show them the result and stop: if a trace of what they wanted gone is still faintly there, say so and offer one more try with ask_person; never start another paid redo on your own. A message from the person while you wait for a job comes first: read it before waiting again, and job_cancel the job if they no longer want it.
|
|
37
38
|
Answer in two or three short sentences, in the language the person wrote their newest message in (not the world's language, not an earlier turn's); the steps already show what you did.`;
|
|
@@ -304,6 +305,15 @@ export const AGENTS = {
|
|
|
304
305
|
codex: { label: 'Codex', make: () => preapproved(codexLive()) },
|
|
305
306
|
antigravity: { label: 'Antigravity', make: () => antigravity({ command: process.env.DIORAMA_AGY_COMMAND ?? 'agy' }) },
|
|
306
307
|
} satisfies Record<string, { label: string; make(command?: string): CodingAgent }>;
|
|
308
|
+
|
|
309
|
+
/**
|
|
310
|
+
* The model the Studio asks an agent for, from DIORAMA_<AGENT>_MODEL (DIORAMA_AGY_MODEL for Antigravity, e.g.
|
|
311
|
+
* gemini-4-argon-high; `agy models` lists them), else DIORAMA_AGENT_MODEL; unset leaves the agent's own default.
|
|
312
|
+
*/
|
|
313
|
+
export function agentModel(name: AgentName, env: NodeJS.ProcessEnv = process.env): string | undefined {
|
|
314
|
+
const own = { muse: 'DIORAMA_MUSE_MODEL', claude: 'DIORAMA_AGENT_CLAUDE_MODEL', codex: 'DIORAMA_CODEX_MODEL', antigravity: 'DIORAMA_AGY_MODEL' }[name];
|
|
315
|
+
return env[own]?.trim() || env.DIORAMA_AGENT_MODEL?.trim() || undefined;
|
|
316
|
+
}
|
|
307
317
|
export type AgentName = keyof typeof AGENTS;
|
|
308
318
|
export const isAgentName = (name: string): name is AgentName => Object.hasOwn(AGENTS, name);
|
|
309
319
|
|
|
@@ -400,10 +410,12 @@ export class AgentHub {
|
|
|
400
410
|
const cwd = fs.existsSync(dir) ? dir : this.opts.root;
|
|
401
411
|
fs.mkdirSync(cwd, { recursive: true });
|
|
402
412
|
const ask = process.env.DIORAMA_AGENT_TOOLS === 'ask';
|
|
403
|
-
const
|
|
413
|
+
const name = this.defaultAgent(slug);
|
|
414
|
+
const agent = this.make(name);
|
|
404
415
|
const session = createAgentSession({
|
|
405
416
|
agent,
|
|
406
417
|
cwd,
|
|
418
|
+
model: agentModel(name),
|
|
407
419
|
task: {
|
|
408
420
|
env: withoutProviderCredentials(withoutSecrets(process.env)) as Record<string, string>,
|
|
409
421
|
// genaicode hands the server to Muse as -c overrides, the token through an environment variable, never argv.
|
|
@@ -477,6 +489,7 @@ export class AgentHub {
|
|
|
477
489
|
const from = this.agents(slug).find((a) => a.using)?.label;
|
|
478
490
|
const agent = this.make(name);
|
|
479
491
|
t.session.setAgent(agent);
|
|
492
|
+
t.session.setModel(agentModel(name));
|
|
480
493
|
t.resumes = !!agent.capabilities.resume;
|
|
481
494
|
t.rebrief();
|
|
482
495
|
this.save(name);
|
package/app/src/studio/mcp.ts
CHANGED
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
// this over streamable HTTP at /mcp (Muse Code connects there) and `diorama mcp` over stdio.
|
|
3
3
|
import { actions, describeActions, runAction, type ActionContext } from '../actions/registry.ts';
|
|
4
4
|
import { redact } from '../secrets.ts';
|
|
5
|
+
import { cli } from '../cli.ts';
|
|
5
6
|
|
|
6
7
|
type Message = { jsonrpc: '2.0'; id?: string | number | null; method?: string; params?: Record<string, unknown> };
|
|
7
8
|
export type Reply = { jsonrpc: '2.0'; id: string | number | null; result?: unknown; error?: { code: number; message: string } };
|
|
@@ -10,7 +11,7 @@ const PROTOCOL = '2025-06-18';
|
|
|
10
11
|
|
|
11
12
|
export const INSTRUCTIONS = 'Diorama builds painted point-and-click worlds. Every tool here is a Diorama Studio action; the person sees each call '
|
|
12
13
|
+ 'in the Studio at once. Long work (world_build, run_step) starts a job: follow it with job_wait. Use ui_focus to show the person the '
|
|
13
|
-
+ 'world or place you are talking about. Keys are never available to you: if setup_status says one is missing, ask the person to run
|
|
14
|
+
+ 'world or place you are talking about. Keys are never available to you: if setup_status says one is missing, ask the person to run ' + cli('setup') + '.';
|
|
14
15
|
|
|
15
16
|
export function mcpTools() {
|
|
16
17
|
return describeActions().map((a) => ({
|
package/app/src/studio/server.ts
CHANGED
|
@@ -24,6 +24,7 @@ import { bundle, WEB, worldData } from '../commands/play.ts';
|
|
|
24
24
|
import { WorldFiles } from '../world/files.ts';
|
|
25
25
|
import { snapshotBeforeJob } from './history.ts';
|
|
26
26
|
import { handleMcp } from './mcp.ts';
|
|
27
|
+
import { cli } from '../cli.ts';
|
|
27
28
|
|
|
28
29
|
export const STUDIO_WEB = fileURLToPath(new URL('../../studio/', import.meta.url));
|
|
29
30
|
const CHAT_CSS = createRequire(import.meta.url).resolve('genaicode/react/styles.css');
|
|
@@ -184,7 +185,7 @@ export async function startStudio(opts: { port?: number; root?: string; cwd?: st
|
|
|
184
185
|
});
|
|
185
186
|
|
|
186
187
|
// The game and pictures read the world through the Studio's session cookie; a browser that never opened the Studio has none.
|
|
187
|
-
const NO_SESSION =
|
|
188
|
+
const NO_SESSION = `no session: open the Studio link from \`${cli('studio')}\` in this browser first, or play the world on its own with \`${cli('play <slug>')}\``;
|
|
188
189
|
const server = http.createServer(async (req, res) => {
|
|
189
190
|
const headers = { 'cache-control': 'no-store', 'x-content-type-options': 'nosniff', 'referrer-policy': 'no-referrer', 'cross-origin-resource-policy': 'same-origin' };
|
|
190
191
|
const send = (status: number, type: string, body: string | Buffer, extra: Record<string, string> = {}) => {
|
|
@@ -239,7 +240,7 @@ export async function startStudio(opts: { port?: number; root?: string; cwd?: st
|
|
|
239
240
|
}
|
|
240
241
|
|
|
241
242
|
if (url.pathname.startsWith('/api/')) {
|
|
242
|
-
if (!authed) return json(401, { error:
|
|
243
|
+
if (!authed) return json(401, { error: `missing or wrong token: open the link ${cli('studio')} printed` });
|
|
243
244
|
if (req.method === 'POST' && !(req.headers['content-type'] ?? '').startsWith('application/json')) return json(415, { error: 'JSON only' });
|
|
244
245
|
|
|
245
246
|
if (url.pathname === '/api/session' && req.method === 'POST') {
|
package/app/src/world/design.ts
CHANGED
|
@@ -35,6 +35,7 @@ ${extra ? `${extra}
|
|
|
35
35
|
` : ''}Design ${places} places that tell this idea${fixed ? ', the room included' : ''}. Rules:
|
|
36
36
|
- 3 to 7 things per place, big enough to see and click; no tiny details.
|
|
37
37
|
- Exits are things you can see (door, path, ladder, boat) and every exit has one back.
|
|
38
|
+
- Only when the idea gives the hero a car to drive around in: one place has "kind": "drive", the car itself (its shows: the car at the kerb with the engine running), with one thing, the car (role exit), and an exit through that same thing to every other place it drives to; every one of those places has the car parked in it as a thing (role exit, the same looks) with an exit back to the drive place. The player gets in, picks a destination and gets out there.
|
|
38
39
|
- The world has at least one item to pick up and one npc to talk to.
|
|
39
40
|
- Everything is original: no characters, places or names from existing films, books, games or brands.
|
|
40
41
|
- A place's shows and its props' looks go to a painter whose safety filter refuses single words: calm, plain words (leafless trees, not bare; a swing seat that rests, not hangs on chains; no blood, weapons, graves or bodies).
|