cozyclay 1.2.0 → 1.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (72) hide show
  1. package/CHANGELOG.md +85 -0
  2. package/README.md +31 -0
  3. package/THIRD_PARTY_NOTICES.md +37 -1
  4. package/bin/cozyclay.mjs +51 -2
  5. package/dist/ai-camera-control/index.html +406 -0
  6. package/dist/app/index.html +5 -5
  7. package/dist/assets/app-B3U5aut1.js +4811 -0
  8. package/dist/assets/app-BrRF0wso.css +1 -0
  9. package/dist/assets/vision_bundle-jFkh-fIS.js +41 -0
  10. package/dist/fonts/InstrumentSerif-OFL.txt +93 -0
  11. package/dist/fonts/Inter-OFL.txt +92 -0
  12. package/dist/fonts/README.md +15 -0
  13. package/dist/index.html +60 -16
  14. package/dist/sitemap.xml +7 -1
  15. package/mcp/LIVE-PROTOCOL.md +63 -0
  16. package/mcp/README.md +142 -0
  17. package/mcp/ardy-prompts.mjs +170 -0
  18. package/mcp/live-hub.mjs +105 -0
  19. package/mcp/package.json +24 -0
  20. package/mcp/server.mjs +1394 -0
  21. package/package.json +122 -90
  22. package/src/App.jsx +2957 -514
  23. package/src/ardy/cskel27.js +7 -2
  24. package/src/ardy/ik.js +25 -15
  25. package/src/ardy/npz.js +64 -3
  26. package/src/ardy/playback.js +31 -1
  27. package/src/ardy/prompt-clips.js +7 -2
  28. package/src/ardy/retime.js +211 -0
  29. package/src/ardy/timeline-coordinates.js +13 -0
  30. package/src/ardy/timeline.jsx +124 -5
  31. package/src/ardy/to-cskel27.js +34 -12
  32. package/src/ardy/trim.js +33 -0
  33. package/src/asset-pane.jsx +36 -0
  34. package/src/dualview.jsx +14 -8
  35. package/src/hierarchy-model.js +95 -13
  36. package/src/hierarchy-panel.jsx +139 -6
  37. package/src/live-control.js +122 -0
  38. package/src/matte-editor.js +543 -0
  39. package/src/matte.js +503 -0
  40. package/src/multimodel-ingest.js +344 -0
  41. package/src/object-gizmo.jsx +43 -15
  42. package/src/planview.jsx +43 -32
  43. package/src/pose-extract/detector.js +75 -0
  44. package/src/pose-extract/index.js +3 -0
  45. package/src/pose-extract/take.js +87 -0
  46. package/src/pose-extract/video-frames.js +91 -0
  47. package/src/pose-thumbs.js +152 -0
  48. package/src/posestudio.jsx +361 -11
  49. package/src/project-browser.jsx +135 -0
  50. package/src/project.js +289 -0
  51. package/src/props.jsx +69 -3
  52. package/src/room.jsx +14 -35
  53. package/src/scene-asset-cache.js +125 -0
  54. package/src/scene-assets.js +288 -0
  55. package/src/scene-objects.js +245 -7
  56. package/src/scenes.js +207 -26
  57. package/src/shot-authoring.js +55 -13
  58. package/src/styles.css +1296 -129
  59. package/tools/ardy/BRIDGE.md +3 -2
  60. package/tools/ardy/README.md +9 -5
  61. package/tools/ardy/bridge.mjs +57 -1
  62. package/tools/ardy/bvh-cskel27.mjs +1209 -0
  63. package/tools/ardy/cclay_constrained_generate.py +123 -11
  64. package/tools/ardy/cclay_sequence_generate.py +49 -0
  65. package/tools/ardy/extract.mjs +367 -0
  66. package/tools/ardy/footage.mjs +462 -0
  67. package/tools/ardy/npz.mjs +74 -9
  68. package/tools/ardy/run-on-box.sh +25 -0
  69. package/tools/ardy/run-sequence-on-box.sh +15 -0
  70. package/tools/ardy/runners/remote.mjs +8 -2
  71. package/dist/assets/app-Cgpk2hwX.js +0 -4803
  72. package/dist/assets/app-DgZvaAE1.css +0 -1
package/mcp/server.mjs ADDED
@@ -0,0 +1,1394 @@
1
+ #!/usr/bin/env node
2
+ /**
3
+ * cozyclay-mcp — an MCP surface over CozyClay's authoring core.
4
+ *
5
+ * This server owns NO geometry, NO film vocabulary and NO prompt text. Every
6
+ * answer it gives is computed by the same modules the studio renders with,
7
+ * imported straight from the published `cozyclay` package:
8
+ *
9
+ * shot.js geometry -> film vocabulary -> prompt
10
+ * scenes.js the scene document + its stage envelope
11
+ * scene-objects.js the set: create/update/remove/normalise
12
+ * cuts.js shots on a timeline
13
+ * camera-move.js two framings -> a named camera move
14
+ * project.js the .cclayproject envelope
15
+ *
16
+ * Running inside the repo, those imports are relative: this server always
17
+ * speaks the working tree's own vocabulary, so a change to shot.js is visible
18
+ * here on the next start with nothing to publish or reinstall.
19
+ *
20
+ * Keeping the maths on the other side of that import is the whole design: the
21
+ * studio and this server can never disagree about what a 35mm medium shot is,
22
+ * because there is only one implementation of it.
23
+ *
24
+ * State is one in-memory scene document plus one camera. `save_project` writes
25
+ * the real `.cclayproject` envelope, so anything authored here opens in the
26
+ * studio, and anything authored in the studio opens here.
27
+ */
28
+ import { readFile, writeFile } from "node:fs/promises";
29
+ import { resolve } from "node:path";
30
+
31
+ import { createServer } from "node:http";
32
+ import { randomUUID } from "node:crypto";
33
+ import { fileURLToPath } from "node:url";
34
+
35
+ import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
36
+ import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js";
37
+ import { StreamableHTTPServerTransport } from "@modelcontextprotocol/sdk/server/streamableHttp.js";
38
+ import { StdioClientTransport } from "@modelcontextprotocol/sdk/client/stdio.js";
39
+ import { z } from "zod";
40
+
41
+ import { startLiveHub } from "./live-hub.mjs";
42
+ import { BLOCK_MAX_SECONDS, PROMPT_GUIDE, normalizePhases, splitLongBeat } from "./ardy-prompts.mjs";
43
+
44
+ import {
45
+ CAMERA_MOVES,
46
+ IMAGE_MODELS,
47
+ VIDEO_MODELS,
48
+ composePrompt,
49
+ deriveShot,
50
+ focalMmToFov,
51
+ fovToFocalMm,
52
+ nearestPrime,
53
+ slateLine,
54
+ } from "../src/shot.js";
55
+ import {
56
+ CHARACTER_MODEL_IDS,
57
+ activeScene,
58
+ addScene,
59
+ createCharacterEntry,
60
+ createSceneDocument,
61
+ readSceneDocument,
62
+ serializeSceneDocument,
63
+ } from "../src/scenes.js";
64
+ import {
65
+ OBJECT_LIBRARY,
66
+ createSceneObject,
67
+ objectSize,
68
+ removeSceneObject,
69
+ setSceneObjectParent,
70
+ updateSceneObject,
71
+ } from "../src/scene-objects.js";
72
+ import { classifyMove, captureFraming, moveSlate } from "../src/camera-move.js";
73
+ import { createProjectDocument, readProjectDocument } from "../src/project.js";
74
+
75
+ /* ------------------------------- state ---------------------------------- */
76
+
77
+ /** The authoring state. One scene document, one camera, one project name. */
78
+ const state = {
79
+ doc: createSceneDocument(),
80
+ name: "Untitled",
81
+ camera: { x: 0, y: 1.6, z: 4.5, focalMm: 35 },
82
+ /** which character the camera frames against; null means the first of the cast */
83
+ focus: null,
84
+ /** framing snapshot taken by `mark_camera_move`, consumed by `describe_camera_move` */
85
+ markedFraming: null,
86
+ };
87
+
88
+ let liveHub = null;
89
+
90
+ const scene = () => activeScene(state.doc.scenes, state.doc.activeSceneId);
91
+ const stage = () => scene().stage;
92
+
93
+ /** The cast of the active scene. A v3 stage carries an unbounded `characters`
94
+ * list, so "character A/B" is simply index 0/1 of this array. */
95
+ const cast = () => stage().characters;
96
+
97
+ /** Resolve a character by id, by the A/B/C letter the studio labels them with,
98
+ * or by 1-based slot. Returns null when nothing matches. */
99
+ const findCharacter = (ref) => {
100
+ const list = cast();
101
+ if (ref === undefined || ref === null || ref === "") return list[0] ?? null;
102
+ const key = String(ref).trim();
103
+ const byId = list.find((c) => c.id === key);
104
+ if (byId) return byId;
105
+ if (/^[A-Za-z]$/.test(key)) return list[key.toUpperCase().charCodeAt(0) - 65] ?? null;
106
+ const n = Number(key);
107
+ if (Number.isInteger(n) && n >= 1) return list[n - 1] ?? null;
108
+ return null;
109
+ };
110
+
111
+ /** The studio labels the cast A, B, C… by position. */
112
+ const letterFor = (character) => String.fromCharCode(65 + cast().indexOf(character));
113
+
114
+ /** What to say when a character reference does not resolve. */
115
+ const castHint = () =>
116
+ `Cast: ${cast()
117
+ .map((c, i) => `${String.fromCharCode(65 + i)}=${c.id}`)
118
+ .join(", ")}`;
119
+
120
+ /** The camera's vertical FOV, derived from the focal length the user set. */
121
+ const fov = () => focalMmToFov(state.camera.focalMm);
122
+
123
+ /** The subject `deriveShot` frames against: the framed character, which is the
124
+ * first of the cast unless `focus_character` moved it. */
125
+ const subject = () => {
126
+ const a = findCharacter(state.focus) ?? cast()[0];
127
+ return { x: a.x, z: a.z, rot: a.rot };
128
+ };
129
+
130
+ /** Yaw/pitch that aim the camera at the framing pivot — what captureFraming wants. */
131
+ const aimAtSubject = () => {
132
+ const s = subject();
133
+ const dx = state.camera.x - s.x;
134
+ const dz = state.camera.z - s.z;
135
+ const dy = state.camera.y - 1.3; // FRAMING_PIVOT_Y
136
+ const horizontal = Math.hypot(dx, dz);
137
+ return {
138
+ yaw: (Math.atan2(dx, dz) * 180) / Math.PI,
139
+ pitch: (-Math.atan2(dy, Math.max(horizontal, 1e-6)) * 180) / Math.PI,
140
+ };
141
+ };
142
+
143
+ const framing = () => {
144
+ const { yaw, pitch } = aimAtSubject();
145
+ return captureFraming({
146
+ pos: { x: state.camera.x, y: state.camera.y, z: state.camera.z },
147
+ yaw,
148
+ pitch,
149
+ fovDeg: (fov() * 180) / Math.PI,
150
+ });
151
+ };
152
+
153
+ const currentShot = () => deriveShot(state.camera, subject(), fov());
154
+
155
+ const modelById = (id) =>
156
+ [...VIDEO_MODELS, ...IMAGE_MODELS].find((m) => m.id === id) ?? null;
157
+
158
+ /** Copy the protocol's deliberately small live description into the existing
159
+ * scene shape. Formatting and film vocabulary below then remain exactly the
160
+ * same code paths as memory-only mode. */
161
+ const applyLiveDescription = (description) => {
162
+ if (!description || typeof description !== "object") throw new Error("Live editor returned an invalid scene description.");
163
+ const sc = scene();
164
+ if (typeof description.sceneName === "string" && description.sceneName) sc.name = description.sceneName;
165
+ if (description.camera && typeof description.camera === "object") {
166
+ for (const key of ["x", "y", "z", "focalMm"]) {
167
+ if (Number.isFinite(description.camera[key])) state.camera[key] = description.camera[key];
168
+ }
169
+ }
170
+ if (Array.isArray(description.characters) && description.characters.length) {
171
+ const prior = new Map(stage().characters.map((character) => [character.id, character]));
172
+ stage().characters = description.characters.map((character, index) =>
173
+ createCharacterEntry({ ...prior.get(character.id), ...character }, index),
174
+ );
175
+ if (state.focus && !stage().characters.some((character) => character.id === state.focus)) state.focus = null;
176
+ }
177
+ if (Array.isArray(description.objects)) {
178
+ const prior = new Map(sc.objects.map((object) => [object.id, object]));
179
+ sc.objects = description.objects.map((object) => {
180
+ const previous = prior.get(object.id);
181
+ const kind = OBJECT_LIBRARY.find(({ label }) => object.name === label || object.name?.startsWith(`${label} `))?.kind;
182
+ const defaults = previous ?? (kind ? createSceneObject(kind, sc.objects, object) : null);
183
+ // The editor is the source of truth for anything it reports; the
184
+ // library defaults only fill what the frame omits. Defaulting AFTER
185
+ // the spread would reset a reported scale back to 1 and make every
186
+ // prop measure 1x1x1 no matter how it was actually built.
187
+ return {
188
+ ...defaults,
189
+ footprint: defaults?.footprint ?? { width: 1, depth: 1 },
190
+ height: defaults?.height ?? 1,
191
+ scaleX: defaults?.scaleX ?? 1,
192
+ scaleY: defaults?.scaleY ?? 1,
193
+ scaleZ: defaults?.scaleZ ?? 1,
194
+ ...object,
195
+ };
196
+ });
197
+ }
198
+ };
199
+
200
+ const refreshLiveDescription = async () => {
201
+ if (!liveHub?.connected) return false;
202
+ applyLiveDescription(await liveHub.command("describe", {}));
203
+ return true;
204
+ };
205
+
206
+ const liveError = (error) => text(`Live editor error: ${error.message}`);
207
+
208
+ /* ------------------------------ formatting ------------------------------- */
209
+
210
+ const round = (n, places = 2) => Number(n.toFixed(places));
211
+ const metres = (n) => `${round(n)}m`;
212
+
213
+ const text = (body) => ({ content: [{ type: "text", text: body }] });
214
+
215
+ /** A scene rendered the way a crew would read it, not as JSON. */
216
+ function sceneReport() {
217
+ const sc = scene();
218
+ const st = sc.stage;
219
+ const shot = currentShot();
220
+ const lines = [
221
+ `Project: ${state.name}`,
222
+ `Scene: ${sc.name} (${state.doc.scenes.length} scene${state.doc.scenes.length === 1 ? "" : "s"} in project)`,
223
+ "",
224
+ "CAMERA",
225
+ ` position x ${round(state.camera.x)} y ${round(state.camera.y)} z ${round(state.camera.z)}`,
226
+ ` lens ${state.camera.focalMm}mm (nearest prime ${nearestPrime(fov())}mm)`,
227
+ ` framing ${slateLine(shot)}`,
228
+ ` distance ${metres(shot.distance)} to the subject's centre of mass`,
229
+ "",
230
+ `CAST (${st.characters.length})`,
231
+ ];
232
+ const framed = findCharacter(state.focus) ?? st.characters[0];
233
+ for (const [index, c] of st.characters.entries()) {
234
+ const letter = String.fromCharCode(65 + index);
235
+ lines.push(
236
+ ` ${letter} ${c.id} "${c.subject}" at x ${round(c.x)}, z ${round(c.z)}, ` +
237
+ `facing ${round(c.rot, 1)}deg [${c.model}]` +
238
+ `${c.pose ? " posed" : ""}${c.hidden ? " hidden" : ""}` +
239
+ `${c === framed ? " <- framed" : ""}`,
240
+ );
241
+ }
242
+
243
+ lines.push("", `SET (${sc.objects.length} object${sc.objects.length === 1 ? "" : "s"})`);
244
+ if (sc.objects.length === 0) {
245
+ lines.push(" empty — add with place_object");
246
+ } else {
247
+ for (const o of sc.objects) {
248
+ const size = objectSize(o);
249
+ lines.push(
250
+ ` ${o.id} ${o.name} at x ${round(o.x)}, y ${round(o.y)}, z ${round(o.z)}` +
251
+ ` yaw ${round(o.rot, 1)}deg size ${round(size.width)}x${round(size.height)}x${round(size.depth)}m`,
252
+ );
253
+ }
254
+ }
255
+ return lines.join("\n");
256
+ }
257
+
258
+ /** The shot, described in the vocabulary a director and an image model share. */
259
+ function shotReport() {
260
+ const shot = currentShot();
261
+ return [
262
+ slateLine(shot),
263
+ "",
264
+ `size ${shot.sizeLabel} — the subject fills ${Math.round(shot.screenFraction * 100)}% of frame height`,
265
+ `view ${shot.viewPhrase}`,
266
+ `level ${shot.levelPhrase}`,
267
+ `lens ${shot.focalMm}mm (exact ${round(shot.exactFocalMm, 1)}mm)`,
268
+ `distance ${metres(shot.distance)}`,
269
+ `elevation ${round(shot.elevationDeg, 1)}deg`,
270
+ "",
271
+ `texture guidance: ${shot.sizeContext}`,
272
+ ].join("\n");
273
+ }
274
+
275
+ /* ------------------------------- server ---------------------------------- */
276
+
277
+ /** Tool registrations are counted as they happen so the HTTP status page can
278
+ * report the real number without reaching into SDK internals. */
279
+ let registeredTools = 0;
280
+
281
+ const server = new McpServer(
282
+ { name: "cozyclay-mcp", version: "0.1.0" },
283
+ {
284
+ instructions:
285
+ "CozyClay previs. Block a 3D scene, place the camera, then read the shot back as film " +
286
+ "vocabulary (shot size, angle, lens) and render it into an AI image/video prompt. " +
287
+ "Call describe_scene first to see the current state. Coordinates are metres: x is right, " +
288
+ "z is toward the camera's default position, y is height above the floor. Rotations are " +
289
+ "degrees of yaw. Save with save_project to a .cclayproject file the CozyClay studio opens.",
290
+ },
291
+ );
292
+
293
+ const registerTool = (name, config, handler) => {
294
+ registeredTools += 1;
295
+ return server.registerTool(name, config, handler);
296
+ };
297
+
298
+ registerTool(
299
+ "describe_scene",
300
+ {
301
+ title: "Describe the scene",
302
+ description:
303
+ "Read the whole authoring state: camera, lens, current framing, cast positions and every " +
304
+ "object in the set. Call this before changing anything, and after, to confirm the result.",
305
+ inputSchema: {},
306
+ },
307
+ async () => {
308
+ try {
309
+ await refreshLiveDescription();
310
+ return text(sceneReport());
311
+ } catch (error) {
312
+ return liveError(error);
313
+ }
314
+ },
315
+ );
316
+
317
+ registerTool(
318
+ "live_status",
319
+ {
320
+ title: "Live editor status",
321
+ description: "Report whether a CozyClay editor is connected to live mode.",
322
+ inputSchema: {},
323
+ },
324
+ async () => text(liveHub?.connected ? "Live editor connected." : "No live editor connected; using in-memory state."),
325
+ );
326
+
327
+ registerTool(
328
+ "describe_shot",
329
+ {
330
+ title: "Describe the current shot",
331
+ description:
332
+ "Turn the current camera geometry into film vocabulary — shot size, angle on the subject, " +
333
+ "camera level, and the nearest real prime lens. This is what the camera is actually seeing.",
334
+ inputSchema: {},
335
+ },
336
+ async () => {
337
+ try {
338
+ await refreshLiveDescription();
339
+ return text(shotReport());
340
+ } catch (error) {
341
+ return liveError(error);
342
+ }
343
+ },
344
+ );
345
+
346
+ registerTool(
347
+ "set_camera",
348
+ {
349
+ title: "Set the camera",
350
+ description:
351
+ "Move the camera and/or change the lens. Every field is optional — omitted fields keep " +
352
+ "their current value. The camera always aims at the subject. Use focal_mm to change " +
353
+ "framing without moving (longer = tighter), or move x/y/z to change the angle.",
354
+ inputSchema: {
355
+ x: z.number().optional().describe("world x in metres (right)"),
356
+ y: z.number().optional().describe("lens height above the floor in metres"),
357
+ z: z.number().optional().describe("world z in metres (toward default camera side)"),
358
+ focal_mm: z
359
+ .number()
360
+ .min(8)
361
+ .max(300)
362
+ .optional()
363
+ .describe("focal length on a full-frame 24mm-tall sensor, e.g. 24, 35, 50, 85"),
364
+ },
365
+ },
366
+ async ({ x, y, z: zPos, focal_mm }) => {
367
+ if (liveHub?.connected) {
368
+ try {
369
+ await liveHub.command("set_camera", { x, y, z: zPos, focalMm: focal_mm });
370
+ await refreshLiveDescription();
371
+ return text(`Camera set.\n\n${shotReport()}`);
372
+ } catch (error) {
373
+ return liveError(error);
374
+ }
375
+ }
376
+ if (x !== undefined) state.camera.x = x;
377
+ if (y !== undefined) state.camera.y = y;
378
+ if (zPos !== undefined) state.camera.z = zPos;
379
+ if (focal_mm !== undefined) state.camera.focalMm = focal_mm;
380
+ return text(`Camera set.\n\n${shotReport()}`);
381
+ },
382
+ );
383
+
384
+ registerTool(
385
+ "frame_shot",
386
+ {
387
+ title: "Frame a shot by intent",
388
+ description:
389
+ "Place the camera by describing the shot you want instead of doing the trigonometry. " +
390
+ "Chooses a distance and height that actually produce the requested size and level, " +
391
+ "orbiting to the requested side of the subject.",
392
+ inputSchema: {
393
+ size: z
394
+ .enum([
395
+ "extreme close-up",
396
+ "close-up",
397
+ "medium close-up",
398
+ "medium shot",
399
+ "medium-wide shot",
400
+ "wide shot",
401
+ "extreme wide shot",
402
+ ])
403
+ .describe("how much of the frame the subject fills"),
404
+ view: z
405
+ .enum(["front", "front three-quarter", "profile", "rear three-quarter", "back"])
406
+ .default("front three-quarter")
407
+ .describe("which side of the subject the camera sits on"),
408
+ level: z
409
+ .enum(["ground", "low", "hip", "eye", "high", "overhead"])
410
+ .default("eye")
411
+ .describe("how high the lens rides"),
412
+ side: z.enum(["left", "right"]).default("right").describe("camera left or camera right"),
413
+ focal_mm: z.number().min(8).max(300).default(35).describe("lens to frame with"),
414
+ },
415
+ },
416
+ async ({ size, view, level, side, focal_mm }) => {
417
+ if (liveHub?.connected) {
418
+ try {
419
+ await refreshLiveDescription();
420
+ } catch (error) {
421
+ return liveError(error);
422
+ }
423
+ }
424
+ // Midpoints of shot.js's SIZE_TABLE bands, so the label that comes back is
425
+ // the label that was asked for rather than whatever sits on a boundary.
426
+ const FRACTION = {
427
+ "extreme close-up": 3.4,
428
+ "close-up": 2.2,
429
+ "medium close-up": 1.375,
430
+ "medium shot": 0.975,
431
+ "medium-wide shot": 0.66,
432
+ "wide shot": 0.41,
433
+ "extreme wide shot": 0.2,
434
+ };
435
+ // Heights that land mid-band in shot.js's LEVEL_TABLE.
436
+ const HEIGHT = { ground: 0.3, low: 0.7, hip: 1.1, eye: 1.65, high: 2.1, overhead: 2.8 };
437
+ // Angle off the subject's facing direction, in degrees.
438
+ const ANGLE = { front: 0, "front three-quarter": 40, profile: 90, "rear three-quarter": 140, back: 180 };
439
+
440
+ const s = subject();
441
+ let lensMm = focal_mm;
442
+ // Invert deriveShot's screenFraction: distance that yields the target size.
443
+ const distanceFor = (mm) => 1.8 / (2 * FRACTION[size] * Math.tan(focalMmToFov(mm) / 2));
444
+
445
+ // Size and level can physically conflict: an extreme close-up on a wide
446
+ // lens sits half a metre from the pivot, which no overhead rig can also
447
+ // satisfy. Size is the stronger request (it is the shot), so the lens is
448
+ // lengthened until the requested level fits, exactly as a crew would swap
449
+ // glass rather than abandon the close-up.
450
+ const neededDy = Math.abs(HEIGHT[level] - 1.3);
451
+ const MIN_HORIZONTAL = 0.25;
452
+ const needed = Math.hypot(neededDy, MIN_HORIZONTAL);
453
+ if (distanceFor(lensMm) < needed) {
454
+ for (const mm of [50, 85, 100, 135, 180, 240, 300]) {
455
+ if (mm <= lensMm) continue;
456
+ lensMm = mm;
457
+ if (distanceFor(mm) >= needed) break;
458
+ }
459
+ }
460
+ const distance = distanceFor(lensMm);
461
+ let camY = HEIGHT[level];
462
+ // deriveShot measures distance in 3D to the framing pivot, so the height
463
+ // offset has to come out of the requested distance. A very tight shot from
464
+ // a very high or low lens can ask for more vertical offset than the whole
465
+ // distance allows; when that happens the size is what was actually asked
466
+ // for, so the lens is pulled toward the pivot rather than the shot widened.
467
+ let dy = camY - 1.3;
468
+ const maxDy = Math.sqrt(Math.max(distance * distance - MIN_HORIZONTAL * MIN_HORIZONTAL, 0));
469
+ if (Math.abs(dy) > maxDy) {
470
+ dy = Math.sign(dy) * maxDy;
471
+ camY = 1.3 + dy;
472
+ }
473
+ const horizontal = Math.sqrt(Math.max(distance * distance - dy * dy, MIN_HORIZONTAL * MIN_HORIZONTAL));
474
+
475
+ // deriveShot calls it camera-right when cross(facing, toCamera) >= 0, which
476
+ // is the negative yaw direction here — so camera-left orbits by +angle.
477
+ const sign = side === "right" ? -1 : 1;
478
+ const theta = ((s.rot + sign * ANGLE[view]) * Math.PI) / 180;
479
+
480
+ const nextCamera = {
481
+ x: s.x + Math.sin(theta) * horizontal,
482
+ z: s.z + Math.cos(theta) * horizontal,
483
+ y: camY,
484
+ focalMm: lensMm,
485
+ };
486
+ if (liveHub?.connected) {
487
+ try {
488
+ await liveHub.command("set_camera", nextCamera);
489
+ await refreshLiveDescription();
490
+ } catch (error) {
491
+ return liveError(error);
492
+ }
493
+ } else {
494
+ Object.assign(state.camera, nextCamera);
495
+ }
496
+
497
+ const note =
498
+ lensMm !== focal_mm
499
+ ? `Note: ${focal_mm}mm could not hold a ${size} from ${level} level — the lens would have to be ` +
500
+ `inside the subject. Went to ${lensMm}mm to keep the size and the angle.\n\n`
501
+ : "";
502
+ return text(`${note}Framed.\n\n${shotReport()}`);
503
+ },
504
+ );
505
+
506
+ registerTool(
507
+ "add_character",
508
+ {
509
+ title: "Add a character to the cast",
510
+ description:
511
+ "Put another character in the scene. The cast is unbounded — each one gets its own " +
512
+ "letter (A, B, C…), position and prompt description.",
513
+ inputSchema: {
514
+ subject: z.string().describe('prompt description, e.g. "a courier holding a package"'),
515
+ x: z.number().default(0).describe("floor position x in metres"),
516
+ z: z.number().default(0).describe("floor position z in metres"),
517
+ facing: z.number().default(0).describe("yaw in degrees; 0 faces the default camera"),
518
+ model: z
519
+ .enum(CHARACTER_MODEL_IDS)
520
+ .optional()
521
+ .describe("which mannequin to use"),
522
+ },
523
+ },
524
+ async ({ subject: desc, x, z: zPos, facing, model }) => {
525
+ if (liveHub?.connected) {
526
+ try {
527
+ const result = await liveHub.command("add_character", { subject: desc, x, z: zPos, rot: facing });
528
+ await refreshLiveDescription();
529
+ return text(`Added ${result?.id ?? "character"}.\n\n${sceneReport()}`);
530
+ } catch (error) {
531
+ return liveError(error);
532
+ }
533
+ }
534
+ const st = stage();
535
+ const index = st.characters.length;
536
+ // The default scene already owns "char-a", and createCharacterEntry only
537
+ // falls back to `char-<n>` when no id is supplied, so pick the first id
538
+ // the cast is not already using instead of assuming a naming scheme.
539
+ const taken = new Set(st.characters.map((c) => c.id));
540
+ let id = `char-${String.fromCharCode(97 + index)}`;
541
+ for (let n = index + 1; taken.has(id); n += 1) id = `char-${n}`;
542
+ const entry = createCharacterEntry({ id, subject: desc, x, z: zPos, rot: facing, model }, index);
543
+ st.characters = [...st.characters, entry];
544
+ return text(`Added ${String.fromCharCode(65 + index)} (${entry.id}).\n\n${sceneReport()}`);
545
+ },
546
+ );
547
+
548
+ registerTool(
549
+ "place_character",
550
+ {
551
+ title: "Move or re-describe a character",
552
+ description:
553
+ "Change a character already in the cast. Every field is optional; omitted fields keep " +
554
+ "their value. Use add_character to introduce a new one.",
555
+ inputSchema: {
556
+ character: z
557
+ .string()
558
+ .default("A")
559
+ .describe('which character — a letter ("A"), a slot number ("2") or an id ("char-a")'),
560
+ x: z.number().optional().describe("floor position x in metres"),
561
+ z: z.number().optional().describe("floor position z in metres"),
562
+ facing: z.number().optional().describe("yaw in degrees; 0 faces the default camera"),
563
+ subject: z.string().optional().describe("prompt description"),
564
+ hidden: z.boolean().optional().describe("hide without removing from the cast"),
565
+ },
566
+ },
567
+ async ({ character, x, z: zPos, facing, subject: desc, hidden }) => {
568
+ if (liveHub?.connected) {
569
+ try {
570
+ await liveHub.command("update_character", { ref: character, x, z: zPos, rot: facing, subject: desc, hidden });
571
+ await refreshLiveDescription();
572
+ return text(`Character updated.\n\n${sceneReport()}`);
573
+ } catch (error) {
574
+ return liveError(error);
575
+ }
576
+ }
577
+ const target = findCharacter(character);
578
+ if (!target) return text(`No character "${character}". ${castHint()}`);
579
+ if (x !== undefined) target.x = x;
580
+ if (zPos !== undefined) target.z = zPos;
581
+ if (facing !== undefined) target.rot = facing;
582
+ if (desc !== undefined) target.subject = desc;
583
+ if (hidden !== undefined) target.hidden = hidden;
584
+ return text(`Character ${letterFor(target)} updated.\n\n${sceneReport()}`);
585
+ },
586
+ );
587
+
588
+ registerTool(
589
+ "remove_character",
590
+ {
591
+ title: "Remove a character",
592
+ description: "Take a character out of the cast. The last remaining character cannot be removed.",
593
+ inputSchema: {
594
+ character: z.string().describe('which character — letter, slot number or id'),
595
+ },
596
+ },
597
+ async ({ character }) => {
598
+ if (liveHub?.connected) {
599
+ try {
600
+ await liveHub.command("remove_character", { ref: character });
601
+ await refreshLiveDescription();
602
+ return text(`Character removed.\n\n${sceneReport()}`);
603
+ } catch (error) {
604
+ return liveError(error);
605
+ }
606
+ }
607
+ const st = stage();
608
+ const target = findCharacter(character);
609
+ if (!target) return text(`No character "${character}". ${castHint()}`);
610
+ if (st.characters.length === 1) return text("The scene needs at least one character.");
611
+ const letter = letterFor(target);
612
+ st.characters = st.characters.filter((c) => c !== target);
613
+ if (state.focus && !findCharacter(state.focus)) state.focus = null;
614
+ return text(`Removed ${letter}.\n\n${sceneReport()}`);
615
+ },
616
+ );
617
+
618
+ registerTool(
619
+ "focus_character",
620
+ {
621
+ title: "Choose who the camera frames",
622
+ description:
623
+ "Pick which character the shot is measured against. describe_shot, frame_shot and " +
624
+ "render_prompt all frame this character. Defaults to the first of the cast.",
625
+ inputSchema: {
626
+ character: z.string().describe('which character — letter, slot number or id'),
627
+ },
628
+ },
629
+ async ({ character }) => {
630
+ try {
631
+ await refreshLiveDescription();
632
+ } catch (error) {
633
+ return liveError(error);
634
+ }
635
+ const target = findCharacter(character);
636
+ if (!target) return text(`No character "${character}". ${castHint()}`);
637
+ state.focus = target.id;
638
+ return text(`Framing ${letterFor(target)} "${target.subject}".\n\n${shotReport()}`);
639
+ },
640
+ );
641
+
642
+ registerTool(
643
+ "place_object",
644
+ {
645
+ title: "Place an object in the set",
646
+ description:
647
+ `Add a prop to the set. Available kinds: ${OBJECT_LIBRARY.map((o) => o.kind).join(", ")}. ` +
648
+ "Returns the object id, which update_object and remove_object take.",
649
+ inputSchema: {
650
+ kind: z
651
+ .enum(OBJECT_LIBRARY.map((o) => o.kind))
652
+ .describe("what to place"),
653
+ x: z.number().default(0).describe("floor position x in metres"),
654
+ z: z.number().default(0).describe("floor position z in metres"),
655
+ y: z.number().optional().describe("height above the floor; 0 stands on the deck"),
656
+ facing: z.number().optional().describe("yaw in degrees"),
657
+ },
658
+ },
659
+ async ({ kind, x, z: zPos, y, facing }) => {
660
+ if (liveHub?.connected) {
661
+ try {
662
+ const result = await liveHub.command("place_object", { kind, x, z: zPos, y, rot: facing });
663
+ await refreshLiveDescription();
664
+ return text(`Placed object as ${result?.id ?? "unknown"}.\n\n${sceneReport()}`);
665
+ } catch (error) {
666
+ return liveError(error);
667
+ }
668
+ }
669
+ const sc = scene();
670
+ const placement = { x, z: zPos };
671
+ if (y !== undefined) placement.y = y;
672
+ if (facing !== undefined) placement.rot = facing;
673
+ const object = createSceneObject(kind, sc.objects, placement);
674
+ sc.objects = [...sc.objects, object];
675
+ return text(`Placed ${object.name} as ${object.id}.\n\n${sceneReport()}`);
676
+ },
677
+ );
678
+
679
+
680
+ registerTool(
681
+ "group_objects",
682
+ {
683
+ title: "Group props so they move as one",
684
+ description:
685
+ "Attach objects to a parent object. The parent then carries them whenever it moves — in " +
686
+ "the studio's gizmo as well as through update_object — so a set piece assembled from " +
687
+ "primitives can be positioned as a single thing. Rotation and scale stay per-object. " +
688
+ "Pass parent: null to detach.",
689
+ inputSchema: {
690
+ parent: z.string().nullable().describe("object id to attach to, or null to detach"),
691
+ children: z.array(z.string()).min(1).describe("object ids to attach or detach"),
692
+ },
693
+ },
694
+ async ({ parent, children }) => {
695
+ if (liveHub?.connected) {
696
+ try {
697
+ await liveHub.command(parent === null ? "ungroup_objects" : "group_objects", { parent, children });
698
+ await refreshLiveDescription();
699
+ return text(
700
+ (parent === null
701
+ ? `Detached ${children.length} object(s).`
702
+ : `Grouped ${children.length} object(s) under ${parent} — move ${parent} and they follow.`) +
703
+ `\n\n${sceneReport()}`,
704
+ );
705
+ } catch (error) {
706
+ return liveError(error);
707
+ }
708
+ }
709
+ const sc = scene();
710
+ if (parent !== null && !sc.objects.some((o) => o.id === parent)) return text(`No object "${parent}".`);
711
+ for (const child of children) {
712
+ if (!sc.objects.some((o) => o.id === child)) return text(`No object "${child}".`);
713
+ }
714
+ sc.objects = children.reduce((acc, child) => setSceneObjectParent(acc, child, parent), sc.objects);
715
+ return text(
716
+ (parent === null ? `Detached ${children.length} object(s).` : `Grouped ${children.length} object(s) under ${parent}.`) +
717
+ `\n\n${sceneReport()}`,
718
+ );
719
+ },
720
+ );
721
+
722
+ registerTool(
723
+ "set_prompt_blocks",
724
+ {
725
+ title: "Author the motion beats on the timeline",
726
+ description:
727
+ "Write Prompt Blocks onto the timeline WITHOUT generating — the beats and their frame " +
728
+ "ranges, so a schedule can be read and revised before any GPU time is spent. Hit " +
729
+ "'Generate all N blocks' in the studio, or call generate_motion, when it reads right.\n\n" +
730
+ PROMPT_GUIDE,
731
+ inputSchema: {
732
+ beats: z
733
+ .array(
734
+ z.object({
735
+ text: z.string().min(3).describe("the beat, in ARDY's sentence shape"),
736
+ seconds: z
737
+ .number()
738
+ .min(0.5)
739
+ .max(20)
740
+ .describe(`how long this beat holds; over ${BLOCK_MAX_SECONDS}s it becomes chained blocks`),
741
+ }),
742
+ )
743
+ .min(1)
744
+ .max(8)
745
+ .describe("beats in order; each one becomes a contiguous block"),
746
+ },
747
+ },
748
+ async ({ beats }) => {
749
+ if (!liveHub?.connected) {
750
+ return text("Prompt Blocks live on the studio timeline — open the editor and try again.");
751
+ }
752
+ const normalized = normalizePhases(beats.map((b) => b.text));
753
+ // The timeline runs on a 24 fps production clock.
754
+ const TIMELINE_FPS = 24;
755
+ let cursor = 0;
756
+ let chained = 0;
757
+ const blocks = [];
758
+ for (const [i, textValue] of normalized.texts.entries()) {
759
+ const whole = beats[Math.min(normalized.sources[i], beats.length - 1)].seconds ?? 2;
760
+ const spans = splitLongBeat(whole);
761
+ if (spans.length > 1) chained += spans.length - 1;
762
+ for (const span of spans) {
763
+ const frames = Math.max(1, Math.round(span * TIMELINE_FPS));
764
+ blocks.push({ startFrame: cursor, endFrame: cursor + frames, text: textValue });
765
+ cursor += frames;
766
+ }
767
+ }
768
+ try {
769
+ await liveHub.command("set_prompt_blocks", { blocks });
770
+ } catch (error) {
771
+ return liveError(error);
772
+ }
773
+ const rewrites = normalized.notes
774
+ .map((notes, i) => (notes.length ? ` ${i + 1}. ${blocks[i].text} ← ${notes.join("; ")}` : null))
775
+ .filter(Boolean);
776
+ return text(
777
+ `${blocks.length} block(s) on the timeline (${(cursor / TIMELINE_FPS).toFixed(1)}s total):\n` +
778
+ blocks.map((b) => ` ${b.startFrame}-${b.endFrame}f ${b.text}`).join("\n") +
779
+ (chained > 0 ? `\n (${chained} block(s) chained to keep every block within ${BLOCK_MAX_SECONDS}s)` : "") +
780
+ (rewrites.length ? `\n\nRewritten for ARDY:\n${rewrites.join("\n")}` : "") +
781
+ "\n\nGenerate them from the studio's Prompt Blocks panel, or with generate_motion.",
782
+ );
783
+ },
784
+ );
785
+
786
+ registerTool(
787
+ "generate_motion",
788
+ {
789
+ title: "Generate character motion (ARDY)",
790
+ description:
791
+ "Generate a multi-phase motion clip through the ARDY bridge and, when a live editor is " +
792
+ "connected, load it onto the active character so it appears on the timeline.\n\n" +
793
+ PROMPT_GUIDE,
794
+ inputSchema: {
795
+ phases: z
796
+ .array(
797
+ z.union([
798
+ z.string().min(3),
799
+ z.object({
800
+ text: z.string().min(3),
801
+ seconds: z.number().min(0.5).max(30).describe("how long THIS beat holds"),
802
+ }),
803
+ ]),
804
+ )
805
+ .min(2)
806
+ .max(8)
807
+ .describe(
808
+ "one beat per phase, in order. Write each as ARDY writes them: " +
809
+ '"A person walks in a circle." — subject, one action, present tense, ' +
810
+ "full stop. Give a plain string to share the clip evenly, or " +
811
+ "{ text, seconds } to hold a beat for a specific time.",
812
+ ),
813
+ seconds: z
814
+ .number()
815
+ .min(2)
816
+ .max(60)
817
+ .default(9)
818
+ .describe("total clip length; ignored when every phase carries its own seconds"),
819
+ seed: z.number().int().optional().describe("generation seed"),
820
+ motion_url: z
821
+ .string()
822
+ .regex(/^\/ardy\/motions\/[0-9]+-[0-9a-f]{6}$/)
823
+ .optional()
824
+ .describe("reuse an already-generated clip instead of generating again"),
825
+ },
826
+ },
827
+ async ({ phases: rawPhases, seconds, seed, motion_url }) => {
828
+ const phases = rawPhases.map((p) => (typeof p === "string" ? p : p.text));
829
+ const phaseSeconds = rawPhases.map((p) => (typeof p === "string" ? null : p.seconds));
830
+ const timed = phaseSeconds.some((s) => s !== null);
831
+ const bridge = process.env.COZYCLAY_BRIDGE ?? "http://127.0.0.1:5181";
832
+ try {
833
+ const health = await fetch(`${bridge}/ardy/health`, { signal: AbortSignal.timeout(4000) }).then((r) => r.json());
834
+ if (!health.ok) throw new Error("bridge unhealthy");
835
+ } catch {
836
+ return text("The ARDY bridge is not running on " + bridge + ". Start the studio with `npm run dev` (it launches the bridge) and try again.");
837
+ }
838
+
839
+ // Every phase is rewritten into ARDY's own sentence shape before it is
840
+ // ever sent; a compound beat is split into the extra phase it was hiding.
841
+ const normalized = normalizePhases(phases);
842
+ const prompts = normalized.texts.filter(Boolean);
843
+ if (prompts.length < 2) return text("Give at least two distinct motion beats.");
844
+
845
+ // ARDY Core is 20 fps; segments must tile 0..clipFrames exactly, and each
846
+ // one needs at least 3 frames.
847
+ const ARDY_FPS = 20;
848
+ let segments;
849
+ let clipFrames;
850
+ let chained = 0;
851
+ if (timed) {
852
+ // A beat that split into pieces shares its time between them, so an
853
+ // explicit "6 seconds of walking" stays 6 seconds however it was phrased.
854
+ const pieceCount = normalized.sources.reduce((acc, source) => {
855
+ acc[source] = (acc[source] ?? 0) + 1;
856
+ return acc;
857
+ }, {});
858
+ segments = [];
859
+ let cursor = 0;
860
+ for (const [i, prompt] of prompts.entries()) {
861
+ const whole = phaseSeconds[normalized.sources[i]] ?? seconds / phases.length;
862
+ const share = whole / pieceCount[normalized.sources[i]];
863
+ // The studio caps a block at BLOCK_MAX_SECONDS and refuses to generate
864
+ // a longer one, so a long beat becomes consecutive blocks here rather
865
+ // than something the UI would reject.
866
+ const spans = splitLongBeat(share);
867
+ if (spans.length > 1) chained += spans.length - 1;
868
+ for (const span of spans) {
869
+ const startFrame = cursor;
870
+ cursor += Math.max(3, Math.round(span * ARDY_FPS));
871
+ segments.push({ startFrame, endFrame: cursor, prompt });
872
+ }
873
+ }
874
+ clipFrames = cursor;
875
+ } else {
876
+ clipFrames = Math.floor(seconds * ARDY_FPS);
877
+ // Even division must also respect the cap: enough blocks that no single
878
+ // one exceeds it, distributed evenly across the clip.
879
+ const perPhase = seconds / prompts.length;
880
+ const piecesPer = Math.ceil(perPhase / BLOCK_MAX_SECONDS);
881
+ if (piecesPer > 1) chained = prompts.length * (piecesPer - 1);
882
+ const total = prompts.length * piecesPer;
883
+ const per = Math.floor(clipFrames / total);
884
+ segments = [];
885
+ for (const [i, prompt] of prompts.entries()) {
886
+ for (let k = 0; k < piecesPer; k += 1) {
887
+ const index = i * piecesPer + k;
888
+ segments.push({
889
+ startFrame: index * per,
890
+ endFrame: index === total - 1 ? clipFrames : (index + 1) * per,
891
+ prompt,
892
+ });
893
+ }
894
+ }
895
+ }
896
+ const clipSeconds = clipFrames / ARDY_FPS;
897
+ const rewrites = normalized.notes
898
+ .map((notes, i) => (notes.length ? ` ${i + 1}. ${prompts[i]} ← ${notes.join("; ")}` : null))
899
+ .filter(Boolean);
900
+ const chainNote = chained > 0 ? `\n (${chained} block(s) chained to keep every block within ${BLOCK_MAX_SECONDS}s)` : "";
901
+ const promptNote =
902
+ rewrites.length || chainNote
903
+ ? (rewrites.length ? `\n\nRewritten for ARDY:\n${rewrites.join("\n")}` : "\n") +
904
+ (normalized.expanded ? "\n (a compound beat became its own phase)" : "") +
905
+ (normalized.dropped > 0 ? `\n (${normalized.dropped} beat(s) past the 8-phase limit were dropped)` : "") +
906
+ chainNote
907
+ : "";
908
+
909
+ const deliver = async (motionUrl, note) => {
910
+ const summary = `${note} ${clipSeconds.toFixed(1)}s / ${clipFrames} frames — ${segments.length} phases:\n` +
911
+ segments
912
+ .map((s) => ` ${s.startFrame}-${s.endFrame} (${((s.endFrame - s.startFrame) / ARDY_FPS).toFixed(1)}s) ${s.prompt}`)
913
+ .join("\n") +
914
+ `\nmotion: ${motionUrl}${promptNote}`;
915
+ if (!liveHub?.connected) return text(`${summary}\n\nNo live editor connected — open the studio and it can load this URL.`);
916
+ try {
917
+ await liveHub.command("load_motion", { url: motionUrl, prompt: prompts.join(" "), blocks: segments });
918
+ return text(`${summary}\n\nLoaded onto the active character with ${segments.length} prompt blocks on the timeline — press play.`);
919
+ } catch (error) {
920
+ return text(`${summary}\n\nGenerated, but the editor did not take it: ${error.message}`);
921
+ }
922
+ };
923
+
924
+ if (motion_url) return deliver(motion_url, "Reusing");
925
+ // segments run the autoregressive sequence generator, which is
926
+ // incompatible with pose pinning — the bridge requires posePin:false.
927
+ const body = { prompt: prompts.join(" "), duration: clipSeconds, segments, posePin: false };
928
+ if (seed !== undefined) body.seed = seed;
929
+
930
+ const res = await fetch(`${bridge}/ardy/generate`, {
931
+ method: "POST",
932
+ headers: { "content-type": "application/json" },
933
+ body: JSON.stringify(body),
934
+ });
935
+ if (!res.ok) return text(`Generation refused (HTTP ${res.status}): ${await res.text()}`);
936
+
937
+ // The bridge streams ndjson progress lines and ends with a done/error event.
938
+ const reader = res.body.getReader();
939
+ const decoder = new TextDecoder();
940
+ let buffer = "";
941
+ let done = null;
942
+ let lastProgress = "";
943
+ for (;;) {
944
+ const chunk = await reader.read();
945
+ if (chunk.done) break;
946
+ buffer += decoder.decode(chunk.value, { stream: true });
947
+ let nl = buffer.indexOf("\n");
948
+ while (nl !== -1) {
949
+ const line = buffer.slice(0, nl).trim();
950
+ buffer = buffer.slice(nl + 1);
951
+ if (line) {
952
+ const event = JSON.parse(line);
953
+ if (event.event === "error") return text(`Generation failed: ${event.message ?? "generator error"}`);
954
+ if (event.event === "done") done = event;
955
+ else lastProgress = event.message ?? event.event ?? lastProgress;
956
+ }
957
+ nl = buffer.indexOf("\n");
958
+ }
959
+ if (done) break;
960
+ }
961
+ if (!done?.motionUrl) return text(`Generation ended without a motion (last progress: ${lastProgress || "none"}).`);
962
+ return deliver(done.motionUrl, "Generated");
963
+ },
964
+ );
965
+
966
+ registerTool(
967
+ "update_object",
968
+ {
969
+ title: "Move, rotate or scale an object",
970
+ description:
971
+ "Change an object already in the set. Every field is optional; omitted fields are left " +
972
+ "alone. Transforms go through the same clamp/snap path the studio's gizmo uses.",
973
+ inputSchema: {
974
+ id: z.string().describe("object id from place_object or describe_scene"),
975
+ x: z.number().optional(),
976
+ y: z.number().optional(),
977
+ z: z.number().optional(),
978
+ facing: z.number().optional().describe("yaw in degrees"),
979
+ tilt: z.number().optional().describe("pitch in degrees (rotation about x)"),
980
+ roll: z.number().optional().describe("roll in degrees (rotation about z)"),
981
+ scale: z.number().positive().optional().describe("uniform scale factor"),
982
+ scale_x: z.number().positive().optional().describe("width scale; overrides `scale` on this axis"),
983
+ scale_y: z.number().positive().optional().describe("height scale; overrides `scale` on this axis"),
984
+ scale_z: z.number().positive().optional().describe("depth scale; overrides `scale` on this axis"),
985
+ color: z
986
+ .string()
987
+ .regex(/^#[0-9a-fA-F]{6}$/)
988
+ .optional()
989
+ .describe("hex colour, e.g. #d9b18c"),
990
+ },
991
+ },
992
+ async ({ id, x, y, z: zPos, facing, tilt, roll, scale, scale_x, scale_y, scale_z, color }) => {
993
+ if (liveHub?.connected) {
994
+ try {
995
+ await liveHub.command("update_object", {
996
+ id, x, y, z: zPos, rot: facing, rotX: tilt, rotZ: roll,
997
+ scale, scaleX: scale_x, scaleY: scale_y, scaleZ: scale_z, color,
998
+ });
999
+ await refreshLiveDescription();
1000
+ return text(`Updated ${id}.\n\n${sceneReport()}`);
1001
+ } catch (error) {
1002
+ return liveError(error);
1003
+ }
1004
+ }
1005
+ const sc = scene();
1006
+ if (!sc.objects.some((o) => o.id === id)) {
1007
+ return text(`No object "${id}" in this scene. Call describe_scene for the current ids.`);
1008
+ }
1009
+ const patch = {};
1010
+ if (x !== undefined) patch.x = x;
1011
+ if (y !== undefined) patch.y = y;
1012
+ if (zPos !== undefined) patch.z = zPos;
1013
+ if (facing !== undefined) patch.rot = facing;
1014
+ if (tilt !== undefined) patch.rotX = tilt;
1015
+ if (roll !== undefined) patch.rotZ = roll;
1016
+ if (scale !== undefined) {
1017
+ patch.scaleX = scale;
1018
+ patch.scaleY = scale;
1019
+ patch.scaleZ = scale;
1020
+ }
1021
+ if (scale_x !== undefined) patch.scaleX = scale_x;
1022
+ if (scale_y !== undefined) patch.scaleY = scale_y;
1023
+ if (scale_z !== undefined) patch.scaleZ = scale_z;
1024
+ if (color !== undefined) patch.color = color;
1025
+ sc.objects = updateSceneObject(sc.objects, id, patch);
1026
+ return text(`Updated ${id}.\n\n${sceneReport()}`);
1027
+ },
1028
+ );
1029
+
1030
+ registerTool(
1031
+ "remove_object",
1032
+ {
1033
+ title: "Remove an object",
1034
+ description: "Take a prop out of the set.",
1035
+ inputSchema: { id: z.string().describe("object id") },
1036
+ },
1037
+ async ({ id }) => {
1038
+ if (liveHub?.connected) {
1039
+ try {
1040
+ await liveHub.command("remove_object", { id });
1041
+ await refreshLiveDescription();
1042
+ return text(`Removed ${id}.\n\n${sceneReport()}`);
1043
+ } catch (error) {
1044
+ return liveError(error);
1045
+ }
1046
+ }
1047
+ const sc = scene();
1048
+ if (!sc.objects.some((o) => o.id === id)) {
1049
+ return text(`No object "${id}" in this scene.`);
1050
+ }
1051
+ sc.objects = removeSceneObject(sc.objects, id);
1052
+ return text(`Removed ${id}.\n\n${sceneReport()}`);
1053
+ },
1054
+ );
1055
+
1056
+ registerTool(
1057
+ "render_prompt",
1058
+ {
1059
+ title: "Render the AI prompt for this shot",
1060
+ description:
1061
+ "Turn the current camera, cast and set into a prompt for an AI image or video model. " +
1062
+ "The prompt carries the real framing — shot size, lens, angle and level — so the " +
1063
+ "generated frame matches the blocking.",
1064
+ inputSchema: {
1065
+ mode: z.enum(["image", "video"]).default("video").describe("still or moving"),
1066
+ model: z
1067
+ .string()
1068
+ .optional()
1069
+ .describe(
1070
+ `target model id. video: ${VIDEO_MODELS.map((m) => m.id).join(", ")}. ` +
1071
+ `image: ${IMAGE_MODELS.map((m) => m.id).join(", ")}`,
1072
+ ),
1073
+ environment: z
1074
+ .string()
1075
+ .describe('the real setting, e.g. "a rain-slicked Seoul side street at night"'),
1076
+ style: z
1077
+ .string()
1078
+ .default("cinematic film still, natural light")
1079
+ .describe('look and grade, e.g. "shot on 35mm film, warm practical light"'),
1080
+ camera_move: z
1081
+ .string()
1082
+ .default(CAMERA_MOVES[0])
1083
+ .describe(`camera move. Known: ${CAMERA_MOVES.filter((m) => m !== "Custom…").join(", ")}`),
1084
+ pose_phrase: z.string().default("").describe("what character A is doing"),
1085
+ pose2_phrase: z.string().default("").describe("what character B is doing"),
1086
+ },
1087
+ },
1088
+ async ({ mode, model, environment, style, camera_move, pose_phrase, pose2_phrase }) => {
1089
+ if (liveHub?.connected) {
1090
+ try {
1091
+ await refreshLiveDescription();
1092
+ } catch (error) {
1093
+ return liveError(error);
1094
+ }
1095
+ }
1096
+ const st = stage();
1097
+ const known = CAMERA_MOVES.includes(camera_move);
1098
+ // composePrompt frames two subjects: the one the camera is on, then the
1099
+ // next visible member of the cast.
1100
+ const framed = findCharacter(state.focus) ?? st.characters[0];
1101
+ const other = st.characters.find((c) => c !== framed && !c.hidden) ?? null;
1102
+ const prompt = composePrompt({
1103
+ mode,
1104
+ model: modelById(model) ?? undefined,
1105
+ shot: currentShot(),
1106
+ subject: framed.subject,
1107
+ subject2: other?.subject ?? null,
1108
+ posePhrase: pose_phrase,
1109
+ pose2Phrase: other ? pose2_phrase : "",
1110
+ environment,
1111
+ style,
1112
+ cameraMove: known ? camera_move : "Custom…",
1113
+ customMove: known ? "" : camera_move,
1114
+ hasCharSheet: st.hasCharSheet === true,
1115
+ hasEnvSheet: false,
1116
+ });
1117
+ return text(`${slateLine(currentShot())}\n\n${prompt}`);
1118
+ },
1119
+ );
1120
+
1121
+ registerTool(
1122
+ "mark_camera_move",
1123
+ {
1124
+ title: "Mark the start of a camera move",
1125
+ description:
1126
+ "Snapshot the current framing as the A position of a camera move. Then move the camera " +
1127
+ "and call describe_camera_move to have the move named in film vocabulary.",
1128
+ inputSchema: {},
1129
+ },
1130
+ async () => {
1131
+ try {
1132
+ await refreshLiveDescription();
1133
+ } catch (error) {
1134
+ return liveError(error);
1135
+ }
1136
+ state.markedFraming = framing();
1137
+ return text(`Marked A position: ${slateLine(currentShot())}\n\nNow move the camera, then call describe_camera_move.`);
1138
+ },
1139
+ );
1140
+
1141
+ registerTool(
1142
+ "describe_camera_move",
1143
+ {
1144
+ title: "Name the camera move",
1145
+ description:
1146
+ "Compare the marked A position against the camera's current B position and name the move " +
1147
+ "the way a crew would — dolly in, crane down, arc left, push, and so on.",
1148
+ inputSchema: {
1149
+ duration_s: z.number().positive().default(3).describe("how long the move takes, in seconds"),
1150
+ },
1151
+ },
1152
+ async ({ duration_s }) => {
1153
+ try {
1154
+ await refreshLiveDescription();
1155
+ } catch (error) {
1156
+ return liveError(error);
1157
+ }
1158
+ if (!state.markedFraming) {
1159
+ return text("No A position marked. Call mark_camera_move first, then move the camera.");
1160
+ }
1161
+ const move = classifyMove(state.markedFraming, framing(), subject(), { durationS: duration_s });
1162
+ return text(
1163
+ [
1164
+ moveSlate(move),
1165
+ "",
1166
+ `from ${slateLine(deriveShot(state.markedFraming.pos, subject(), (state.markedFraming.fovDeg * Math.PI) / 180))}`,
1167
+ `to ${slateLine(currentShot())}`,
1168
+ `over ${duration_s}s`,
1169
+ ].join("\n"),
1170
+ );
1171
+ },
1172
+ );
1173
+
1174
+ registerTool(
1175
+ "add_scene",
1176
+ {
1177
+ title: "Add a scene",
1178
+ description:
1179
+ "Add another scene to the project and make it active. This remains MCP memory-only while " +
1180
+ "an editor is connected; use load_scenes via open_project to replace the live document.",
1181
+ inputSchema: { name: z.string().default("SCENE 02").describe("scene name") },
1182
+ },
1183
+ async ({ name }) => {
1184
+ state.doc.scenes = addScene(state.doc.scenes, name);
1185
+ state.doc.activeSceneId = state.doc.scenes[state.doc.scenes.length - 1].id;
1186
+ const note = liveHub?.connected ? " (memory-only; the connected editor was not changed)" : "";
1187
+ return text(`Added "${name}"${note}.\n\n${sceneReport()}`);
1188
+ },
1189
+ );
1190
+
1191
+ registerTool(
1192
+ "switch_scene",
1193
+ {
1194
+ title: "Switch the active scene",
1195
+ description:
1196
+ "Make a different scene active. This remains MCP memory-only while an editor is connected; " +
1197
+ "everything else operates on the active scene.",
1198
+ inputSchema: { name: z.string().describe("scene name to switch to") },
1199
+ },
1200
+ async ({ name }) => {
1201
+ const target = state.doc.scenes.find((s) => s.name.toLowerCase() === name.toLowerCase());
1202
+ if (!target) {
1203
+ return text(`No scene "${name}". Have: ${state.doc.scenes.map((s) => s.name).join(", ")}`);
1204
+ }
1205
+ state.doc.activeSceneId = target.id;
1206
+ const note = liveHub?.connected ? " (memory-only; the connected editor was not changed)" : "";
1207
+ return text(`Switched to "${target.name}"${note}.\n\n${sceneReport()}`);
1208
+ },
1209
+ );
1210
+
1211
+ registerTool(
1212
+ "open_project",
1213
+ {
1214
+ title: "Open a .cclayproject file",
1215
+ description:
1216
+ "Load a project authored in the CozyClay studio (or saved here). Replaces the current state.",
1217
+ inputSchema: { path: z.string().describe("path to a .cclayproject file") },
1218
+ },
1219
+ async ({ path }) => {
1220
+ const full = resolve(path);
1221
+ let raw;
1222
+ try {
1223
+ raw = await readFile(full, "utf8");
1224
+ } catch (error) {
1225
+ return text(`Could not read ${full}: ${error.message}`);
1226
+ }
1227
+ const result = readProjectDocument(raw);
1228
+ if (!result.ok) return text(`Not a usable project file (${result.reason}): ${full}`);
1229
+
1230
+ // Round-trip through readSceneDocument so an older document is migrated to
1231
+ // the current stage shape rather than trusted as-is.
1232
+ const scenes = readSceneDocument(serializeSceneDocument(result.project.scenesDocument));
1233
+ if (!scenes.document) return text(`That project was written by a newer CozyClay: ${full}`);
1234
+ state.doc = scenes.document;
1235
+ state.name = result.project.name;
1236
+ state.focus = null;
1237
+ state.markedFraming = null;
1238
+ if (liveHub?.connected) {
1239
+ try {
1240
+ const live = await liveHub.command("load_scenes", { document: state.doc });
1241
+ if (typeof live?.sceneName === "string" && live.sceneName) scene().name = live.sceneName;
1242
+ await refreshLiveDescription();
1243
+ } catch (error) {
1244
+ return liveError(error);
1245
+ }
1246
+ }
1247
+ return text(`Opened ${full}.\n\n${sceneReport()}`);
1248
+ },
1249
+ );
1250
+
1251
+ registerTool(
1252
+ "save_project",
1253
+ {
1254
+ title: "Save a .cclayproject file",
1255
+ description:
1256
+ "Write the current state as a .cclayproject file. The CozyClay studio opens this file " +
1257
+ "directly, so a scene blocked here can be finished in the UI.",
1258
+ inputSchema: {
1259
+ path: z.string().describe("destination path, ending in .cclayproject"),
1260
+ name: z.string().optional().describe("project name recorded in the file"),
1261
+ },
1262
+ },
1263
+ async ({ path, name }) => {
1264
+ if (name) state.name = name;
1265
+ if (liveHub?.connected) {
1266
+ try {
1267
+ await refreshLiveDescription();
1268
+ } catch (error) {
1269
+ return liveError(error);
1270
+ }
1271
+ }
1272
+ const full = resolve(path);
1273
+ const project = createProjectDocument({
1274
+ scenesDocument: state.doc,
1275
+ workspaceLayout: null,
1276
+ customPoses: [],
1277
+ name: state.name,
1278
+ });
1279
+ try {
1280
+ await writeFile(full, JSON.stringify(project, null, "\t"), "utf8");
1281
+ } catch (error) {
1282
+ return text(`Could not write ${full}: ${error.message}`);
1283
+ }
1284
+ return text(`Saved "${state.name}" to ${full} (${state.doc.scenes.length} scene(s)).`);
1285
+ },
1286
+ );
1287
+
1288
+ /* -------------------------------- start ---------------------------------- */
1289
+
1290
+ /**
1291
+ * stdio is the default because that is how an MCP client launches a local
1292
+ * server. `--http` is for driving it by hand: a long-lived endpoint on
1293
+ * loopback that survives across client restarts and can be curled.
1294
+ */
1295
+ /** Counted as the tools are registered, so the status page cannot drift. */
1296
+ const TOOL_COUNT = registeredTools;
1297
+
1298
+ const httpFlag = process.argv.indexOf("--http");
1299
+ const livePortFlag = process.argv.indexOf("--live-port");
1300
+ const livePort = Number(
1301
+ livePortFlag === -1 ? process.env.COZYCLAY_LIVE_PORT ?? 5184 : process.argv[livePortFlag + 1],
1302
+ );
1303
+ if (!Number.isInteger(livePort) || livePort < 1 || livePort > 65535) throw new Error("--live-port must be a valid TCP port.");
1304
+
1305
+ if (httpFlag === -1) {
1306
+ liveHub = await startLiveHub(livePort);
1307
+ await server.connect(new StdioServerTransport());
1308
+ } else {
1309
+ // Tools execute in the stdio children, not this HTTP front. Each child tries
1310
+ // to own the one editor port; the winner is live and later sessions see the
1311
+ // port occupied and deliberately remain memory-only rather than sharing state.
1312
+ const requestedHttpPort = process.argv[httpFlag + 1];
1313
+ const port = Number(
1314
+ requestedHttpPort && !requestedHttpPort.startsWith("--") ? requestedHttpPort : process.env.COZYCLAY_MCP_PORT ?? 5183,
1315
+ );
1316
+ if (!Number.isInteger(port) || port < 1 || port > 65535) throw new Error("--http must use a valid TCP port.");
1317
+
1318
+ // One MCP session per client, each backed by its own stdio child of this
1319
+ // same file. A single shared transport would let the first client's
1320
+ // initialize claim the server and refuse every later one; sharing one
1321
+ // server object would also mean two clients silently editing one scene.
1322
+ // A child process per session keeps each client's scene its own, and reuses
1323
+ // the stdio path that the tools already run on.
1324
+ const sessions = new Map();
1325
+
1326
+ const openSession = async () => {
1327
+ const transport = new StreamableHTTPServerTransport({
1328
+ sessionIdGenerator: () => randomUUID(),
1329
+ onsessioninitialized: (id) => sessions.set(id, { transport, child }),
1330
+ });
1331
+ const child = new StdioClientTransport({
1332
+ command: process.execPath,
1333
+ args: [fileURLToPath(import.meta.url), "--live-port", String(livePort)],
1334
+ });
1335
+
1336
+ // Splice the two transports together: the browser-facing session and the
1337
+ // child's stdio pipe just forward each other's messages verbatim.
1338
+ transport.onmessage = (message) => child.send(message);
1339
+ child.onmessage = (message) => transport.send(message);
1340
+ transport.onclose = () => {
1341
+ if (transport.sessionId) sessions.delete(transport.sessionId);
1342
+ child.close().catch(() => {});
1343
+ };
1344
+ child.onclose = () => transport.close().catch(() => {});
1345
+
1346
+ await child.start();
1347
+ await transport.start();
1348
+ return transport;
1349
+ };
1350
+
1351
+ const http = createServer((req, res) => {
1352
+ const path = (req.url ?? "/").split("?")[0];
1353
+
1354
+ // A browser hitting the port should get something legible rather than a
1355
+ // protocol error, so the root is a plain status page.
1356
+ if (path === "/" && req.method === "GET") {
1357
+ res.writeHead(200, { "content-type": "text/plain; charset=utf-8" });
1358
+ res.end(
1359
+ [
1360
+ "CozyClay MCP server",
1361
+ "",
1362
+ `endpoint http://127.0.0.1:${port}/mcp`,
1363
+ "transport Streamable HTTP",
1364
+ `tools ${TOOL_COUNT}`,
1365
+ "",
1366
+ "Point an MCP client at the endpoint above:",
1367
+ ' { "mcpServers": { "cozyclay": { "url": ' +
1368
+ `"http://127.0.0.1:${port}/mcp" } } }`,
1369
+ "",
1370
+ ].join("\n"),
1371
+ );
1372
+ return;
1373
+ }
1374
+
1375
+ if (path === "/mcp") {
1376
+ const existing = sessions.get(req.headers["mcp-session-id"])?.transport;
1377
+ const ready = existing ? Promise.resolve(existing) : openSession();
1378
+ ready
1379
+ .then((transport) => transport.handleRequest(req, res))
1380
+ .catch((error) => {
1381
+ if (!res.headersSent) res.writeHead(500, { "content-type": "application/json" });
1382
+ res.end(JSON.stringify({ error: String(error?.message ?? error) }));
1383
+ });
1384
+ return;
1385
+ }
1386
+
1387
+ res.writeHead(404, { "content-type": "text/plain; charset=utf-8" });
1388
+ res.end(`not found — the MCP endpoint is /mcp\n`);
1389
+ });
1390
+
1391
+ http.listen(port, "127.0.0.1", () => {
1392
+ console.log(`CozyClay MCP on http://127.0.0.1:${port}/mcp (${TOOL_COUNT} tools)`);
1393
+ });
1394
+ }