cozyclay 2.0.2 → 2.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +61 -0
- package/README.md +17 -2
- package/THIRD_PARTY_NOTICES.md +8 -0
- package/bin/agent/agent-routes.mjs +211 -36
- package/bin/agent/agent-runner.mjs +98 -4
- package/bin/agent/codex-client.mjs +2 -1
- package/bin/agent/fal-contract.mjs +0 -17
- package/bin/agent/h3-preservation.mjs +6 -8
- package/bin/agent/provider-keys.mjs +1 -1
- package/bin/agent/providers.mjs +124 -3
- package/bin/agent/script-tool.mjs +74 -0
- package/bin/agent/session-store.mjs +5 -5
- package/bin/agent/studio-prompt.mjs +2 -1
- package/bin/agent/studio-rebase.mjs +314 -0
- package/bin/agent/studio-tools.mjs +21 -14
- package/bin/agent/supervisor.mjs +311 -0
- package/bin/agent/video-adapters.mjs +0 -7
- package/bin/codex-auth.mjs +4 -4
- package/bin/live/cli.mjs +23 -8
- package/bin/live/client.mjs +1 -1
- package/bin/live/studio.mjs +5 -2
- package/bin/mcp-runtime.mjs +2 -2
- package/bin/telemetry-state.mjs +3 -3
- package/dist/app/index.html +6 -6
- package/dist/assets/analytics-C9pMpW7s.js +1 -0
- package/dist/assets/app-B_2XB9rw.css +1 -0
- package/dist/assets/app-DAuvKPP8.js +4920 -0
- package/dist/assets/{demo-ByeYp-dY.js → demo-CvKuBgy5.js} +1 -1
- package/dist/assets/{first-shot-handoff-DgT28ZiV.js → first-shot-handoff-msLI71g1.js} +1 -1
- package/dist/assets/{landing-BaXkpFip.js → landing-ChUvB_zL.js} +1 -1
- package/dist/assets/scene-asset-sync-DKkpjKBV.js +19 -0
- package/dist/assets/{shot-prompt-CbqWcWYi.css → scene-asset-sync-DjJ_cJ2R.css} +1 -1
- package/dist/assets/{ticket-BcmsnHn5.js → ticket-BPk2Ni95.js} +1 -1
- package/dist/assets/workflow-jJRBR94Z.js +179 -0
- package/dist/cozyclay-package.json +1 -1
- package/dist/index.html +328 -222
- package/dist/llms.txt +14 -0
- package/dist/sitemap.xml +1 -1
- package/dist/workflow/index.html +4 -4
- package/mcp/LIVE-PROTOCOL.md +23 -2
- package/mcp/README.md +37 -2
- package/mcp/ardy-prompts.mjs +1 -1
- package/mcp/image-file.mjs +132 -0
- package/mcp/live-hub.mjs +2 -2
- package/mcp/package.json +1 -0
- package/mcp/runtime/package-lock.json +7 -0
- package/mcp/runtime/package.json +1 -0
- package/mcp/server.mjs +0 -6
- package/mcp/tool-handlers.mjs +289 -87
- package/package.json +11 -2
- package/src/App.jsx +813 -278
- package/src/agent-image-client.js +94 -0
- package/src/analytics.js +47 -3
- package/src/app-stage.jsx +129 -2745
- package/src/ardy/auto-physics.js +6 -16
- package/src/ardy/collision-blockers.js +3 -7
- package/src/ardy/fix-collisions.js +5 -35
- package/src/ardy/ik-drag.js +86 -5
- package/src/ardy/ik-key-json.js +3 -1
- package/src/ardy/ik.js +15 -13
- package/src/ardy/motion-calibration.js +1 -1
- package/src/ardy/npz.js +0 -8
- package/src/ardy/physics-panel.jsx +3 -2
- package/src/ardy/physics-review.js +3 -2
- package/src/ardy/physics-support.js +1 -1
- package/src/ardy/physics-surface.js +1 -1
- package/src/ardy/platform-fit-panel.jsx +2 -1
- package/src/ardy/playback.js +1 -5
- package/src/ardy/range-pin.js +2 -2
- package/src/ardy/root-drop.js +0 -1
- package/src/ardy/timeline.css +184 -6
- package/src/ardy/timeline.jsx +121 -36
- package/src/ardy/to-cskel27.js +0 -2
- package/src/ardy/waypoints.js +0 -3
- package/src/ardy-bridge-contract.js +281 -0
- package/src/asset-pane.jsx +19 -9
- package/src/auto-color.js +1 -1
- package/src/burn-in.js +1 -1
- package/src/camera-block.js +1 -1
- package/src/camera-follow.js +75 -32
- package/src/camera-move.js +0 -6
- package/src/camera-tutorial.jsx +6 -7
- package/src/character-kind.js +79 -0
- package/src/command-bus.js +76 -17
- package/src/commands/cast.js +31 -7
- package/src/commands/elements/object.js +1 -1
- package/src/commands/elements/shot.js +1 -1
- package/src/commands/elements.js +1 -1
- package/src/commands/export.js +3 -3
- package/src/commands/motion.js +2 -2
- package/src/commands/objects.js +105 -10
- package/src/commands/project.js +15 -2
- package/src/commands/sculpt-images.js +53 -0
- package/src/commands/shot.js +88 -10
- package/src/commands/view.js +10 -4
- package/src/coplanar-depth.js +216 -0
- package/src/cuts.js +7 -6
- package/src/domains/cast.js +60 -43
- package/src/domains/motion/motion-intent-codec.js +50 -0
- package/src/domains/motion/motion-range-pin-rules.js +23 -0
- package/src/domains/motion/motion-video-capture.js +664 -0
- package/src/domains/motion.js +253 -742
- package/src/domains/objects.js +106 -5
- package/src/domains/scenes.js +79 -28
- package/src/domains/shots.js +33 -7
- package/src/error-boundary.jsx +1 -1
- package/src/execution-telemetry.js +5 -5
- package/src/facing-marks.js +14 -1
- package/src/first-success-guide.jsx +35 -4
- package/src/gizmo-claim.js +3 -1
- package/src/grid-floor.jsx +1 -4
- package/src/hierarchy-model.js +48 -35
- package/src/hierarchy-panel.css +1 -0
- package/src/hierarchy-panel.jsx +54 -102
- package/src/history.js +0 -2
- package/src/i2v-motion-client.js +1 -4
- package/src/ime.js +6 -0
- package/src/keyframe-pack.js +15 -6
- package/src/line-edit.js +4 -4
- package/src/live-control.js +6 -6
- package/src/matte-editor.js +2 -8
- package/src/matte.js +0 -19
- package/src/mesh-graph-clone.js +16 -2
- package/src/mesh-sniff.js +0 -1
- package/src/monotone-cubic.js +40 -0
- package/src/motion-store.js +4 -4
- package/src/motion-trail.js +5 -11
- package/src/multimodel-ingest.js +1 -3
- package/src/object-catalog.jsx +7 -5
- package/src/object-gizmo.jsx +45 -31
- package/src/object-path.js +310 -67
- package/src/object-travel.js +224 -0
- package/src/otio.js +8 -3
- package/src/panels/CharacterTransformPanel.jsx +24 -6
- package/src/panels/ObjectTransformPanel.jsx +4 -3
- package/src/panels/PoseImagePanel.jsx +205 -0
- package/src/panels/PromptBlocksPanel.jsx +3 -1
- package/src/panels/ProxyFigurePanel.jsx +56 -0
- package/src/panels/ReferenceImageField.jsx +1 -1
- package/src/panels/RigControlPanel.jsx +24 -3
- package/src/panels/RigPanel.jsx +2 -1
- package/src/panels/SculptPanel.jsx +405 -0
- package/src/panels/SubjectBox.jsx +1 -1
- package/src/panels/SubjectsPanel.jsx +0 -2
- package/src/panels/VideoCapturePanel.jsx +2 -1
- package/src/panels/pose-image.css +42 -0
- package/src/panels/pose.css +3 -0
- package/src/panels/sculpt.css +162 -0
- package/src/plan-diagram.js +184 -0
- package/src/planview.jsx +127 -80
- package/src/playground.js +3 -5
- package/src/pose-fade.js +77 -0
- package/src/pose-gizmo.css +23 -0
- package/src/pose-gizmo.js +96 -0
- package/src/posestudio.jsx +134 -83
- package/src/project-browser.css +53 -0
- package/src/project-browser.jsx +49 -4
- package/src/project.js +55 -24
- package/src/props.jsx +177 -36
- package/src/proxy-figure.jsx +69 -0
- package/src/range-pin-panel.jsx +0 -1
- package/src/receipt-findings.js +20 -0
- package/src/render-passes.js +0 -1
- package/src/resource-status.jsx +1 -1
- package/src/root-path.js +48 -0
- package/src/route-owner.js +52 -0
- package/src/scene-asset-cache.js +1 -2
- package/src/scene-assets.js +12 -4
- package/src/scene-fbx.js +1 -1
- package/src/scene-mesh-cache.js +0 -1
- package/src/scene-objects.js +319 -19
- package/src/scenes.js +58 -9
- package/src/sculpt-blueprint.js +65 -0
- package/src/sculpt-compare.js +384 -0
- package/src/sculpt-geometry.js +141 -0
- package/src/sculpt-mode.js +168 -0
- package/src/sculpt-recipe.js +385 -0
- package/src/sculpt-run.js +351 -0
- package/src/sculpt-stage.jsx +221 -0
- package/src/settings-menu.jsx +1 -1
- package/src/shell/BoardSlot.jsx +341 -0
- package/src/shell/BottomDock.jsx +60 -16
- package/src/shell/CompositionGuideButton.jsx +20 -0
- package/src/shell/DetailsSlot.jsx +26 -6
- package/src/shell/MenuBar.jsx +78 -6
- package/src/shell/OutlinerSlot.jsx +3 -1
- package/src/shell/ShotPreviewHeader.jsx +17 -0
- package/src/shell/TopBar.jsx +18 -3
- package/src/shell/ViewportToolbar.jsx +36 -31
- package/src/shell/dock.css +239 -0
- package/src/shell/glass-regions.css +3 -2
- package/src/shell/log-store.js +4 -4
- package/src/shell/mode.css +20 -0
- package/src/shell/studio-shell-context.js +4 -1
- package/src/shell/viewport.css +12 -0
- package/src/shot-authoring.js +69 -6
- package/src/stable-items.js +1 -1
- package/src/stage-attach-frames.js +169 -0
- package/src/stage-canvas-rigs.jsx +402 -0
- package/src/stage-capture.jsx +216 -0
- package/src/stage-characters.jsx +302 -0
- package/src/stage-editor-gizmos.jsx +1229 -0
- package/src/stage-labels-ko.js +204 -0
- package/src/stage-startup-defaults.js +182 -0
- package/src/starter-scenes.js +1 -1
- package/src/storyboard.js +24 -5
- package/src/studio-actions.js +42 -9
- package/src/studio-agent-commands.js +213 -23
- package/src/studio-agent-context.js +11 -9
- package/src/studio-agent-motion.js +2 -1
- package/src/studio-agent-protocol.js +60 -24
- package/src/studio-app-binding.js +56 -23
- package/src/studio-contact-sheet.js +3 -3
- package/src/studio-elements.js +11 -2
- package/src/studio-geometry-facts.js +114 -0
- package/src/styles.css +92 -9
- package/src/ui-scrub.js +4 -4
- package/src/ui.jsx +6 -4
- package/src/use-asset-shelf.js +60 -0
- package/src/use-camera-tutorial.js +50 -0
- package/src/use-embed-bridge.js +12 -0
- package/src/use-first-success-prompt.js +7 -0
- package/src/use-i2v-capture-modal.js +15 -0
- package/src/use-sculpt-mode.js +95 -0
- package/src/workflow/AgentPanel.jsx +106 -18
- package/src/workflow/WorkflowBuilder.jsx +1 -1
- package/src/workflow/agent-client.js +113 -45
- package/src/workflow/agent-panel.css +35 -0
- package/src/workflow/attachment-image.js +1 -1
- package/src/workflow/canvas-commands.js +3 -3
- package/src/workflow/cozy-scene-node.js +3 -3
- package/src/workflow/local-workflow.js +2 -2
- package/src/workflow/motion-input.js +3 -3
- package/src/workflow/node-schema.js +1 -1
- package/src/workflow/scene-asset-sync.js +2 -9
- package/src/workflow/shot-prompt-node.js +1 -1
- package/src/workflow/video-contract.js +1 -1
- package/src/workflow/workflow-resources.js +0 -8
- package/src/workflow-send.js +87 -0
- package/tools/ardy/__pycache__/cclay_gvhmr_worker.cpython-313.pyc +0 -0
- package/tools/check-eslint.mjs +57 -0
- package/tools/check-knip.mjs +60 -0
- package/tools/check-loc.mjs +54 -0
- package/tools/demo-worker/api-client.mjs +7 -13
- package/tools/demo-worker/generate.mjs +2 -4
- package/tools/eslint-baseline.json +23 -0
- package/tools/gt-render/browser.mjs +23 -0
- package/tools/headless-studio.mjs +85 -0
- package/tools/kimodo/effector-constraints.mjs +4 -4
- package/tools/kimodo/generate.mjs +1 -1
- package/tools/knip-baseline.json +331 -0
- package/tools/loc-baseline.json +126 -0
- package/tools/process-supervisor.mjs +1 -1
- package/tools/projflow/replay.mjs +4 -4
- package/tools/projflow/runner.mjs +3 -3
- package/tools/qa/studio-control-count.mjs +2 -0
- package/tools/run-tests.mjs +95 -7
- package/dist/assets/analytics-MVZOEW8O.js +0 -1
- package/dist/assets/app-BxBLC6GT.js +0 -4876
- package/dist/assets/app-CJdxvnrl.css +0 -1
- package/dist/assets/shot-prompt-hN0j_O-p.js +0 -17
- package/dist/assets/workflow-B5XZwVfe.js +0 -179
package/mcp/tool-handlers.mjs
CHANGED
|
@@ -30,6 +30,8 @@ import { z } from "zod";
|
|
|
30
30
|
|
|
31
31
|
import { MAX_COMMAND_TIMEOUT_MS } from "./live-hub.mjs";
|
|
32
32
|
import { readMeshFromPath } from "./mesh-file.mjs";
|
|
33
|
+
import { encodePng, readImageFromPath } from "./image-file.mjs";
|
|
34
|
+
import { rankingRow, scoreSculptAgainstReference } from "../src/sculpt-compare.js";
|
|
33
35
|
import { DEFAULT_POSE } from "../src/poses.js";
|
|
34
36
|
import { generationArgs } from '../src/motion/generation.js';
|
|
35
37
|
import { BLOCK_MAX_SECONDS, PROMPT_GUIDE, normalizePhases, splitLongBeat, tileClipFrames } from "./ardy-prompts.mjs";
|
|
@@ -47,31 +49,37 @@ import {
|
|
|
47
49
|
slateLine,
|
|
48
50
|
} from "../src/shot.js";
|
|
49
51
|
import {
|
|
50
|
-
|
|
52
|
+
CHARACTER_KIND_IDS,
|
|
53
|
+
POSTURES,
|
|
51
54
|
activeScene,
|
|
55
|
+
isProxyFigure,
|
|
52
56
|
addScene,
|
|
53
57
|
createCharacterEntry,
|
|
54
58
|
createSceneDocument,
|
|
55
59
|
readSceneDocument,
|
|
56
60
|
serializeSceneDocument,
|
|
57
61
|
} from "../src/scenes.js";
|
|
62
|
+
import { normalizeSculptRecipe, patchSculptRecipe } from "../src/sculpt-recipe.js";
|
|
58
63
|
import {
|
|
59
64
|
OBJECT_LIBRARY,
|
|
65
|
+
SCULPT_KIND,
|
|
60
66
|
createSceneObject,
|
|
67
|
+
createSculptObject,
|
|
61
68
|
objectSize,
|
|
62
69
|
removeSceneObject,
|
|
63
70
|
setSceneObjectParent,
|
|
64
71
|
updateSceneObject,
|
|
65
72
|
} from "../src/scene-objects.js";
|
|
66
73
|
import { classifyMove, captureFraming, moveSlate } from "../src/camera-move.js";
|
|
67
|
-
import { createProjectDocument, readProjectDocument } from "../src/project.js";
|
|
68
|
-
|
|
69
|
-
/* ------------------------------- state ---------------------------------- */
|
|
74
|
+
import { DEFAULT_PREVIS_MODE, PREVIS_MODES, createProjectDocument, readProjectDocument } from "../src/project.js";
|
|
75
|
+
import { kindRefusal } from "../src/character-kind.js";
|
|
70
76
|
|
|
71
77
|
/** The authoring state. One scene document, one camera, one project name. */
|
|
72
78
|
export const state = {
|
|
73
79
|
doc: createSceneDocument(),
|
|
74
80
|
name: "Untitled",
|
|
81
|
+
/** the project's previs mode ("storyboard" | "animation"); the live editor reports it with every describe */
|
|
82
|
+
previsMode: DEFAULT_PREVIS_MODE,
|
|
75
83
|
camera: { x: 0, y: 1.6, z: 4.5, focalMm: 35 },
|
|
76
84
|
timeline: { currentFrame: 0, frameCount: 360, fps: 24 },
|
|
77
85
|
/** which character the camera frames against; null means the first of the cast */
|
|
@@ -97,7 +105,7 @@ export const liveWorkspace = new AsyncLocalStorage();
|
|
|
97
105
|
const liveWorkspaceTools = new Set([
|
|
98
106
|
"describe_scene", "describe_shot", "render_prompt", "mark_camera_move", "describe_camera_move", "save_project",
|
|
99
107
|
"set_camera", "frame_shot", "add_character", "place_character", "remove_character",
|
|
100
|
-
"focus_character", "place_object", "import_mesh", "group_objects", "set_prompt_blocks", "generate_motion", "update_object",
|
|
108
|
+
"focus_character", "place_object", "import_mesh", "sculpt_object", "compare_sculpt", "group_objects", "set_prompt_blocks", "generate_motion", "update_object",
|
|
101
109
|
"remove_object", "apply_batch", "add_scene", "switch_scene", "open_project", "capture_frame", "load_motion",
|
|
102
110
|
"studio_commands", "studio_run",
|
|
103
111
|
]);
|
|
@@ -153,6 +161,8 @@ const TOOL_ANNOTATIONS = Object.freeze({
|
|
|
153
161
|
focus_character: { readOnlyHint: false, destructiveHint: true, idempotentHint: true, openWorldHint: false },
|
|
154
162
|
place_object: { readOnlyHint: false, destructiveHint: false, idempotentHint: false, openWorldHint: false },
|
|
155
163
|
import_mesh: { readOnlyHint: false, destructiveHint: false, idempotentHint: false, openWorldHint: true },
|
|
164
|
+
sculpt_object: { readOnlyHint: false, destructiveHint: false, idempotentHint: false, openWorldHint: false },
|
|
165
|
+
compare_sculpt: { readOnlyHint: true, destructiveHint: false, idempotentHint: true, openWorldHint: true },
|
|
156
166
|
group_objects: { readOnlyHint: false, destructiveHint: true, idempotentHint: true, openWorldHint: false },
|
|
157
167
|
set_prompt_blocks: { readOnlyHint: false, destructiveHint: true, idempotentHint: true, openWorldHint: false },
|
|
158
168
|
load_motion: { readOnlyHint: false, destructiveHint: true, idempotentHint: true, openWorldHint: false },
|
|
@@ -176,6 +186,15 @@ const TOOL_ANNOTATIONS = Object.freeze({
|
|
|
176
186
|
/** The document identity a Studio command is admitted in. */
|
|
177
187
|
const STUDIO_IDENTITY_KEYS = ["workspaceId", "documentEpoch", "sceneId", "sceneEpoch"];
|
|
178
188
|
|
|
189
|
+
/** The Studio context behind an inspect answer. inspect_studio answers the
|
|
190
|
+
* revision and document identity, not the whole context (older editors embedded
|
|
191
|
+
* it); entities, selection and the mode are read apart, at that identity. */
|
|
192
|
+
const contextOf = async (inspected, workspaceHandle = liveWorkspace.getStore()) => {
|
|
193
|
+
if (inspected?.context) return inspected.context;
|
|
194
|
+
if (!inspected?.host || !Number.isSafeInteger(inspected.revision?.scene)) return undefined;
|
|
195
|
+
const read = await liveHub.command("read_studio_context", { host: inspected.host }, workspaceHandle);
|
|
196
|
+
return read?.context ?? read;
|
|
197
|
+
};
|
|
179
198
|
/** One registered editor command through the editor's own bus, admitted at
|
|
180
199
|
* the open document and its current revision like the agent's run_action. Its
|
|
181
200
|
* declaration (read from the editor, never from this server) sets the hub
|
|
@@ -183,15 +202,18 @@ const STUDIO_IDENTITY_KEYS = ["workspaceId", "documentEpoch", "sceneId", "sceneE
|
|
|
183
202
|
const executeStudioCommand = async ({ action, args, expectedRevision, commandId, timeoutMs, confirmationToken, inspected }) => {
|
|
184
203
|
const workspaceHandle = liveWorkspace.getStore();
|
|
185
204
|
inspected ??= await liveHub.command("inspect_studio", { scope: "actions", ids: [action] }, workspaceHandle);
|
|
186
|
-
const context = inspected
|
|
205
|
+
const context = await contextOf(inspected, workspaceHandle);
|
|
187
206
|
if (!context?.host || !Number.isSafeInteger(context.revision?.scene)) throw new Error("The editor did not return a Studio context to admit this command against.");
|
|
207
|
+
// Admission stays at the revision the inspect answered: an edit that lands
|
|
208
|
+
// between that inspect and the context read is the editor's to refuse (STALE_SCENE).
|
|
209
|
+
const inspectedRevision = Number.isSafeInteger(inspected?.revision?.scene) ? inspected.revision.scene : context.revision.scene;
|
|
188
210
|
const declared = inspected.actions?.find((row) => row.id === action);
|
|
189
211
|
const receipt = await liveHub.command("run_action", {
|
|
190
212
|
name: "run_action",
|
|
191
213
|
args: { action, args: typeof args === "function" ? await args(context) : args ?? {}, ...(confirmationToken ? { confirmationToken } : {}) },
|
|
192
214
|
commandId: commandId ?? randomUUID(),
|
|
193
215
|
host: Object.fromEntries(STUDIO_IDENTITY_KEYS.map((key) => [key, context.host[key]])),
|
|
194
|
-
expectedRevision: expectedRevision ??
|
|
216
|
+
expectedRevision: expectedRevision ?? inspectedRevision,
|
|
195
217
|
}, workspaceHandle, { timeoutMs: timeoutMs ?? (declared?.timeoutMs === undefined ? undefined : Math.min(MAX_COMMAND_TIMEOUT_MS, declared.timeoutMs + (declared.generation ? 5000 : 0))) });
|
|
196
218
|
return receipt;
|
|
197
219
|
};
|
|
@@ -202,6 +224,27 @@ const studioResult = (...receipts) => ({
|
|
|
202
224
|
});
|
|
203
225
|
const runStudioCommand = async options => studioResult(await executeStudioCommand(options));
|
|
204
226
|
|
|
227
|
+
// The tool list is fixed when the server starts, so a tool the project's mode
|
|
228
|
+
// or the target's kind cannot run answers a structured refusal instead of acting.
|
|
229
|
+
const refusal = (code, reason) => ({ content: [{ type: "text", text: JSON.stringify({ ok: false, code, reason }) }], isError: true });
|
|
230
|
+
|
|
231
|
+
/** NOT_IN_MODE in a storyboard project; TARGET_NOT_READY when `feature` is given
|
|
232
|
+
* and the target (`characterId`, else the active character) is a capsule figure.
|
|
233
|
+
* Null when the tool may run. `context` is the connected editor's Studio context;
|
|
234
|
+
* without one the in-memory project answers. */
|
|
235
|
+
const previsRefusal = (toolName, { feature, context, characterId } = {}) => {
|
|
236
|
+
if ((context ? context.scene?.previsMode : state.previsMode) === "storyboard") {
|
|
237
|
+
return refusal("NOT_IN_MODE", `This project is a Storyboard - ${toolName} is available in Animation projects.`);
|
|
238
|
+
}
|
|
239
|
+
if (!feature) return null;
|
|
240
|
+
const proxy = context
|
|
241
|
+
? context.entities?.find((entity) => entity.id === (characterId ?? context.activeCharacterId))?.characterKind === "proxy"
|
|
242
|
+
: isProxyFigure(findCharacter(characterId ?? state.focus));
|
|
243
|
+
return proxy ? refusal("TARGET_NOT_READY", kindRefusal(feature)) : null;
|
|
244
|
+
};
|
|
245
|
+
/** The Studio context and one command's declaration, as executeStudioCommand admits it. */
|
|
246
|
+
const inspectAction = (action) => liveHub.command("inspect_studio", { scope: "actions", ids: [action] }, liveWorkspace.getStore());
|
|
247
|
+
|
|
205
248
|
const scene = () => activeScene(state.doc.scenes, state.doc.activeSceneId);
|
|
206
249
|
const stage = () => scene().stage;
|
|
207
250
|
|
|
@@ -299,6 +342,7 @@ const applyLiveDescription = (description) => {
|
|
|
299
342
|
if (!parsed.document) throw new Error("Live editor returned an invalid scene document.");
|
|
300
343
|
state.doc = parsed.document;
|
|
301
344
|
}
|
|
345
|
+
if (PREVIS_MODES.includes(description.previsMode)) state.previsMode = description.previsMode;
|
|
302
346
|
const sc = scene();
|
|
303
347
|
if (typeof description.sceneName === "string" && description.sceneName) sc.name = description.sceneName;
|
|
304
348
|
if (description.camera && typeof description.camera === "object") {
|
|
@@ -339,11 +383,11 @@ const applyLiveDescription = (description) => {
|
|
|
339
383
|
// A") defeats the name match, and a record without a renderer survives
|
|
340
384
|
// the save but cannot be drawn after the load.
|
|
341
385
|
const kind =
|
|
342
|
-
(typeof object.renderer === "string" && OBJECT_LIBRARY.some((entry) => entry.kind === object.renderer)
|
|
386
|
+
(typeof object.renderer === "string" && (object.renderer === SCULPT_KIND || OBJECT_LIBRARY.some((entry) => entry.kind === object.renderer))
|
|
343
387
|
? object.renderer
|
|
344
388
|
: null) ??
|
|
345
389
|
OBJECT_LIBRARY.find(({ label }) => object.name === label || object.name?.startsWith(`${label} `))?.kind;
|
|
346
|
-
const defaults = previous ?? (kind ? createSceneObject(kind, sc.objects, object) : null);
|
|
390
|
+
const defaults = previous ?? (kind === SCULPT_KIND ? createSculptObject({ recipe: object.recipe }, sc.objects, object).object ?? null : kind ? createSceneObject(kind, sc.objects, object) : null);
|
|
347
391
|
// The editor is the source of truth for anything it reports; the
|
|
348
392
|
// library defaults only fill what the frame omits. Defaulting AFTER
|
|
349
393
|
// the spread would reset a reported scale back to 1 and make every
|
|
@@ -372,7 +416,64 @@ const liveError = (error) => ({
|
|
|
372
416
|
isError: true,
|
|
373
417
|
});
|
|
374
418
|
|
|
375
|
-
|
|
419
|
+
/** capture_frame's checked frame as a tool result; sculpt_object { capture: true } reuses it. */
|
|
420
|
+
const captureFrameResult = async (maxInlineBytes) => {
|
|
421
|
+
if (!liveHub?.connected) return liveError(new Error(noLiveEditor("capture_frame requires a connected CozyClay editor with a renderable shot camera.")));
|
|
422
|
+
try {
|
|
423
|
+
const workspaceHandle = liveWorkspace.getStore();
|
|
424
|
+
const frame = await liveHub.command("capture_frame", {}, workspaceHandle);
|
|
425
|
+
const beforeHash = createHash("sha256").update(frame?.authoredStateBefore ?? "").digest("hex");
|
|
426
|
+
const afterHash = createHash("sha256").update(frame?.authoredStateAfter ?? "").digest("hex");
|
|
427
|
+
if (beforeHash !== afterHash) {
|
|
428
|
+
throw new Error(`capture_frame changed authored editor state; capture was rejected (${beforeHash} -> ${afterHash}).`);
|
|
429
|
+
}
|
|
430
|
+
if (!frame || frame.width !== 640 || frame.height !== 360 || frame.mimeType !== "image/png" || typeof frame.data !== "string") {
|
|
431
|
+
throw new Error("Live editor returned an invalid capture payload.");
|
|
432
|
+
}
|
|
433
|
+
const bytes = Buffer.from(frame.data, "base64");
|
|
434
|
+
if (bytes.length === 0 || bytes.length !== frame.byteSize || bytes.length > MAX_CAPTURE_BYTES) {
|
|
435
|
+
throw new Error(`Live editor returned an invalid compressed image size (maximum ${MAX_CAPTURE_BYTES} bytes).`);
|
|
436
|
+
}
|
|
437
|
+
if (frame.assertions?.renderable !== true || frame.assertions?.blackFrame === true) {
|
|
438
|
+
throw new Error("Live editor rejected the capture as non-renderable or black.");
|
|
439
|
+
}
|
|
440
|
+
const metadata = {
|
|
441
|
+
width: frame.width,
|
|
442
|
+
height: frame.height,
|
|
443
|
+
mimeType: frame.mimeType,
|
|
444
|
+
encoding: frame.encoding,
|
|
445
|
+
byteSize: frame.byteSize,
|
|
446
|
+
assertions: frame.assertions,
|
|
447
|
+
stateHashBefore: beforeHash,
|
|
448
|
+
stateHashAfter: afterHash,
|
|
449
|
+
};
|
|
450
|
+
if (bytes.length > maxInlineBytes) {
|
|
451
|
+
while (captureArtifacts.length >= MAX_CAPTURE_ARTIFACTS) {
|
|
452
|
+
const oldest = captureArtifacts.shift();
|
|
453
|
+
await unlink(oldest).catch(() => {});
|
|
454
|
+
}
|
|
455
|
+
const path = join(tmpdir(), `cozyclay-capture-${randomUUID()}.png`);
|
|
456
|
+
await writeFile(path, bytes, { mode: 0o600 });
|
|
457
|
+
captureArtifacts.push(path);
|
|
458
|
+
const expiry = setTimeout(() => {
|
|
459
|
+
const index = captureArtifacts.indexOf(path);
|
|
460
|
+
if (index >= 0) captureArtifacts.splice(index, 1);
|
|
461
|
+
void unlink(path).catch(() => {});
|
|
462
|
+
}, CAPTURE_ARTIFACT_TTL_MS);
|
|
463
|
+
expiry.unref?.();
|
|
464
|
+
return text(JSON.stringify({ ...metadata, artifact: { path, width: frame.width, height: frame.height, byteSize: bytes.length } }));
|
|
465
|
+
}
|
|
466
|
+
return {
|
|
467
|
+
content: [
|
|
468
|
+
{ type: "text", text: JSON.stringify({ ...metadata, image: { transport: "inline", width: frame.width, height: frame.height, byteSize: bytes.length } }) },
|
|
469
|
+
{ type: "image", data: frame.data, mimeType: frame.mimeType },
|
|
470
|
+
],
|
|
471
|
+
};
|
|
472
|
+
} catch (error) {
|
|
473
|
+
return liveError(error);
|
|
474
|
+
}
|
|
475
|
+
};
|
|
476
|
+
|
|
376
477
|
|
|
377
478
|
const round = (n, places = 2) => Number(n.toFixed(places));
|
|
378
479
|
const metres = (n) => `${round(n)}m`;
|
|
@@ -393,6 +494,7 @@ function sceneReport({ characterCursor = 0, objectCursor = 0, limit = 50 } = {})
|
|
|
393
494
|
const lines = [
|
|
394
495
|
`Project: ${state.name}`,
|
|
395
496
|
`Scene: ${sc.name} (${state.doc.scenes.length} scene${state.doc.scenes.length === 1 ? "" : "s"} in project)`,
|
|
497
|
+
`previsMode: ${state.previsMode}`,
|
|
396
498
|
"",
|
|
397
499
|
"CAMERA",
|
|
398
500
|
` position x ${round(state.camera.x)} y ${round(state.camera.y)} z ${round(state.camera.z)}`,
|
|
@@ -411,7 +513,7 @@ function sceneReport({ characterCursor = 0, objectCursor = 0, limit = 50 } = {})
|
|
|
411
513
|
`facing ${round(c.rot, 1)}deg [${c.model}]` +
|
|
412
514
|
`${c.pose ? " posed" : ""}${c.hidden ? " hidden" : ""}` +
|
|
413
515
|
`${c === framed ? " <- framed" : ""}`,
|
|
414
|
-
` model: ${c.model} pose: ${JSON.stringify(c.pose ?? null)} tint: ${c.tint ?? null} scale: ${c.scale ?? 1}`,
|
|
516
|
+
` model: ${c.model} kind: ${isProxyFigure(c) ? "proxy" : "rig"}${isProxyFigure(c) ? ` posture: ${c.posture ?? "stand"}` : ""} pose: ${JSON.stringify(c.pose ?? null)} tint: ${c.tint ?? null} scale: ${c.scale ?? 1}`,
|
|
415
517
|
` motionRef: ${JSON.stringify(c.motionRef ?? null)}`,
|
|
416
518
|
` layer: ${JSON.stringify({ waypoints: c.layer?.waypoints ?? [], promptClips: c.layer?.promptClips ?? [] })}`,
|
|
417
519
|
);
|
|
@@ -427,13 +529,13 @@ function sceneReport({ characterCursor = 0, objectCursor = 0, limit = 50 } = {})
|
|
|
427
529
|
` ${object.id} ${object.name} at x ${round(object.x)}, y ${round(object.y)}, z ${round(object.z)}` +
|
|
428
530
|
` yaw ${round(object.rot, 1)}deg size ${round(size.width)}x${round(size.height)}x${round(size.depth)}m` +
|
|
429
531
|
`${object.hidden ? " hidden" : ""}`,
|
|
430
|
-
` rotX: ${round(object.rotX ?? 0, 1)} rotZ: ${round(object.rotZ ?? 0, 1)} color: ${object.color ?? null} parent: ${object.parent ?? null}`,
|
|
532
|
+
` rotX: ${round(object.rotX ?? 0, 1)} rotZ: ${round(object.rotZ ?? 0, 1)} color: ${object.color ?? null}${object.opacity < 1 ? ` opacity: ${round(object.opacity, 2)}` : ""} parent: ${object.parent ?? null}`,
|
|
431
533
|
);
|
|
432
534
|
if (object.path?.points?.length >= 2) {
|
|
433
535
|
const points = object.path.points;
|
|
434
536
|
const first = points[0];
|
|
435
537
|
const last = points[points.length - 1];
|
|
436
|
-
lines.push(` path: ${points.length} pts (${round(first.x, 1)},${round(first.z, 1)}) → (${round(last.x, 1)},${round(last.z, 1)}) speed: ${object.path.speed || "fills take"}${object.path.extend ? " keeps going" : ""}${object.path.loop ? " loops" : ""}`);
|
|
538
|
+
lines.push(` path: ${points.length} pts (${round(first.x, 1)},${round(first.z, 1)}) → (${round(last.x, 1)},${round(last.z, 1)}) speed: ${object.path.speed || "fills take"}${object.path.extend ? " keeps going" : ""}${object.path.loop ? " loops" : ""}${object.path.marks?.length ? ` marks: ${object.path.marks.map((mark) => (typeof mark === "number" ? round(mark, 2) : `${round(mark.t, 2)}${mark.bank ? ` bank ${round(mark.bank, 1)}` : ""}${mark.pitch ? ` pitch ${round(mark.pitch, 1)}` : ""}`)).join(", ")}` : ""}`);
|
|
437
539
|
}
|
|
438
540
|
}
|
|
439
541
|
}
|
|
@@ -484,11 +586,9 @@ function shotReport() {
|
|
|
484
586
|
].join("\n");
|
|
485
587
|
}
|
|
486
588
|
|
|
487
|
-
/* --------------------------------- tools --------------------------------- */
|
|
488
|
-
|
|
489
589
|
const studioAliasTools = new Set([
|
|
490
590
|
"set_camera", "frame_shot", "place_object", "update_object", "remove_object",
|
|
491
|
-
"import_mesh", "group_objects", "add_scene", "switch_scene", "apply_batch", "open_project",
|
|
591
|
+
"import_mesh", "sculpt_object", "group_objects", "add_scene", "switch_scene", "apply_batch", "open_project",
|
|
492
592
|
"add_character", "place_character", "remove_character", "set_prompt_blocks", "load_motion", "generate_motion",
|
|
493
593
|
]);
|
|
494
594
|
const studioAdmissionSchema = {
|
|
@@ -616,60 +716,7 @@ export const createToolHandlers = ({ projectRootPromise } = {}) => {
|
|
|
616
716
|
},
|
|
617
717
|
},
|
|
618
718
|
async ({ max_inline_bytes }) => {
|
|
619
|
-
|
|
620
|
-
try {
|
|
621
|
-
const workspaceHandle = liveWorkspace.getStore();
|
|
622
|
-
const frame = await liveHub.command("capture_frame", {}, workspaceHandle);
|
|
623
|
-
const beforeHash = createHash("sha256").update(frame?.authoredStateBefore ?? "").digest("hex");
|
|
624
|
-
const afterHash = createHash("sha256").update(frame?.authoredStateAfter ?? "").digest("hex");
|
|
625
|
-
if (beforeHash !== afterHash) {
|
|
626
|
-
throw new Error(`capture_frame changed authored editor state; capture was rejected (${beforeHash} -> ${afterHash}).`);
|
|
627
|
-
}
|
|
628
|
-
if (!frame || frame.width !== 640 || frame.height !== 360 || frame.mimeType !== "image/png" || typeof frame.data !== "string") {
|
|
629
|
-
throw new Error("Live editor returned an invalid capture payload.");
|
|
630
|
-
}
|
|
631
|
-
const bytes = Buffer.from(frame.data, "base64");
|
|
632
|
-
if (bytes.length === 0 || bytes.length !== frame.byteSize || bytes.length > MAX_CAPTURE_BYTES) {
|
|
633
|
-
throw new Error(`Live editor returned an invalid compressed image size (maximum ${MAX_CAPTURE_BYTES} bytes).`);
|
|
634
|
-
}
|
|
635
|
-
if (frame.assertions?.renderable !== true || frame.assertions?.blackFrame === true) {
|
|
636
|
-
throw new Error("Live editor rejected the capture as non-renderable or black.");
|
|
637
|
-
}
|
|
638
|
-
const metadata = {
|
|
639
|
-
width: frame.width,
|
|
640
|
-
height: frame.height,
|
|
641
|
-
mimeType: frame.mimeType,
|
|
642
|
-
encoding: frame.encoding,
|
|
643
|
-
byteSize: frame.byteSize,
|
|
644
|
-
assertions: frame.assertions,
|
|
645
|
-
stateHashBefore: beforeHash,
|
|
646
|
-
stateHashAfter: afterHash,
|
|
647
|
-
};
|
|
648
|
-
if (bytes.length > max_inline_bytes) {
|
|
649
|
-
while (captureArtifacts.length >= MAX_CAPTURE_ARTIFACTS) {
|
|
650
|
-
const oldest = captureArtifacts.shift();
|
|
651
|
-
await unlink(oldest).catch(() => {});
|
|
652
|
-
}
|
|
653
|
-
const path = join(tmpdir(), `cozyclay-capture-${randomUUID()}.png`);
|
|
654
|
-
await writeFile(path, bytes, { mode: 0o600 });
|
|
655
|
-
captureArtifacts.push(path);
|
|
656
|
-
const expiry = setTimeout(() => {
|
|
657
|
-
const index = captureArtifacts.indexOf(path);
|
|
658
|
-
if (index >= 0) captureArtifacts.splice(index, 1);
|
|
659
|
-
void unlink(path).catch(() => {});
|
|
660
|
-
}, CAPTURE_ARTIFACT_TTL_MS);
|
|
661
|
-
expiry.unref?.();
|
|
662
|
-
return text(JSON.stringify({ ...metadata, artifact: { path, width: frame.width, height: frame.height, byteSize: bytes.length } }));
|
|
663
|
-
}
|
|
664
|
-
return {
|
|
665
|
-
content: [
|
|
666
|
-
{ type: "text", text: JSON.stringify({ ...metadata, image: { transport: "inline", width: frame.width, height: frame.height, byteSize: bytes.length } }) },
|
|
667
|
-
{ type: "image", data: frame.data, mimeType: frame.mimeType },
|
|
668
|
-
],
|
|
669
|
-
};
|
|
670
|
-
} catch (error) {
|
|
671
|
-
return liveError(error);
|
|
672
|
-
}
|
|
719
|
+
return captureFrameResult(max_inline_bytes);
|
|
673
720
|
},
|
|
674
721
|
),
|
|
675
722
|
|
|
@@ -860,23 +907,29 @@ export const createToolHandlers = ({ projectRootPromise } = {}) => {
|
|
|
860
907
|
title: "Add a character to the cast",
|
|
861
908
|
description:
|
|
862
909
|
"Unlike place_character, add_character adds a new cast member instead of changing an existing one. " +
|
|
863
|
-
"The cast is unbounded — each one gets its own letter (A, B, C…), position and prompt description."
|
|
910
|
+
"The cast is unbounded — each one gets its own letter (A, B, C…), position and prompt description. " +
|
|
911
|
+
'model "proxy-figure" adds a capsule figure: a rigless stand-in for blocking that holds a posture (stand, sit or lie) ' +
|
|
912
|
+
"and refuses motion, takes, IK and posing with TARGET_NOT_READY.",
|
|
864
913
|
inputSchema: {
|
|
865
914
|
subject: z.string().describe('prompt description, e.g. "a courier holding a package"'),
|
|
866
915
|
x: z.number().default(0).describe("floor position x in metres"),
|
|
867
916
|
z: z.number().default(0).describe("floor position z in metres"),
|
|
868
917
|
facing: z.number().default(0).describe("yaw in degrees; 0 faces the default camera"),
|
|
869
918
|
model: z
|
|
870
|
-
.enum(
|
|
919
|
+
.enum(CHARACTER_KIND_IDS)
|
|
920
|
+
.optional()
|
|
921
|
+
.describe('which mannequin to use; "proxy-figure" is a rigless capsule figure for blocking'),
|
|
922
|
+
posture: z
|
|
923
|
+
.enum(POSTURES)
|
|
871
924
|
.optional()
|
|
872
|
-
.describe("
|
|
925
|
+
.describe("capsule figure posture (stand, sit or lie); stand when omitted"),
|
|
873
926
|
},
|
|
874
927
|
},
|
|
875
|
-
async ({ subject: desc, x, z: zPos, facing, model, ...admission }) => {
|
|
928
|
+
async ({ subject: desc, x, z: zPos, facing, model, posture, ...admission }) => {
|
|
876
929
|
if (liveHub?.connected) {
|
|
877
930
|
try {
|
|
878
931
|
return await runStudioCommand({ ...admission, action: "character.add", args: {
|
|
879
|
-
character: { subject: desc, x, z: zPos, rot: facing, model, pose: DEFAULT_POSE },
|
|
932
|
+
character: { subject: desc, x, z: zPos, rot: facing, model, posture, pose: DEFAULT_POSE },
|
|
880
933
|
} });
|
|
881
934
|
} catch (error) {
|
|
882
935
|
return liveError(error);
|
|
@@ -890,7 +943,7 @@ export const createToolHandlers = ({ projectRootPromise } = {}) => {
|
|
|
890
943
|
const taken = new Set(st.characters.map((c) => c.id));
|
|
891
944
|
let id = `char-${String.fromCharCode(97 + index)}`;
|
|
892
945
|
for (let n = index + 1; taken.has(id); n += 1) id = `char-${n}`;
|
|
893
|
-
const entry = createCharacterEntry({ id, subject: desc, x, z: zPos, rot: facing, model }, index);
|
|
946
|
+
const entry = createCharacterEntry({ id, subject: desc, x, z: zPos, rot: facing, model, posture }, index);
|
|
894
947
|
st.characters = [...st.characters, entry];
|
|
895
948
|
return text(`Added ${String.fromCharCode(65 + index)} (${entry.id}).\n\n${sceneReport()}`);
|
|
896
949
|
},
|
|
@@ -1005,7 +1058,7 @@ export const createToolHandlers = ({ projectRootPromise } = {}) => {
|
|
|
1005
1058
|
{
|
|
1006
1059
|
title: "Place an object in the set",
|
|
1007
1060
|
description:
|
|
1008
|
-
`Unlike update_object and
|
|
1061
|
+
`Unlike update_object, import_mesh and sculpt_object, place_object adds a catalog prop instead of changing an existing one, loading a GLB, OBJ or FBX from disk, or building one from a recipe. Available kinds: ${OBJECT_LIBRARY.map((o) => o.kind).join(", ")}. ` +
|
|
1009
1062
|
"It returns the object id, which update_object and remove_object take.",
|
|
1010
1063
|
inputSchema: {
|
|
1011
1064
|
kind: z
|
|
@@ -1105,6 +1158,132 @@ export const createToolHandlers = ({ projectRootPromise } = {}) => {
|
|
|
1105
1158
|
},
|
|
1106
1159
|
),
|
|
1107
1160
|
|
|
1161
|
+
tool(
|
|
1162
|
+
"sculpt_object",
|
|
1163
|
+
{
|
|
1164
|
+
title: "Sculpt a clay prop from a recipe of soft parts",
|
|
1165
|
+
description:
|
|
1166
|
+
"Unlike place_object, import_mesh and compare_sculpt, sculpt_object builds or changes a clay prop from a recipe of parts you write, typically after reading a reference image. It needs no file and no catalogue entry. " +
|
|
1167
|
+
"recipe = { version: 1, parts: [{ id, shape, size: [x, y, z], position?, rotation?, color?, parent?, mirror?, roundness?, taper?, border? }] }. " +
|
|
1168
|
+
"Units are metres and degrees. The floor is y = 0, front is +z, and the subject's own left is +x. size is the full extent; position is the part's centre, in its parent part's frame when parent is set. " +
|
|
1169
|
+
"shape: blob (soft superellipsoid: roundness 0 is an ellipsoid, 1 is a rounded box; taper narrows the top when positive and the bottom when negative), box (roundness bevels the edges), " +
|
|
1170
|
+
"cylinder (taper 1 is a cone), torus (size [outer, outer, tube]), frame (a rectangular ring in x/y with a border width, for glasses rims, windows and picture frames). " +
|
|
1171
|
+
"mirror: true adds a copy reflected across x = 0 of the parent frame, with its children; author the subject's left (+x) side once. color is #rrggbb. At most 48 parts after mirrors. " +
|
|
1172
|
+
"To fix a sculpt, pass its id with edits instead of a whole recipe: [{ set: partId, ...fields }, { add: part }, { remove: partId }] (parent: null and mirror: false clear; remove takes the part's children). " +
|
|
1173
|
+
"capture: true also returns the editor's frame (connected editor only), so one call is one round. " +
|
|
1174
|
+
"Workflow: measure the reference's outline landmarks (limb tips, head top, feet as fractions of height) and place parts on them, compare_sculpt against the reference image, then rank 3-6 candidate edits with compare_sculpt { candidates } and apply the winner. Add small identity details (eyes, rims, plates) last as children of their parent part. " +
|
|
1175
|
+
"A recipe or edit that cannot be drawn is refused with the path of the bad field.",
|
|
1176
|
+
inputSchema: {
|
|
1177
|
+
recipe: z.object({}).passthrough().optional().describe("{ version: 1, parts: [...] }; the whole recipe, for a new sculpt or a full rewrite"),
|
|
1178
|
+
edits: z.array(z.object({}).passthrough()).min(1).max(48).optional().describe("part-level changes to the sculpt named by id"),
|
|
1179
|
+
id: z.string().min(1).optional().describe("an existing sculpt's id to re-sculpt or patch in place (keeps its position and turn)"),
|
|
1180
|
+
name: z.string().min(1).optional().describe("display name, e.g. 'Nerd turtle'"),
|
|
1181
|
+
x: z.number().optional().describe("floor x in metres"),
|
|
1182
|
+
z: z.number().optional().describe("floor z in metres"),
|
|
1183
|
+
y: z.number().optional().describe("height above the floor in metres"),
|
|
1184
|
+
facing: z.number().optional().describe("yaw in degrees"),
|
|
1185
|
+
parent: z.string().optional().describe("object id to attach the new sculpt to"),
|
|
1186
|
+
capture: z.boolean().optional().describe("true returns capture_frame's image with the result (connected editor only)"),
|
|
1187
|
+
},
|
|
1188
|
+
},
|
|
1189
|
+
async ({ recipe, edits, id, name, x, z: zPos, y, facing, parent, capture, ...admission }) => {
|
|
1190
|
+
if ((recipe === undefined) === (edits === undefined)) return { content: [{ type: "text", text: "Give exactly one of recipe (whole) or edits (part by part)." }], isError: true };
|
|
1191
|
+
if (edits !== undefined && id === undefined) return { content: [{ type: "text", text: "edits patch an existing sculpt: give its id." }], isError: true };
|
|
1192
|
+
const placement = {};
|
|
1193
|
+
if (x !== undefined) placement.x = x;
|
|
1194
|
+
if (zPos !== undefined) placement.z = zPos;
|
|
1195
|
+
if (y !== undefined) placement.y = y;
|
|
1196
|
+
if (facing !== undefined) placement.rot = facing;
|
|
1197
|
+
if (liveHub?.connected) {
|
|
1198
|
+
let result;
|
|
1199
|
+
try {
|
|
1200
|
+
result = await runStudioCommand({ ...admission, action: "object.sculpt", args: { ...(recipe === undefined ? { edits } : { recipe }), ...(id === undefined ? {} : { id }), ...(name === undefined ? {} : { name }), ...(Object.keys(placement).length ? { placement } : {}), ...(parent === undefined ? {} : { parent }) } });
|
|
1201
|
+
} catch (error) {
|
|
1202
|
+
return liveError(error);
|
|
1203
|
+
}
|
|
1204
|
+
if (capture !== true || result.isError) return result;
|
|
1205
|
+
// The sculpt is applied either way; a failed capture is reported beside it, not as a failure.
|
|
1206
|
+
const frame = await captureFrameResult(1_000_000);
|
|
1207
|
+
return { content: [...result.content, ...frame.content] };
|
|
1208
|
+
}
|
|
1209
|
+
if (capture === true) return liveError(new Error(noLiveEditor("capture: true needs a connected CozyClay editor; call sculpt_object without it, or compare_sculpt.")));
|
|
1210
|
+
const sc = scene();
|
|
1211
|
+
if (parent !== undefined && !sc.objects.some((o) => o.id === parent)) {
|
|
1212
|
+
return text(`No object "${parent}" to attach to. Call describe_scene for the current ids.`);
|
|
1213
|
+
}
|
|
1214
|
+
if (id !== undefined) {
|
|
1215
|
+
const target = sc.objects.find((o) => o.id === id);
|
|
1216
|
+
if (!target) return text(`No object "${id}". Call describe_scene for the current ids.`);
|
|
1217
|
+
if (target.renderer !== SCULPT_KIND) return { content: [{ type: "text", text: `${id} is a ${target.renderer}, not a sculpt; omit id to create a new sculpt.` }], isError: true };
|
|
1218
|
+
const next = edits !== undefined ? patchSculptRecipe(target.recipe, edits) : normalizeSculptRecipe(recipe);
|
|
1219
|
+
if (!next.ok) return { content: [{ type: "text", text: `Recipe refused at ${next.error}` }], isError: true };
|
|
1220
|
+
sc.objects = updateSceneObject(sc.objects, id, { ...placement, ...(name === undefined ? {} : { name }), recipe: next.recipe });
|
|
1221
|
+
const row = sc.objects.find((o) => o.id === id);
|
|
1222
|
+
const verb = edits !== undefined ? `Patched ${next.changed.join(", ")} on` : "Re-sculpted";
|
|
1223
|
+
return text(`${verb} ${row.name} as ${row.id}: ${row.footprint.width} x ${row.footprint.depth} m, ${row.height} m tall.\n\n${sceneReport()}`);
|
|
1224
|
+
}
|
|
1225
|
+
const made = createSculptObject({ recipe, name }, sc.objects, placement);
|
|
1226
|
+
if (made.error) return { content: [{ type: "text", text: `Recipe refused at ${made.error}` }], isError: true };
|
|
1227
|
+
sc.objects = [...sc.objects, made.object];
|
|
1228
|
+
if (parent !== undefined) sc.objects = setSceneObjectParent(sc.objects, made.object.id, parent);
|
|
1229
|
+
const row = sc.objects.find((o) => o.id === made.object.id);
|
|
1230
|
+
return text(`Sculpted ${row.name} as ${row.id}: ${row.footprint.width} x ${row.footprint.depth} m, ${row.height} m tall${parent !== undefined ? ` under ${parent}` : ""}.\n\n${sceneReport()}`);
|
|
1231
|
+
},
|
|
1232
|
+
),
|
|
1233
|
+
|
|
1234
|
+
tool(
|
|
1235
|
+
"compare_sculpt",
|
|
1236
|
+
{
|
|
1237
|
+
title: "Compare a sculpt's silhouette with its reference image",
|
|
1238
|
+
description:
|
|
1239
|
+
"Unlike capture_frame and sculpt_object, compare_sculpt scores a sculpt's shape against a reference image instead of rendering the shot or changing the sculpt. " +
|
|
1240
|
+
"Give the sculpt's id and the path of the reference (PNG or JPEG on this machine; a flat background or transparency). It reads the reference's subject as a mask, draws the sculpt's silhouette from a view (yaw: the camera's azimuth, 0 in front, positive toward the subject's own left; elevation: degrees above), " +
|
|
1241
|
+
"fits it to the subject's height and position, and returns IoU, the width-to-height delta, which height band is missing area (top, middle, bottom) and which parts stick out, plus a sheet: reference | overlay (grey both, red missing, blue extra). " +
|
|
1242
|
+
"Omit yaw to try -45..45 and keep the best. Pass candidates (up to 12 labelled edit sets) to score several fixes in one call without applying any; the answer ranks them against the current sculpt. Shape only: colour is not scored. Works without an editor.",
|
|
1243
|
+
inputSchema: {
|
|
1244
|
+
id: z.string().min(1).describe("the sculpt's object id"),
|
|
1245
|
+
reference: z.string().min(1).describe("filesystem path to the reference PNG or JPEG"),
|
|
1246
|
+
yaw: z.number().min(-180).max(180).optional().describe("reference camera azimuth in degrees; omitted tries -45..45"),
|
|
1247
|
+
elevation: z.number().min(-60).max(60).optional().describe("reference camera elevation in degrees; default 0"),
|
|
1248
|
+
candidates: z.array(z.object({ label: z.string().min(1).max(60), edits: z.array(z.object({}).passthrough()).min(1).max(48) })).min(1).max(12).optional()
|
|
1249
|
+
.describe("alternative edit sets to score side by side without applying any; returns them ranked by IoU with the winner's sheet"),
|
|
1250
|
+
},
|
|
1251
|
+
},
|
|
1252
|
+
async ({ id, reference, yaw, elevation = 0, candidates }) => {
|
|
1253
|
+
const workspaceHandle = liveWorkspace.getStore();
|
|
1254
|
+
if (liveHub?.connected) {
|
|
1255
|
+
try { await refreshLiveDescription(workspaceHandle); } catch (error) { return liveError(error); }
|
|
1256
|
+
}
|
|
1257
|
+
const target = scene().objects.find((o) => o.id === id);
|
|
1258
|
+
if (!target) return text(`No object "${id}". Call describe_scene for the current ids.`);
|
|
1259
|
+
if (target.renderer !== SCULPT_KIND || !target.recipe) return { content: [{ type: "text", text: `${id} is a ${target.renderer}, not a sculpt.` }], isError: true };
|
|
1260
|
+
let image;
|
|
1261
|
+
try { image = await readImageFromPath(reference); } catch (error) { return { content: [{ type: "text", text: error.message }], isError: true }; }
|
|
1262
|
+
let result;
|
|
1263
|
+
try {
|
|
1264
|
+
const scored = scoreSculptAgainstReference(target.recipe, image, { yaw, elevation, candidates });
|
|
1265
|
+
result = scored.result;
|
|
1266
|
+
if (candidates) {
|
|
1267
|
+
const { ranked, refused } = scored;
|
|
1268
|
+
const best = ranked[0];
|
|
1269
|
+
const png = encodePng(best.result.sheet.width, best.result.sheet.height, best.result.sheet.data);
|
|
1270
|
+
const summary = { id, view: best.result.report.view, ranking: ranked.map((row) => rankingRow(row, result)), ...(refused.length ? { refused } : {}),
|
|
1271
|
+
note: `Nothing was applied. Apply the winner with sculpt_object { id: "${id}", edits }.` };
|
|
1272
|
+
return { content: [{ type: "text", text: JSON.stringify(summary) }, { type: "image", data: png.toString("base64"), mimeType: "image/png" }] };
|
|
1273
|
+
}
|
|
1274
|
+
} catch (error) {
|
|
1275
|
+
return { content: [{ type: "text", text: error.message }], isError: true };
|
|
1276
|
+
}
|
|
1277
|
+
const png = encodePng(result.sheet.width, result.sheet.height, result.sheet.data);
|
|
1278
|
+
return {
|
|
1279
|
+
content: [
|
|
1280
|
+
{ type: "text", text: JSON.stringify({ id, ...result.report }) },
|
|
1281
|
+
{ type: "image", data: png.toString("base64"), mimeType: "image/png" },
|
|
1282
|
+
],
|
|
1283
|
+
};
|
|
1284
|
+
},
|
|
1285
|
+
),
|
|
1286
|
+
|
|
1108
1287
|
tool(
|
|
1109
1288
|
"group_objects",
|
|
1110
1289
|
{
|
|
@@ -1169,7 +1348,7 @@ export const createToolHandlers = ({ projectRootPromise } = {}) => {
|
|
|
1169
1348
|
},
|
|
1170
1349
|
async ({ beats, ...admission }) => {
|
|
1171
1350
|
if (!liveHub?.connected) {
|
|
1172
|
-
return text("Prompt Blocks live on the studio timeline — open the editor and try again.");
|
|
1351
|
+
return previsRefusal("set_prompt_blocks") ?? text("Prompt Blocks live on the studio timeline — open the editor and try again.");
|
|
1173
1352
|
}
|
|
1174
1353
|
const normalized = normalizePhases(beats.map((b) => b.text));
|
|
1175
1354
|
// The timeline runs on a 24 fps production clock.
|
|
@@ -1186,7 +1365,9 @@ export const createToolHandlers = ({ projectRootPromise } = {}) => {
|
|
|
1186
1365
|
}
|
|
1187
1366
|
}
|
|
1188
1367
|
try {
|
|
1189
|
-
|
|
1368
|
+
const inspected = await inspectAction("character.setPromptBlocks");
|
|
1369
|
+
return previsRefusal("set_prompt_blocks", { context: await contextOf(inspected) }) ??
|
|
1370
|
+
await runStudioCommand({ ...admission, inspected, action: "character.setPromptBlocks", args: context => ({
|
|
1190
1371
|
characterId: context.activeCharacterId, blocks,
|
|
1191
1372
|
}) });
|
|
1192
1373
|
} catch (error) {
|
|
@@ -1213,8 +1394,12 @@ export const createToolHandlers = ({ projectRootPromise } = {}) => {
|
|
|
1213
1394
|
if (!motionUrlPattern.test(args.url)) {
|
|
1214
1395
|
throw new Error(`Unsupported motion url "${args.url}". Use /ardy/motions/<id> or /ardy/assembled/<name>.npz.`);
|
|
1215
1396
|
}
|
|
1216
|
-
|
|
1217
|
-
|
|
1397
|
+
const inspected = liveHub?.connected ? await inspectAction('motion.replace') : undefined;
|
|
1398
|
+
const characterId = args.character === undefined ? inspected?.context?.activeCharacterId : await liveCharacterId(args.character);
|
|
1399
|
+
const refused = previsRefusal("load_motion", { feature: "take", context: await contextOf(inspected), characterId });
|
|
1400
|
+
if (refused) return refused;
|
|
1401
|
+
return runStudioCommand({ ...args, inspected, action: 'motion.replace', args: context => ({
|
|
1402
|
+
characterId: args.character === undefined ? context.activeCharacterId : characterId,
|
|
1218
1403
|
url: args.url, prompt: args.prompt ?? '',
|
|
1219
1404
|
}) });
|
|
1220
1405
|
},
|
|
@@ -1274,10 +1459,13 @@ export const createToolHandlers = ({ projectRootPromise } = {}) => {
|
|
|
1274
1459
|
},
|
|
1275
1460
|
},
|
|
1276
1461
|
async ({ phases, seconds = 9, seed, motion_url, drop, ...admission }) => {
|
|
1277
|
-
if (!liveHub?.connected) return text(noLiveEditor("Motion requires a connected CozyClay editor."));
|
|
1462
|
+
if (!liveHub?.connected) return previsRefusal("generate_motion", { feature: "motion" }) ?? text(noLiveEditor("Motion requires a connected CozyClay editor."));
|
|
1278
1463
|
const normalized = normalizePhases(phases.map(phase => typeof phase === 'string' ? phase : phase.text));
|
|
1279
1464
|
const beats = normalized.texts.map((text, index) => ({ text, seconds: phases[normalized.sources[index]]?.seconds ?? seconds / phases.length })).filter(beat => beat.text);
|
|
1280
|
-
|
|
1465
|
+
const action = motion_url ? 'motion.replace' : 'motion.generate', inspected = await inspectAction(action);
|
|
1466
|
+
const refused = previsRefusal("generate_motion", { feature: "motion", context: await contextOf(inspected) });
|
|
1467
|
+
if (refused) return refused;
|
|
1468
|
+
return runStudioCommand({ ...admission, inspected, action, args: context => {
|
|
1281
1469
|
const generated = generationArgs({ characterId: context.activeCharacterId, source: { kind: 'generate', beats, ...(seed === undefined ? {} : { seed }) } });
|
|
1282
1470
|
return motion_url ? { characterId: generated.characterId, url: motion_url, prompt: beats.map(beat => beat.text).join(' '),
|
|
1283
1471
|
blocks: generated.blocks.map(({ startFrame, endFrame, text: prompt }) => ({ startFrame, endFrame, prompt })), ...(drop ? { drop } : {}) }
|
|
@@ -1292,7 +1480,7 @@ export const createToolHandlers = ({ projectRootPromise } = {}) => {
|
|
|
1292
1480
|
title: "Move, rotate or scale an object",
|
|
1293
1481
|
description:
|
|
1294
1482
|
"Unlike place_object, update_object changes an existing prop instead of adding one. " +
|
|
1295
|
-
"Every field is optional; omitted fields are left alone. Transforms go through the same clamp/snap path the studio's gizmo uses. hidden shows or hides the prop without deleting it.",
|
|
1483
|
+
"Every field is optional; omitted fields are left alone. Transforms go through the same clamp/snap path the studio's gizmo uses. hidden shows or hides the prop without deleting it. opacity makes it see-through (glass, windows, water).",
|
|
1296
1484
|
inputSchema: {
|
|
1297
1485
|
id: z.string().describe("object id from place_object or describe_scene"),
|
|
1298
1486
|
x: z.number().optional(),
|
|
@@ -1321,6 +1509,14 @@ export const createToolHandlers = ({ projectRootPromise } = {}) => {
|
|
|
1321
1509
|
face_travel: z.boolean().optional().describe("turn to face the direction of travel (default true)"),
|
|
1322
1510
|
loop: z.boolean().optional(),
|
|
1323
1511
|
extend: z.boolean().optional().describe("keep going in the final direction after the route ends"),
|
|
1512
|
+
marks: z
|
|
1513
|
+
.array(z.union([
|
|
1514
|
+
z.number().min(0).max(1),
|
|
1515
|
+
z.object({ t: z.number().min(0).max(1), bank: z.number().min(-90).max(90).optional(), pitch: z.number().min(-90).max(90).optional() }),
|
|
1516
|
+
]))
|
|
1517
|
+
.max(6)
|
|
1518
|
+
.optional()
|
|
1519
|
+
.describe("dots along the route: an arc fraction 0-1, or { t, bank, pitch } (degrees, +-90) to lean there; level again at the neighbouring marks and ends"),
|
|
1324
1520
|
})
|
|
1325
1521
|
.nullable()
|
|
1326
1522
|
.optional()
|
|
@@ -1328,20 +1524,21 @@ export const createToolHandlers = ({ projectRootPromise } = {}) => {
|
|
|
1328
1524
|
height: z.number().positive().optional().describe("cutout or mesh height in metres"),
|
|
1329
1525
|
clay: z.boolean().optional().describe("mesh only: replace file materials with matte clay"),
|
|
1330
1526
|
hidden: z.boolean().optional().describe("true hides the prop without deleting it"),
|
|
1527
|
+
opacity: z.number().min(0.05).max(1).optional().describe("1 is solid; lower is see-through, e.g. 0.3 for clear glass"),
|
|
1331
1528
|
},
|
|
1332
1529
|
},
|
|
1333
|
-
async ({ id, x, y, z: zPos, facing, tilt, roll, scale, scale_x, scale_y, scale_z, color, name, path, height, clay, hidden, ...admission }) => {
|
|
1530
|
+
async ({ id, x, y, z: zPos, facing, tilt, roll, scale, scale_x, scale_y, scale_z, color, name, path, height, clay, hidden, opacity, ...admission }) => {
|
|
1334
1531
|
const travelPath = path === null
|
|
1335
1532
|
? null
|
|
1336
1533
|
: path
|
|
1337
|
-
? { points: path.points.map((point) => ({ x: point.x, y: point.y ?? 0, z: point.z })), speed: path.speed ?? 0, faceTravel: path.face_travel !== false, loop: path.loop === true, extend: path.extend === true }
|
|
1534
|
+
? { points: path.points.map((point) => ({ x: point.x, y: point.y ?? 0, z: point.z })), speed: path.speed ?? 0, faceTravel: path.face_travel !== false, loop: path.loop === true, extend: path.extend === true, ...(path.marks ? { marks: path.marks } : {}) }
|
|
1338
1535
|
: undefined;
|
|
1339
1536
|
if (liveHub?.connected) {
|
|
1340
1537
|
try {
|
|
1341
1538
|
return await runStudioCommand({ ...admission, action: "object.update", args: { id, patch: {
|
|
1342
1539
|
x, y, z: zPos, rot: facing, rotX: tilt, rotZ: roll,
|
|
1343
1540
|
scaleX: scale_x ?? scale, scaleY: scale_y ?? scale, scaleZ: scale_z ?? scale,
|
|
1344
|
-
color, name, path: travelPath, height, clay, hidden,
|
|
1541
|
+
color, name, path: travelPath, height, clay, hidden, opacity,
|
|
1345
1542
|
} } });
|
|
1346
1543
|
} catch (error) {
|
|
1347
1544
|
return liveError(error);
|
|
@@ -1371,6 +1568,7 @@ export const createToolHandlers = ({ projectRootPromise } = {}) => {
|
|
|
1371
1568
|
if (height !== undefined) patch.height = height;
|
|
1372
1569
|
if (clay !== undefined) patch.clay = clay;
|
|
1373
1570
|
if (hidden !== undefined) patch.hidden = hidden;
|
|
1571
|
+
if (opacity !== undefined) patch.opacity = opacity;
|
|
1374
1572
|
sc.objects = updateSceneObject(sc.objects, id, patch);
|
|
1375
1573
|
return text(`Updated ${id}.\n\n${sceneReport()}`);
|
|
1376
1574
|
},
|
|
@@ -1522,6 +1720,9 @@ export const createToolHandlers = ({ projectRootPromise } = {}) => {
|
|
|
1522
1720
|
} catch (error) {
|
|
1523
1721
|
return liveError(error);
|
|
1524
1722
|
}
|
|
1723
|
+
// describe carries the editor's previsMode into state.
|
|
1724
|
+
const refused = previsRefusal("mark_camera_move");
|
|
1725
|
+
if (refused) return refused;
|
|
1525
1726
|
state.markedFraming = framing();
|
|
1526
1727
|
return text(`Marked A position: ${slateLine(currentShot())}\n\nNow move the camera, then call describe_camera_move.`);
|
|
1527
1728
|
},
|
|
@@ -1676,6 +1877,7 @@ export const createToolHandlers = ({ projectRootPromise } = {}) => {
|
|
|
1676
1877
|
}
|
|
1677
1878
|
}
|
|
1678
1879
|
state.doc = nextDocument;
|
|
1880
|
+
state.previsMode = result.project.previsMode ?? DEFAULT_PREVIS_MODE;
|
|
1679
1881
|
state.name = result.project.name;
|
|
1680
1882
|
state.focus = null;
|
|
1681
1883
|
state.focusLocked = false;
|