cozyclay 2.0.2 → 2.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +56 -0
- package/README.md +17 -2
- package/THIRD_PARTY_NOTICES.md +8 -0
- package/bin/agent/agent-routes.mjs +211 -36
- package/bin/agent/agent-runner.mjs +98 -4
- package/bin/agent/codex-client.mjs +2 -1
- package/bin/agent/fal-contract.mjs +0 -17
- package/bin/agent/h3-preservation.mjs +6 -8
- package/bin/agent/provider-keys.mjs +1 -1
- package/bin/agent/providers.mjs +124 -3
- package/bin/agent/script-tool.mjs +74 -0
- package/bin/agent/session-store.mjs +5 -5
- package/bin/agent/studio-prompt.mjs +2 -1
- package/bin/agent/studio-rebase.mjs +314 -0
- package/bin/agent/studio-tools.mjs +21 -14
- package/bin/agent/supervisor.mjs +311 -0
- package/bin/agent/video-adapters.mjs +0 -7
- package/bin/codex-auth.mjs +4 -4
- package/bin/live/cli.mjs +2 -3
- package/bin/live/client.mjs +1 -1
- package/bin/live/studio.mjs +5 -2
- package/bin/mcp-runtime.mjs +2 -2
- package/bin/telemetry-state.mjs +3 -3
- package/dist/app/index.html +6 -6
- package/dist/assets/analytics-B41FmPYI.js +1 -0
- package/dist/assets/app-Do-Og5Pu.css +1 -0
- package/dist/assets/app-DwP-tl2x.js +4920 -0
- package/dist/assets/{demo-ByeYp-dY.js → demo-Dt0qZfS7.js} +1 -1
- package/dist/assets/{first-shot-handoff-DgT28ZiV.js → first-shot-handoff-BlT33CGf.js} +1 -1
- package/dist/assets/{landing-BaXkpFip.js → landing-BSesrCpL.js} +1 -1
- package/dist/assets/scene-asset-sync-BCWDJB1d.js +19 -0
- package/dist/assets/{shot-prompt-CbqWcWYi.css → scene-asset-sync-DjJ_cJ2R.css} +1 -1
- package/dist/assets/{ticket-BcmsnHn5.js → ticket-jqMhqL88.js} +1 -1
- package/dist/assets/workflow-Br-K7Srs.js +179 -0
- package/dist/cozyclay-package.json +1 -1
- package/dist/index.html +328 -222
- package/dist/sitemap.xml +1 -1
- package/dist/workflow/index.html +4 -4
- package/mcp/LIVE-PROTOCOL.md +23 -2
- package/mcp/README.md +37 -2
- package/mcp/ardy-prompts.mjs +1 -1
- package/mcp/image-file.mjs +132 -0
- package/mcp/live-hub.mjs +2 -2
- package/mcp/package.json +1 -0
- package/mcp/runtime/package-lock.json +7 -0
- package/mcp/runtime/package.json +1 -0
- package/mcp/server.mjs +0 -6
- package/mcp/tool-handlers.mjs +289 -87
- package/package.json +11 -2
- package/src/App.jsx +789 -263
- package/src/agent-image-client.js +94 -0
- package/src/analytics.js +47 -3
- package/src/app-stage.jsx +129 -2745
- package/src/ardy/auto-physics.js +6 -16
- package/src/ardy/collision-blockers.js +3 -7
- package/src/ardy/fix-collisions.js +5 -35
- package/src/ardy/ik-drag.js +86 -5
- package/src/ardy/ik-key-json.js +3 -1
- package/src/ardy/ik.js +15 -13
- package/src/ardy/motion-calibration.js +1 -1
- package/src/ardy/npz.js +0 -8
- package/src/ardy/physics-panel.jsx +3 -2
- package/src/ardy/physics-review.js +3 -2
- package/src/ardy/physics-support.js +1 -1
- package/src/ardy/physics-surface.js +1 -1
- package/src/ardy/platform-fit-panel.jsx +2 -1
- package/src/ardy/playback.js +1 -5
- package/src/ardy/range-pin.js +2 -2
- package/src/ardy/root-drop.js +0 -1
- package/src/ardy/timeline.css +184 -6
- package/src/ardy/timeline.jsx +113 -28
- package/src/ardy/to-cskel27.js +0 -2
- package/src/ardy/waypoints.js +0 -3
- package/src/ardy-bridge-contract.js +281 -0
- package/src/asset-pane.jsx +19 -9
- package/src/auto-color.js +1 -1
- package/src/burn-in.js +1 -1
- package/src/camera-block.js +1 -1
- package/src/camera-follow.js +75 -32
- package/src/camera-move.js +0 -6
- package/src/camera-tutorial.jsx +6 -7
- package/src/character-kind.js +79 -0
- package/src/command-bus.js +76 -17
- package/src/commands/cast.js +31 -7
- package/src/commands/elements/object.js +1 -1
- package/src/commands/elements/shot.js +1 -1
- package/src/commands/elements.js +1 -1
- package/src/commands/export.js +3 -3
- package/src/commands/motion.js +2 -2
- package/src/commands/objects.js +105 -10
- package/src/commands/project.js +15 -2
- package/src/commands/sculpt-images.js +53 -0
- package/src/commands/shot.js +88 -10
- package/src/commands/view.js +10 -4
- package/src/coplanar-depth.js +216 -0
- package/src/cuts.js +7 -6
- package/src/domains/cast.js +60 -43
- package/src/domains/motion/motion-intent-codec.js +50 -0
- package/src/domains/motion/motion-range-pin-rules.js +23 -0
- package/src/domains/motion/motion-video-capture.js +664 -0
- package/src/domains/motion.js +253 -742
- package/src/domains/objects.js +106 -5
- package/src/domains/scenes.js +79 -28
- package/src/domains/shots.js +33 -7
- package/src/error-boundary.jsx +1 -1
- package/src/execution-telemetry.js +5 -5
- package/src/facing-marks.js +14 -1
- package/src/first-success-guide.jsx +35 -4
- package/src/gizmo-claim.js +3 -1
- package/src/grid-floor.jsx +1 -4
- package/src/hierarchy-model.js +48 -35
- package/src/hierarchy-panel.css +1 -0
- package/src/hierarchy-panel.jsx +54 -102
- package/src/history.js +0 -2
- package/src/i2v-motion-client.js +1 -4
- package/src/ime.js +6 -0
- package/src/keyframe-pack.js +15 -6
- package/src/line-edit.js +4 -4
- package/src/live-control.js +6 -6
- package/src/matte-editor.js +2 -8
- package/src/matte.js +0 -19
- package/src/mesh-graph-clone.js +16 -2
- package/src/mesh-sniff.js +0 -1
- package/src/monotone-cubic.js +40 -0
- package/src/motion-store.js +4 -4
- package/src/motion-trail.js +5 -11
- package/src/multimodel-ingest.js +1 -3
- package/src/object-catalog.jsx +7 -5
- package/src/object-gizmo.jsx +45 -31
- package/src/object-path.js +310 -67
- package/src/object-travel.js +224 -0
- package/src/otio.js +8 -3
- package/src/panels/CharacterTransformPanel.jsx +24 -6
- package/src/panels/ObjectTransformPanel.jsx +4 -3
- package/src/panels/PoseImagePanel.jsx +205 -0
- package/src/panels/PromptBlocksPanel.jsx +3 -1
- package/src/panels/ProxyFigurePanel.jsx +56 -0
- package/src/panels/ReferenceImageField.jsx +1 -1
- package/src/panels/RigControlPanel.jsx +24 -3
- package/src/panels/RigPanel.jsx +2 -1
- package/src/panels/SculptPanel.jsx +405 -0
- package/src/panels/SubjectBox.jsx +1 -1
- package/src/panels/SubjectsPanel.jsx +0 -2
- package/src/panels/VideoCapturePanel.jsx +2 -1
- package/src/panels/pose-image.css +42 -0
- package/src/panels/pose.css +3 -0
- package/src/panels/sculpt.css +162 -0
- package/src/plan-diagram.js +184 -0
- package/src/planview.jsx +127 -80
- package/src/playground.js +3 -5
- package/src/pose-fade.js +77 -0
- package/src/pose-gizmo.css +23 -0
- package/src/pose-gizmo.js +96 -0
- package/src/posestudio.jsx +134 -83
- package/src/project-browser.css +53 -0
- package/src/project-browser.jsx +49 -4
- package/src/project.js +55 -24
- package/src/props.jsx +177 -36
- package/src/proxy-figure.jsx +69 -0
- package/src/range-pin-panel.jsx +0 -1
- package/src/receipt-findings.js +20 -0
- package/src/render-passes.js +0 -1
- package/src/resource-status.jsx +1 -1
- package/src/root-path.js +48 -0
- package/src/route-owner.js +52 -0
- package/src/scene-asset-cache.js +1 -2
- package/src/scene-assets.js +12 -4
- package/src/scene-fbx.js +1 -1
- package/src/scene-mesh-cache.js +0 -1
- package/src/scene-objects.js +319 -19
- package/src/scenes.js +58 -9
- package/src/sculpt-blueprint.js +65 -0
- package/src/sculpt-compare.js +384 -0
- package/src/sculpt-geometry.js +141 -0
- package/src/sculpt-mode.js +168 -0
- package/src/sculpt-recipe.js +385 -0
- package/src/sculpt-run.js +351 -0
- package/src/sculpt-stage.jsx +221 -0
- package/src/settings-menu.jsx +1 -1
- package/src/shell/BoardSlot.jsx +341 -0
- package/src/shell/BottomDock.jsx +58 -14
- package/src/shell/DetailsSlot.jsx +26 -6
- package/src/shell/MenuBar.jsx +78 -6
- package/src/shell/OutlinerSlot.jsx +3 -1
- package/src/shell/TopBar.jsx +17 -3
- package/src/shell/ViewportToolbar.jsx +18 -3
- package/src/shell/dock.css +239 -0
- package/src/shell/glass-regions.css +3 -2
- package/src/shell/log-store.js +4 -4
- package/src/shell/mode.css +20 -0
- package/src/shell/studio-shell-context.js +4 -1
- package/src/shot-authoring.js +69 -6
- package/src/stable-items.js +1 -1
- package/src/stage-attach-frames.js +169 -0
- package/src/stage-canvas-rigs.jsx +402 -0
- package/src/stage-capture.jsx +216 -0
- package/src/stage-characters.jsx +302 -0
- package/src/stage-editor-gizmos.jsx +1229 -0
- package/src/stage-labels-ko.js +204 -0
- package/src/stage-startup-defaults.js +182 -0
- package/src/starter-scenes.js +1 -1
- package/src/storyboard.js +24 -5
- package/src/studio-actions.js +42 -9
- package/src/studio-agent-commands.js +186 -23
- package/src/studio-agent-context.js +11 -9
- package/src/studio-agent-motion.js +2 -1
- package/src/studio-agent-protocol.js +56 -23
- package/src/studio-app-binding.js +52 -23
- package/src/studio-contact-sheet.js +3 -3
- package/src/studio-elements.js +11 -2
- package/src/studio-geometry-facts.js +114 -0
- package/src/styles.css +92 -9
- package/src/ui-scrub.js +4 -4
- package/src/ui.jsx +6 -4
- package/src/use-asset-shelf.js +60 -0
- package/src/use-camera-tutorial.js +50 -0
- package/src/use-embed-bridge.js +12 -0
- package/src/use-first-success-prompt.js +7 -0
- package/src/use-i2v-capture-modal.js +15 -0
- package/src/use-sculpt-mode.js +95 -0
- package/src/workflow/AgentPanel.jsx +106 -18
- package/src/workflow/WorkflowBuilder.jsx +1 -1
- package/src/workflow/agent-client.js +113 -45
- package/src/workflow/agent-panel.css +35 -0
- package/src/workflow/attachment-image.js +1 -1
- package/src/workflow/canvas-commands.js +3 -3
- package/src/workflow/cozy-scene-node.js +3 -3
- package/src/workflow/local-workflow.js +2 -2
- package/src/workflow/motion-input.js +3 -3
- package/src/workflow/node-schema.js +1 -1
- package/src/workflow/scene-asset-sync.js +2 -9
- package/src/workflow/shot-prompt-node.js +1 -1
- package/src/workflow/video-contract.js +1 -1
- package/src/workflow/workflow-resources.js +0 -8
- package/src/workflow-send.js +87 -0
- package/tools/ardy/__pycache__/cclay_gvhmr_worker.cpython-313.pyc +0 -0
- package/tools/check-eslint.mjs +57 -0
- package/tools/check-knip.mjs +60 -0
- package/tools/check-loc.mjs +54 -0
- package/tools/demo-worker/api-client.mjs +7 -13
- package/tools/demo-worker/generate.mjs +2 -4
- package/tools/eslint-baseline.json +23 -0
- package/tools/kimodo/effector-constraints.mjs +4 -4
- package/tools/kimodo/generate.mjs +1 -1
- package/tools/knip-baseline.json +331 -0
- package/tools/loc-baseline.json +126 -0
- package/tools/process-supervisor.mjs +1 -1
- package/tools/projflow/replay.mjs +4 -4
- package/tools/projflow/runner.mjs +3 -3
- package/tools/qa/studio-control-count.mjs +2 -0
- package/tools/run-tests.mjs +94 -7
- package/dist/assets/analytics-MVZOEW8O.js +0 -1
- package/dist/assets/app-BxBLC6GT.js +0 -4876
- package/dist/assets/app-CJdxvnrl.css +0 -1
- package/dist/assets/shot-prompt-hN0j_O-p.js +0 -17
- package/dist/assets/workflow-B5XZwVfe.js +0 -179
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,61 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## Unreleased
|
|
4
|
+
|
|
5
|
+
## 2.1.0 - 2026-10-12
|
|
6
|
+
|
|
7
|
+
Storyboard and Animation modes. A project is now either a storyboard (a row of
|
|
8
|
+
still panels) or an animation (moving shots); both are on in a normal launch.
|
|
9
|
+
|
|
10
|
+
Modes
|
|
11
|
+
|
|
12
|
+
- The start screen has a Storyboard / Animation control, the project keeps its
|
|
13
|
+
mode through save and reopen, and the top bar shows it as a badge. A
|
|
14
|
+
storyboard project hides the Generate group and Motion mode (`4`); motion
|
|
15
|
+
commands are refused with a reason instead of failing quietly. (#649, #650)
|
|
16
|
+
- The bottom dock has a **Board** tab in storyboard projects: one card per
|
|
17
|
+
panel with a greybox thumbnail, caption, hold, duplicate, delete and
|
|
18
|
+
drag-to-reorder. Each panel can keep its own cast placement. (#640, #641)
|
|
19
|
+
- **Stylize** turns a panel into a finished picture through the image route
|
|
20
|
+
and keeps it in the project beside the greybox; **Send to Workflow** puts
|
|
21
|
+
the stylized image or the greybox on a Workflow canvas with a connected
|
|
22
|
+
Video node. (#643, #645)
|
|
23
|
+
|
|
24
|
+
Capsule figures
|
|
25
|
+
|
|
26
|
+
- A rigless **Capsule figure** joins the Assets tile list for blocking: stand,
|
|
27
|
+
sit or lie, set in Details. It follows root paths in the viewport and in
|
|
28
|
+
exports. Rig-only tools (Pose, IK, motion, take import, physics) refuse it
|
|
29
|
+
and say why. (#634, #635)
|
|
30
|
+
|
|
31
|
+
Storyboard exports
|
|
32
|
+
|
|
33
|
+
- The Export menu of a storyboard lists **Contact sheet (PNG)**, **Panel pack
|
|
34
|
+
(zip)**, **All panels (zip)**, **Animatic (mp4)**, **Cut list (OTIO)** and
|
|
35
|
+
**Depth + normal (PNG)**. File ▸ **Export as Animation project…** turns the
|
|
36
|
+
panels into an Animation project. Keyframe pack and shot video stay
|
|
37
|
+
Animation-only.
|
|
38
|
+
|
|
39
|
+
Agent and MCP
|
|
40
|
+
|
|
41
|
+
- In a storyboard project the Studio Agent adds one panel per request, frames
|
|
42
|
+
it and never generates motion; the Studio context and `describe_scene` now
|
|
43
|
+
report the project mode and each character's kind. (#637, #642)
|
|
44
|
+
- MCP motion tools answer `NOT_IN_MODE` in a storyboard and
|
|
45
|
+
`TARGET_NOT_READY` on a capsule figure; `add_character` takes a model and
|
|
46
|
+
posture. (#652)
|
|
47
|
+
|
|
48
|
+
Analytics and docs
|
|
49
|
+
|
|
50
|
+
- Three counts-only events: `storyboard:panel_created` (`source`),
|
|
51
|
+
`storyboard:panel_stylized` (`outcome`) and `cast:proxy_added`
|
|
52
|
+
(`surface`); `scene:created` carries `previs_mode`, and the export
|
|
53
|
+
`export_kind` vocabulary gains `contact_sheet`, `panel_pack`, `animatic`
|
|
54
|
+
and `animation_project`. Captions, names and prompts are never properties.
|
|
55
|
+
Disclosed in `docs/privacy.md` and `docs/analytics-queries.md`. (#653)
|
|
56
|
+
- README has a **Modes** section; the Agent rules, the Studio IA and the
|
|
57
|
+
capsule-figure notes are updated. (#653)
|
|
58
|
+
|
|
3
59
|
## 2.0.2
|
|
4
60
|
|
|
5
61
|
The first-run audit release: `npx cozyclay` is exercised end to end as a new
|
package/README.md
CHANGED
|
@@ -52,7 +52,7 @@ https://github.com/user-attachments/assets/1d0113e5-6922-443d-affc-1bdabc666247
|
|
|
52
52
|
|
|
53
53
|
| | |
|
|
54
54
|
| --- | --- |
|
|
55
|
-
| **Stage a scene** | Create primitives and set pieces, then move, rotate and scale them with the transform strip's gizmo. Grid snapping is a preference, not a law — hold `Ctrl` mid-drag to invert it. A bird's-eye Top-View drives 2D root waypoints for character paths. **View ▾** on the viewport bar holds the reference grid and Auto Color — Blender's random viewport color, so twenty grey blockout boxes stay tellable apart without touching the colors you authored (captures include the display colors while it is on). |
|
|
55
|
+
| **Stage a scene** | Create primitives and set pieces, then move, rotate and scale them with the transform strip's gizmo. Cast people from **Assets › Characters**: a rigged character you can pose and animate, or a **Capsule figure** — a rigless stand-in with one posture (stand, sit or lie) chosen in Details, for blocking where nobody needs to move their limbs. Capsule figures follow root paths and per-panel placements like any character but refuse pose, IK and motion tools, with the reason in the tooltip. Grid snapping is a preference, not a law — hold `Ctrl` mid-drag to invert it. A bird's-eye Top-View drives 2D root waypoints for character paths. **View ▾** on the viewport bar holds the reference grid and Auto Color — Blender's random viewport color, so twenty grey blockout boxes stay tellable apart without touching the colors you authored (captures include the display colors while it is on). |
|
|
56
56
|
| **Fly the camera** | Right-drag flies (WASD walks, Q/E cranes), middle-drag pans, Alt+drag orbits the selection, click selects, `F` frames — the muscle memory you already have from a 3D editor. Fly, pan and orbit lock the pointer for the hold, so the view can turn past the window edge. Selecting the camera switches to Camera mode. **Look through** in the Shot monitor puts you behind the shot camera with the same bindings; click the on-screen **Shot camera** indicator or press `Esc` to return to the free camera. |
|
|
57
57
|
| **Cut and move the camera** | Add shots on the timeline, draw a dolly rail on the Top-View, set speed, height and crane, and preview the move through the shot camera. Each shot carries a **Target model** (Seedance 2.5, Kling 2, Veo 3, self-hosted MiniMax-H3) and is flagged when the cut runs past that model's limits. |
|
|
58
58
|
| **Export for AI video** | One **Export ▾** menu: a keyframe pack (first/last frame, clip, camera JSON, prompt, README) as a zip, an mp4 of the shot, depth + normal conditioning passes, a storyboard contact sheet, and an OTIO cut list. Seedance 2.5 reads the pack's greybox clip as its white-model reference video, and MiniMax H3, Wan 3.0, LTX Desktop and fal render-to-real accept the same clip. The **Shot Prompt** turns the framing into a structured prompt for the model you picked. |
|
|
@@ -61,6 +61,21 @@ https://github.com/user-attachments/assets/1d0113e5-6922-443d-affc-1bdabc666247
|
|
|
61
61
|
| **Capture motion from video or a photo** | Drop a clip or a still: the GPU box runs [GVHMR](https://github.com/zju3dv/GVHMR) and the result is retargeted onto the character with stabilisation, contact correction and a quality gate. |
|
|
62
62
|
| **Direct it with an AI** | Open the **Agent panel** (`Cmd/Ctrl+B`), sign in with your ChatGPT account, and ask for a shot in plain language; the conversation survives reloads and you can paste a reference screenshot into it. Or connect Claude — or any MCP client — or drive the Studio from a terminal with `cclay live`. All three place the cast, frame "a low wide profile", generate multi-phase motion, and the viewport moves in front of you. See [AI control](#ai-control). |
|
|
63
63
|
|
|
64
|
+
## Modes
|
|
65
|
+
|
|
66
|
+
A project is either a **Storyboard** or an **Animation**. Pick it on the start screen (the badge in the top bar shows which one you are in); a project keeps its mode when saved and reopened.
|
|
67
|
+
|
|
68
|
+
| | Storyboard | Animation |
|
|
69
|
+
| --- | --- | --- |
|
|
70
|
+
| **Choose it when** | You want a row of still panels: framing, who stands where, a caption and a hold per panel | You want moving shots: motion on a timeline, camera moves, takes |
|
|
71
|
+
| **Timeline** | The **Board** dock tab: one card per panel with a greybox thumbnail, caption, hold, duplicate, delete and drag-to-reorder | Shots on the timeline, with motion and Prompt Blocks |
|
|
72
|
+
| **Cast** | Rigged characters or capsule figures; each panel can keep its own placement ("Use stage placement" clears it) | Rigged characters with Pose and Motion mode; capsule figures for blocking |
|
|
73
|
+
| **Motion tools** | Hidden: no Generate group, no Motion mode (`4`); the agent has no `generate_motion` and the MCP motion tools answer `NOT_IN_MODE` | Available |
|
|
74
|
+
| **Agent** | **Agent · Storyboard** builds one panel per request and never generates motion | The full Studio tool set |
|
|
75
|
+
| **Exports** | **Stylize** a panel through the image route, **Send to Workflow** (the stylized image or the greybox) from its card, plus the Export ▾ menu: **Contact sheet (PNG)**, **Panel pack (zip)**, **All panels (zip)**, **Animatic (mp4)**, **Cut list (OTIO)** and **Depth + normal (PNG)**; File ▸ **Export as Animation project…** turns the panels into an Animation. There is no keyframe pack or shot video in a storyboard | Export ▾: keyframe pack, mp4, depth + normal passes, contact sheet, OTIO cut list |
|
|
76
|
+
|
|
77
|
+
Start in Storyboard when you are deciding what the shots are, then open the idea as an Animation when it needs motion.
|
|
78
|
+
|
|
64
79
|
## Quick start
|
|
65
80
|
|
|
66
81
|
You need Node.js 22.19 or newer, npm or bun, and a Chromium-based browser.
|
|
@@ -182,7 +197,7 @@ Every control's home is recorded in [`docs/studio-ui-ia.md`](docs/studio-ui-ia.m
|
|
|
182
197
|
| --- | --- |
|
|
183
198
|
| [`docs/kimodo-setup.md`](docs/kimodo-setup.md) | Installing Kimodo: the route table, CUDA over SSH, local MLX/Metal/CPU generation and what each route supports |
|
|
184
199
|
| [`docs/agent-cli.md`](docs/agent-cli.md) | Driving the Studio from a terminal with `cclay live` |
|
|
185
|
-
| [`docs/agent-prompt-to-scene.md`](docs/agent-prompt-to-scene.md) | The ask / infer / omit rule the Agent follows when turning a video prompt into blocking |
|
|
200
|
+
| [`docs/agent-prompt-to-scene.md`](docs/agent-prompt-to-scene.md) | The ask / infer / omit rule the Agent follows when turning a video prompt into blocking, and the one-panel-per-request rule in storyboards |
|
|
186
201
|
| [`mcp/README.md`](mcp/README.md) | MCP tools, transports, and the live-control protocol |
|
|
187
202
|
| [`docs/studio-ui-ia.md`](docs/studio-ui-ia.md) | Where every Studio control lives and why |
|
|
188
203
|
| [`docs/privacy.md`](docs/privacy.md) | The full analytics disclosure: events, identifiers, opt-out, internal QA marking |
|
package/THIRD_PARTY_NOTICES.md
CHANGED
|
@@ -165,6 +165,14 @@ Their transitive runtime dependencies, which the agent sidecar also loads, are:
|
|
|
165
165
|
- `ignore` — License: MIT — https://github.com/kaelzhang/node-ignore
|
|
166
166
|
- `yaml` — License: ISC — https://github.com/eemeli/yaml
|
|
167
167
|
|
|
168
|
+
## jpeg-js
|
|
169
|
+
|
|
170
|
+
The MCP runtime decodes JPEG reference images for `compare_sculpt` with `jpeg-js`, on the local machine.
|
|
171
|
+
|
|
172
|
+
- Copyright (c) 2014, Eugene Ware and Adobe
|
|
173
|
+
- License: BSD-3-Clause
|
|
174
|
+
- Source: https://github.com/jpeg-js/jpeg-js
|
|
175
|
+
|
|
168
176
|
## CozyClay license scope
|
|
169
177
|
|
|
170
178
|
The CozyClay combined work in this repository is distributed under AGPL-3.0-or-later, subject to the transition details in `LICENSING.md`. That license does not replace or relicense Three.js, ARDY, ARDY model checkpoints, the bundled fonts, the character rigs, or any other third-party component.
|
|
@@ -10,6 +10,7 @@ import { createAgentTools, SYSTEM_PROMPT, pickWorkspace } from "./agent-tools.mj
|
|
|
10
10
|
import { createVideoAdapters, validateFalVideoRequest } from "./video-adapters.mjs";
|
|
11
11
|
import { createSessionStore, transcriptFromHistory } from "./session-store.mjs";
|
|
12
12
|
import { createAgentRunner } from "./agent-runner.mjs";
|
|
13
|
+
import { STUDIO_SYSTEM_PROMPT_STORYBOARD } from "./studio-prompt.mjs";
|
|
13
14
|
|
|
14
15
|
// Values the codex backend accepts for reasoning.effort (its own 400 lists them).
|
|
15
16
|
export const REASONING_EFFORTS = ["none", "minimal", "low", "medium", "high", "xhigh", "max", "ultra"];
|
|
@@ -19,7 +20,7 @@ const json = (res, status, value) => {
|
|
|
19
20
|
res.end(JSON.stringify(value));
|
|
20
21
|
};
|
|
21
22
|
|
|
22
|
-
|
|
23
|
+
function allowAgentOrigin(req, port) {
|
|
23
24
|
return [`http://127.0.0.1:${port}`, `http://${"local" + "host"}:${port}`].includes(req.headers.origin)
|
|
24
25
|
|| (req.headers.origin === undefined && req.method === "GET"
|
|
25
26
|
&& [`127.0.0.1:${port}`, `localhost:${port}`].includes(req.headers.host));
|
|
@@ -27,6 +28,8 @@ export function allowAgentOrigin(req, port) {
|
|
|
27
28
|
|
|
28
29
|
// Two 1920x1080 PNG data URLs (frame + reference) fit comfortably in this.
|
|
29
30
|
const IMAGE_BODY_LIMIT = 24 * 1024 * 1024;
|
|
31
|
+
// The sizes gpt-image-2 accepts; omitted means the backend's own choice.
|
|
32
|
+
const IMAGE_SIZES = ["auto", "1024x1024", "1536x1024", "1024x1536"];
|
|
30
33
|
|
|
31
34
|
// Attached scene references (#167): identity sheets and the environment
|
|
32
35
|
// reference. Capped because every one of them is another full image the
|
|
@@ -41,6 +44,10 @@ const ATTACHMENTS_MAX = 4;
|
|
|
41
44
|
const ATTACHMENT_MAX_CHARS = 6_000_000;
|
|
42
45
|
const TURN_BODY_LIMIT = ATTACHMENTS_MAX * ATTACHMENT_MAX_CHARS + 64 * 1024;
|
|
43
46
|
|
|
47
|
+
// How long GET /agent/turn/<id>/supervisor holds a request for a review that
|
|
48
|
+
// is still running before it answers 204 (#728).
|
|
49
|
+
const SUPERVISOR_WAIT_MS = 180_000;
|
|
50
|
+
|
|
44
51
|
/** Reject anything that is not a short list of inline {dataUrl, name?} images.
|
|
45
52
|
* The Studio envelope validates its own copy; this is the legacy body's. */
|
|
46
53
|
function validAttachments(attachments) {
|
|
@@ -60,7 +67,6 @@ const agentToolCategory = (name) => {
|
|
|
60
67
|
// Studio families are authored edits or reads; the analytics vocabulary has
|
|
61
68
|
// no Studio-specific category, so a write says so and everything else does not.
|
|
62
69
|
if (["arrange_objects", "arrange_characters", "patch_elements", "frame_shot", "undo_edit"].includes(name)) return "scene_write";
|
|
63
|
-
if (["inspect_studio", "operate_studio", "verify_result", "generate_motion"].includes(name)) return "other";
|
|
64
70
|
if (name === "run_workflow") return "workflow_run";
|
|
65
71
|
if (name === "describe_workflow" || name === "focus_workflow_node") return "workflow_read";
|
|
66
72
|
if (["add_workflow_node", "update_workflow_node", "remove_workflow_node", "connect_workflow_nodes", "disconnect_workflow_nodes", "set_workflow_node_output"].includes(name)) return "workflow_write";
|
|
@@ -75,6 +81,55 @@ const agentFailureCode = (error, signal, tool = false) => advisory(() => {
|
|
|
75
81
|
if (error?.status === 429) return "rate_limited";
|
|
76
82
|
return tool ? "tool_failed" : "upstream";
|
|
77
83
|
}, tool ? "tool_failed" : "upstream");
|
|
84
|
+
/**
|
|
85
|
+
* One user message is one editor transaction: every Studio turn whose editor
|
|
86
|
+
* reports a previsMode opens agent.turn.begin and closes agent.turn.finish, so
|
|
87
|
+
* all of its mutations land as ONE history entry and one receipt frame. A
|
|
88
|
+
* context without a previsMode comes from an editor that predates turn
|
|
89
|
+
* transactions and runs unwrapped. rollback() cancels the open transaction
|
|
90
|
+
* (restoring the pre-turn document) and reopens it for the rest of the turn.
|
|
91
|
+
*/
|
|
92
|
+
const agentTurn = ({ hub, context, turnId, admission }) => {
|
|
93
|
+
const wrapped = Boolean(context.scene?.previsMode);
|
|
94
|
+
const host = Object.fromEntries(["workspaceId", "documentEpoch", "sceneId", "sceneEpoch"].map(key => [key, context.host[key]]));
|
|
95
|
+
const control = async action => {
|
|
96
|
+
const receipt = await hub.command("run_action", {
|
|
97
|
+
name: "run_action", args: { action, args: { turnId } }, turnId,
|
|
98
|
+
commandId: randomUUID(), host, expectedRevision: admission.revision,
|
|
99
|
+
}, context.host.workspaceHandle);
|
|
100
|
+
if (!receipt.ok) throw Object.assign(new Error(receipt.message), { code: receipt.code });
|
|
101
|
+
return receipt;
|
|
102
|
+
};
|
|
103
|
+
const open = async () => { await control("agent.turn.begin"); admission.turnId = turnId; };
|
|
104
|
+
return {
|
|
105
|
+
async rollback() {
|
|
106
|
+
if (admission.turnId !== turnId) return null;
|
|
107
|
+
delete admission.turnId;
|
|
108
|
+
const receipt = await control("agent.turn.cancel");
|
|
109
|
+
await admission.refresh();
|
|
110
|
+
await open();
|
|
111
|
+
return receipt;
|
|
112
|
+
},
|
|
113
|
+
run: start => async function* (input) {
|
|
114
|
+
if (!wrapped) { yield* start(input); return; }
|
|
115
|
+
await open();
|
|
116
|
+
let terminal;
|
|
117
|
+
try {
|
|
118
|
+
for await (const frame of start(input)) {
|
|
119
|
+
if (frame.type === "done") terminal = frame; else yield frame;
|
|
120
|
+
}
|
|
121
|
+
} finally {
|
|
122
|
+
// A rollback whose reopen failed has already closed the transaction.
|
|
123
|
+
if (admission.turnId === turnId) {
|
|
124
|
+
delete admission.turnId;
|
|
125
|
+
const receipt = await control("agent.turn.finish");
|
|
126
|
+
if (receipt.authored) yield { type: "receipt", receipt };
|
|
127
|
+
}
|
|
128
|
+
}
|
|
129
|
+
if (terminal) yield terminal;
|
|
130
|
+
},
|
|
131
|
+
};
|
|
132
|
+
};
|
|
78
133
|
// These existing canvas commands only return a node/edge after publishing a
|
|
79
134
|
// new insertion. Read/focus, generic accepted responses, update no-ops and run
|
|
80
135
|
// outputs (which can echo old values) are deliberately not application proof.
|
|
@@ -134,6 +189,8 @@ function validReferences(references) {
|
|
|
134
189
|
* owns the geometry, each character sheet owns one performer's look and the
|
|
135
190
|
* environment reference owns the location.
|
|
136
191
|
*/
|
|
192
|
+
export const FRAME_COMPOSITION_GUIDANCE = "Use the first image (a clay blocking frame) as the layout: keep its composition and camera angle, and keep the pose but make it look natural. Render it as:";
|
|
193
|
+
|
|
137
194
|
export function referenceGuidance(references = []) {
|
|
138
195
|
const list = Array.isArray(references) ? references : [];
|
|
139
196
|
if (!list.length) return "";
|
|
@@ -143,6 +200,8 @@ export function referenceGuidance(references = []) {
|
|
|
143
200
|
lines.push(`Character ${entry.name || "reference"}: match the identity, face, hair and wardrobe from the attached character sheet.`);
|
|
144
201
|
} else if (entry.role === "environment") {
|
|
145
202
|
lines.push("Environment: take the location look, materials, palette and lighting from the attached environment reference.");
|
|
203
|
+
} else if (entry.role === "reference") {
|
|
204
|
+
lines.push(`Reference ${entry.name || "image"}: use it for the look the prompt asks for (subject, wardrobe, style); keep the exact pose, body position and camera framing of the first image.`);
|
|
146
205
|
}
|
|
147
206
|
}
|
|
148
207
|
return `\n${lines.join("\n")}`;
|
|
@@ -174,7 +233,7 @@ function quotaEvent(codex, headers) {
|
|
|
174
233
|
// one line — without it neither the log nor the panel can say why a turn died.
|
|
175
234
|
const UPSTREAM_DETAIL_MAX = 200;
|
|
176
235
|
export function sanitizeUpstreamDetail(body) {
|
|
177
|
-
let message
|
|
236
|
+
let message;
|
|
178
237
|
try {
|
|
179
238
|
const parsed = JSON.parse(body);
|
|
180
239
|
const candidate = parsed?.error?.message ?? parsed?.error ?? parsed?.message ?? parsed?.detail;
|
|
@@ -285,7 +344,10 @@ function liveToolsRuntime(mcpRuntime) {
|
|
|
285
344
|
});
|
|
286
345
|
}
|
|
287
346
|
|
|
288
|
-
|
|
347
|
+
// `supervisor` (#728) defaults off only under an injected scripted provider:
|
|
348
|
+
// a test's script is an exact queue of replies, and a review would take one
|
|
349
|
+
// meant for a later turn. The real sidecar never injects one.
|
|
350
|
+
export function createAgentHandler({ auth = defaultAuth, codex, models, codexBaseUrl, cliproxyBaseUrl, env, fauxProvider, handlers, liveHub, mcpRuntime, port, studioRuntime, clock = Date.now, setIntervalImpl = setInterval, clearIntervalImpl = clearInterval, sessionStore: injectedSessionStore, supervisor = !fauxProvider } = {}) {
|
|
289
351
|
const requestContext = new AsyncLocalStorage();
|
|
290
352
|
codex ||= defaultClient(auth, requestContext);
|
|
291
353
|
const runtime = handlers !== undefined || liveHub !== undefined ? Promise.resolve({ handlers: handlers ?? [], liveHub }) : liveToolsRuntime(mcpRuntime);
|
|
@@ -347,6 +409,9 @@ export function createAgentHandler({ auth = defaultAuth, codex, models, codexBas
|
|
|
347
409
|
return owner;
|
|
348
410
|
};
|
|
349
411
|
const emitStudioEvent = (turnId, event) => {
|
|
412
|
+
// The supervisor note lands after the turn ended (#728): it joins the turn's
|
|
413
|
+
// record, terminal or not, but never resurrects one that was retired.
|
|
414
|
+
if (event.type === "supervisor" && !studioEvents.has(turnId)) return;
|
|
350
415
|
const record = studioEvents.get(turnId) || { next: 0, events: [], listeners: new Set(), terminal: false };
|
|
351
416
|
// Execution telemetry is advisory and is validated key-by-key in the
|
|
352
417
|
// browser: it carries no replay cursor, is never retained for a resume,
|
|
@@ -377,10 +442,10 @@ export function createAgentHandler({ auth = defaultAuth, codex, models, codexBas
|
|
|
377
442
|
});
|
|
378
443
|
|
|
379
444
|
const studioIdentity = host => Object.fromEntries(["workspaceId", "documentEpoch", "sceneId", "sceneEpoch"].map(key => [key, host[key]]));
|
|
380
|
-
const writeStudioStream = (res, record, after = 0
|
|
445
|
+
const writeStudioStream = (res, record, after = 0) => {
|
|
381
446
|
res.writeHead(200, { "content-type": "text/event-stream; charset=utf-8", "cache-control": "no-cache", connection: "keep-alive" }); res.flushHeaders?.();
|
|
382
447
|
let cursor = after; const send = event => {
|
|
383
|
-
if (res.destroyed) return;
|
|
448
|
+
if (res.destroyed || res.writableEnded) return;
|
|
384
449
|
if (event.eventSeq === undefined) { res.write(`data: ${JSON.stringify(event)}\n\n`); return; }
|
|
385
450
|
if (event.eventSeq > cursor) { cursor = event.eventSeq; res.write(`data: ${JSON.stringify(event)}\n\n`); }
|
|
386
451
|
};
|
|
@@ -401,8 +466,8 @@ export function createAgentHandler({ auth = defaultAuth, codex, models, codexBas
|
|
|
401
466
|
};
|
|
402
467
|
const handleStudioTurn = async (req, res, value, path) => {
|
|
403
468
|
const { StudioProtocolError, validateStudioContextFreshness } = await import("../../src/studio-agent-protocol.js");
|
|
404
|
-
const [{ createStudioTools }, { encodeStudioContext }] = await Promise.all([
|
|
405
|
-
import("./studio-tools.mjs"), import("../../src/studio-agent-context.js"),
|
|
469
|
+
const [{ createStudioTools }, { encodeStudioContext }, { createScriptTool }, { runSupervisor, shouldSupervise }] = await Promise.all([
|
|
470
|
+
import("./studio-tools.mjs"), import("../../src/studio-agent-context.js"), import("./script-tool.mjs"), import("./supervisor.mjs"),
|
|
406
471
|
]);
|
|
407
472
|
const hubDeps = await runtime; const hub = hubDeps.liveHub || liveHub;
|
|
408
473
|
if (path === "/agent/stop") {
|
|
@@ -436,43 +501,65 @@ export function createAgentHandler({ auth = defaultAuth, codex, models, codexBas
|
|
|
436
501
|
let session = studioSessions.get(value.sessionId);
|
|
437
502
|
const suppliedOwner = parseCookies(req).studio_owner;
|
|
438
503
|
if (session && suppliedOwner && session.owner !== suppliedOwner) throw new StudioProtocolError("AUTH_REQUIRED", "Studio session owner mismatch.");
|
|
504
|
+
const existing = session?.turns.get(value.turnId);
|
|
505
|
+
if (existing) { session.updatedAt = clock(); writeStudioStream(res, existing, 0); return true; }
|
|
506
|
+
let current;
|
|
507
|
+
try { current = studioRuntime?.readContext ? await studioRuntime.readContext(value.context.host) : await authoritativeStudioContext(value, hub); }
|
|
508
|
+
catch (error) { if (error instanceof StudioProtocolError) throw error; if (error?.code) throw new StudioProtocolError(error.code, error.message); throw new StudioProtocolError("CAPABILITY_MISSING", "The connected editor does not expose authoritative Studio context."); }
|
|
509
|
+
validateStudioContextFreshness(value.context, current);
|
|
510
|
+
// The session, and the owner token only this turn's response can hand
|
|
511
|
+
// the browser, exist only once the turn is admitted. Minting them before
|
|
512
|
+
// a refusal (the editor still reconnecting after a restart) left a
|
|
513
|
+
// session whose owner the browser never received: its cookie from the
|
|
514
|
+
// previous server then failed every retry with "owner mismatch".
|
|
439
515
|
if (!session) {
|
|
440
516
|
studioOwner(req, value.sessionId, true);
|
|
441
517
|
let persisted = null;
|
|
442
|
-
try { persisted = sessionStore.read(value.sessionId); } catch {
|
|
518
|
+
try { persisted = sessionStore.read(value.sessionId); } catch { /* an unreadable history starts the session fresh */ }
|
|
443
519
|
session = { owner: studioOwnerTokens.get(value.sessionId), history: persisted?.history ?? [], persistedItems: persisted?.history?.length ?? 0, meta: persisted?.meta ?? null, turns: new Map(), controller: null, activeJobId: null, activeJobTurnId: null, motionJobIds: new Set(Array.isArray(persisted?.meta?.motionJobIds) ? persisted.meta.motionJobIds.filter(id => typeof id === "string") : []), host: null, updatedAt: clock() };
|
|
444
520
|
studioSessions.set(value.sessionId, session);
|
|
445
521
|
}
|
|
446
522
|
session.updatedAt = clock();
|
|
447
|
-
const existing = session.turns.get(value.turnId);
|
|
448
|
-
if (existing) { writeStudioStream(res, existing, 0, req); return true; }
|
|
449
|
-
let current;
|
|
450
|
-
try { current = studioRuntime?.readContext ? await studioRuntime.readContext(value.context.host) : await authoritativeStudioContext(value, hub); }
|
|
451
|
-
catch (error) { if (error instanceof StudioProtocolError) throw error; if (error?.code) throw new StudioProtocolError(error.code, error.message); throw new StudioProtocolError("CAPABILITY_MISSING", "The connected editor does not expose authoritative Studio context."); }
|
|
452
|
-
validateStudioContextFreshness(value.context, current);
|
|
453
523
|
if (studioRuntime?.handleTurn) { await studioRuntime.handleTurn(value, req, res); return true; }
|
|
454
524
|
const record = { next: 0, events: [], listeners: new Set(), terminal: false }; studioEvents.set(value.turnId, record); session.turns.set(value.turnId, record); session.host = studioIdentity(value.context.host);
|
|
455
525
|
res.setHeader("set-cookie", `studio_owner=${encodeURIComponent(session.owner)}; Path=/agent; HttpOnly; SameSite=Strict`);
|
|
456
|
-
const close = writeStudioStream(res, record, 0
|
|
457
|
-
|
|
526
|
+
const close = writeStudioStream(res, record, 0);
|
|
527
|
+
// What the supervisor reviews once the turn is over: every frame but text
|
|
528
|
+
// and telemetry, and the last stretch of text the model wrote.
|
|
529
|
+
const turnFrames = [];
|
|
530
|
+
let replySegment = "", lastReply = "";
|
|
531
|
+
const send = event => {
|
|
532
|
+
if (event.type === "text.delta") { replySegment += event.text ?? ""; lastReply = replySegment; }
|
|
533
|
+
else if (!["execution_telemetry", "execution_tool_started"].includes(event.type)) {
|
|
534
|
+
if (event.type === "tool.start") replySegment = "";
|
|
535
|
+
turnFrames.push(event);
|
|
536
|
+
}
|
|
537
|
+
emitStudioEvent(value.turnId, event);
|
|
538
|
+
};
|
|
458
539
|
// The Studio turn is measured exactly like the Workflow turn, correlated by
|
|
459
540
|
// the turn id the host already owns (the frozen envelope carries no room
|
|
460
541
|
// for a second one).
|
|
461
542
|
const telemetry = createTurnTelemetry(send, value.turnId);
|
|
462
543
|
let turnOutcome = "succeeded", turnFailureCode = null, toolFailed = false;
|
|
463
|
-
const controller = new AbortController(); session.controller = controller;
|
|
544
|
+
const controller = new AbortController(); session.controller = controller; session.activeTurnId = value.turnId;
|
|
464
545
|
// Admission is the document identity plus the exact scene revision each
|
|
465
546
|
// command expects. Per-entity tokens are deliberately absent: the revision
|
|
466
547
|
// already moves whenever any authored content changes, and a turn's second
|
|
467
548
|
// edit to the same entity would otherwise carry a token the first edit retired.
|
|
468
549
|
const admission = {
|
|
469
550
|
commandId: () => randomUUID(), host: studioIdentity(value.context.host), revision: value.context.revision.scene,
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
551
|
+
// Read at the admitted host: a scene action may have opened another
|
|
552
|
+
// scene of this workspace during the turn (studio-tools adopts it).
|
|
553
|
+
snapshot: async () => {
|
|
473
554
|
const host = { ...value.context.host, ...admission.host };
|
|
474
|
-
|
|
555
|
+
return studioRuntime?.readContext ? await studioRuntime.readContext(host) : await authoritativeStudioContext({ ...value, context: { ...value.context, host } }, hub);
|
|
556
|
+
},
|
|
557
|
+
// Re-admits at the live revision and hands back the context it read, which
|
|
558
|
+
// the tools compare against what the agent last saw (studio-rebase.mjs).
|
|
559
|
+
refresh: async () => {
|
|
560
|
+
const refreshed = await admission.snapshot();
|
|
475
561
|
admission.revision = refreshed.revision.scene;
|
|
562
|
+
return refreshed;
|
|
476
563
|
},
|
|
477
564
|
};
|
|
478
565
|
// One motion generation per user message, whichever path starts it:
|
|
@@ -504,7 +591,7 @@ export function createAgentHandler({ auth = defaultAuth, codex, models, codexBas
|
|
|
504
591
|
if (session.activeJobId === jobId) { session.activeJobId = null; session.activeJobTurnId = null; }
|
|
505
592
|
}
|
|
506
593
|
};
|
|
507
|
-
const tools = createStudioTools({ liveHub: hub, workspaceHandle: value.context.host.workspaceHandle, session: { signal: controller.signal, admission, generation, onJob, actionIndex: current?.actionIndex ?? [] }, resolveImage: async (id, correlation) => hub.command("resolve_studio_image", { imageId: id, ...correlation }, value.context.host.workspaceHandle) });
|
|
594
|
+
const tools = createStudioTools({ liveHub: hub, workspaceHandle: value.context.host.workspaceHandle, previsMode: value.context.scene?.previsMode, session: { signal: controller.signal, admission, generation, onJob, actionIndex: current?.actionIndex ?? [], baseline: value.context }, resolveImage: async (id, correlation) => hub.command("resolve_studio_image", { imageId: id, ...correlation }, value.context.host.workspaceHandle) });
|
|
508
595
|
const motion = async args => {
|
|
509
596
|
// Private artifact IDs belonged to the retired sidecar runtime. Reuse
|
|
510
597
|
// retained editor takes through motion.loadVersion, not another installer.
|
|
@@ -525,12 +612,35 @@ export function createAgentHandler({ auth = defaultAuth, codex, models, codexBas
|
|
|
525
612
|
return { ...result, visualStatus: visual.visualStatus, imageId: visual.imageId, revision: visual.revision, receiptId: visual.receiptId, ...(visual.dataUrl ? { dataUrl: visual.dataUrl } : {}) };
|
|
526
613
|
},
|
|
527
614
|
}));
|
|
615
|
+
const turnTransaction = agentTurn({ hub, context: current, turnId: value.turnId, admission });
|
|
616
|
+
const script = createScriptTool({ tools: modelTools, signal: controller.signal, emit: send });
|
|
617
|
+
const scriptTool = {
|
|
618
|
+
...script,
|
|
619
|
+
// A script that fails after it edited the scene rolls the turn back, so
|
|
620
|
+
// the model retries from the pre-turn document, never from half a script.
|
|
621
|
+
handler: async (args, options) => {
|
|
622
|
+
const result = await script.handler(args, options);
|
|
623
|
+
if (result.ok || !result.calls.some(call => call.ok && Number.isSafeInteger(call.revision?.after) && call.revision.after !== call.revision.before)) return result;
|
|
624
|
+
const rollback = await turnTransaction.rollback();
|
|
625
|
+
return rollback ? { ...result, rollback: { status: "rolled_back", receiptId: rollback.receiptId, revision: rollback.revision, affectedIds: rollback.affectedIds.slice(0, 8) } } : result;
|
|
626
|
+
},
|
|
627
|
+
};
|
|
528
628
|
let frameObservation;
|
|
529
629
|
if (value.attachFrame) {
|
|
530
630
|
const captured = await hub.command("capture_framing_png", {}, value.context.host.workspaceHandle);
|
|
531
631
|
if (!captured?.dataUrl?.startsWith("data:image/")) throw new StudioProtocolError("TARGET_NOT_READY", "The current frame has no image bytes.");
|
|
532
632
|
const match = /^data:([^;]+);base64,(.*)$/.exec(captured.dataUrl);
|
|
533
633
|
frameObservation = { data: match[2], mimeType: match[1], revision: captured.revision, receiptId: captured.receiptId };
|
|
634
|
+
} else if (value.context.scene?.characterCount >= 2 || (value.context.shots?.length ?? 0) >= 1 || /샷|컷|카메라|커버리지|OTS|투샷|프레임|shot|camera|coverage|two-shot|framing/i.test(value.text || "")) {
|
|
635
|
+
// A staging or coverage request also sees the top-view plan; without it the turn just runs.
|
|
636
|
+
try {
|
|
637
|
+
const captured = await hub.command("capture_plan_png", {}, value.context.host.workspaceHandle);
|
|
638
|
+
const match = /^data:([^;]+);base64,(.*)$/.exec(captured?.dataUrl ?? "");
|
|
639
|
+
if (!match) throw new Error("The plan capture has no image bytes.");
|
|
640
|
+
frameObservation = { kind: "plan", data: match[2], mimeType: match[1], revision: captured.revision ?? value.context.revision?.scene ?? null, receiptId: captured.receiptId };
|
|
641
|
+
} catch (error) {
|
|
642
|
+
console.warn(`[agent] plan observation skipped: ${error.message}`);
|
|
643
|
+
}
|
|
534
644
|
}
|
|
535
645
|
const telemetryTools = new Map();
|
|
536
646
|
const emitFrame = frame => {
|
|
@@ -545,6 +655,9 @@ export function createAgentHandler({ auth = defaultAuth, codex, models, codexBas
|
|
|
545
655
|
toolFailed = true;
|
|
546
656
|
if (typeof frame.error === "string" && frame.error.startsWith("Validation failed for tool ")) frame.error = "INVALID_ARGUMENT: Unexpected field.";
|
|
547
657
|
}
|
|
658
|
+
// A command the sidecar re-sent once (stale revision, busy gesture) is
|
|
659
|
+
// marked on the frame itself; the receipt carries autoRebased and a warning.
|
|
660
|
+
if (frame.ok && frame.result?.autoRebased === true) frame.autoRebased = true;
|
|
548
661
|
send(frame);
|
|
549
662
|
if (record) {
|
|
550
663
|
record.execution.executed(frame.ok ? "succeeded" : (controller.signal.aborted ? "cancelled" : "failed"));
|
|
@@ -560,10 +673,15 @@ export function createAgentHandler({ auth = defaultAuth, codex, models, codexBas
|
|
|
560
673
|
if (frame.type === "done") {
|
|
561
674
|
if (turnOutcome === "succeeded" && !toolFailed) telemetry.finished("succeeded", null);
|
|
562
675
|
else telemetry.finished(turnOutcome, turnFailureCode || "tool_failed");
|
|
676
|
+
// Marked before the done frame leaves, so a client that asks for the
|
|
677
|
+
// review the moment its stream ends finds it pending, not absent.
|
|
678
|
+
// The author's own switch (value.supervisor === false) skips this turn's review.
|
|
679
|
+
if (supervisor && process.env.COZYCLAY_SUPERVISOR !== "off" && value.supervisor !== false && !controller.signal.aborted && shouldSupervise(turnFrames)) record.supervisor = "pending";
|
|
563
680
|
}
|
|
564
681
|
send(frame);
|
|
565
682
|
};
|
|
566
683
|
const persistenceMeta = { sceneName: value.context?.scene?.name ?? value.context?.sceneName ?? null, firstText: value.text, motionJobIds: [...session.motionJobIds] };
|
|
684
|
+
const modelKey = value.model || (fauxProvider ? `${fauxProvider.provider?.id || fauxProvider.provider || "faux"}/scripted` : "gpt-6-astra");
|
|
567
685
|
let runner = studioRunners.get(value.sessionId);
|
|
568
686
|
if (!runner) {
|
|
569
687
|
runner = createAgentRunner({ models: await ensureWorkflowModels(), fauxProvider, sessionStore, clock, codexBaseUrl, cliproxyBaseUrl, auth, env });
|
|
@@ -571,10 +689,10 @@ export function createAgentHandler({ auth = defaultAuth, codex, models, codexBas
|
|
|
571
689
|
}
|
|
572
690
|
try {
|
|
573
691
|
if (!session.modelSession) session.modelSession = await runner.openSession(value.sessionId, { surface: "studio" });
|
|
574
|
-
for await (const frame of session.modelSession.start({
|
|
575
|
-
surface: "studio", sessionId: value.sessionId, model:
|
|
692
|
+
for await (const frame of turnTransaction.run(input => session.modelSession.start(input))({
|
|
693
|
+
surface: "studio", sessionId: value.sessionId, model: modelKey, effort: value.effort,
|
|
576
694
|
text: value.text, attachments: value.attachments, contextText: encodeStudioContext(value.context), frameObservation,
|
|
577
|
-
context: value.context, tools: modelTools, signal: controller.signal, emit: send,
|
|
695
|
+
context: value.context, tools: [...modelTools, scriptTool], systemPrompt: value.context.scene?.previsMode === "storyboard" ? STUDIO_SYSTEM_PROMPT_STORYBOARD : undefined, signal: controller.signal, emit: send,
|
|
578
696
|
meta: persistenceMeta,
|
|
579
697
|
quotaEvent: headers => codex?.parseQuotaHeaders ? quotaEvent(codex, headers) : { type: "quota", plan: null, primary: { usedPercent: null, windowMinutes: null, resetAt: null }, credits: { has: null } },
|
|
580
698
|
})) emitFrame(frame);
|
|
@@ -587,7 +705,25 @@ export function createAgentHandler({ auth = defaultAuth, codex, models, codexBas
|
|
|
587
705
|
}
|
|
588
706
|
if (controller.signal.aborted) { turnOutcome = "cancelled"; turnFailureCode = "aborted"; }
|
|
589
707
|
if (!record.terminal) emitFrame({ type: "done" });
|
|
590
|
-
close(); if (!res.writableEnded) res.end(); session.controller = null;
|
|
708
|
+
close(); if (!res.writableEnded) res.end(); session.controller = null; session.activeTurnId = null;
|
|
709
|
+
// The review starts only now, after the stream has ended, and only ever
|
|
710
|
+
// appends one note to the record: it neither delays nor steers the turn.
|
|
711
|
+
if (record.supervisor === "pending") void (async () => {
|
|
712
|
+
const startedAt = Date.now();
|
|
713
|
+
let frame;
|
|
714
|
+
try {
|
|
715
|
+
frame = await runSupervisor({ registry: await ensureWorkflowModels(), modelKey, effort: process.env.COZYCLAY_SUPERVISOR_EFFORT || "high", turnId: value.turnId, request: value.text, frames: turnFrames, reply: lastReply,
|
|
716
|
+
hub, workspaceHandle: value.context.host.workspaceHandle, admission, baseline: value.context, actions: current?.actionIndex ?? [] });
|
|
717
|
+
} catch (error) {
|
|
718
|
+
frame = { type: "supervisor", turnId: value.turnId, verdict: "unavailable", model: modelKey, effort: null, elapsedMs: Date.now() - startedAt, summary: "", items: [], issues: [], fixes: [], droppedFixes: [], counts: { blocker: 0, concern: 0, note: 0, fixes: 0 }, reason: "The supervisor could not start." };
|
|
719
|
+
}
|
|
720
|
+
// COZYCLAY_SUPERVISOR_AUTOFIX=blocker (#743) only marks a note whose
|
|
721
|
+
// blockers carry fixes; turns stay the panel's to start, never the sidecar's.
|
|
722
|
+
if (process.env.COZYCLAY_SUPERVISOR_AUTOFIX === "blocker" && frame.fixes?.some(fix => fix.severity === "blocker")) frame = { ...frame, autofix: { eligible: true, severity: "blocker" } };
|
|
723
|
+
record.supervisor = "done";
|
|
724
|
+
emitStudioEvent(value.turnId, frame);
|
|
725
|
+
})();
|
|
726
|
+
return true;
|
|
591
727
|
};
|
|
592
728
|
const handle = async (req, res, path = new URL(req.url, "http://127.0.0.1").pathname) => {
|
|
593
729
|
if (!path.startsWith("/agent/")) return false;
|
|
@@ -601,7 +737,34 @@ export function createAgentHandler({ auth = defaultAuth, codex, models, codexBas
|
|
|
601
737
|
if (!record || !session || parseCookies(req).studio_owner !== session.owner) { json(res, 403, { error: { code: "AUTH_REQUIRED", message: "Studio event stream is not owned by this session." } }); return true; }
|
|
602
738
|
const after = Number(new URL(req.url, "http://127.0.0.1").searchParams.get("after") || 0);
|
|
603
739
|
if (!Number.isSafeInteger(after) || after < 0) { json(res, 400, { error: "invalid cursor" }); return true; }
|
|
604
|
-
writeStudioStream(res, record, after
|
|
740
|
+
writeStudioStream(res, record, after); return true;
|
|
741
|
+
}
|
|
742
|
+
// The supervisor note for one Studio turn (#728): 200 with the note once it
|
|
743
|
+
// exists, held up to SUPERVISOR_WAIT_MS while it is pending, 204 when the
|
|
744
|
+
// turn was never reviewed or the wait ran out.
|
|
745
|
+
if (path.startsWith("/agent/turn/") && path.endsWith("/supervisor") && req.method === "GET") {
|
|
746
|
+
if (!await hasAnyCredential()) { json(res, 401, { error: { code: "AUTH_REQUIRED", message: "Sign in or configure a provider key." } }); return true; }
|
|
747
|
+
const turnId = decodeURIComponent(path.slice("/agent/turn/".length, -"/supervisor".length));
|
|
748
|
+
const session = [...studioSessions.values()].find(candidate => candidate.turns.has(turnId)); const record = studioEvents.get(turnId);
|
|
749
|
+
if (!record || !session) { json(res, 404, { error: "turn not found" }); return true; }
|
|
750
|
+
if (parseCookies(req).studio_owner !== session.owner) { json(res, 403, { error: { code: "AUTH_REQUIRED", message: "Studio supervisor note is not owned by this session." } }); return true; }
|
|
751
|
+
const noContent = () => { res.writeHead(204, { "cache-control": "no-store" }); res.end(); };
|
|
752
|
+
const existing = record.events.findLast(event => event.type === "supervisor");
|
|
753
|
+
if (existing) { json(res, 200, existing); return true; }
|
|
754
|
+
if (record.supervisor !== "pending") { noContent(); return true; }
|
|
755
|
+
await new Promise(resolve => {
|
|
756
|
+
let timer;
|
|
757
|
+
const settle = note => {
|
|
758
|
+
clearTimeout(timer); record.listeners.delete(listener); res.off("close", onClose);
|
|
759
|
+
if (note !== undefined && !res.writableEnded && !res.destroyed) { if (note) json(res, 200, note); else noContent(); }
|
|
760
|
+
resolve();
|
|
761
|
+
};
|
|
762
|
+
const listener = event => { if (event.type === "supervisor") settle(event); };
|
|
763
|
+
const onClose = () => settle(undefined);
|
|
764
|
+
record.listeners.add(listener); res.once("close", onClose);
|
|
765
|
+
timer = setTimeout(() => settle(null), SUPERVISOR_WAIT_MS);
|
|
766
|
+
});
|
|
767
|
+
return true;
|
|
605
768
|
}
|
|
606
769
|
if (path.startsWith("/agent/turn/") && path.endsWith("/steer") && req.method === "POST") {
|
|
607
770
|
const turnId = decodeURIComponent(path.slice("/agent/turn/".length, -"/steer".length));
|
|
@@ -610,12 +773,15 @@ export function createAgentHandler({ auth = defaultAuth, codex, models, codexBas
|
|
|
610
773
|
value = await readBody(req, TURN_BODY_LIMIT);
|
|
611
774
|
if (!value || typeof value.text !== "string" || !value.text.trim() || !validAttachments(value.attachments)) throw new Error("Invalid request.");
|
|
612
775
|
} catch { json(res, 400, { error: "invalid request" }); return true; }
|
|
613
|
-
// Studio
|
|
614
|
-
//
|
|
615
|
-
|
|
616
|
-
const
|
|
776
|
+
// Studio turns are keyed in studioSessions, never in the Workflow
|
|
777
|
+
// `sessions` map, so that lookup alone tells them apart. Only the cookie
|
|
778
|
+
// owner of the Studio session may steer its running turn.
|
|
779
|
+
const studio = [...studioSessions.values()].find(candidate => candidate.turns.has(turnId));
|
|
780
|
+
if (studio && parseCookies(req).studio_owner !== studio.owner) { json(res, 403, { error: { code: "AUTH_REQUIRED", message: "Studio turn is not owned by this session." } }); return true; }
|
|
781
|
+
const session = studio ?? [...sessions.values()].find(candidate => candidate.turnId === turnId);
|
|
617
782
|
if (!session) { json(res, 404, { error: "turn not found" }); return true; }
|
|
618
|
-
|
|
783
|
+
const active = studio ? studio.activeTurnId === turnId && studio.controller && !studio.controller.signal.aborted : session.running;
|
|
784
|
+
if (!active || !session.modelSession) { json(res, 409, { error: { code: "NO_ACTIVE_TURN", message: "This turn already ended." } }); return true; }
|
|
619
785
|
const images = (Array.isArray(value.attachments) ? value.attachments : []).map(attachment => {
|
|
620
786
|
const match = /^data:([^;]+);base64,(.*)$/.exec(attachment?.dataUrl || "");
|
|
621
787
|
return match ? { type: "image", data: match[2], mimeType: match[1] } : null;
|
|
@@ -678,7 +844,7 @@ export function createAgentHandler({ auth = defaultAuth, codex, models, codexBas
|
|
|
678
844
|
let value;
|
|
679
845
|
try {
|
|
680
846
|
value = await readBody(req, IMAGE_BODY_LIMIT);
|
|
681
|
-
if (typeof value.prompt !== "string" || !value.prompt.trim() || typeof value.imageDataUrl !== "string" || !value.imageDataUrl.startsWith("data:image/") || (value.referenceDataUrl !== undefined && (typeof value.referenceDataUrl !== "string" || !value.referenceDataUrl.startsWith("data:image/"))) || !validReferences(value.references) || (value.quality !== undefined && !["auto", "low", "medium", "high"].includes(value.quality))) throw new Error("Invalid request.");
|
|
847
|
+
if (typeof value.prompt !== "string" || !value.prompt.trim() || typeof value.imageDataUrl !== "string" || !value.imageDataUrl.startsWith("data:image/") || (value.referenceDataUrl !== undefined && (typeof value.referenceDataUrl !== "string" || !value.referenceDataUrl.startsWith("data:image/"))) || !validReferences(value.references) || (value.composition !== undefined && !["frame", "sheet"].includes(value.composition)) || (value.quality !== undefined && !["auto", "low", "medium", "high"].includes(value.quality)) || (value.size !== undefined && !IMAGE_SIZES.includes(value.size))) throw new Error("Invalid request.");
|
|
682
848
|
} catch { json(res, 400, { error: "invalid request" }); return true; }
|
|
683
849
|
if (!await auth.getAccessToken()) { json(res, 401, { error: { code: "auth", message: "Sign in with ChatGPT in the Agent panel." } }); return true; }
|
|
684
850
|
try {
|
|
@@ -687,8 +853,17 @@ export function createAgentHandler({ auth = defaultAuth, codex, models, codexBas
|
|
|
687
853
|
// Scene references (#167) are appended after the frame/reference pair,
|
|
688
854
|
// and the prompt says what each attachment is for.
|
|
689
855
|
const references = Array.isArray(value.references) ? value.references : [];
|
|
690
|
-
|
|
691
|
-
|
|
856
|
+
// composition "frame": the attached frame was framed by hand (Pose mode), so the shot
|
|
857
|
+
// camera's description would contradict it; the frame alone owns camera and pose.
|
|
858
|
+
// composition "sheet" (Sculpt blueprint): a model sheet, not a shot, so no scene or
|
|
859
|
+
// camera guidance is added; the prompt is sent as written.
|
|
860
|
+
const prompt = value.composition === "sheet"
|
|
861
|
+
? value.prompt
|
|
862
|
+
: value.composition === "frame"
|
|
863
|
+
? `${FRAME_COMPOSITION_GUIDANCE}\n${value.prompt}${referenceGuidance(references)}`
|
|
864
|
+
: `${value.prompt}${await renderGuidance(value.prompt)}${referenceGuidance(references)}`;
|
|
865
|
+
const { composition: _composition, ...request } = value;
|
|
866
|
+
const result = await codex.editImage({ ...request, prompt, extraImages: references.map((entry) => entry.dataUrl) });
|
|
692
867
|
json(res, 200, { dataUrl: `data:image/png;base64,${result.pngBase64}`, width: result.width, height: result.height });
|
|
693
868
|
} catch (error) { json(res, error.status === 401 ? 401 : 502, { error: errorInfo(error) }); }
|
|
694
869
|
return true;
|