cozyclay 2.0.2 → 2.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (256) hide show
  1. package/CHANGELOG.md +56 -0
  2. package/README.md +17 -2
  3. package/THIRD_PARTY_NOTICES.md +8 -0
  4. package/bin/agent/agent-routes.mjs +211 -36
  5. package/bin/agent/agent-runner.mjs +98 -4
  6. package/bin/agent/codex-client.mjs +2 -1
  7. package/bin/agent/fal-contract.mjs +0 -17
  8. package/bin/agent/h3-preservation.mjs +6 -8
  9. package/bin/agent/provider-keys.mjs +1 -1
  10. package/bin/agent/providers.mjs +124 -3
  11. package/bin/agent/script-tool.mjs +74 -0
  12. package/bin/agent/session-store.mjs +5 -5
  13. package/bin/agent/studio-prompt.mjs +2 -1
  14. package/bin/agent/studio-rebase.mjs +314 -0
  15. package/bin/agent/studio-tools.mjs +21 -14
  16. package/bin/agent/supervisor.mjs +311 -0
  17. package/bin/agent/video-adapters.mjs +0 -7
  18. package/bin/codex-auth.mjs +4 -4
  19. package/bin/live/cli.mjs +2 -3
  20. package/bin/live/client.mjs +1 -1
  21. package/bin/live/studio.mjs +5 -2
  22. package/bin/mcp-runtime.mjs +2 -2
  23. package/bin/telemetry-state.mjs +3 -3
  24. package/dist/app/index.html +6 -6
  25. package/dist/assets/analytics-B41FmPYI.js +1 -0
  26. package/dist/assets/app-Do-Og5Pu.css +1 -0
  27. package/dist/assets/app-DwP-tl2x.js +4920 -0
  28. package/dist/assets/{demo-ByeYp-dY.js → demo-Dt0qZfS7.js} +1 -1
  29. package/dist/assets/{first-shot-handoff-DgT28ZiV.js → first-shot-handoff-BlT33CGf.js} +1 -1
  30. package/dist/assets/{landing-BaXkpFip.js → landing-BSesrCpL.js} +1 -1
  31. package/dist/assets/scene-asset-sync-BCWDJB1d.js +19 -0
  32. package/dist/assets/{shot-prompt-CbqWcWYi.css → scene-asset-sync-DjJ_cJ2R.css} +1 -1
  33. package/dist/assets/{ticket-BcmsnHn5.js → ticket-jqMhqL88.js} +1 -1
  34. package/dist/assets/workflow-Br-K7Srs.js +179 -0
  35. package/dist/cozyclay-package.json +1 -1
  36. package/dist/index.html +328 -222
  37. package/dist/sitemap.xml +1 -1
  38. package/dist/workflow/index.html +4 -4
  39. package/mcp/LIVE-PROTOCOL.md +23 -2
  40. package/mcp/README.md +37 -2
  41. package/mcp/ardy-prompts.mjs +1 -1
  42. package/mcp/image-file.mjs +132 -0
  43. package/mcp/live-hub.mjs +2 -2
  44. package/mcp/package.json +1 -0
  45. package/mcp/runtime/package-lock.json +7 -0
  46. package/mcp/runtime/package.json +1 -0
  47. package/mcp/server.mjs +0 -6
  48. package/mcp/tool-handlers.mjs +289 -87
  49. package/package.json +11 -2
  50. package/src/App.jsx +789 -263
  51. package/src/agent-image-client.js +94 -0
  52. package/src/analytics.js +47 -3
  53. package/src/app-stage.jsx +129 -2745
  54. package/src/ardy/auto-physics.js +6 -16
  55. package/src/ardy/collision-blockers.js +3 -7
  56. package/src/ardy/fix-collisions.js +5 -35
  57. package/src/ardy/ik-drag.js +86 -5
  58. package/src/ardy/ik-key-json.js +3 -1
  59. package/src/ardy/ik.js +15 -13
  60. package/src/ardy/motion-calibration.js +1 -1
  61. package/src/ardy/npz.js +0 -8
  62. package/src/ardy/physics-panel.jsx +3 -2
  63. package/src/ardy/physics-review.js +3 -2
  64. package/src/ardy/physics-support.js +1 -1
  65. package/src/ardy/physics-surface.js +1 -1
  66. package/src/ardy/platform-fit-panel.jsx +2 -1
  67. package/src/ardy/playback.js +1 -5
  68. package/src/ardy/range-pin.js +2 -2
  69. package/src/ardy/root-drop.js +0 -1
  70. package/src/ardy/timeline.css +184 -6
  71. package/src/ardy/timeline.jsx +113 -28
  72. package/src/ardy/to-cskel27.js +0 -2
  73. package/src/ardy/waypoints.js +0 -3
  74. package/src/ardy-bridge-contract.js +281 -0
  75. package/src/asset-pane.jsx +19 -9
  76. package/src/auto-color.js +1 -1
  77. package/src/burn-in.js +1 -1
  78. package/src/camera-block.js +1 -1
  79. package/src/camera-follow.js +75 -32
  80. package/src/camera-move.js +0 -6
  81. package/src/camera-tutorial.jsx +6 -7
  82. package/src/character-kind.js +79 -0
  83. package/src/command-bus.js +76 -17
  84. package/src/commands/cast.js +31 -7
  85. package/src/commands/elements/object.js +1 -1
  86. package/src/commands/elements/shot.js +1 -1
  87. package/src/commands/elements.js +1 -1
  88. package/src/commands/export.js +3 -3
  89. package/src/commands/motion.js +2 -2
  90. package/src/commands/objects.js +105 -10
  91. package/src/commands/project.js +15 -2
  92. package/src/commands/sculpt-images.js +53 -0
  93. package/src/commands/shot.js +88 -10
  94. package/src/commands/view.js +10 -4
  95. package/src/coplanar-depth.js +216 -0
  96. package/src/cuts.js +7 -6
  97. package/src/domains/cast.js +60 -43
  98. package/src/domains/motion/motion-intent-codec.js +50 -0
  99. package/src/domains/motion/motion-range-pin-rules.js +23 -0
  100. package/src/domains/motion/motion-video-capture.js +664 -0
  101. package/src/domains/motion.js +253 -742
  102. package/src/domains/objects.js +106 -5
  103. package/src/domains/scenes.js +79 -28
  104. package/src/domains/shots.js +33 -7
  105. package/src/error-boundary.jsx +1 -1
  106. package/src/execution-telemetry.js +5 -5
  107. package/src/facing-marks.js +14 -1
  108. package/src/first-success-guide.jsx +35 -4
  109. package/src/gizmo-claim.js +3 -1
  110. package/src/grid-floor.jsx +1 -4
  111. package/src/hierarchy-model.js +48 -35
  112. package/src/hierarchy-panel.css +1 -0
  113. package/src/hierarchy-panel.jsx +54 -102
  114. package/src/history.js +0 -2
  115. package/src/i2v-motion-client.js +1 -4
  116. package/src/ime.js +6 -0
  117. package/src/keyframe-pack.js +15 -6
  118. package/src/line-edit.js +4 -4
  119. package/src/live-control.js +6 -6
  120. package/src/matte-editor.js +2 -8
  121. package/src/matte.js +0 -19
  122. package/src/mesh-graph-clone.js +16 -2
  123. package/src/mesh-sniff.js +0 -1
  124. package/src/monotone-cubic.js +40 -0
  125. package/src/motion-store.js +4 -4
  126. package/src/motion-trail.js +5 -11
  127. package/src/multimodel-ingest.js +1 -3
  128. package/src/object-catalog.jsx +7 -5
  129. package/src/object-gizmo.jsx +45 -31
  130. package/src/object-path.js +310 -67
  131. package/src/object-travel.js +224 -0
  132. package/src/otio.js +8 -3
  133. package/src/panels/CharacterTransformPanel.jsx +24 -6
  134. package/src/panels/ObjectTransformPanel.jsx +4 -3
  135. package/src/panels/PoseImagePanel.jsx +205 -0
  136. package/src/panels/PromptBlocksPanel.jsx +3 -1
  137. package/src/panels/ProxyFigurePanel.jsx +56 -0
  138. package/src/panels/ReferenceImageField.jsx +1 -1
  139. package/src/panels/RigControlPanel.jsx +24 -3
  140. package/src/panels/RigPanel.jsx +2 -1
  141. package/src/panels/SculptPanel.jsx +405 -0
  142. package/src/panels/SubjectBox.jsx +1 -1
  143. package/src/panels/SubjectsPanel.jsx +0 -2
  144. package/src/panels/VideoCapturePanel.jsx +2 -1
  145. package/src/panels/pose-image.css +42 -0
  146. package/src/panels/pose.css +3 -0
  147. package/src/panels/sculpt.css +162 -0
  148. package/src/plan-diagram.js +184 -0
  149. package/src/planview.jsx +127 -80
  150. package/src/playground.js +3 -5
  151. package/src/pose-fade.js +77 -0
  152. package/src/pose-gizmo.css +23 -0
  153. package/src/pose-gizmo.js +96 -0
  154. package/src/posestudio.jsx +134 -83
  155. package/src/project-browser.css +53 -0
  156. package/src/project-browser.jsx +49 -4
  157. package/src/project.js +55 -24
  158. package/src/props.jsx +177 -36
  159. package/src/proxy-figure.jsx +69 -0
  160. package/src/range-pin-panel.jsx +0 -1
  161. package/src/receipt-findings.js +20 -0
  162. package/src/render-passes.js +0 -1
  163. package/src/resource-status.jsx +1 -1
  164. package/src/root-path.js +48 -0
  165. package/src/route-owner.js +52 -0
  166. package/src/scene-asset-cache.js +1 -2
  167. package/src/scene-assets.js +12 -4
  168. package/src/scene-fbx.js +1 -1
  169. package/src/scene-mesh-cache.js +0 -1
  170. package/src/scene-objects.js +319 -19
  171. package/src/scenes.js +58 -9
  172. package/src/sculpt-blueprint.js +65 -0
  173. package/src/sculpt-compare.js +384 -0
  174. package/src/sculpt-geometry.js +141 -0
  175. package/src/sculpt-mode.js +168 -0
  176. package/src/sculpt-recipe.js +385 -0
  177. package/src/sculpt-run.js +351 -0
  178. package/src/sculpt-stage.jsx +221 -0
  179. package/src/settings-menu.jsx +1 -1
  180. package/src/shell/BoardSlot.jsx +341 -0
  181. package/src/shell/BottomDock.jsx +58 -14
  182. package/src/shell/DetailsSlot.jsx +26 -6
  183. package/src/shell/MenuBar.jsx +78 -6
  184. package/src/shell/OutlinerSlot.jsx +3 -1
  185. package/src/shell/TopBar.jsx +17 -3
  186. package/src/shell/ViewportToolbar.jsx +18 -3
  187. package/src/shell/dock.css +239 -0
  188. package/src/shell/glass-regions.css +3 -2
  189. package/src/shell/log-store.js +4 -4
  190. package/src/shell/mode.css +20 -0
  191. package/src/shell/studio-shell-context.js +4 -1
  192. package/src/shot-authoring.js +69 -6
  193. package/src/stable-items.js +1 -1
  194. package/src/stage-attach-frames.js +169 -0
  195. package/src/stage-canvas-rigs.jsx +402 -0
  196. package/src/stage-capture.jsx +216 -0
  197. package/src/stage-characters.jsx +302 -0
  198. package/src/stage-editor-gizmos.jsx +1229 -0
  199. package/src/stage-labels-ko.js +204 -0
  200. package/src/stage-startup-defaults.js +182 -0
  201. package/src/starter-scenes.js +1 -1
  202. package/src/storyboard.js +24 -5
  203. package/src/studio-actions.js +42 -9
  204. package/src/studio-agent-commands.js +186 -23
  205. package/src/studio-agent-context.js +11 -9
  206. package/src/studio-agent-motion.js +2 -1
  207. package/src/studio-agent-protocol.js +56 -23
  208. package/src/studio-app-binding.js +52 -23
  209. package/src/studio-contact-sheet.js +3 -3
  210. package/src/studio-elements.js +11 -2
  211. package/src/studio-geometry-facts.js +114 -0
  212. package/src/styles.css +92 -9
  213. package/src/ui-scrub.js +4 -4
  214. package/src/ui.jsx +6 -4
  215. package/src/use-asset-shelf.js +60 -0
  216. package/src/use-camera-tutorial.js +50 -0
  217. package/src/use-embed-bridge.js +12 -0
  218. package/src/use-first-success-prompt.js +7 -0
  219. package/src/use-i2v-capture-modal.js +15 -0
  220. package/src/use-sculpt-mode.js +95 -0
  221. package/src/workflow/AgentPanel.jsx +106 -18
  222. package/src/workflow/WorkflowBuilder.jsx +1 -1
  223. package/src/workflow/agent-client.js +113 -45
  224. package/src/workflow/agent-panel.css +35 -0
  225. package/src/workflow/attachment-image.js +1 -1
  226. package/src/workflow/canvas-commands.js +3 -3
  227. package/src/workflow/cozy-scene-node.js +3 -3
  228. package/src/workflow/local-workflow.js +2 -2
  229. package/src/workflow/motion-input.js +3 -3
  230. package/src/workflow/node-schema.js +1 -1
  231. package/src/workflow/scene-asset-sync.js +2 -9
  232. package/src/workflow/shot-prompt-node.js +1 -1
  233. package/src/workflow/video-contract.js +1 -1
  234. package/src/workflow/workflow-resources.js +0 -8
  235. package/src/workflow-send.js +87 -0
  236. package/tools/ardy/__pycache__/cclay_gvhmr_worker.cpython-313.pyc +0 -0
  237. package/tools/check-eslint.mjs +57 -0
  238. package/tools/check-knip.mjs +60 -0
  239. package/tools/check-loc.mjs +54 -0
  240. package/tools/demo-worker/api-client.mjs +7 -13
  241. package/tools/demo-worker/generate.mjs +2 -4
  242. package/tools/eslint-baseline.json +23 -0
  243. package/tools/kimodo/effector-constraints.mjs +4 -4
  244. package/tools/kimodo/generate.mjs +1 -1
  245. package/tools/knip-baseline.json +331 -0
  246. package/tools/loc-baseline.json +126 -0
  247. package/tools/process-supervisor.mjs +1 -1
  248. package/tools/projflow/replay.mjs +4 -4
  249. package/tools/projflow/runner.mjs +3 -3
  250. package/tools/qa/studio-control-count.mjs +2 -0
  251. package/tools/run-tests.mjs +94 -7
  252. package/dist/assets/analytics-MVZOEW8O.js +0 -1
  253. package/dist/assets/app-BxBLC6GT.js +0 -4876
  254. package/dist/assets/app-CJdxvnrl.css +0 -1
  255. package/dist/assets/shot-prompt-hN0j_O-p.js +0 -17
  256. package/dist/assets/workflow-B5XZwVfe.js +0 -179
package/CHANGELOG.md CHANGED
@@ -1,5 +1,61 @@
1
1
  # Changelog
2
2
 
3
+ ## Unreleased
4
+
5
+ ## 2.1.0 - 2026-10-12
6
+
7
+ Storyboard and Animation modes. A project is now either a storyboard (a row of
8
+ still panels) or an animation (moving shots); both are on in a normal launch.
9
+
10
+ Modes
11
+
12
+ - The start screen has a Storyboard / Animation control, the project keeps its
13
+ mode through save and reopen, and the top bar shows it as a badge. A
14
+ storyboard project hides the Generate group and Motion mode (`4`); motion
15
+ commands are refused with a reason instead of failing quietly. (#649, #650)
16
+ - The bottom dock has a **Board** tab in storyboard projects: one card per
17
+ panel with a greybox thumbnail, caption, hold, duplicate, delete and
18
+ drag-to-reorder. Each panel can keep its own cast placement. (#640, #641)
19
+ - **Stylize** turns a panel into a finished picture through the image route
20
+ and keeps it in the project beside the greybox; **Send to Workflow** puts
21
+ the stylized image or the greybox on a Workflow canvas with a connected
22
+ Video node. (#643, #645)
23
+
24
+ Capsule figures
25
+
26
+ - A rigless **Capsule figure** joins the Assets tile list for blocking: stand,
27
+ sit or lie, set in Details. It follows root paths in the viewport and in
28
+ exports. Rig-only tools (Pose, IK, motion, take import, physics) refuse it
29
+ and say why. (#634, #635)
30
+
31
+ Storyboard exports
32
+
33
+ - The Export menu of a storyboard lists **Contact sheet (PNG)**, **Panel pack
34
+ (zip)**, **All panels (zip)**, **Animatic (mp4)**, **Cut list (OTIO)** and
35
+ **Depth + normal (PNG)**. File ▸ **Export as Animation project…** turns the
36
+ panels into an Animation project. Keyframe pack and shot video stay
37
+ Animation-only.
38
+
39
+ Agent and MCP
40
+
41
+ - In a storyboard project the Studio Agent adds one panel per request, frames
42
+ it and never generates motion; the Studio context and `describe_scene` now
43
+ report the project mode and each character's kind. (#637, #642)
44
+ - MCP motion tools answer `NOT_IN_MODE` in a storyboard and
45
+ `TARGET_NOT_READY` on a capsule figure; `add_character` takes a model and
46
+ posture. (#652)
47
+
48
+ Analytics and docs
49
+
50
+ - Three counts-only events: `storyboard:panel_created` (`source`),
51
+ `storyboard:panel_stylized` (`outcome`) and `cast:proxy_added`
52
+ (`surface`); `scene:created` carries `previs_mode`, and the export
53
+ `export_kind` vocabulary gains `contact_sheet`, `panel_pack`, `animatic`
54
+ and `animation_project`. Captions, names and prompts are never properties.
55
+ Disclosed in `docs/privacy.md` and `docs/analytics-queries.md`. (#653)
56
+ - README has a **Modes** section; the Agent rules, the Studio IA and the
57
+ capsule-figure notes are updated. (#653)
58
+
3
59
  ## 2.0.2
4
60
 
5
61
  The first-run audit release: `npx cozyclay` is exercised end to end as a new
package/README.md CHANGED
@@ -52,7 +52,7 @@ https://github.com/user-attachments/assets/1d0113e5-6922-443d-affc-1bdabc666247
52
52
 
53
53
  | | |
54
54
  | --- | --- |
55
- | **Stage a scene** | Create primitives and set pieces, then move, rotate and scale them with the transform strip's gizmo. Grid snapping is a preference, not a law — hold `Ctrl` mid-drag to invert it. A bird's-eye Top-View drives 2D root waypoints for character paths. **View ▾** on the viewport bar holds the reference grid and Auto Color — Blender's random viewport color, so twenty grey blockout boxes stay tellable apart without touching the colors you authored (captures include the display colors while it is on). |
55
+ | **Stage a scene** | Create primitives and set pieces, then move, rotate and scale them with the transform strip's gizmo. Cast people from **Assets › Characters**: a rigged character you can pose and animate, or a **Capsule figure** — a rigless stand-in with one posture (stand, sit or lie) chosen in Details, for blocking where nobody needs to move their limbs. Capsule figures follow root paths and per-panel placements like any character but refuse pose, IK and motion tools, with the reason in the tooltip. Grid snapping is a preference, not a law — hold `Ctrl` mid-drag to invert it. A bird's-eye Top-View drives 2D root waypoints for character paths. **View ▾** on the viewport bar holds the reference grid and Auto Color — Blender's random viewport color, so twenty grey blockout boxes stay tellable apart without touching the colors you authored (captures include the display colors while it is on). |
56
56
  | **Fly the camera** | Right-drag flies (WASD walks, Q/E cranes), middle-drag pans, Alt+drag orbits the selection, click selects, `F` frames — the muscle memory you already have from a 3D editor. Fly, pan and orbit lock the pointer for the hold, so the view can turn past the window edge. Selecting the camera switches to Camera mode. **Look through** in the Shot monitor puts you behind the shot camera with the same bindings; click the on-screen **Shot camera** indicator or press `Esc` to return to the free camera. |
57
57
  | **Cut and move the camera** | Add shots on the timeline, draw a dolly rail on the Top-View, set speed, height and crane, and preview the move through the shot camera. Each shot carries a **Target model** (Seedance 2.5, Kling 2, Veo 3, self-hosted MiniMax-H3) and is flagged when the cut runs past that model's limits. |
58
58
  | **Export for AI video** | One **Export ▾** menu: a keyframe pack (first/last frame, clip, camera JSON, prompt, README) as a zip, an mp4 of the shot, depth + normal conditioning passes, a storyboard contact sheet, and an OTIO cut list. Seedance 2.5 reads the pack's greybox clip as its white-model reference video, and MiniMax H3, Wan 3.0, LTX Desktop and fal render-to-real accept the same clip. The **Shot Prompt** turns the framing into a structured prompt for the model you picked. |
@@ -61,6 +61,21 @@ https://github.com/user-attachments/assets/1d0113e5-6922-443d-affc-1bdabc666247
61
61
  | **Capture motion from video or a photo** | Drop a clip or a still: the GPU box runs [GVHMR](https://github.com/zju3dv/GVHMR) and the result is retargeted onto the character with stabilisation, contact correction and a quality gate. |
62
62
  | **Direct it with an AI** | Open the **Agent panel** (`Cmd/Ctrl+B`), sign in with your ChatGPT account, and ask for a shot in plain language; the conversation survives reloads and you can paste a reference screenshot into it. Or connect Claude — or any MCP client — or drive the Studio from a terminal with `cclay live`. All three place the cast, frame "a low wide profile", generate multi-phase motion, and the viewport moves in front of you. See [AI control](#ai-control). |
63
63
 
64
+ ## Modes
65
+
66
+ A project is either a **Storyboard** or an **Animation**. Pick it on the start screen (the badge in the top bar shows which one you are in); a project keeps its mode when saved and reopened.
67
+
68
+ | | Storyboard | Animation |
69
+ | --- | --- | --- |
70
+ | **Choose it when** | You want a row of still panels: framing, who stands where, a caption and a hold per panel | You want moving shots: motion on a timeline, camera moves, takes |
71
+ | **Timeline** | The **Board** dock tab: one card per panel with a greybox thumbnail, caption, hold, duplicate, delete and drag-to-reorder | Shots on the timeline, with motion and Prompt Blocks |
72
+ | **Cast** | Rigged characters or capsule figures; each panel can keep its own placement ("Use stage placement" clears it) | Rigged characters with Pose and Motion mode; capsule figures for blocking |
73
+ | **Motion tools** | Hidden: no Generate group, no Motion mode (`4`); the agent has no `generate_motion` and the MCP motion tools answer `NOT_IN_MODE` | Available |
74
+ | **Agent** | **Agent · Storyboard** builds one panel per request and never generates motion | The full Studio tool set |
75
+ | **Exports** | **Stylize** a panel through the image route, **Send to Workflow** (the stylized image or the greybox) from its card, plus the Export ▾ menu: **Contact sheet (PNG)**, **Panel pack (zip)**, **All panels (zip)**, **Animatic (mp4)**, **Cut list (OTIO)** and **Depth + normal (PNG)**; File ▸ **Export as Animation project…** turns the panels into an Animation. There is no keyframe pack or shot video in a storyboard | Export ▾: keyframe pack, mp4, depth + normal passes, contact sheet, OTIO cut list |
76
+
77
+ Start in Storyboard when you are deciding what the shots are, then open the idea as an Animation when it needs motion.
78
+
64
79
  ## Quick start
65
80
 
66
81
  You need Node.js 22.19 or newer, npm or bun, and a Chromium-based browser.
@@ -182,7 +197,7 @@ Every control's home is recorded in [`docs/studio-ui-ia.md`](docs/studio-ui-ia.m
182
197
  | --- | --- |
183
198
  | [`docs/kimodo-setup.md`](docs/kimodo-setup.md) | Installing Kimodo: the route table, CUDA over SSH, local MLX/Metal/CPU generation and what each route supports |
184
199
  | [`docs/agent-cli.md`](docs/agent-cli.md) | Driving the Studio from a terminal with `cclay live` |
185
- | [`docs/agent-prompt-to-scene.md`](docs/agent-prompt-to-scene.md) | The ask / infer / omit rule the Agent follows when turning a video prompt into blocking |
200
+ | [`docs/agent-prompt-to-scene.md`](docs/agent-prompt-to-scene.md) | The ask / infer / omit rule the Agent follows when turning a video prompt into blocking, and the one-panel-per-request rule in storyboards |
186
201
  | [`mcp/README.md`](mcp/README.md) | MCP tools, transports, and the live-control protocol |
187
202
  | [`docs/studio-ui-ia.md`](docs/studio-ui-ia.md) | Where every Studio control lives and why |
188
203
  | [`docs/privacy.md`](docs/privacy.md) | The full analytics disclosure: events, identifiers, opt-out, internal QA marking |
@@ -165,6 +165,14 @@ Their transitive runtime dependencies, which the agent sidecar also loads, are:
165
165
  - `ignore` — License: MIT — https://github.com/kaelzhang/node-ignore
166
166
  - `yaml` — License: ISC — https://github.com/eemeli/yaml
167
167
 
168
+ ## jpeg-js
169
+
170
+ The MCP runtime decodes JPEG reference images for `compare_sculpt` with `jpeg-js`, on the local machine.
171
+
172
+ - Copyright (c) 2014, Eugene Ware and Adobe
173
+ - License: BSD-3-Clause
174
+ - Source: https://github.com/jpeg-js/jpeg-js
175
+
168
176
  ## CozyClay license scope
169
177
 
170
178
  The CozyClay combined work in this repository is distributed under AGPL-3.0-or-later, subject to the transition details in `LICENSING.md`. That license does not replace or relicense Three.js, ARDY, ARDY model checkpoints, the bundled fonts, the character rigs, or any other third-party component.
@@ -10,6 +10,7 @@ import { createAgentTools, SYSTEM_PROMPT, pickWorkspace } from "./agent-tools.mj
10
10
  import { createVideoAdapters, validateFalVideoRequest } from "./video-adapters.mjs";
11
11
  import { createSessionStore, transcriptFromHistory } from "./session-store.mjs";
12
12
  import { createAgentRunner } from "./agent-runner.mjs";
13
+ import { STUDIO_SYSTEM_PROMPT_STORYBOARD } from "./studio-prompt.mjs";
13
14
 
14
15
  // Values the codex backend accepts for reasoning.effort (its own 400 lists them).
15
16
  export const REASONING_EFFORTS = ["none", "minimal", "low", "medium", "high", "xhigh", "max", "ultra"];
@@ -19,7 +20,7 @@ const json = (res, status, value) => {
19
20
  res.end(JSON.stringify(value));
20
21
  };
21
22
 
22
- export function allowAgentOrigin(req, port) {
23
+ function allowAgentOrigin(req, port) {
23
24
  return [`http://127.0.0.1:${port}`, `http://${"local" + "host"}:${port}`].includes(req.headers.origin)
24
25
  || (req.headers.origin === undefined && req.method === "GET"
25
26
  && [`127.0.0.1:${port}`, `localhost:${port}`].includes(req.headers.host));
@@ -27,6 +28,8 @@ export function allowAgentOrigin(req, port) {
27
28
 
28
29
  // Two 1920x1080 PNG data URLs (frame + reference) fit comfortably in this.
29
30
  const IMAGE_BODY_LIMIT = 24 * 1024 * 1024;
31
+ // The sizes gpt-image-2 accepts; omitted means the backend's own choice.
32
+ const IMAGE_SIZES = ["auto", "1024x1024", "1536x1024", "1024x1536"];
30
33
 
31
34
  // Attached scene references (#167): identity sheets and the environment
32
35
  // reference. Capped because every one of them is another full image the
@@ -41,6 +44,10 @@ const ATTACHMENTS_MAX = 4;
41
44
  const ATTACHMENT_MAX_CHARS = 6_000_000;
42
45
  const TURN_BODY_LIMIT = ATTACHMENTS_MAX * ATTACHMENT_MAX_CHARS + 64 * 1024;
43
46
 
47
+ // How long GET /agent/turn/<id>/supervisor holds a request for a review that
48
+ // is still running before it answers 204 (#728).
49
+ const SUPERVISOR_WAIT_MS = 180_000;
50
+
44
51
  /** Reject anything that is not a short list of inline {dataUrl, name?} images.
45
52
  * The Studio envelope validates its own copy; this is the legacy body's. */
46
53
  function validAttachments(attachments) {
@@ -60,7 +67,6 @@ const agentToolCategory = (name) => {
60
67
  // Studio families are authored edits or reads; the analytics vocabulary has
61
68
  // no Studio-specific category, so a write says so and everything else does not.
62
69
  if (["arrange_objects", "arrange_characters", "patch_elements", "frame_shot", "undo_edit"].includes(name)) return "scene_write";
63
- if (["inspect_studio", "operate_studio", "verify_result", "generate_motion"].includes(name)) return "other";
64
70
  if (name === "run_workflow") return "workflow_run";
65
71
  if (name === "describe_workflow" || name === "focus_workflow_node") return "workflow_read";
66
72
  if (["add_workflow_node", "update_workflow_node", "remove_workflow_node", "connect_workflow_nodes", "disconnect_workflow_nodes", "set_workflow_node_output"].includes(name)) return "workflow_write";
@@ -75,6 +81,55 @@ const agentFailureCode = (error, signal, tool = false) => advisory(() => {
75
81
  if (error?.status === 429) return "rate_limited";
76
82
  return tool ? "tool_failed" : "upstream";
77
83
  }, tool ? "tool_failed" : "upstream");
84
+ /**
85
+ * One user message is one editor transaction: every Studio turn whose editor
86
+ * reports a previsMode opens agent.turn.begin and closes agent.turn.finish, so
87
+ * all of its mutations land as ONE history entry and one receipt frame. A
88
+ * context without a previsMode comes from an editor that predates turn
89
+ * transactions and runs unwrapped. rollback() cancels the open transaction
90
+ * (restoring the pre-turn document) and reopens it for the rest of the turn.
91
+ */
92
+ const agentTurn = ({ hub, context, turnId, admission }) => {
93
+ const wrapped = Boolean(context.scene?.previsMode);
94
+ const host = Object.fromEntries(["workspaceId", "documentEpoch", "sceneId", "sceneEpoch"].map(key => [key, context.host[key]]));
95
+ const control = async action => {
96
+ const receipt = await hub.command("run_action", {
97
+ name: "run_action", args: { action, args: { turnId } }, turnId,
98
+ commandId: randomUUID(), host, expectedRevision: admission.revision,
99
+ }, context.host.workspaceHandle);
100
+ if (!receipt.ok) throw Object.assign(new Error(receipt.message), { code: receipt.code });
101
+ return receipt;
102
+ };
103
+ const open = async () => { await control("agent.turn.begin"); admission.turnId = turnId; };
104
+ return {
105
+ async rollback() {
106
+ if (admission.turnId !== turnId) return null;
107
+ delete admission.turnId;
108
+ const receipt = await control("agent.turn.cancel");
109
+ await admission.refresh();
110
+ await open();
111
+ return receipt;
112
+ },
113
+ run: start => async function* (input) {
114
+ if (!wrapped) { yield* start(input); return; }
115
+ await open();
116
+ let terminal;
117
+ try {
118
+ for await (const frame of start(input)) {
119
+ if (frame.type === "done") terminal = frame; else yield frame;
120
+ }
121
+ } finally {
122
+ // A rollback whose reopen failed has already closed the transaction.
123
+ if (admission.turnId === turnId) {
124
+ delete admission.turnId;
125
+ const receipt = await control("agent.turn.finish");
126
+ if (receipt.authored) yield { type: "receipt", receipt };
127
+ }
128
+ }
129
+ if (terminal) yield terminal;
130
+ },
131
+ };
132
+ };
78
133
  // These existing canvas commands only return a node/edge after publishing a
79
134
  // new insertion. Read/focus, generic accepted responses, update no-ops and run
80
135
  // outputs (which can echo old values) are deliberately not application proof.
@@ -134,6 +189,8 @@ function validReferences(references) {
134
189
  * owns the geometry, each character sheet owns one performer's look and the
135
190
  * environment reference owns the location.
136
191
  */
192
+ export const FRAME_COMPOSITION_GUIDANCE = "Use the first image (a clay blocking frame) as the layout: keep its composition and camera angle, and keep the pose but make it look natural. Render it as:";
193
+
137
194
  export function referenceGuidance(references = []) {
138
195
  const list = Array.isArray(references) ? references : [];
139
196
  if (!list.length) return "";
@@ -143,6 +200,8 @@ export function referenceGuidance(references = []) {
143
200
  lines.push(`Character ${entry.name || "reference"}: match the identity, face, hair and wardrobe from the attached character sheet.`);
144
201
  } else if (entry.role === "environment") {
145
202
  lines.push("Environment: take the location look, materials, palette and lighting from the attached environment reference.");
203
+ } else if (entry.role === "reference") {
204
+ lines.push(`Reference ${entry.name || "image"}: use it for the look the prompt asks for (subject, wardrobe, style); keep the exact pose, body position and camera framing of the first image.`);
146
205
  }
147
206
  }
148
207
  return `\n${lines.join("\n")}`;
@@ -174,7 +233,7 @@ function quotaEvent(codex, headers) {
174
233
  // one line — without it neither the log nor the panel can say why a turn died.
175
234
  const UPSTREAM_DETAIL_MAX = 200;
176
235
  export function sanitizeUpstreamDetail(body) {
177
- let message = null;
236
+ let message;
178
237
  try {
179
238
  const parsed = JSON.parse(body);
180
239
  const candidate = parsed?.error?.message ?? parsed?.error ?? parsed?.message ?? parsed?.detail;
@@ -285,7 +344,10 @@ function liveToolsRuntime(mcpRuntime) {
285
344
  });
286
345
  }
287
346
 
288
- export function createAgentHandler({ auth = defaultAuth, codex, models, codexBaseUrl, cliproxyBaseUrl, env, fauxProvider, handlers, liveHub, mcpRuntime, port, studioRuntime, clock = Date.now, setIntervalImpl = setInterval, clearIntervalImpl = clearInterval, sessionStore: injectedSessionStore } = {}) {
347
+ // `supervisor` (#728) defaults off only under an injected scripted provider:
348
+ // a test's script is an exact queue of replies, and a review would take one
349
+ // meant for a later turn. The real sidecar never injects one.
350
+ export function createAgentHandler({ auth = defaultAuth, codex, models, codexBaseUrl, cliproxyBaseUrl, env, fauxProvider, handlers, liveHub, mcpRuntime, port, studioRuntime, clock = Date.now, setIntervalImpl = setInterval, clearIntervalImpl = clearInterval, sessionStore: injectedSessionStore, supervisor = !fauxProvider } = {}) {
289
351
  const requestContext = new AsyncLocalStorage();
290
352
  codex ||= defaultClient(auth, requestContext);
291
353
  const runtime = handlers !== undefined || liveHub !== undefined ? Promise.resolve({ handlers: handlers ?? [], liveHub }) : liveToolsRuntime(mcpRuntime);
@@ -347,6 +409,9 @@ export function createAgentHandler({ auth = defaultAuth, codex, models, codexBas
347
409
  return owner;
348
410
  };
349
411
  const emitStudioEvent = (turnId, event) => {
412
+ // The supervisor note lands after the turn ended (#728): it joins the turn's
413
+ // record, terminal or not, but never resurrects one that was retired.
414
+ if (event.type === "supervisor" && !studioEvents.has(turnId)) return;
350
415
  const record = studioEvents.get(turnId) || { next: 0, events: [], listeners: new Set(), terminal: false };
351
416
  // Execution telemetry is advisory and is validated key-by-key in the
352
417
  // browser: it carries no replay cursor, is never retained for a resume,
@@ -377,10 +442,10 @@ export function createAgentHandler({ auth = defaultAuth, codex, models, codexBas
377
442
  });
378
443
 
379
444
  const studioIdentity = host => Object.fromEntries(["workspaceId", "documentEpoch", "sceneId", "sceneEpoch"].map(key => [key, host[key]]));
380
- const writeStudioStream = (res, record, after = 0, req = null) => {
445
+ const writeStudioStream = (res, record, after = 0) => {
381
446
  res.writeHead(200, { "content-type": "text/event-stream; charset=utf-8", "cache-control": "no-cache", connection: "keep-alive" }); res.flushHeaders?.();
382
447
  let cursor = after; const send = event => {
383
- if (res.destroyed) return;
448
+ if (res.destroyed || res.writableEnded) return;
384
449
  if (event.eventSeq === undefined) { res.write(`data: ${JSON.stringify(event)}\n\n`); return; }
385
450
  if (event.eventSeq > cursor) { cursor = event.eventSeq; res.write(`data: ${JSON.stringify(event)}\n\n`); }
386
451
  };
@@ -401,8 +466,8 @@ export function createAgentHandler({ auth = defaultAuth, codex, models, codexBas
401
466
  };
402
467
  const handleStudioTurn = async (req, res, value, path) => {
403
468
  const { StudioProtocolError, validateStudioContextFreshness } = await import("../../src/studio-agent-protocol.js");
404
- const [{ createStudioTools }, { encodeStudioContext }] = await Promise.all([
405
- import("./studio-tools.mjs"), import("../../src/studio-agent-context.js"),
469
+ const [{ createStudioTools }, { encodeStudioContext }, { createScriptTool }, { runSupervisor, shouldSupervise }] = await Promise.all([
470
+ import("./studio-tools.mjs"), import("../../src/studio-agent-context.js"), import("./script-tool.mjs"), import("./supervisor.mjs"),
406
471
  ]);
407
472
  const hubDeps = await runtime; const hub = hubDeps.liveHub || liveHub;
408
473
  if (path === "/agent/stop") {
@@ -436,43 +501,65 @@ export function createAgentHandler({ auth = defaultAuth, codex, models, codexBas
436
501
  let session = studioSessions.get(value.sessionId);
437
502
  const suppliedOwner = parseCookies(req).studio_owner;
438
503
  if (session && suppliedOwner && session.owner !== suppliedOwner) throw new StudioProtocolError("AUTH_REQUIRED", "Studio session owner mismatch.");
504
+ const existing = session?.turns.get(value.turnId);
505
+ if (existing) { session.updatedAt = clock(); writeStudioStream(res, existing, 0); return true; }
506
+ let current;
507
+ try { current = studioRuntime?.readContext ? await studioRuntime.readContext(value.context.host) : await authoritativeStudioContext(value, hub); }
508
+ catch (error) { if (error instanceof StudioProtocolError) throw error; if (error?.code) throw new StudioProtocolError(error.code, error.message); throw new StudioProtocolError("CAPABILITY_MISSING", "The connected editor does not expose authoritative Studio context."); }
509
+ validateStudioContextFreshness(value.context, current);
510
+ // The session, and the owner token only this turn's response can hand
511
+ // the browser, exist only once the turn is admitted. Minting them before
512
+ // a refusal (the editor still reconnecting after a restart) left a
513
+ // session whose owner the browser never received: its cookie from the
514
+ // previous server then failed every retry with "owner mismatch".
439
515
  if (!session) {
440
516
  studioOwner(req, value.sessionId, true);
441
517
  let persisted = null;
442
- try { persisted = sessionStore.read(value.sessionId); } catch { persisted = null; }
518
+ try { persisted = sessionStore.read(value.sessionId); } catch { /* an unreadable history starts the session fresh */ }
443
519
  session = { owner: studioOwnerTokens.get(value.sessionId), history: persisted?.history ?? [], persistedItems: persisted?.history?.length ?? 0, meta: persisted?.meta ?? null, turns: new Map(), controller: null, activeJobId: null, activeJobTurnId: null, motionJobIds: new Set(Array.isArray(persisted?.meta?.motionJobIds) ? persisted.meta.motionJobIds.filter(id => typeof id === "string") : []), host: null, updatedAt: clock() };
444
520
  studioSessions.set(value.sessionId, session);
445
521
  }
446
522
  session.updatedAt = clock();
447
- const existing = session.turns.get(value.turnId);
448
- if (existing) { writeStudioStream(res, existing, 0, req); return true; }
449
- let current;
450
- try { current = studioRuntime?.readContext ? await studioRuntime.readContext(value.context.host) : await authoritativeStudioContext(value, hub); }
451
- catch (error) { if (error instanceof StudioProtocolError) throw error; if (error?.code) throw new StudioProtocolError(error.code, error.message); throw new StudioProtocolError("CAPABILITY_MISSING", "The connected editor does not expose authoritative Studio context."); }
452
- validateStudioContextFreshness(value.context, current);
453
523
  if (studioRuntime?.handleTurn) { await studioRuntime.handleTurn(value, req, res); return true; }
454
524
  const record = { next: 0, events: [], listeners: new Set(), terminal: false }; studioEvents.set(value.turnId, record); session.turns.set(value.turnId, record); session.host = studioIdentity(value.context.host);
455
525
  res.setHeader("set-cookie", `studio_owner=${encodeURIComponent(session.owner)}; Path=/agent; HttpOnly; SameSite=Strict`);
456
- const close = writeStudioStream(res, record, 0, req);
457
- const send = event => emitStudioEvent(value.turnId, event);
526
+ const close = writeStudioStream(res, record, 0);
527
+ // What the supervisor reviews once the turn is over: every frame but text
528
+ // and telemetry, and the last stretch of text the model wrote.
529
+ const turnFrames = [];
530
+ let replySegment = "", lastReply = "";
531
+ const send = event => {
532
+ if (event.type === "text.delta") { replySegment += event.text ?? ""; lastReply = replySegment; }
533
+ else if (!["execution_telemetry", "execution_tool_started"].includes(event.type)) {
534
+ if (event.type === "tool.start") replySegment = "";
535
+ turnFrames.push(event);
536
+ }
537
+ emitStudioEvent(value.turnId, event);
538
+ };
458
539
  // The Studio turn is measured exactly like the Workflow turn, correlated by
459
540
  // the turn id the host already owns (the frozen envelope carries no room
460
541
  // for a second one).
461
542
  const telemetry = createTurnTelemetry(send, value.turnId);
462
543
  let turnOutcome = "succeeded", turnFailureCode = null, toolFailed = false;
463
- const controller = new AbortController(); session.controller = controller;
544
+ const controller = new AbortController(); session.controller = controller; session.activeTurnId = value.turnId;
464
545
  // Admission is the document identity plus the exact scene revision each
465
546
  // command expects. Per-entity tokens are deliberately absent: the revision
466
547
  // already moves whenever any authored content changes, and a turn's second
467
548
  // edit to the same entity would otherwise carry a token the first edit retired.
468
549
  const admission = {
469
550
  commandId: () => randomUUID(), host: studioIdentity(value.context.host), revision: value.context.revision.scene,
470
- refresh: async () => {
471
- // Read at the admitted host: a scene action may have opened another
472
- // scene of this workspace during the turn (studio-tools adopts it).
551
+ // Read at the admitted host: a scene action may have opened another
552
+ // scene of this workspace during the turn (studio-tools adopts it).
553
+ snapshot: async () => {
473
554
  const host = { ...value.context.host, ...admission.host };
474
- const refreshed = studioRuntime?.readContext ? await studioRuntime.readContext(host) : await authoritativeStudioContext({ ...value, context: { ...value.context, host } }, hub);
555
+ return studioRuntime?.readContext ? await studioRuntime.readContext(host) : await authoritativeStudioContext({ ...value, context: { ...value.context, host } }, hub);
556
+ },
557
+ // Re-admits at the live revision and hands back the context it read, which
558
+ // the tools compare against what the agent last saw (studio-rebase.mjs).
559
+ refresh: async () => {
560
+ const refreshed = await admission.snapshot();
475
561
  admission.revision = refreshed.revision.scene;
562
+ return refreshed;
476
563
  },
477
564
  };
478
565
  // One motion generation per user message, whichever path starts it:
@@ -504,7 +591,7 @@ export function createAgentHandler({ auth = defaultAuth, codex, models, codexBas
504
591
  if (session.activeJobId === jobId) { session.activeJobId = null; session.activeJobTurnId = null; }
505
592
  }
506
593
  };
507
- const tools = createStudioTools({ liveHub: hub, workspaceHandle: value.context.host.workspaceHandle, session: { signal: controller.signal, admission, generation, onJob, actionIndex: current?.actionIndex ?? [] }, resolveImage: async (id, correlation) => hub.command("resolve_studio_image", { imageId: id, ...correlation }, value.context.host.workspaceHandle) });
594
+ const tools = createStudioTools({ liveHub: hub, workspaceHandle: value.context.host.workspaceHandle, previsMode: value.context.scene?.previsMode, session: { signal: controller.signal, admission, generation, onJob, actionIndex: current?.actionIndex ?? [], baseline: value.context }, resolveImage: async (id, correlation) => hub.command("resolve_studio_image", { imageId: id, ...correlation }, value.context.host.workspaceHandle) });
508
595
  const motion = async args => {
509
596
  // Private artifact IDs belonged to the retired sidecar runtime. Reuse
510
597
  // retained editor takes through motion.loadVersion, not another installer.
@@ -525,12 +612,35 @@ export function createAgentHandler({ auth = defaultAuth, codex, models, codexBas
525
612
  return { ...result, visualStatus: visual.visualStatus, imageId: visual.imageId, revision: visual.revision, receiptId: visual.receiptId, ...(visual.dataUrl ? { dataUrl: visual.dataUrl } : {}) };
526
613
  },
527
614
  }));
615
+ const turnTransaction = agentTurn({ hub, context: current, turnId: value.turnId, admission });
616
+ const script = createScriptTool({ tools: modelTools, signal: controller.signal, emit: send });
617
+ const scriptTool = {
618
+ ...script,
619
+ // A script that fails after it edited the scene rolls the turn back, so
620
+ // the model retries from the pre-turn document, never from half a script.
621
+ handler: async (args, options) => {
622
+ const result = await script.handler(args, options);
623
+ if (result.ok || !result.calls.some(call => call.ok && Number.isSafeInteger(call.revision?.after) && call.revision.after !== call.revision.before)) return result;
624
+ const rollback = await turnTransaction.rollback();
625
+ return rollback ? { ...result, rollback: { status: "rolled_back", receiptId: rollback.receiptId, revision: rollback.revision, affectedIds: rollback.affectedIds.slice(0, 8) } } : result;
626
+ },
627
+ };
528
628
  let frameObservation;
529
629
  if (value.attachFrame) {
530
630
  const captured = await hub.command("capture_framing_png", {}, value.context.host.workspaceHandle);
531
631
  if (!captured?.dataUrl?.startsWith("data:image/")) throw new StudioProtocolError("TARGET_NOT_READY", "The current frame has no image bytes.");
532
632
  const match = /^data:([^;]+);base64,(.*)$/.exec(captured.dataUrl);
533
633
  frameObservation = { data: match[2], mimeType: match[1], revision: captured.revision, receiptId: captured.receiptId };
634
+ } else if (value.context.scene?.characterCount >= 2 || (value.context.shots?.length ?? 0) >= 1 || /샷|컷|카메라|커버리지|OTS|투샷|프레임|shot|camera|coverage|two-shot|framing/i.test(value.text || "")) {
635
+ // A staging or coverage request also sees the top-view plan; without it the turn just runs.
636
+ try {
637
+ const captured = await hub.command("capture_plan_png", {}, value.context.host.workspaceHandle);
638
+ const match = /^data:([^;]+);base64,(.*)$/.exec(captured?.dataUrl ?? "");
639
+ if (!match) throw new Error("The plan capture has no image bytes.");
640
+ frameObservation = { kind: "plan", data: match[2], mimeType: match[1], revision: captured.revision ?? value.context.revision?.scene ?? null, receiptId: captured.receiptId };
641
+ } catch (error) {
642
+ console.warn(`[agent] plan observation skipped: ${error.message}`);
643
+ }
534
644
  }
535
645
  const telemetryTools = new Map();
536
646
  const emitFrame = frame => {
@@ -545,6 +655,9 @@ export function createAgentHandler({ auth = defaultAuth, codex, models, codexBas
545
655
  toolFailed = true;
546
656
  if (typeof frame.error === "string" && frame.error.startsWith("Validation failed for tool ")) frame.error = "INVALID_ARGUMENT: Unexpected field.";
547
657
  }
658
+ // A command the sidecar re-sent once (stale revision, busy gesture) is
659
+ // marked on the frame itself; the receipt carries autoRebased and a warning.
660
+ if (frame.ok && frame.result?.autoRebased === true) frame.autoRebased = true;
548
661
  send(frame);
549
662
  if (record) {
550
663
  record.execution.executed(frame.ok ? "succeeded" : (controller.signal.aborted ? "cancelled" : "failed"));
@@ -560,10 +673,15 @@ export function createAgentHandler({ auth = defaultAuth, codex, models, codexBas
560
673
  if (frame.type === "done") {
561
674
  if (turnOutcome === "succeeded" && !toolFailed) telemetry.finished("succeeded", null);
562
675
  else telemetry.finished(turnOutcome, turnFailureCode || "tool_failed");
676
+ // Marked before the done frame leaves, so a client that asks for the
677
+ // review the moment its stream ends finds it pending, not absent.
678
+ // The author's own switch (value.supervisor === false) skips this turn's review.
679
+ if (supervisor && process.env.COZYCLAY_SUPERVISOR !== "off" && value.supervisor !== false && !controller.signal.aborted && shouldSupervise(turnFrames)) record.supervisor = "pending";
563
680
  }
564
681
  send(frame);
565
682
  };
566
683
  const persistenceMeta = { sceneName: value.context?.scene?.name ?? value.context?.sceneName ?? null, firstText: value.text, motionJobIds: [...session.motionJobIds] };
684
+ const modelKey = value.model || (fauxProvider ? `${fauxProvider.provider?.id || fauxProvider.provider || "faux"}/scripted` : "gpt-6-astra");
567
685
  let runner = studioRunners.get(value.sessionId);
568
686
  if (!runner) {
569
687
  runner = createAgentRunner({ models: await ensureWorkflowModels(), fauxProvider, sessionStore, clock, codexBaseUrl, cliproxyBaseUrl, auth, env });
@@ -571,10 +689,10 @@ export function createAgentHandler({ auth = defaultAuth, codex, models, codexBas
571
689
  }
572
690
  try {
573
691
  if (!session.modelSession) session.modelSession = await runner.openSession(value.sessionId, { surface: "studio" });
574
- for await (const frame of session.modelSession.start({
575
- surface: "studio", sessionId: value.sessionId, model: value.model || (fauxProvider ? `${fauxProvider.provider?.id || fauxProvider.provider || "faux"}/scripted` : "gpt-6-astra"), effort: value.effort,
692
+ for await (const frame of turnTransaction.run(input => session.modelSession.start(input))({
693
+ surface: "studio", sessionId: value.sessionId, model: modelKey, effort: value.effort,
576
694
  text: value.text, attachments: value.attachments, contextText: encodeStudioContext(value.context), frameObservation,
577
- context: value.context, tools: modelTools, signal: controller.signal, emit: send,
695
+ context: value.context, tools: [...modelTools, scriptTool], systemPrompt: value.context.scene?.previsMode === "storyboard" ? STUDIO_SYSTEM_PROMPT_STORYBOARD : undefined, signal: controller.signal, emit: send,
578
696
  meta: persistenceMeta,
579
697
  quotaEvent: headers => codex?.parseQuotaHeaders ? quotaEvent(codex, headers) : { type: "quota", plan: null, primary: { usedPercent: null, windowMinutes: null, resetAt: null }, credits: { has: null } },
580
698
  })) emitFrame(frame);
@@ -587,7 +705,25 @@ export function createAgentHandler({ auth = defaultAuth, codex, models, codexBas
587
705
  }
588
706
  if (controller.signal.aborted) { turnOutcome = "cancelled"; turnFailureCode = "aborted"; }
589
707
  if (!record.terminal) emitFrame({ type: "done" });
590
- close(); if (!res.writableEnded) res.end(); session.controller = null; return true;
708
+ close(); if (!res.writableEnded) res.end(); session.controller = null; session.activeTurnId = null;
709
+ // The review starts only now, after the stream has ended, and only ever
710
+ // appends one note to the record: it neither delays nor steers the turn.
711
+ if (record.supervisor === "pending") void (async () => {
712
+ const startedAt = Date.now();
713
+ let frame;
714
+ try {
715
+ frame = await runSupervisor({ registry: await ensureWorkflowModels(), modelKey, effort: process.env.COZYCLAY_SUPERVISOR_EFFORT || "high", turnId: value.turnId, request: value.text, frames: turnFrames, reply: lastReply,
716
+ hub, workspaceHandle: value.context.host.workspaceHandle, admission, baseline: value.context, actions: current?.actionIndex ?? [] });
717
+ } catch (error) {
718
+ frame = { type: "supervisor", turnId: value.turnId, verdict: "unavailable", model: modelKey, effort: null, elapsedMs: Date.now() - startedAt, summary: "", items: [], issues: [], fixes: [], droppedFixes: [], counts: { blocker: 0, concern: 0, note: 0, fixes: 0 }, reason: "The supervisor could not start." };
719
+ }
720
+ // COZYCLAY_SUPERVISOR_AUTOFIX=blocker (#743) only marks a note whose
721
+ // blockers carry fixes; turns stay the panel's to start, never the sidecar's.
722
+ if (process.env.COZYCLAY_SUPERVISOR_AUTOFIX === "blocker" && frame.fixes?.some(fix => fix.severity === "blocker")) frame = { ...frame, autofix: { eligible: true, severity: "blocker" } };
723
+ record.supervisor = "done";
724
+ emitStudioEvent(value.turnId, frame);
725
+ })();
726
+ return true;
591
727
  };
592
728
  const handle = async (req, res, path = new URL(req.url, "http://127.0.0.1").pathname) => {
593
729
  if (!path.startsWith("/agent/")) return false;
@@ -601,7 +737,34 @@ export function createAgentHandler({ auth = defaultAuth, codex, models, codexBas
601
737
  if (!record || !session || parseCookies(req).studio_owner !== session.owner) { json(res, 403, { error: { code: "AUTH_REQUIRED", message: "Studio event stream is not owned by this session." } }); return true; }
602
738
  const after = Number(new URL(req.url, "http://127.0.0.1").searchParams.get("after") || 0);
603
739
  if (!Number.isSafeInteger(after) || after < 0) { json(res, 400, { error: "invalid cursor" }); return true; }
604
- writeStudioStream(res, record, after, req); return true;
740
+ writeStudioStream(res, record, after); return true;
741
+ }
742
+ // The supervisor note for one Studio turn (#728): 200 with the note once it
743
+ // exists, held up to SUPERVISOR_WAIT_MS while it is pending, 204 when the
744
+ // turn was never reviewed or the wait ran out.
745
+ if (path.startsWith("/agent/turn/") && path.endsWith("/supervisor") && req.method === "GET") {
746
+ if (!await hasAnyCredential()) { json(res, 401, { error: { code: "AUTH_REQUIRED", message: "Sign in or configure a provider key." } }); return true; }
747
+ const turnId = decodeURIComponent(path.slice("/agent/turn/".length, -"/supervisor".length));
748
+ const session = [...studioSessions.values()].find(candidate => candidate.turns.has(turnId)); const record = studioEvents.get(turnId);
749
+ if (!record || !session) { json(res, 404, { error: "turn not found" }); return true; }
750
+ if (parseCookies(req).studio_owner !== session.owner) { json(res, 403, { error: { code: "AUTH_REQUIRED", message: "Studio supervisor note is not owned by this session." } }); return true; }
751
+ const noContent = () => { res.writeHead(204, { "cache-control": "no-store" }); res.end(); };
752
+ const existing = record.events.findLast(event => event.type === "supervisor");
753
+ if (existing) { json(res, 200, existing); return true; }
754
+ if (record.supervisor !== "pending") { noContent(); return true; }
755
+ await new Promise(resolve => {
756
+ let timer;
757
+ const settle = note => {
758
+ clearTimeout(timer); record.listeners.delete(listener); res.off("close", onClose);
759
+ if (note !== undefined && !res.writableEnded && !res.destroyed) { if (note) json(res, 200, note); else noContent(); }
760
+ resolve();
761
+ };
762
+ const listener = event => { if (event.type === "supervisor") settle(event); };
763
+ const onClose = () => settle(undefined);
764
+ record.listeners.add(listener); res.once("close", onClose);
765
+ timer = setTimeout(() => settle(null), SUPERVISOR_WAIT_MS);
766
+ });
767
+ return true;
605
768
  }
606
769
  if (path.startsWith("/agent/turn/") && path.endsWith("/steer") && req.method === "POST") {
607
770
  const turnId = decodeURIComponent(path.slice("/agent/turn/".length, -"/steer".length));
@@ -610,12 +773,15 @@ export function createAgentHandler({ auth = defaultAuth, codex, models, codexBas
610
773
  value = await readBody(req, TURN_BODY_LIMIT);
611
774
  if (!value || typeof value.text !== "string" || !value.text.trim() || !validAttachments(value.attachments)) throw new Error("Invalid request.");
612
775
  } catch { json(res, 400, { error: "invalid request" }); return true; }
613
- // Studio turnIds are frozen envelopes; their events are keyed in studioSessions,
614
- // never in the Workflow `sessions` map, so that lookup alone tells them apart.
615
- if ([...studioSessions.values()].some(candidate => candidate.turns.has(turnId))) { json(res, 409, { error: { code: "STEER_UNSUPPORTED", message: "Steering a Studio turn is not supported." } }); return true; }
616
- const session = [...sessions.values()].find(candidate => candidate.turnId === turnId);
776
+ // Studio turns are keyed in studioSessions, never in the Workflow
777
+ // `sessions` map, so that lookup alone tells them apart. Only the cookie
778
+ // owner of the Studio session may steer its running turn.
779
+ const studio = [...studioSessions.values()].find(candidate => candidate.turns.has(turnId));
780
+ if (studio && parseCookies(req).studio_owner !== studio.owner) { json(res, 403, { error: { code: "AUTH_REQUIRED", message: "Studio turn is not owned by this session." } }); return true; }
781
+ const session = studio ?? [...sessions.values()].find(candidate => candidate.turnId === turnId);
617
782
  if (!session) { json(res, 404, { error: "turn not found" }); return true; }
618
- if (!session.running || !session.modelSession) { json(res, 409, { error: { code: "NO_ACTIVE_TURN", message: "This turn already ended." } }); return true; }
783
+ const active = studio ? studio.activeTurnId === turnId && studio.controller && !studio.controller.signal.aborted : session.running;
784
+ if (!active || !session.modelSession) { json(res, 409, { error: { code: "NO_ACTIVE_TURN", message: "This turn already ended." } }); return true; }
619
785
  const images = (Array.isArray(value.attachments) ? value.attachments : []).map(attachment => {
620
786
  const match = /^data:([^;]+);base64,(.*)$/.exec(attachment?.dataUrl || "");
621
787
  return match ? { type: "image", data: match[2], mimeType: match[1] } : null;
@@ -678,7 +844,7 @@ export function createAgentHandler({ auth = defaultAuth, codex, models, codexBas
678
844
  let value;
679
845
  try {
680
846
  value = await readBody(req, IMAGE_BODY_LIMIT);
681
- if (typeof value.prompt !== "string" || !value.prompt.trim() || typeof value.imageDataUrl !== "string" || !value.imageDataUrl.startsWith("data:image/") || (value.referenceDataUrl !== undefined && (typeof value.referenceDataUrl !== "string" || !value.referenceDataUrl.startsWith("data:image/"))) || !validReferences(value.references) || (value.quality !== undefined && !["auto", "low", "medium", "high"].includes(value.quality))) throw new Error("Invalid request.");
847
+ if (typeof value.prompt !== "string" || !value.prompt.trim() || typeof value.imageDataUrl !== "string" || !value.imageDataUrl.startsWith("data:image/") || (value.referenceDataUrl !== undefined && (typeof value.referenceDataUrl !== "string" || !value.referenceDataUrl.startsWith("data:image/"))) || !validReferences(value.references) || (value.composition !== undefined && !["frame", "sheet"].includes(value.composition)) || (value.quality !== undefined && !["auto", "low", "medium", "high"].includes(value.quality)) || (value.size !== undefined && !IMAGE_SIZES.includes(value.size))) throw new Error("Invalid request.");
682
848
  } catch { json(res, 400, { error: "invalid request" }); return true; }
683
849
  if (!await auth.getAccessToken()) { json(res, 401, { error: { code: "auth", message: "Sign in with ChatGPT in the Agent panel." } }); return true; }
684
850
  try {
@@ -687,8 +853,17 @@ export function createAgentHandler({ auth = defaultAuth, codex, models, codexBas
687
853
  // Scene references (#167) are appended after the frame/reference pair,
688
854
  // and the prompt says what each attachment is for.
689
855
  const references = Array.isArray(value.references) ? value.references : [];
690
- const prompt = `${value.prompt}${await renderGuidance(value.prompt)}${referenceGuidance(references)}`;
691
- const result = await codex.editImage({ ...value, prompt, extraImages: references.map((entry) => entry.dataUrl) });
856
+ // composition "frame": the attached frame was framed by hand (Pose mode), so the shot
857
+ // camera's description would contradict it; the frame alone owns camera and pose.
858
+ // composition "sheet" (Sculpt blueprint): a model sheet, not a shot, so no scene or
859
+ // camera guidance is added; the prompt is sent as written.
860
+ const prompt = value.composition === "sheet"
861
+ ? value.prompt
862
+ : value.composition === "frame"
863
+ ? `${FRAME_COMPOSITION_GUIDANCE}\n${value.prompt}${referenceGuidance(references)}`
864
+ : `${value.prompt}${await renderGuidance(value.prompt)}${referenceGuidance(references)}`;
865
+ const { composition: _composition, ...request } = value;
866
+ const result = await codex.editImage({ ...request, prompt, extraImages: references.map((entry) => entry.dataUrl) });
692
867
  json(res, 200, { dataUrl: `data:image/png;base64,${result.pngBase64}`, width: result.width, height: result.height });
693
868
  } catch (error) { json(res, error.status === 401 ? 401 : 502, { error: errorInfo(error) }); }
694
869
  return true;