cozyclay 1.8.0 → 1.9.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (156) hide show
  1. package/CHANGELOG.md +90 -0
  2. package/README.md +89 -142
  3. package/THIRD_PARTY_NOTICES.md +48 -0
  4. package/bin/agent/agent-routes.mjs +619 -119
  5. package/bin/agent/agent-runner.mjs +513 -0
  6. package/bin/agent/agent-tools.mjs +11 -19
  7. package/bin/agent/codex-client.mjs +6 -150
  8. package/bin/agent/credential-store.mjs +93 -0
  9. package/bin/agent/fal-contract.mjs +25 -0
  10. package/bin/agent/h3-preservation.mjs +0 -4
  11. package/bin/agent/motion-runtime.mjs +304 -0
  12. package/bin/agent/pi-tools.mjs +64 -0
  13. package/bin/agent/provider-keys.mjs +56 -0
  14. package/bin/agent/providers.mjs +344 -0
  15. package/bin/agent/schema-to-typebox.mjs +58 -0
  16. package/bin/agent/secure-file.mjs +19 -0
  17. package/bin/agent/session-store.mjs +324 -0
  18. package/bin/agent/studio-prompt.mjs +1 -0
  19. package/bin/agent/studio-tools.mjs +53 -0
  20. package/bin/agent/video-adapters.mjs +60 -9
  21. package/bin/codex-auth.mjs +37 -13
  22. package/bin/cozyclay.mjs +42 -10
  23. package/bin/live/cli.mjs +474 -0
  24. package/bin/live/client.mjs +216 -0
  25. package/bin/live/studio.mjs +85 -0
  26. package/bin/live/workspace.mjs +69 -0
  27. package/bin/live-endpoint.mjs +52 -0
  28. package/bin/mcp-runtime.mjs +4 -0
  29. package/bin/telemetry-state.mjs +10 -0
  30. package/dist/app/index.html +6 -5
  31. package/dist/assets/analytics-B1hnH66c.js +1 -0
  32. package/dist/assets/app-BWusbqgO.js +4861 -0
  33. package/dist/assets/app-qKDo4PBX.css +1 -0
  34. package/dist/assets/{demo-BENMp24T.js → demo-B-eg1CT8.js} +1 -1
  35. package/dist/assets/first-shot-handoff-CGgnvNu8.js +1 -0
  36. package/dist/assets/landing-IbRp3FnQ.js +1 -0
  37. package/dist/assets/shot-prompt-CHUtw6af.js +17 -0
  38. package/dist/assets/shot-prompt-C_g2BHVc.css +1 -0
  39. package/dist/assets/{ticket-BDieXjZs.js → ticket-BgIzOBOf.js} +1 -1
  40. package/dist/assets/workflow-CVwzMhz2.js +179 -0
  41. package/dist/assets/{workflow-CeQxy8Uq.css → workflow-DsudxKHF.css} +1 -1
  42. package/dist/cozyclay-package.json +1 -1
  43. package/dist/greybox-to-video/index.html +19 -0
  44. package/dist/index.html +15 -118
  45. package/dist/privacy/index.html +57 -3
  46. package/dist/sitemap.xml +3 -3
  47. package/dist/workflow/index.html +5 -5
  48. package/mcp/LIVE-PROTOCOL.md +89 -16
  49. package/mcp/README.md +12 -3
  50. package/mcp/live-hub.mjs +364 -35
  51. package/mcp/package.json +2 -2
  52. package/mcp/server.mjs +47 -21
  53. package/mcp/tool-handlers.mjs +113 -37
  54. package/package.json +8 -3
  55. package/src/App.jsx +2367 -735
  56. package/src/analytics.js +361 -27
  57. package/src/app-stage.jsx +111 -96
  58. package/src/ardy/client.js +19 -9
  59. package/src/ardy/foot-lock.js +230 -0
  60. package/src/ardy/ground.js +52 -0
  61. package/src/ardy/ik.js +25 -7
  62. package/src/ardy/motion-calibration.js +0 -4
  63. package/src/ardy/npz.js +7 -1
  64. package/src/ardy/physics-refine.js +1 -1
  65. package/src/ardy/physics-review.js +102 -52
  66. package/src/ardy/playback.js +0 -8
  67. package/src/asset-pane.jsx +97 -8
  68. package/src/asset-shelf.js +10 -3
  69. package/src/camera-tutorial.jsx +311 -7
  70. package/src/controls.jsx +91 -8
  71. package/src/dualview.jsx +7 -7
  72. package/src/execution-telemetry.js +97 -0
  73. package/src/facing-marks.js +72 -0
  74. package/src/fal-motion-client.js +80 -0
  75. package/src/fal-motion-studio.jsx +180 -0
  76. package/src/first-shot-handoff.js +44 -0
  77. package/src/hierarchy-panel.jsx +20 -2
  78. package/src/keyframe-pack.js +1 -0
  79. package/src/live-control.js +257 -11
  80. package/src/mesh-graph-clone.js +23 -0
  81. package/src/mesh-sniff.js +82 -0
  82. package/src/model-presets.js +8 -3
  83. package/src/motion-readiness-ui.jsx +67 -0
  84. package/src/motion-readiness.js +29 -0
  85. package/src/motion-resources.js +177 -0
  86. package/src/motion-store.js +65 -0
  87. package/src/multimodel-ingest.js +9 -0
  88. package/src/offscreen-export.js +125 -59
  89. package/src/part-colours.js +25 -15
  90. package/src/posestudio.jsx +2 -1
  91. package/src/project-resources.js +286 -0
  92. package/src/project.js +219 -32
  93. package/src/props.jsx +99 -2
  94. package/src/render-passes.js +17 -5
  95. package/src/resource-status.jsx +174 -0
  96. package/src/result-modal.jsx +23 -7
  97. package/src/scene-asset-cache.js +19 -2
  98. package/src/scene-assets.js +73 -12
  99. package/src/scene-fbx.js +75 -0
  100. package/src/scene-history.js +8 -4
  101. package/src/scene-mesh-cache.js +135 -0
  102. package/src/scene-mesh.js +390 -0
  103. package/src/scene-objects.js +146 -41
  104. package/src/scenes.js +55 -21
  105. package/src/semantic-edit.js +98 -0
  106. package/src/settings-menu.jsx +29 -1
  107. package/src/shot-authoring.js +4 -1
  108. package/src/shot.js +4 -0
  109. package/src/studio-agent-commands.js +561 -0
  110. package/src/studio-agent-context.js +113 -0
  111. package/src/studio-agent-motion.js +440 -0
  112. package/src/studio-agent-protocol.js +508 -0
  113. package/src/studio-elements.js +78 -0
  114. package/src/styles.css +792 -3
  115. package/src/tutorial-analytics.js +86 -0
  116. package/src/use-semantic-state.js +12 -0
  117. package/src/workflow/AgentPanel.jsx +668 -186
  118. package/src/workflow/CozySceneNode.jsx +31 -6
  119. package/src/workflow/WorkflowBuilder.jsx +222 -118
  120. package/src/workflow/agent-client.js +1254 -53
  121. package/src/workflow/agent-panel.css +464 -2
  122. package/src/workflow/attachment-image.js +163 -0
  123. package/src/workflow/canvas-commands.js +16 -5
  124. package/src/workflow/cozy-scene-node.css +14 -3
  125. package/src/workflow/cozy-scene-node.js +22 -0
  126. package/src/workflow/keyframe-pack-request.js +16 -2
  127. package/src/workflow/main.jsx +3 -0
  128. package/src/workflow/node-schema.js +1 -1
  129. package/src/workflow/scene-asset-sync.js +2 -2
  130. package/src/workflow/video-contract.js +25 -0
  131. package/src/workflow/workflow-resources.js +275 -0
  132. package/src/workflow/workflow.css +2 -0
  133. package/tools/ardy/README.md +13 -0
  134. package/tools/ardy/__pycache__/cclay_gvhmr_worker.cpython-313.pyc +0 -0
  135. package/tools/ardy/bridge.mjs +28 -47
  136. package/tools/ardy/bvh-cskel27.mjs +0 -2
  137. package/tools/ardy/cclay_gvhmr_worker.py +7 -1
  138. package/tools/ardy/extract.mjs +6 -2
  139. package/tools/ardy/motion-stabilize.mjs +109 -4
  140. package/tools/ardy/runners/gvhmr-worker.mjs +11 -2
  141. package/tools/dev/pages/privacy.html +57 -3
  142. package/tools/dev-full.mjs +5 -2
  143. package/tools/kimodo/edit.mjs +1 -6
  144. package/tools/kimodo/generate.mjs +84 -35
  145. package/tools/kimodo/local-output.mjs +265 -0
  146. package/tools/kimodo/mlx-generate.py +111 -0
  147. package/tools/kimodo/pose-constraints.mjs +0 -2
  148. package/tools/kimodo/runner.mjs +46 -8
  149. package/tools/kimodo/setup-on-box.sh +288 -32
  150. package/tools/run-tests.mjs +72 -4
  151. package/dist/assets/analytics-Dc9ooyn1.js +0 -1
  152. package/dist/assets/app-CHc6SS4r.js +0 -4859
  153. package/dist/assets/app-q2B32RKM.css +0 -1
  154. package/dist/assets/shot-prompt-CuqtYxB7.css +0 -1
  155. package/dist/assets/shot-prompt-F2C_YN3J.js +0 -15
  156. package/dist/assets/workflow-BHvP2fxY.js +0 -179
@@ -1,8 +1,14 @@
1
1
  import { AsyncLocalStorage } from "node:async_hooks";
2
+ import { randomBytes, randomUUID } from "node:crypto";
3
+ import { basename } from "node:path";
2
4
  import * as defaultAuth from "../codex-auth.mjs";
5
+ import { publishLiveEndpoint, removeLiveEndpoint } from "../live-endpoint.mjs";
3
6
  import { createCodexClient } from "./codex-client.mjs";
4
- import { createAgentTools, agentToolSchemas, SYSTEM_PROMPT, pickWorkspace } from "./agent-tools.mjs";
5
- import { createVideoAdapters } from "./video-adapters.mjs";
7
+ import { createAgentTools, SYSTEM_PROMPT, pickWorkspace } from "./agent-tools.mjs";
8
+
9
+ import { createVideoAdapters, validateFalVideoRequest } from "./video-adapters.mjs";
10
+ import { createSessionStore, transcriptFromHistory } from "./session-store.mjs";
11
+ import { createAgentRunner } from "./agent-runner.mjs";
6
12
 
7
13
  // Values the codex backend accepts for reasoning.effort (its own 400 lists them).
8
14
  export const REASONING_EFFORTS = ["none", "minimal", "low", "medium", "high", "xhigh", "max", "ultra"];
@@ -26,6 +32,91 @@ const IMAGE_BODY_LIMIT = 24 * 1024 * 1024;
26
32
  // backend has to read, and a shot with seven of them is a prompt nobody wrote.
27
33
  const IMAGE_REFERENCES_MAX = 6;
28
34
 
35
+ // Pictures pasted or dropped into the composer (#367) ride in the turn body,
36
+ // so the chat routes need room for them. Everything else about the small chat
37
+ // limit stays: this is exactly four attachments of the size the turn envelope
38
+ // admits, plus the envelope itself.
39
+ const ATTACHMENTS_MAX = 4;
40
+ const ATTACHMENT_MAX_CHARS = 6_000_000;
41
+ const TURN_BODY_LIMIT = ATTACHMENTS_MAX * ATTACHMENT_MAX_CHARS + 64 * 1024;
42
+
43
+ /** Reject anything that is not a short list of inline {dataUrl, name?} images.
44
+ * The Studio envelope validates its own copy; this is the legacy body's. */
45
+ function validAttachments(attachments) {
46
+ if (attachments === undefined) return true;
47
+ if (!Array.isArray(attachments) || !attachments.length || attachments.length > ATTACHMENTS_MAX) return false;
48
+ return attachments.every((entry) => entry && typeof entry === "object" && !Array.isArray(entry)
49
+ && typeof entry.dataUrl === "string" && entry.dataUrl.length <= ATTACHMENT_MAX_CHARS && /^data:image\/(png|jpeg|webp);base64,/.test(entry.dataUrl)
50
+ && (entry.name === undefined || (typeof entry.name === "string" && entry.name.length <= 120)));
51
+ }
52
+
53
+ // Keep this local relay self-contained: minimal sidecar installs omit src/.
54
+ const advisory = (read, fallback) => { try { return read(); } catch { return fallback; } };
55
+ const telemetryId = () => advisory(() => randomBytes(16).toString("hex"), null);
56
+ const telemetryNow = () => advisory(() => { const value = performance.now(); return Number.isFinite(value) ? value : NaN; }, NaN);
57
+ const bucketMs = (ms) => !Number.isFinite(ms) || ms < 1000 ? "lt1s" : ms < 3000 ? "1-3s" : ms < 10000 ? "3-10s" : ms < 30000 ? "10-30s" : "gte30s";
58
+ const agentToolCategory = (name) => {
59
+ // Studio families are authored edits or reads; the analytics vocabulary has
60
+ // no Studio-specific category, so a write says so and everything else does not.
61
+ if (["arrange_objects", "arrange_characters", "patch_elements", "frame_shot", "undo_edit"].includes(name)) return "scene_write";
62
+ if (["inspect_studio", "operate_studio", "verify_result", "generate_motion"].includes(name)) return "other";
63
+ if (name === "run_workflow") return "workflow_run";
64
+ if (name === "describe_workflow" || name === "focus_workflow_node") return "workflow_read";
65
+ if (["add_workflow_node", "update_workflow_node", "remove_workflow_node", "connect_workflow_nodes", "disconnect_workflow_nodes", "set_workflow_node_output"].includes(name)) return "workflow_write";
66
+ if (name === "capture_blocking_frame") return "frame_capture";
67
+ if (name === "render_from_frame") return "image_generate";
68
+ if (name === "place_image_in_scene" || name === "add_reference_node") return "scene_write";
69
+ return "other";
70
+ };
71
+ const agentFailureCode = (error, signal, tool = false) => advisory(() => {
72
+ if (signal.aborted || error?.name === "AbortError") return "aborted";
73
+ if (error?.status === 401 || error?.code === "unauthorized") return "auth";
74
+ if (error?.status === 429) return "rate_limited";
75
+ return tool ? "tool_failed" : "upstream";
76
+ }, tool ? "tool_failed" : "upstream");
77
+ // These existing canvas commands only return a node/edge after publishing a
78
+ // new insertion. Read/focus, generic accepted responses, update no-ops and run
79
+ // outputs (which can echo old values) are deliberately not application proof.
80
+ const appliedCanvasResult = (name, result) => {
81
+ if (name === "add_workflow_node" || name === "add_reference_node") return typeof result?.node?.id === "string" && result.node.id.length > 0;
82
+ if (name === "connect_workflow_nodes") return typeof result?.edge?.id === "string" && result.edge.id.length > 0;
83
+ return false;
84
+ };
85
+
86
+ /**
87
+ * One execution-telemetry emitter, shared by both surfaces. The turn that owns
88
+ * the correlation id owns its frames: a turn without one stays unobserved
89
+ * rather than emitting frames nothing can correlate.
90
+ */
91
+ function createTurnTelemetry(send, turnId) {
92
+ const turnStartedAt = telemetryNow();
93
+ let applied = false;
94
+ const emit = (event, props, toolId) => advisory(() => {
95
+ if (turnId) send({ type: "execution_telemetry", event, props, ...(toolId ? { telemetry_id: toolId } : {}) });
96
+ });
97
+ return {
98
+ /** One tool execution: the started frame now, its outcome when it ends. */
99
+ toolStarted(category) {
100
+ const startedAt = telemetryNow();
101
+ const toolId = telemetryId();
102
+ advisory(() => { if (turnId && toolId) send({ type: "execution_tool_started", turn_id: turnId, telemetry_id: toolId, tool_category: category }); });
103
+ return {
104
+ elapsedMs: () => Math.round(telemetryNow() - startedAt),
105
+ executed: (outcome) => {
106
+ if (toolId) emit("agent:tool_executed", { turn_id: turnId, tool_category: category, outcome, duration_bucket: bucketMs(telemetryNow() - startedAt) }, toolId);
107
+ },
108
+ };
109
+ },
110
+ /** The first proof that the scene actually changed, once per turn. */
111
+ applied: () => advisory(() => { if (!applied) { applied = true; emit("agent:result_applied", { turn_id: turnId }); } }),
112
+ finished: (outcome, failureCode) => emit(`agent:turn_${outcome}`, {
113
+ turn_id: turnId,
114
+ duration_bucket: bucketMs(telemetryNow() - turnStartedAt),
115
+ ...(outcome !== "succeeded" ? { failure_code: failureCode } : {}),
116
+ }),
117
+ };
118
+ }
119
+
29
120
  /** Reject anything that is not a list of {role, name?, dataUrl} inline images. */
30
121
  function validReferences(references) {
31
122
  if (references === undefined) return true;
@@ -77,14 +168,43 @@ function quotaEvent(codex, headers) {
77
168
  };
78
169
  }
79
170
 
171
+ // Backend errors may echo credentials or image inputs, so their bodies are
172
+ // never forwarded. Only a structured message survives, redacted and clipped to
173
+ // one line — without it neither the log nor the panel can say why a turn died.
174
+ const UPSTREAM_DETAIL_MAX = 200;
175
+ export function sanitizeUpstreamDetail(body) {
176
+ let message = null;
177
+ try {
178
+ const parsed = JSON.parse(body);
179
+ const candidate = parsed?.error?.message ?? parsed?.error ?? parsed?.message ?? parsed?.detail;
180
+ message = typeof candidate === "string" ? candidate : null;
181
+ } catch { return null; }
182
+ if (!message) return null;
183
+ const cleaned = message
184
+ .replace(/data:[^\s"']+/gi, "[image]")
185
+ .replace(/\b(?:AIza[\w-]{20,}|(?:sk-or-|sk-ant-|sk-|Bearer\s+|eyJ)[\w.\-+/=]+)/gi, "[redacted]")
186
+ .replace(/\s+/g, " ")
187
+ .trim();
188
+ if (!cleaned) return null;
189
+ return cleaned.length > UPSTREAM_DETAIL_MAX ? `${cleaned.slice(0, UPSTREAM_DETAIL_MAX - 1)}\u2026` : cleaned;
190
+ }
191
+
80
192
  function errorInfo(error, quota) {
81
- if (error?.status === 401 || error?.code === "unauthorized") return { code: "auth", message: "Authentication required. Sign in again." };
82
- if (error?.status === 429) return { code: "rate_limit", message: "Rate limit exceeded.", resetAt: quota?.primary.resetAt ?? null };
83
- if (error?.code === "entitlement") return { code: "entitlement", message: "This account cannot generate images." };
84
- if (error?.code === "overloaded") return { code: "overloaded", message: "The model service is overloaded right now. Try again in a moment." };
85
- if (error?.code === "server_error") return { code: "overloaded", message: "The model service hit an internal error. Try again in a moment." };
86
- // Backend errors may echo credentials or image inputs. Never forward their bodies.
87
- return { code: "upstream", message: "The model or live editor could not complete this turn." };
193
+ const status = Number.isInteger(error?.status) ? error.status : null;
194
+ const detail = typeof error?.detail === "string" && error.detail ? error.detail : null;
195
+ // The status and the backend's own sanitized words are the only things that
196
+ // can tell an author WHY the turn died; a generic sentence cannot.
197
+ const explain = (message) => ({
198
+ message: status === null ? message : `${status} \u2014 ${detail || message}`,
199
+ ...(status === null ? {} : { status }),
200
+ ...(detail ? { detail } : {}),
201
+ });
202
+ if (error?.status === 401 || error?.code === "unauthorized") return { code: "auth", ...explain("Authentication required. Sign in again.") };
203
+ if (error?.status === 429) return { code: "rate_limit", ...explain("Rate limit exceeded."), resetAt: quota?.primary.resetAt ?? null };
204
+ if (error?.code === "entitlement") return { code: "entitlement", ...explain("This account cannot generate images.") };
205
+ if (error?.code === "overloaded") return { code: "overloaded", ...explain("The model service is overloaded right now. Try again in a moment.") };
206
+ if (error?.code === "server_error") return { code: "overloaded", ...explain("The model service hit an internal error. Try again in a moment.") };
207
+ return { code: "upstream", ...explain("The model or live editor could not complete this turn.") };
88
208
  }
89
209
 
90
210
  /** Use the existing client and its request queue, retaining failure headers that
@@ -98,18 +218,25 @@ function defaultClient(auth, requestContext) {
98
218
  requestContext.getStore()?.(response.headers);
99
219
  if (response.ok) return response;
100
220
  const detail = await response.text();
101
- const error = Object.assign(new Error("Codex backend request failed."), { status: response.status, headers: response.headers });
221
+ const error = Object.assign(new Error("Codex backend request failed."), { status: response.status, headers: response.headers, detail: sanitizeUpstreamDetail(detail) });
102
222
  if (url.includes("/images/") && /entitlement|plan/i.test(detail)) error.code = "entitlement";
103
223
  throw error;
104
224
  },
105
225
  });
106
226
  }
107
227
 
228
+ // The launcher and the dev runner both own a hub of their own; the endpoint
229
+ // file names which one a controller has reached.
230
+ const liveHubOwner = () => process.env.COZYCLAY_LIVE_OWNER
231
+ || (basename(process.argv[1] ?? "") === "dev-full.mjs" ? "dev-full" : "cozyclay");
232
+
108
233
  /** Start the optional registry/live dependencies without making signed-out
109
234
  * startup depend on an MCP dependency install. Failures remain visible on use. */
110
235
  function liveToolsRuntime() {
111
236
  return Promise.all([import("../../mcp/tool-handlers.mjs"), import("../../mcp/live-hub.mjs")]).then(async ([registry, { startLiveHub }]) => {
112
- const liveHub = await startLiveHub(Number(process.env.COZYCLAY_LIVE_PORT ?? 5184));
237
+ const owner = liveHubOwner();
238
+ const token = randomBytes(32).toString("hex");
239
+ const liveHub = await startLiveHub(Number(process.env.COZYCLAY_LIVE_PORT ?? 5184), { token, owner });
113
240
  registry.setLiveHub(liveHub);
114
241
  const handlers = registry.createToolHandlers().map((tool) => ({
115
242
  ...tool,
@@ -119,11 +246,22 @@ function liveToolsRuntime() {
119
246
  return liveHub?.connected ? liveHub.runExclusive(tool.name, workspaceHandle, run) : run(workspaceHandle);
120
247
  },
121
248
  }));
249
+ if (liveHub) {
250
+ // A controller runs the same per-workspace wrapper the panel's own tools
251
+ // run, so registry state stays serialized across both surfaces.
252
+ liveHub.serveTool = (name, args, workspaceHandle) => {
253
+ const tool = handlers.find((entry) => entry.name === name);
254
+ if (!tool) throw Object.assign(new Error(`Unknown live tool "${name}".`), { code: "UNKNOWN_TOOL" });
255
+ return tool.handler(args ?? {}, { workspaceHandle });
256
+ };
257
+ publishLiveEndpoint({ port: liveHub.port, token, owner });
258
+ liveHub.server?.once("close", () => removeLiveEndpoint(liveHub.port));
259
+ }
122
260
  return { liveHub, handlers };
123
261
  }).catch((error) => ({ error }));
124
262
  }
125
263
 
126
- export function createAgentHandler({ auth = defaultAuth, codex, handlers, liveHub, port, retryDelayMs = 2000 } = {}) {
264
+ export function createAgentHandler({ auth = defaultAuth, codex, models, codexBaseUrl, cliproxyBaseUrl, env, fauxProvider, handlers, liveHub, port, getBridgeOrigin, studioRuntime, clock = Date.now, setIntervalImpl = setInterval, clearIntervalImpl = clearInterval, sessionStore: injectedSessionStore } = {}) {
127
265
  const requestContext = new AsyncLocalStorage();
128
266
  codex ||= defaultClient(auth, requestContext);
129
267
  const runtime = handlers !== undefined || liveHub !== undefined ? Promise.resolve({ handlers: handlers ?? [], liveHub }) : liveToolsRuntime();
@@ -140,26 +278,380 @@ export function createAgentHandler({ auth = defaultAuth, codex, handlers, liveHu
140
278
  } catch { return ""; }
141
279
  };
142
280
  const sessions = new Map();
143
- const unsubscribe = auth.onAuthChange?.(() => {
281
+ const workflowRunners = new Map();
282
+ const studioRunners = new Map();
283
+ let workflowModels = models;
284
+ let workflowModelsPromise = models ? Promise.resolve(models) : null;
285
+ const ensureWorkflowModels = async () => {
286
+ const pending = workflowModelsPromise ??= import("./providers.mjs").then(({ createModels }) => createModels({ auth, codexBaseUrl, cliproxyBaseUrl, env }));
287
+ try { return await pending; }
288
+ catch (error) {
289
+ // A retired build must not clear a newer identity's initialization.
290
+ if (workflowModelsPromise === pending) workflowModelsPromise = null;
291
+ throw error;
292
+ }
293
+ };
294
+ const hasAnyCredential = async () => {
295
+ try { if (await auth.getAccessToken()) return true; } catch { /* an unavailable ChatGPT token is not a credential */ }
296
+ const registry = await ensureWorkflowModels();
297
+ if (typeof registry?.getAuth !== "function") return false;
298
+ const { PROVIDERS } = await import("./providers.mjs");
299
+ for (const provider of PROVIDERS) {
300
+ try { if (await registry.getAuth(provider.id)) return true; } catch { /* one broken provider does not break the gate */ }
301
+ }
302
+ return false;
303
+ };
304
+ // Tests hand in a store of their own; only the real sidecar writes the
305
+ // author's config dir (#375).
306
+ const sessionStore = injectedSessionStore ?? createSessionStore();
307
+ const studioSessions = new Map();
308
+ const studioEvents = new Map();
309
+ let ownedStudioRuntime = studioRuntime || null;
310
+ const studioOwnerTokens = new Map();
311
+ const parseCookies = req => Object.fromEntries(String(req.headers.cookie || "").split(";").map(part => part.trim().split("=")).filter(([key, value]) => key && value).map(([key, value]) => [key, decodeURIComponent(value)]));
312
+ const pruneStudioSessions = () => {
313
+ const now = clock();
314
+ for (const [id, session] of studioSessions) if (!session.activeJobId && now - session.updatedAt > 600_000) { studioSessions.delete(id); studioOwnerTokens.delete(id); for (const turn of session.turns.keys()) studioEvents.delete(turn); }
315
+ const retired = [...studioSessions.entries()].filter(([, session]) => !session.activeJobId).sort((a, b) => a[1].updatedAt - b[1].updatedAt);
316
+ while (studioSessions.size > 256 && retired.length) { const [id, session] = retired.shift(); studioSessions.delete(id); studioOwnerTokens.delete(id); for (const turn of session.turns.keys()) studioEvents.delete(turn); }
317
+ };
318
+ const studioOwner = (req, sessionId, create = false) => {
319
+ pruneStudioSessions();
320
+ const cookies = parseCookies(req), supplied = cookies.studio_owner;
321
+ let owner = studioOwnerTokens.get(sessionId);
322
+ if (!owner && create) { owner = randomBytes(24).toString("hex"); studioOwnerTokens.set(sessionId, owner); return owner; }
323
+ if (!owner || supplied !== owner) throw Object.assign(new Error("Studio session owner mismatch."), { code: "AUTH_REQUIRED" });
324
+ return owner;
325
+ };
326
+ const studioRuntimeFor = async hub => {
327
+ if (ownedStudioRuntime) return ownedStudioRuntime;
328
+ const { createStudioMotionRuntime } = await import("./motion-runtime.mjs");
329
+ if (!hub) return null;
330
+ ownedStudioRuntime = createStudioMotionRuntime({ liveHub: hub, getBridgeOrigin, clock });
331
+ return ownedStudioRuntime;
332
+ };
333
+ const emitStudioEvent = (turnId, event) => {
334
+ const record = studioEvents.get(turnId) || { next: 0, events: [], listeners: new Set(), terminal: false };
335
+ // Execution telemetry is advisory and is validated key-by-key in the
336
+ // browser: it carries no replay cursor, is never retained for a resume,
337
+ // and a reconnect simply misses the frames it was not there for.
338
+ if (["execution_telemetry", "execution_tool_started"].includes(event.type)) {
339
+ studioEvents.set(turnId, record);
340
+ for (const listener of [...record.listeners]) listener(event);
341
+ return;
342
+ }
343
+ const value = { ...event, eventSeq: ++record.next };
344
+ record.next = value.eventSeq;
345
+ const previous = record.events.at(-1);
346
+ if (value.type === "job.progress" && previous?.type === "job.progress" && previous.jobId === value.jobId) record.events[record.events.length - 1] = value;
347
+ else record.events.push(value);
348
+ if (record.events.length > 256) record.events.splice(0, record.events.length - 256);
349
+ if (["done", "error", "receipt"].includes(value.type)) record.terminal = true;
350
+ studioEvents.set(turnId, record); for (const listener of [...record.listeners]) listener(value);
351
+ };
352
+ const unsubscribe = auth.onAuthChange?.(({ kind }) => {
353
+ if (kind !== "signed_out" && kind !== "replaced") return;
144
354
  for (const session of sessions.values()) session.controller?.abort();
145
- sessions.clear();
355
+ for (const session of studioSessions.values()) session.controller?.abort();
356
+ void ownedStudioRuntime?.dispose?.(); ownedStudioRuntime = studioRuntime || null;
357
+ for (const runner of [...workflowRunners.values(), ...studioRunners.values()]) void runner.close?.();
358
+ workflowRunners.clear(); studioRunners.clear();
359
+ workflowModels = models;
360
+ workflowModelsPromise = models ? Promise.resolve(models) : null;
361
+ sessions.clear(); studioSessions.clear(); studioEvents.clear(); studioOwnerTokens.clear();
146
362
  });
147
363
 
364
+ const studioIdentity = host => Object.fromEntries(["workspaceId", "documentEpoch", "sceneId", "sceneEpoch"].map(key => [key, host[key]]));
365
+ const writeStudioStream = (res, record, after = 0, req = null) => {
366
+ res.writeHead(200, { "content-type": "text/event-stream; charset=utf-8", "cache-control": "no-cache", connection: "keep-alive" }); res.flushHeaders?.();
367
+ let cursor = after; const send = event => {
368
+ if (res.destroyed) return;
369
+ if (event.eventSeq === undefined) { res.write(`data: ${JSON.stringify(event)}\n\n`); return; }
370
+ if (event.eventSeq > cursor) { cursor = event.eventSeq; res.write(`data: ${JSON.stringify(event)}\n\n`); }
371
+ };
372
+ const listener = event => { send(event); if (event.type === "done" && !res.writableEnded) res.end(); }; record.listeners.add(listener); for (const event of record.events) send(event);
373
+ if (record.terminal) { record.listeners.delete(listener); res.end(); return () => {}; }
374
+ const heartbeat = setIntervalImpl(() => { clock(); if (!res.writableEnded && !res.destroyed) res.write(": heartbeat\n\n"); }, 15_000);
375
+ const close = () => { clearIntervalImpl(heartbeat); record.listeners.delete(listener); };
376
+ res.once("close", close); return close;
377
+ };
378
+ const authoritativeStudioContext = async (value, hub) => {
379
+ if (!hub?.workspaceId || !hub?.command) throw Object.assign(new Error("A connected editor is required."), { code: "LIVE_HUB_UNAVAILABLE" });
380
+ let actualHandle;
381
+ try { actualHandle = hub.resolveWorkspace("studio context", value.context.host.workspaceHandle); }
382
+ catch { throw Object.assign(new Error("The submitted Studio handle is stale or unknown."), { code: "LIVE_HUB_UNAVAILABLE" }); }
383
+ if (hub.workspaceId(actualHandle) !== value.context.host.workspaceId) throw Object.assign(new Error("Studio handle belongs to a different workspace."), { code: "STALE_SCENE" });
384
+ const result = await hub.command("read_studio_context", { host: studioIdentity(value.context.host) }, actualHandle);
385
+ return result?.context ?? result;
386
+ };
387
+ const handleStudioTurn = async (req, res, value, path) => {
388
+ const { StudioProtocolError, validateStudioContextFreshness } = await import("../../src/studio-agent-protocol.js");
389
+ const [{ createStudioTools }, { encodeStudioContext }] = await Promise.all([
390
+ import("./studio-tools.mjs"), import("../../src/studio-agent-context.js"),
391
+ ]);
392
+ const hubDeps = await runtime; const hub = hubDeps.liveHub || liveHub;
393
+ if (path === "/agent/stop") {
394
+ const session = studioSessions.get(value.sessionId);
395
+ if (!session || session.owner !== parseCookies(req).studio_owner || !session.turns.has(value.turnId)) throw new StudioProtocolError("AUTH_REQUIRED", "Studio stop is not owned by this session.");
396
+ const jobId = value.jobId ?? session.activeJobId;
397
+ // Explicit ids are checked against every motion job this session admitted.
398
+ // Retired ids remain in the set so a repeat Stop is idempotent, while an id
399
+ // admitted by another session is stale even when this session is idle.
400
+ if (value.jobId && !session.motionJobIds.has(value.jobId)) throw new StudioProtocolError("STALE_TARGET", "Stop does not own that motion job.");
401
+ // An owned retired id is still stale while another motion job is active:
402
+ // this Stop cannot acknowledge the current turn's different job.
403
+ if (value.jobId && session.activeJobId && value.jobId !== session.activeJobId) throw new StudioProtocolError("STALE_TARGET", "Stop does not target the active motion job.");
404
+ // #379 / 16r: a job's id is only "acknowledged" by THIS turn's held motion
405
+ // tool — session.activeJobId now stays set only while that job is genuinely
406
+ // still in flight (or pending an explicit accept), and activeJobTurnId ties
407
+ // it to the turn that admitted it. A retired job from an earlier turn must
408
+ // never make a later, unrelated turn's Stop go quiet.
409
+ const acknowledged = Boolean(session.activeJobId) && session.activeJobTurnId === value.turnId;
410
+ // The runtime is the only thing that knows whether the job was applied.
411
+ // Forwarding its outcome keeps the panel from turning "I could not find
412
+ // out" into "nothing was applied"; a discarded outcome reads as proof.
413
+ let outcome = null;
414
+ if (jobId && ownedStudioRuntime?.stop) outcome = await ownedStudioRuntime.stop(jobId);
415
+ if (jobId && session.activeJobId === jobId) { session.activeJobId = null; session.activeJobTurnId = null; }
416
+ // #379 / 16q: `session.controller.signal` is the SAME signal wired into the
417
+ // active turn's prompt context (`:520 signal: controller.signal`), which
418
+ // pi's `withAbortSignal`/`awaitWithContext` races against every awaited
419
+ // internal step — aborting it here interrupts the in-flight generate_motion
420
+ // tool call's OWN completion machinery before it can settle, well before
421
+ // pi's own graceful `lane.abort()` (below) ever runs. A job the runtime
422
+ // actually acknowledged already resolved the held tool call with its real
423
+ // outcome via `runtime.stop()` above; the only abort this stop still needs
424
+ // is the graceful one that stops the model from being prompted again, so
425
+ // `session.controller` must NOT be touched on that path (no other tool is
426
+ // concurrently in flight while generate_motion holds the turn). A stop
427
+ // for a job this turn never acknowledged (none active, or a retired one
428
+ // from an earlier turn) never resolved anything for THIS turn and stays a
429
+ // plain (loud) abort on both signals.
430
+ if (!acknowledged) session.controller?.abort();
431
+ await session.modelSession?.abort?.("studio stop", acknowledged ? { quiet: true } : undefined);
432
+ json(res, 200, { ok: true, status: jobId ? "stopped" : "detached", ...(outcome ? { outcome: { status: outcome.status ?? null, code: outcome.code ?? null, mutated: outcome.mutated ?? null } } : {}) }); return true;
433
+ }
434
+ if (!studioRuntime && (!hub?.command || !hub?.workspaceId)) throw new StudioProtocolError("CAPABILITY_MISSING", "Studio execution is not installed.");
435
+ if (!value.context.host.workspaceHandle) throw new StudioProtocolError("LIVE_HUB_UNAVAILABLE", "A connected editor handle is required.");
436
+ let session = studioSessions.get(value.sessionId);
437
+ const suppliedOwner = parseCookies(req).studio_owner;
438
+ if (session && suppliedOwner && session.owner !== suppliedOwner) throw new StudioProtocolError("AUTH_REQUIRED", "Studio session owner mismatch.");
439
+ if (!session) {
440
+ studioOwner(req, value.sessionId, true);
441
+ let persisted = null;
442
+ try { persisted = sessionStore.read(value.sessionId); } catch { persisted = null; }
443
+ session = { owner: studioOwnerTokens.get(value.sessionId), history: persisted?.history ?? [], persistedItems: persisted?.history?.length ?? 0, meta: persisted?.meta ?? null, turns: new Map(), controller: null, activeJobId: null, activeJobTurnId: null, motionJobIds: new Set(Array.isArray(persisted?.meta?.motionJobIds) ? persisted.meta.motionJobIds.filter(id => typeof id === "string") : []), generationPrompt: null, host: null, updatedAt: clock() };
444
+ studioSessions.set(value.sessionId, session);
445
+ }
446
+ session.updatedAt = clock();
447
+ const existing = session.turns.get(value.turnId);
448
+ if (existing) { writeStudioStream(res, existing, 0, req); return true; }
449
+ let current;
450
+ try { current = studioRuntime?.readContext ? await studioRuntime.readContext(value.context.host) : await authoritativeStudioContext(value, hub); }
451
+ catch (error) { if (error instanceof StudioProtocolError) throw error; if (error?.code) throw new StudioProtocolError(error.code, error.message); throw new StudioProtocolError("CAPABILITY_MISSING", "The connected editor does not expose authoritative Studio context."); }
452
+ validateStudioContextFreshness(value.context, current);
453
+ if (studioRuntime?.handleTurn) { await studioRuntime.handleTurn(value, req, res); return true; }
454
+ const record = { next: 0, events: [], listeners: new Set(), terminal: false }; studioEvents.set(value.turnId, record); session.turns.set(value.turnId, record); session.host = studioIdentity(value.context.host);
455
+ res.setHeader("set-cookie", `studio_owner=${encodeURIComponent(session.owner)}; Path=/agent; HttpOnly; SameSite=Strict`);
456
+ const close = writeStudioStream(res, record, 0, req);
457
+ const send = event => emitStudioEvent(value.turnId, event);
458
+ // The Studio turn is measured exactly like the Workflow turn, correlated by
459
+ // the turn id the host already owns (the frozen envelope carries no room
460
+ // for a second one).
461
+ const telemetry = createTurnTelemetry(send, value.turnId);
462
+ let turnOutcome = "succeeded", turnFailureCode = null, toolFailed = false;
463
+ const controller = new AbortController(); session.controller = controller;
464
+ // Admission is the document identity plus the exact scene revision each
465
+ // command expects. Per-entity tokens are deliberately absent: the revision
466
+ // already moves whenever any authored content changes, and a turn's second
467
+ // edit to the same entity would otherwise carry a token the first edit retired.
468
+ const admission = {
469
+ commandId: () => randomUUID(), host: studioIdentity(value.context.host), revision: value.context.revision.scene,
470
+ refresh: async () => {
471
+ const refreshed = studioRuntime?.readContext ? await studioRuntime.readContext(value.context.host) : await authoritativeStudioContext(value, hub);
472
+ admission.revision = refreshed.revision.scene;
473
+ },
474
+ };
475
+ const runtimeForJob = await studioRuntimeFor(hub);
476
+ const tools = createStudioTools({ liveHub: hub, workspaceHandle: value.context.host.workspaceHandle, session: { signal: controller.signal, admission }, resolveImage: async (id, correlation) => hub.command("resolve_studio_image", { imageId: id, ...correlation }, value.context.host.workspaceHandle) });
477
+ const motion = async args => {
478
+ if (session.generationPrompt === value.text) throw new StudioProtocolError("AUTH_REQUIRED", "This generation request already has a retained result; start a new explicit request.");
479
+ if (!runtimeForJob) throw new StudioProtocolError("CAPABILITY_MISSING", "Studio motion runtime is unavailable.");
480
+ const character = value.context.entities.find(entity => entity.id === args.characterId && entity.kind === "character");
481
+ if (!character) throw new StudioProtocolError("TARGET_NOT_READY", "The admitted character is unavailable.");
482
+ const commandId = randomUUID(); const host = { ...value.context.host, workspaceHandle: value.context.host.workspaceHandle };
483
+ const admissionResult = runtimeForJob.admit({ hostBinding: host, characterId: args.characterId, targetToken: character.token, turnId: value.turnId, commandId, authorization: { id: randomUUID(), generations: 1 }, source: args.source, repair: args.repair ?? "bounded" });
484
+ session.motionJobIds.add(admissionResult.jobId); persistenceMeta.motionJobIds = [...session.motionJobIds]; session.activeJobId = admissionResult.jobId; session.activeJobTurnId = value.turnId; session.generationPrompt = value.text;
485
+ const unsubscribe = runtimeForJob.subscribe(admissionResult.jobId, event => send({ ...event, sourceEventSeq: event.eventSeq }));
486
+ // Subscription precedes start, including replay of the queued admission event.
487
+ let outcome;
488
+ try { outcome = await runtimeForJob.start(admissionResult.jobId); if (outcome?.ok && outcome.status === "installed") send({ type: "receipt", receipt: outcome }); return outcome; }
489
+ finally {
490
+ unsubscribe();
491
+ // A job still awaiting an explicit accept (review_required) stays "active"
492
+ // for the accept route's ownership check (:678); every other outcome —
493
+ // installed, cancelled, proved-not-applied, failed, or a thrown error —
494
+ // is terminal for this session and must not leak into a later turn's Stop.
495
+ if (session.activeJobId === admissionResult.jobId && outcome?.status !== "review_required") { session.activeJobId = null; session.activeJobTurnId = null; }
496
+ }
497
+ };
498
+ const modelTools = tools.map(tool => ({
499
+ ...tool,
500
+ handler: async args => {
501
+ const result = await (tool.name === "generate_motion" ? motion(args) : tool.handler(args));
502
+ if (!result || !Array.isArray(result.visualRefs) || !result.visualRefs.length) return result;
503
+ const ref = result.visualRefs.find(item => item?.imageId || item?.id);
504
+ if (!ref) return result;
505
+ const visual = await tools.resolveImage(ref.imageId || ref.id, { receiptId: result.receiptId, revision: result.revision });
506
+ return { ...result, visualStatus: visual.visualStatus, imageId: visual.imageId, revision: visual.revision, receiptId: visual.receiptId, ...(visual.dataUrl ? { dataUrl: visual.dataUrl } : {}) };
507
+ },
508
+ }));
509
+ let frameObservation;
510
+ if (value.attachFrame) {
511
+ const captured = await hub.command("capture_framing_png", {}, value.context.host.workspaceHandle);
512
+ if (!captured?.dataUrl?.startsWith("data:image/")) throw new StudioProtocolError("TARGET_NOT_READY", "The current frame has no image bytes.");
513
+ const match = /^data:([^;]+);base64,(.*)$/.exec(captured.dataUrl);
514
+ frameObservation = { data: match[2], mimeType: match[1], revision: captured.revision, receiptId: captured.receiptId };
515
+ }
516
+ const telemetryTools = new Map();
517
+ const emitFrame = frame => {
518
+ if (frame.type === "tool.start") {
519
+ telemetryTools.set(frame.callId, { execution: telemetry.toolStarted(agentToolCategory(frame.name)), name: frame.name });
520
+ return send(frame);
521
+ }
522
+ if (frame.type === "tool.done") {
523
+ const record = telemetryTools.get(frame.callId);
524
+ if (record && frame.ok && frame.result?.ok && frame.result?.authored) telemetry.applied();
525
+ if (!frame.ok) {
526
+ toolFailed = true;
527
+ if (typeof frame.error === "string" && frame.error.startsWith("Validation failed for tool ")) frame.error = "INVALID_ARGUMENT: Unexpected field.";
528
+ }
529
+ send(frame);
530
+ if (record) {
531
+ record.execution.executed(frame.ok ? "succeeded" : (controller.signal.aborted ? "cancelled" : "failed"));
532
+ record.finished = true;
533
+ }
534
+ return;
535
+ }
536
+ if (frame.type === "error") {
537
+ turnOutcome = controller.signal.aborted || frame.code === "aborted" ? "cancelled" : "failed";
538
+ turnFailureCode = controller.signal.aborted ? "aborted" : frame.code;
539
+ return send(frame);
540
+ }
541
+ if (frame.type === "done") {
542
+ if (turnOutcome === "succeeded" && !toolFailed) telemetry.finished("succeeded", null);
543
+ else telemetry.finished(turnOutcome, turnFailureCode || "tool_failed");
544
+ }
545
+ send(frame);
546
+ };
547
+ const persistenceMeta = { sceneName: value.context?.scene?.name ?? value.context?.sceneName ?? null, firstText: value.text, motionJobIds: [...session.motionJobIds] };
548
+ let runner = studioRunners.get(value.sessionId);
549
+ if (!runner) {
550
+ runner = createAgentRunner({ models: await ensureWorkflowModels(), fauxProvider, sessionStore, clock, codexBaseUrl, cliproxyBaseUrl, auth, env });
551
+ studioRunners.set(value.sessionId, runner);
552
+ }
553
+ try {
554
+ if (!session.modelSession) session.modelSession = await runner.openSession(value.sessionId, { surface: "studio" });
555
+ for await (const frame of session.modelSession.start({
556
+ surface: "studio", sessionId: value.sessionId, model: value.model || (fauxProvider ? `${fauxProvider.provider?.id || fauxProvider.provider || "faux"}/scripted` : "gpt-6-astra"), effort: value.effort,
557
+ text: value.text, attachments: value.attachments, contextText: encodeStudioContext(value.context), frameObservation,
558
+ context: value.context, tools: modelTools, signal: controller.signal, emit: send,
559
+ meta: persistenceMeta,
560
+ quotaEvent: headers => codex?.parseQuotaHeaders ? quotaEvent(codex, headers) : { type: "quota", plan: null, primary: { usedPercent: null, windowMinutes: null, resetAt: null }, credits: { has: null } },
561
+ })) emitFrame(frame);
562
+ } catch (error) {
563
+ const info = errorInfo(error);
564
+ turnOutcome = controller.signal.aborted ? "cancelled" : "failed";
565
+ turnFailureCode = controller.signal.aborted ? "aborted" : agentFailureCode(error, controller.signal, toolFailed);
566
+ send({ type: "error", code: controller.signal.aborted ? "aborted" : error.code || "upstream", message: info.status === undefined ? error.message : info.message, ...(info.status === undefined ? {} : { status: info.status }) });
567
+ emitFrame({ type: "done" });
568
+ }
569
+ if (controller.signal.aborted) { turnOutcome = "cancelled"; turnFailureCode = "aborted"; }
570
+ if (!record.terminal) emitFrame({ type: "done" });
571
+ close(); if (!res.writableEnded) res.end(); session.controller = null; return true;
572
+ };
148
573
  const handle = async (req, res, path = new URL(req.url, "http://127.0.0.1").pathname) => {
149
574
  if (!path.startsWith("/agent/")) return false;
150
575
  if (port !== undefined && !allowAgentOrigin(req, typeof port === "function" ? port() : port)) {
151
576
  json(res, 403, { error: "forbidden origin" }); return true;
152
577
  }
578
+ if (path.startsWith("/agent/turn/") && path.endsWith("/events") && req.method === "GET") {
579
+ if (!await hasAnyCredential()) { json(res, 401, { error: { code: "AUTH_REQUIRED", message: "Sign in or configure a provider key." } }); return true; }
580
+ const turnId = decodeURIComponent(path.slice("/agent/turn/".length, -"/events".length));
581
+ const session = [...studioSessions.values()].find(candidate => candidate.turns.has(turnId)); const record = studioEvents.get(turnId);
582
+ if (!record || !session || parseCookies(req).studio_owner !== session.owner) { json(res, 403, { error: { code: "AUTH_REQUIRED", message: "Studio event stream is not owned by this session." } }); return true; }
583
+ const after = Number(new URL(req.url, "http://127.0.0.1").searchParams.get("after") || 0);
584
+ if (!Number.isSafeInteger(after) || after < 0) { json(res, 400, { error: "invalid cursor" }); return true; }
585
+ writeStudioStream(res, record, after, req); return true;
586
+ }
587
+ if (path.startsWith("/agent/turn/") && path.endsWith("/steer") && req.method === "POST") {
588
+ const turnId = decodeURIComponent(path.slice("/agent/turn/".length, -"/steer".length));
589
+ let value;
590
+ try {
591
+ value = await readBody(req, TURN_BODY_LIMIT);
592
+ if (!value || typeof value.text !== "string" || !value.text.trim() || !validAttachments(value.attachments)) throw new Error("Invalid request.");
593
+ } catch { json(res, 400, { error: "invalid request" }); return true; }
594
+ // Studio turnIds are frozen envelopes; their events are keyed in studioSessions,
595
+ // never in the Workflow `sessions` map, so that lookup alone tells them apart.
596
+ if ([...studioSessions.values()].some(candidate => candidate.turns.has(turnId))) { json(res, 409, { error: { code: "STEER_UNSUPPORTED", message: "Steering a Studio turn is not supported." } }); return true; }
597
+ const session = [...sessions.values()].find(candidate => candidate.turnId === turnId);
598
+ if (!session) { json(res, 404, { error: "turn not found" }); return true; }
599
+ if (!session.running || !session.modelSession) { json(res, 409, { error: { code: "NO_ACTIVE_TURN", message: "This turn already ended." } }); return true; }
600
+ const images = (Array.isArray(value.attachments) ? value.attachments : []).map(attachment => {
601
+ const match = /^data:([^;]+);base64,(.*)$/.exec(attachment?.dataUrl || "");
602
+ return match ? { type: "image", data: match[2], mimeType: match[1] } : null;
603
+ }).filter(Boolean);
604
+ await session.modelSession.steer(value.text, images);
605
+ json(res, 200, { ok: true, queued: true }); return true;
606
+ }
607
+ if (path === "/agent/providers" && req.method === "GET") {
608
+ const { providerStatus } = await import("./providers.mjs");
609
+ try { json(res, 200, { providers: providerStatus({ auth }) }); }
610
+ catch { json(res, 500, { error: "provider credentials unavailable" }); }
611
+ return true;
612
+ }
613
+ if (/^\/agent\/providers\/[^/]+$/.test(path) && ["PUT", "DELETE"].includes(req.method)) {
614
+ const { PROVIDERS } = await import("./providers.mjs");
615
+ const keys = await import("./provider-keys.mjs");
616
+ const { createCredentialStore } = await import("./credential-store.mjs");
617
+ const id = path.slice("/agent/providers/".length);
618
+ if (!PROVIDERS.some((provider) => provider.id === id)) { json(res, 404, { error: "provider not found" }); return true; }
619
+ const credentials = createCredentialStore({ auth, keys });
620
+ let key;
621
+ if (req.method === "PUT") {
622
+ if (id === "openai-codex") { json(res, 400, { error: "Use ChatGPT sign-in for OpenAI Codex." }); return true; }
623
+ try {
624
+ const value = await readBody(req);
625
+ if (typeof value?.key !== "string" || !value.key.trim()) throw new Error("Invalid key.");
626
+ key = value.key.trim();
627
+ } catch { json(res, 400, { error: "a non-empty API key is required" }); return true; }
628
+ }
629
+ try {
630
+ if (req.method === "PUT") await credentials.modify(id, async () => ({ type: "api_key", key }));
631
+ else await credentials.delete(id);
632
+ json(res, 200, { ok: true });
633
+ } catch { json(res, 500, { error: "provider credentials could not be saved" }); }
634
+ return true;
635
+ }
636
+ if (path === "/agent/sessions" && req.method === "GET") {
637
+ const surface = new URL(req.url, "http://127.0.0.1").searchParams.get("surface") || undefined;
638
+ try { json(res, 200, { sessions: sessionStore.list({ surface }) }); }
639
+ catch { json(res, 500, { error: "session history unavailable" }); }
640
+ return true;
641
+ }
642
+ if (path.startsWith("/agent/sessions/") && req.method === "GET") {
643
+ const sessionId = decodeURIComponent(path.slice("/agent/sessions/".length));
644
+ try {
645
+ const persisted = sessionStore.read(sessionId);
646
+ if (!persisted) { json(res, 404, { error: "session not found" }); return true; }
647
+ json(res, 200, { sessionId, transcript: transcriptFromHistory(persisted.history), meta: persisted.meta });
648
+ } catch { json(res, 404, { error: "session not found" }); }
649
+ return true;
650
+ }
153
651
  if (path === "/agent/models" && req.method === "GET") {
154
652
  try {
155
- const result = await codex.listModels();
156
- const models = (Array.isArray(result) ? result : result.models).map((model) => {
157
- const id = typeof model === "string" ? model : model.slug || model.id;
158
- const efforts = Array.isArray(model.supported_reasoning_levels) ? model.supported_reasoning_levels.map((level) => (typeof level === "string" ? level : level.effort)).filter(Boolean) : [];
159
- return { id, label: id, efforts, defaultEffort: typeof model.default_reasoning_level === "string" ? model.default_reasoning_level : efforts[0] ?? null };
160
- });
161
- models.sort((a, b) => Number(b.id === "gpt-6-astra") - Number(a.id === "gpt-6-astra"));
162
- json(res, 200, { models });
653
+ const { listAgentModels } = await import("./providers.mjs");
654
+ json(res, 200, await listAgentModels({ auth, codex, models: await ensureWorkflowModels(), env }));
163
655
  } catch (error) { json(res, error.status === 401 ? 401 : 502, { error: errorInfo(error) }); }
164
656
  return true;
165
657
  }
@@ -183,7 +675,7 @@ export function createAgentHandler({ auth = defaultAuth, codex, handlers, liveHu
183
675
  return true;
184
676
  }
185
677
  if (path === "/agent/video/providers" && req.method === "GET") {
186
- json(res, 200, { providers: createVideoAdapters().map((adapter) => ({ id: adapter.id, name: adapter.name, configured: adapter.configured() })) });
678
+ json(res, 200, { providers: createVideoAdapters().map((adapter) => ({ id: adapter.id, name: adapter.name, configured: adapter.configured(), ...(adapter.model ? { model: adapter.model, resolution: adapter.resolution } : {}) })) });
187
679
  return true;
188
680
  }
189
681
  if (path === "/agent/video" && req.method === "POST") {
@@ -191,33 +683,55 @@ export function createAgentHandler({ auth = defaultAuth, codex, handlers, liveHu
191
683
  try {
192
684
  value = await readBody(req, IMAGE_BODY_LIMIT);
193
685
  if (!value || typeof value.provider !== "string" || typeof value.prompt !== "string" || !value.prompt.trim() || typeof value.imageDataUrl !== "string" || !value.imageDataUrl.startsWith("data:image/") || (value.lastFrameDataUrl !== undefined && (typeof value.lastFrameDataUrl !== "string" || !value.lastFrameDataUrl.startsWith("data:image/"))) || !Number.isFinite(Number(value.durationSeconds)) || Number(value.durationSeconds) < 1 || Number(value.durationSeconds) > 15 || typeof value.aspect !== "string" || (value.model !== undefined && typeof value.model !== "string")) throw new Error("Invalid request.");
194
- } catch { json(res, 400, { error: "invalid request" }); return true; }
686
+ if (value.provider === "fal") validateFalVideoRequest({ model: process.env.FAL_MODEL || undefined, durationSeconds: Number(value.durationSeconds), aspect: value.aspect });
687
+ } catch (error) { json(res, error?.code === "fal-invalid-request" ? 422 : 400, { error: error?.message || "invalid request" }); return true; }
195
688
  const adapter = createVideoAdapters().find((entry) => entry.id === value.provider);
196
689
  if (!adapter || !adapter.configured()) { json(res, 409, { error: "video provider is not configured" }); return true; }
197
690
  try {
198
691
  const result = await adapter.generate({ ...value, durationSeconds: Number(value.durationSeconds) });
199
- json(res, 200, { ...(result.mp4Base64 ? { dataUrl: `data:video/mp4;base64,${result.mp4Base64}` } : { url: result.url }), width: result.width, height: result.height, seconds: result.seconds, ...(result.preservation ? { preservation: result.preservation } : {}) });
692
+ json(res, 200, { ...(result.mp4Base64 ? { dataUrl: `data:video/mp4;base64,${result.mp4Base64}` } : { url: result.url }), width: result.width, height: result.height, fps: result.fps, seconds: result.seconds, ...(result.metadataMeasured !== undefined ? { metadataMeasured: result.metadataMeasured } : {}), ...(result.preservation ? { preservation: result.preservation } : {}) });
200
693
  } catch (error) {
201
694
  // A generated H3 take that fails the plate check is unsafe to show as
202
695
  // a locked shot. Keep the distinction visible to the client so it can
203
696
  // ask for a retry instead of silently accepting a drifting set.
204
- const status = error?.code === "h3-preservation-failed" ? 422 : 502;
697
+ const status = ["h3-preservation-failed", "fal-invalid-request"].includes(error?.code) ? 422 : 502;
205
698
  json(res, status, { error: error?.message || "video provider failed", ...(error?.preservation ? { preservation: error.preservation } : {}) });
206
699
  }
207
700
  return true;
208
701
  }
702
+ if (path.startsWith("/agent/jobs/") && path.endsWith("/accept") && req.method === "POST") {
703
+ if (!await hasAnyCredential()) { json(res, 401, { error: { code: "AUTH_REQUIRED", message: "Sign in or configure a provider key." } }); return true; }
704
+ let value; try { value = await readBody(req); } catch { json(res, 400, { error: "invalid request" }); return true; }
705
+ const jobId = decodeURIComponent(path.slice("/agent/jobs/".length, -"/accept".length)); const session = studioSessions.get(value?.sessionId);
706
+ if (!session || session.owner !== parseCookies(req).studio_owner || value.surface !== "studio" || value.explicitUnverifiedAcceptance !== true || !session.turns.has(value.turnId) || session.activeJobId !== jobId) { json(res, 403, { error: { code: "AUTH_REQUIRED", message: "Only the owning Studio UI may accept this candidate." } }); return true; }
707
+ try { const receipt = await ownedStudioRuntime.accept(jobId); const record = studioEvents.get(value.turnId); if (record) { emitStudioEvent(value.turnId, { type: "receipt", receipt }); emitStudioEvent(value.turnId, { type: "done" }); record.terminal = true; } session.activeJobId = null; session.activeJobTurnId = null; json(res, 200, { receipt }); } catch (error) { json(res, 409, { error: { code: error.code || "VERIFICATION_FAILED", message: error.message } }); }
708
+ return true;
709
+ }
209
710
  if (req.method !== "POST" || !["/agent/turn", "/agent/stop"].includes(path)) {
210
711
  json(res, 404, { error: "not found" }); return true;
211
712
  }
212
713
  let value;
213
714
  try {
214
- value = await readBody(req);
215
- if (!value || typeof value.sessionId !== "string" || !value.sessionId
715
+ value = await readBody(req, TURN_BODY_LIMIT);
716
+ if (value?.surface === "studio") {
717
+ // Lazy only for the minimal legacy-sidecar fixture, which omits src/.
718
+ // Actual npm packages include src; there is exactly one validator.
719
+ const protocol = await import("../../src/studio-agent-protocol.js");
720
+ value = path === "/agent/stop" ? protocol.validateStudioStopEnvelope(value) : protocol.validateStudioTurnEnvelope(value);
721
+ } else if (!value || (value.surface !== undefined && value.surface !== "workflow")
722
+ || value.context !== undefined || value.turnId !== undefined || typeof value.sessionId !== "string" || !value.sessionId
216
723
  || (path === "/agent/turn" && (typeof value.text !== "string"
217
724
  || (value.attachFrame !== undefined && typeof value.attachFrame !== "boolean")
725
+ || !validAttachments(value.attachments)
218
726
  || (value.model !== undefined && typeof value.model !== "string")
219
727
  || (value.effort !== undefined && !REASONING_EFFORTS.includes(value.effort))))) throw new Error("Invalid request.");
220
- } catch { json(res, 400, { error: "invalid request" }); return true; }
728
+ } catch (error) {
729
+ json(res, 400, { error: error?.name === "StudioProtocolError" ? error.toJSON() : "invalid request" }); return true;
730
+ }
731
+ if (value.surface === "studio") {
732
+ try { return await handleStudioTurn(req, res, value, path); }
733
+ catch (error) { if (error instanceof (await import("../../src/studio-agent-protocol.js")).StudioProtocolError) json(res, 409, { error: error.toJSON() }); else throw error; return true; }
734
+ }
221
735
  if (path === "/agent/stop") {
222
736
  sessions.get(value.sessionId)?.controller?.abort();
223
737
  json(res, 200, { ok: true }); return true;
@@ -234,115 +748,101 @@ export function createAgentHandler({ auth = defaultAuth, codex, handlers, liveHu
234
748
  sessions.set(value.sessionId, session);
235
749
  const controller = new AbortController();
236
750
  const { signal } = controller;
237
- Object.assign(session, { running: true, controller, signal });
751
+ Object.assign(session, { running: true, controller, signal, emit: send });
238
752
  const disconnect = () => controller.abort();
239
753
  res.once("close", disconnect);
754
+ const turnId = typeof value.turn_id === "string" && /^[a-f0-9]{32}$/.test(value.turn_id) ? value.turn_id : null;
755
+ session.turnId = turnId;
756
+ const telemetry = createTurnTelemetry(send, turnId);
757
+ const telemetryTools = new Map();
758
+ let turnOutcome = "succeeded";
759
+ let turnFailureCode = null;
760
+ let toolFailed = false;
240
761
  let quota;
241
- const observeHeaders = (headers) => {
242
- const next = quotaEvent(codex, headers);
243
- if (!quota) send(next);
244
- quota = next;
762
+ const quotaForResponse = (headers, model) => {
763
+ if (quota) return quota;
764
+ quota = model?.provider === "openai-codex" ? quotaEvent(codex, headers) : {
765
+ type: "quota", plan: null,
766
+ primary: { usedPercent: null, windowMinutes: null, resetAt: null },
767
+ credits: { has: null },
768
+ };
769
+ return quota;
245
770
  };
246
- let refreshed = false;
247
- // A stream that fails with server_is_overloaded usually succeeds on the
248
- // next attempt; retry twice before reporting it.
249
- const retryOverloaded = async (operation, attempts = 2) => {
250
- for (let attempt = 0; ; attempt += 1) {
251
- try { return await operation(); } catch (error) {
252
- if (!["overloaded", "server_error"].includes(error.code) || attempt >= attempts || signal.aborted) throw error;
253
- await new Promise((resolve) => setTimeout(resolve, retryDelayMs * (attempt + 1)));
771
+ const emitFrame = (frame) => {
772
+ if (frame.type === "tool.start") {
773
+ telemetryTools.set(frame.callId, { execution: telemetry.toolStarted(agentToolCategory(frame.name)), name: frame.name });
774
+ return send(frame);
775
+ }
776
+ if (frame.type === "tool.done") {
777
+ send(frame);
778
+ const record = telemetryTools.get(frame.callId);
779
+ if (record) {
780
+ if (frame.ok && appliedCanvasResult(record.name, frame.result)) telemetry.applied();
781
+ record.execution.executed(frame.ok ? "succeeded" : (signal.aborted ? "cancelled" : "failed"));
782
+ record.finished = true;
254
783
  }
784
+ if (!frame.ok) toolFailed = true;
785
+ return;
255
786
  }
256
- };
257
- const retryAuth = async (operation) => {
258
- try { signal.throwIfAborted(); return await operation(); }
259
- catch (error) {
260
- if (error.status !== 401 || refreshed || signal.aborted) throw error;
261
- refreshed = true;
262
- if (!await auth.getAccessToken()) throw error;
263
- signal.throwIfAborted();
264
- return operation();
787
+ if (frame.type === "error") {
788
+ if (frame.code === "truncated") {
789
+ turnOutcome = "unresolved";
790
+ return;
791
+ }
792
+ turnOutcome = signal.aborted || frame.code === "aborted" ? "cancelled" : "failed";
793
+ turnFailureCode = signal.aborted ? "aborted" : frame.code;
794
+ return send(frame);
795
+ }
796
+ if (frame.type === "done") {
797
+ for (const record of telemetryTools.values()) if (!record.finished) {
798
+ record.execution.executed(signal.aborted ? "cancelled" : "failed");
799
+ record.finished = true;
800
+ }
801
+ if (turnOutcome === "unresolved") return send(frame);
802
+ if (turnOutcome === "succeeded" && !toolFailed) telemetry.finished("succeeded", null);
803
+ else telemetry.finished(toolFailed ? "failed" : turnOutcome, toolFailed ? "tool_failed" : turnFailureCode);
265
804
  }
805
+ send(frame);
266
806
  };
267
807
  const turn = async () => {
268
- if (!await auth.getAccessToken()) throw Object.assign(new Error("Authentication required."), { status: 401 });
808
+ if (!await hasAnyCredential()) throw Object.assign(new Error("Authentication required."), { status: 401 });
809
+ workflowModels = await ensureWorkflowModels();
269
810
  const dependencies = await runtime;
270
- session.codex = { editImage: (args) => retryAuth(() => codex.editImage(args)) };
271
- const tools = createAgentTools({ ...dependencies, session, emit: send });
272
- const executeTool = async (item, override) => {
273
- signal.throwIfAborted();
274
- const tool = override ?? tools.find((entry) => entry.name === item.name);
275
- if (!tool) throw new Error("Unknown tool.");
276
- const args = typeof item.arguments === "string" ? JSON.parse(item.arguments) : item.arguments;
277
- send({ type: "tool.start", callId: item.call_id, name: item.name, label: item.name.replaceAll("_", " "), args });
278
- const started = performance.now();
279
- try {
280
- if (dependencies.error) throw dependencies.error;
281
- const result = await tool.handler(args);
282
- signal.throwIfAborted();
283
- send({ type: "tool.done", callId: item.call_id, ok: true, elapsedMs: Math.round(performance.now() - started), result });
284
- return result;
285
- } catch (error) {
286
- if (process.env.COZYCLAY_AGENT_DEBUG) console.error("[agent] tool", item.name, "failed:", error?.message);
287
- send({ type: "tool.done", callId: item.call_id, ok: false, elapsedMs: Math.round(performance.now() - started), error: errorInfo(error).message });
288
- throw error;
289
- }
290
- };
291
- let text = value.text;
292
- if (value.attachFrame) {
293
- const captured = await executeTool({ call_id: "attached-frame", name: "capture_blocking_frame", arguments: {} }, tools.internal.capture);
294
- text += `\nAttached frame imageId: ${captured.imageId}`;
811
+ session.codex = { editImage: (args) => codex.editImage(args) };
812
+ const workflowTools = createAgentTools({ ...dependencies, session, emit: send });
813
+ let workflowRunner = workflowRunners.get(value.sessionId);
814
+ if (!workflowRunner) {
815
+ workflowRunner = createAgentRunner({ models: workflowModels, tools: workflowTools, systemPrompt: SYSTEM_PROMPT, clock, fauxProvider, sessionStore, codexBaseUrl, cliproxyBaseUrl, auth, env, onQuota: quotaForResponse });
816
+ workflowRunners.set(value.sessionId, workflowRunner);
295
817
  }
296
- const history = [...session.history, { role: "user", content: [{ type: "input_text", text }] }];
297
- // runAgentTurn closes over streamResponses and hides its headers. Drive
298
- // that same serial loop here so quotas are observable, and retry only
299
- // the failed request rather than replaying already-executed scene tools.
300
- while (true) {
301
- const output = await retryOverloaded(() => retryAuth(async () => {
302
- const stream = codex.streamResponses({ input: history, tools: agentToolSchemas(tools), instructions: SYSTEM_PROMPT, model: value.model, effort: value.effort, signal });
303
- const headers = stream.headers.then(observeHeaders, () => {});
304
- const items = [];
305
- try {
306
- for await (const event of stream) {
307
- signal.throwIfAborted();
308
- if (event.type === "response.output_text.delta") send({ type: "text.delta", text: event.delta });
309
- if (event.type === "response.output_item.done") items.push(event.item);
310
- if (event.type === "error" || event.type === "response.failed") {
311
- if (process.env.COZYCLAY_AGENT_DEBUG) console.error("[agent] model event:", JSON.stringify(event).slice(0, 600));
312
- const code = event.error?.code ?? event.response?.error?.code;
313
- throw Object.assign(new Error("Model response failed."), code === "server_is_overloaded" ? { code: "overloaded" } : code === "server_error" ? { code: "server_error" } : {});
314
- }
315
- }
316
- await headers;
317
- return items;
318
- } finally { await headers; }
319
- }));
320
- for (const item of output) {
321
- history.push(item); // Preserve reasoning items verbatim.
322
- if (item.type === "function_call") {
323
- const result = await executeTool(item);
324
- history.push({ type: "function_call_output", call_id: item.call_id, output: JSON.stringify(result) });
325
- }
326
- }
327
- if (!output.some((item) => item.type === "function_call")) break;
328
- }
329
- session.history = history;
818
+ if (!session.modelSession) session.modelSession = await workflowRunner.openSession(value.sessionId, { surface: "workflow" });
819
+ const turnInput = { surface: "workflow", sessionId: value.sessionId, text: value.text, model: value.model || "gpt-6-astra", effort: value.effort, attachments: value.attachments, attachFrame: value.attachFrame, signal, emit: send, quotaEvent: quotaForResponse };
820
+ for await (const frame of session.modelSession.start(turnInput)) emitFrame(frame);
330
821
  };
331
- try { await requestContext.run(observeHeaders, turn); }
822
+ try { await turn(); }
332
823
  catch (error) {
333
- if (!signal.aborted) {
334
- if (error.headers) observeHeaders(error.headers);
335
- if (process.env.COZYCLAY_AGENT_DEBUG) console.error("[agent] turn failed:", error?.status, error?.message, String(error?.body ?? "").slice(0, 300));
336
- send({ type: "error", ...errorInfo(error, quota) });
337
- }
338
- } finally {
339
- session.running = false; send({ type: "done" }); res.end(); res.off("close", disconnect);
824
+ turnOutcome = signal.aborted ? "cancelled" : "failed";
825
+ turnFailureCode = signal.aborted ? "aborted" : agentFailureCode(error, signal, toolFailed);
826
+ if (!signal.aborted) emitFrame({ type: "error", ...errorInfo(error, quota) });
827
+ if (!res.writableEnded) emitFrame({ type: "done" });
828
+ }
829
+ finally {
830
+ if (!res.writableEnded) res.end();
831
+ session.running = false;
832
+ res.off("close", disconnect);
340
833
  }
341
834
  return true;
342
835
  };
836
+
343
837
  handle.close = async () => {
344
838
  unsubscribe?.();
345
839
  for (const session of sessions.values()) session.controller?.abort();
840
+ for (const session of studioSessions.values()) session.controller?.abort();
841
+ for (const runner of workflowRunners.values()) await runner.close?.();
842
+ for (const runner of studioRunners.values()) await runner.close?.();
843
+ workflowRunners.clear(); studioRunners.clear();
844
+ if (ownedStudioRuntime?.dispose) await ownedStudioRuntime.dispose();
845
+ studioSessions.clear(); studioEvents.clear(); studioOwnerTokens.clear();
346
846
  sessions.clear();
347
847
  const { liveHub: hub } = await runtime;
348
848
  if (hub?.server) {