cozyclay 1.8.0 → 1.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +90 -0
- package/README.md +89 -142
- package/THIRD_PARTY_NOTICES.md +48 -0
- package/bin/agent/agent-routes.mjs +619 -119
- package/bin/agent/agent-runner.mjs +513 -0
- package/bin/agent/agent-tools.mjs +11 -19
- package/bin/agent/codex-client.mjs +6 -150
- package/bin/agent/credential-store.mjs +93 -0
- package/bin/agent/fal-contract.mjs +25 -0
- package/bin/agent/h3-preservation.mjs +0 -4
- package/bin/agent/motion-runtime.mjs +304 -0
- package/bin/agent/pi-tools.mjs +64 -0
- package/bin/agent/provider-keys.mjs +56 -0
- package/bin/agent/providers.mjs +344 -0
- package/bin/agent/schema-to-typebox.mjs +58 -0
- package/bin/agent/secure-file.mjs +19 -0
- package/bin/agent/session-store.mjs +324 -0
- package/bin/agent/studio-prompt.mjs +1 -0
- package/bin/agent/studio-tools.mjs +53 -0
- package/bin/agent/video-adapters.mjs +60 -9
- package/bin/codex-auth.mjs +37 -13
- package/bin/cozyclay.mjs +42 -10
- package/bin/live/cli.mjs +474 -0
- package/bin/live/client.mjs +216 -0
- package/bin/live/studio.mjs +85 -0
- package/bin/live/workspace.mjs +69 -0
- package/bin/live-endpoint.mjs +52 -0
- package/bin/mcp-runtime.mjs +4 -0
- package/bin/telemetry-state.mjs +10 -0
- package/dist/app/index.html +6 -5
- package/dist/assets/analytics-B1hnH66c.js +1 -0
- package/dist/assets/app-BWusbqgO.js +4861 -0
- package/dist/assets/app-qKDo4PBX.css +1 -0
- package/dist/assets/{demo-BENMp24T.js → demo-B-eg1CT8.js} +1 -1
- package/dist/assets/first-shot-handoff-CGgnvNu8.js +1 -0
- package/dist/assets/landing-IbRp3FnQ.js +1 -0
- package/dist/assets/shot-prompt-CHUtw6af.js +17 -0
- package/dist/assets/shot-prompt-C_g2BHVc.css +1 -0
- package/dist/assets/{ticket-BDieXjZs.js → ticket-BgIzOBOf.js} +1 -1
- package/dist/assets/workflow-CVwzMhz2.js +179 -0
- package/dist/assets/{workflow-CeQxy8Uq.css → workflow-DsudxKHF.css} +1 -1
- package/dist/cozyclay-package.json +1 -1
- package/dist/greybox-to-video/index.html +19 -0
- package/dist/index.html +15 -118
- package/dist/privacy/index.html +57 -3
- package/dist/sitemap.xml +3 -3
- package/dist/workflow/index.html +5 -5
- package/mcp/LIVE-PROTOCOL.md +89 -16
- package/mcp/README.md +12 -3
- package/mcp/live-hub.mjs +364 -35
- package/mcp/package.json +2 -2
- package/mcp/server.mjs +47 -21
- package/mcp/tool-handlers.mjs +113 -37
- package/package.json +8 -3
- package/src/App.jsx +2367 -735
- package/src/analytics.js +361 -27
- package/src/app-stage.jsx +111 -96
- package/src/ardy/client.js +19 -9
- package/src/ardy/foot-lock.js +230 -0
- package/src/ardy/ground.js +52 -0
- package/src/ardy/ik.js +25 -7
- package/src/ardy/motion-calibration.js +0 -4
- package/src/ardy/npz.js +7 -1
- package/src/ardy/physics-refine.js +1 -1
- package/src/ardy/physics-review.js +102 -52
- package/src/ardy/playback.js +0 -8
- package/src/asset-pane.jsx +97 -8
- package/src/asset-shelf.js +10 -3
- package/src/camera-tutorial.jsx +311 -7
- package/src/controls.jsx +91 -8
- package/src/dualview.jsx +7 -7
- package/src/execution-telemetry.js +97 -0
- package/src/facing-marks.js +72 -0
- package/src/fal-motion-client.js +80 -0
- package/src/fal-motion-studio.jsx +180 -0
- package/src/first-shot-handoff.js +44 -0
- package/src/hierarchy-panel.jsx +20 -2
- package/src/keyframe-pack.js +1 -0
- package/src/live-control.js +257 -11
- package/src/mesh-graph-clone.js +23 -0
- package/src/mesh-sniff.js +82 -0
- package/src/model-presets.js +8 -3
- package/src/motion-readiness-ui.jsx +67 -0
- package/src/motion-readiness.js +29 -0
- package/src/motion-resources.js +177 -0
- package/src/motion-store.js +65 -0
- package/src/multimodel-ingest.js +9 -0
- package/src/offscreen-export.js +125 -59
- package/src/part-colours.js +25 -15
- package/src/posestudio.jsx +2 -1
- package/src/project-resources.js +286 -0
- package/src/project.js +219 -32
- package/src/props.jsx +99 -2
- package/src/render-passes.js +17 -5
- package/src/resource-status.jsx +174 -0
- package/src/result-modal.jsx +23 -7
- package/src/scene-asset-cache.js +19 -2
- package/src/scene-assets.js +73 -12
- package/src/scene-fbx.js +75 -0
- package/src/scene-history.js +8 -4
- package/src/scene-mesh-cache.js +135 -0
- package/src/scene-mesh.js +390 -0
- package/src/scene-objects.js +146 -41
- package/src/scenes.js +55 -21
- package/src/semantic-edit.js +98 -0
- package/src/settings-menu.jsx +29 -1
- package/src/shot-authoring.js +4 -1
- package/src/shot.js +4 -0
- package/src/studio-agent-commands.js +561 -0
- package/src/studio-agent-context.js +113 -0
- package/src/studio-agent-motion.js +440 -0
- package/src/studio-agent-protocol.js +508 -0
- package/src/studio-elements.js +78 -0
- package/src/styles.css +792 -3
- package/src/tutorial-analytics.js +86 -0
- package/src/use-semantic-state.js +12 -0
- package/src/workflow/AgentPanel.jsx +668 -186
- package/src/workflow/CozySceneNode.jsx +31 -6
- package/src/workflow/WorkflowBuilder.jsx +222 -118
- package/src/workflow/agent-client.js +1254 -53
- package/src/workflow/agent-panel.css +464 -2
- package/src/workflow/attachment-image.js +163 -0
- package/src/workflow/canvas-commands.js +16 -5
- package/src/workflow/cozy-scene-node.css +14 -3
- package/src/workflow/cozy-scene-node.js +22 -0
- package/src/workflow/keyframe-pack-request.js +16 -2
- package/src/workflow/main.jsx +3 -0
- package/src/workflow/node-schema.js +1 -1
- package/src/workflow/scene-asset-sync.js +2 -2
- package/src/workflow/video-contract.js +25 -0
- package/src/workflow/workflow-resources.js +275 -0
- package/src/workflow/workflow.css +2 -0
- package/tools/ardy/README.md +13 -0
- package/tools/ardy/__pycache__/cclay_gvhmr_worker.cpython-313.pyc +0 -0
- package/tools/ardy/bridge.mjs +28 -47
- package/tools/ardy/bvh-cskel27.mjs +0 -2
- package/tools/ardy/cclay_gvhmr_worker.py +7 -1
- package/tools/ardy/extract.mjs +6 -2
- package/tools/ardy/motion-stabilize.mjs +109 -4
- package/tools/ardy/runners/gvhmr-worker.mjs +11 -2
- package/tools/dev/pages/privacy.html +57 -3
- package/tools/dev-full.mjs +5 -2
- package/tools/kimodo/edit.mjs +1 -6
- package/tools/kimodo/generate.mjs +84 -35
- package/tools/kimodo/local-output.mjs +265 -0
- package/tools/kimodo/mlx-generate.py +111 -0
- package/tools/kimodo/pose-constraints.mjs +0 -2
- package/tools/kimodo/runner.mjs +46 -8
- package/tools/kimodo/setup-on-box.sh +288 -32
- package/tools/run-tests.mjs +72 -4
- package/dist/assets/analytics-Dc9ooyn1.js +0 -1
- package/dist/assets/app-CHc6SS4r.js +0 -4859
- package/dist/assets/app-q2B32RKM.css +0 -1
- package/dist/assets/shot-prompt-CuqtYxB7.css +0 -1
- package/dist/assets/shot-prompt-F2C_YN3J.js +0 -15
- package/dist/assets/workflow-BHvP2fxY.js +0 -179
|
@@ -1,8 +1,14 @@
|
|
|
1
1
|
import { AsyncLocalStorage } from "node:async_hooks";
|
|
2
|
+
import { randomBytes, randomUUID } from "node:crypto";
|
|
3
|
+
import { basename } from "node:path";
|
|
2
4
|
import * as defaultAuth from "../codex-auth.mjs";
|
|
5
|
+
import { publishLiveEndpoint, removeLiveEndpoint } from "../live-endpoint.mjs";
|
|
3
6
|
import { createCodexClient } from "./codex-client.mjs";
|
|
4
|
-
import { createAgentTools,
|
|
5
|
-
|
|
7
|
+
import { createAgentTools, SYSTEM_PROMPT, pickWorkspace } from "./agent-tools.mjs";
|
|
8
|
+
|
|
9
|
+
import { createVideoAdapters, validateFalVideoRequest } from "./video-adapters.mjs";
|
|
10
|
+
import { createSessionStore, transcriptFromHistory } from "./session-store.mjs";
|
|
11
|
+
import { createAgentRunner } from "./agent-runner.mjs";
|
|
6
12
|
|
|
7
13
|
// Values the codex backend accepts for reasoning.effort (its own 400 lists them).
|
|
8
14
|
export const REASONING_EFFORTS = ["none", "minimal", "low", "medium", "high", "xhigh", "max", "ultra"];
|
|
@@ -26,6 +32,91 @@ const IMAGE_BODY_LIMIT = 24 * 1024 * 1024;
|
|
|
26
32
|
// backend has to read, and a shot with seven of them is a prompt nobody wrote.
|
|
27
33
|
const IMAGE_REFERENCES_MAX = 6;
|
|
28
34
|
|
|
35
|
+
// Pictures pasted or dropped into the composer (#367) ride in the turn body,
|
|
36
|
+
// so the chat routes need room for them. Everything else about the small chat
|
|
37
|
+
// limit stays: this is exactly four attachments of the size the turn envelope
|
|
38
|
+
// admits, plus the envelope itself.
|
|
39
|
+
const ATTACHMENTS_MAX = 4;
|
|
40
|
+
const ATTACHMENT_MAX_CHARS = 6_000_000;
|
|
41
|
+
const TURN_BODY_LIMIT = ATTACHMENTS_MAX * ATTACHMENT_MAX_CHARS + 64 * 1024;
|
|
42
|
+
|
|
43
|
+
/** Reject anything that is not a short list of inline {dataUrl, name?} images.
|
|
44
|
+
* The Studio envelope validates its own copy; this is the legacy body's. */
|
|
45
|
+
function validAttachments(attachments) {
|
|
46
|
+
if (attachments === undefined) return true;
|
|
47
|
+
if (!Array.isArray(attachments) || !attachments.length || attachments.length > ATTACHMENTS_MAX) return false;
|
|
48
|
+
return attachments.every((entry) => entry && typeof entry === "object" && !Array.isArray(entry)
|
|
49
|
+
&& typeof entry.dataUrl === "string" && entry.dataUrl.length <= ATTACHMENT_MAX_CHARS && /^data:image\/(png|jpeg|webp);base64,/.test(entry.dataUrl)
|
|
50
|
+
&& (entry.name === undefined || (typeof entry.name === "string" && entry.name.length <= 120)));
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
// Keep this local relay self-contained: minimal sidecar installs omit src/.
|
|
54
|
+
const advisory = (read, fallback) => { try { return read(); } catch { return fallback; } };
|
|
55
|
+
const telemetryId = () => advisory(() => randomBytes(16).toString("hex"), null);
|
|
56
|
+
const telemetryNow = () => advisory(() => { const value = performance.now(); return Number.isFinite(value) ? value : NaN; }, NaN);
|
|
57
|
+
const bucketMs = (ms) => !Number.isFinite(ms) || ms < 1000 ? "lt1s" : ms < 3000 ? "1-3s" : ms < 10000 ? "3-10s" : ms < 30000 ? "10-30s" : "gte30s";
|
|
58
|
+
const agentToolCategory = (name) => {
|
|
59
|
+
// Studio families are authored edits or reads; the analytics vocabulary has
|
|
60
|
+
// no Studio-specific category, so a write says so and everything else does not.
|
|
61
|
+
if (["arrange_objects", "arrange_characters", "patch_elements", "frame_shot", "undo_edit"].includes(name)) return "scene_write";
|
|
62
|
+
if (["inspect_studio", "operate_studio", "verify_result", "generate_motion"].includes(name)) return "other";
|
|
63
|
+
if (name === "run_workflow") return "workflow_run";
|
|
64
|
+
if (name === "describe_workflow" || name === "focus_workflow_node") return "workflow_read";
|
|
65
|
+
if (["add_workflow_node", "update_workflow_node", "remove_workflow_node", "connect_workflow_nodes", "disconnect_workflow_nodes", "set_workflow_node_output"].includes(name)) return "workflow_write";
|
|
66
|
+
if (name === "capture_blocking_frame") return "frame_capture";
|
|
67
|
+
if (name === "render_from_frame") return "image_generate";
|
|
68
|
+
if (name === "place_image_in_scene" || name === "add_reference_node") return "scene_write";
|
|
69
|
+
return "other";
|
|
70
|
+
};
|
|
71
|
+
const agentFailureCode = (error, signal, tool = false) => advisory(() => {
|
|
72
|
+
if (signal.aborted || error?.name === "AbortError") return "aborted";
|
|
73
|
+
if (error?.status === 401 || error?.code === "unauthorized") return "auth";
|
|
74
|
+
if (error?.status === 429) return "rate_limited";
|
|
75
|
+
return tool ? "tool_failed" : "upstream";
|
|
76
|
+
}, tool ? "tool_failed" : "upstream");
|
|
77
|
+
// These existing canvas commands only return a node/edge after publishing a
|
|
78
|
+
// new insertion. Read/focus, generic accepted responses, update no-ops and run
|
|
79
|
+
// outputs (which can echo old values) are deliberately not application proof.
|
|
80
|
+
const appliedCanvasResult = (name, result) => {
|
|
81
|
+
if (name === "add_workflow_node" || name === "add_reference_node") return typeof result?.node?.id === "string" && result.node.id.length > 0;
|
|
82
|
+
if (name === "connect_workflow_nodes") return typeof result?.edge?.id === "string" && result.edge.id.length > 0;
|
|
83
|
+
return false;
|
|
84
|
+
};
|
|
85
|
+
|
|
86
|
+
/**
|
|
87
|
+
* One execution-telemetry emitter, shared by both surfaces. The turn that owns
|
|
88
|
+
* the correlation id owns its frames: a turn without one stays unobserved
|
|
89
|
+
* rather than emitting frames nothing can correlate.
|
|
90
|
+
*/
|
|
91
|
+
function createTurnTelemetry(send, turnId) {
|
|
92
|
+
const turnStartedAt = telemetryNow();
|
|
93
|
+
let applied = false;
|
|
94
|
+
const emit = (event, props, toolId) => advisory(() => {
|
|
95
|
+
if (turnId) send({ type: "execution_telemetry", event, props, ...(toolId ? { telemetry_id: toolId } : {}) });
|
|
96
|
+
});
|
|
97
|
+
return {
|
|
98
|
+
/** One tool execution: the started frame now, its outcome when it ends. */
|
|
99
|
+
toolStarted(category) {
|
|
100
|
+
const startedAt = telemetryNow();
|
|
101
|
+
const toolId = telemetryId();
|
|
102
|
+
advisory(() => { if (turnId && toolId) send({ type: "execution_tool_started", turn_id: turnId, telemetry_id: toolId, tool_category: category }); });
|
|
103
|
+
return {
|
|
104
|
+
elapsedMs: () => Math.round(telemetryNow() - startedAt),
|
|
105
|
+
executed: (outcome) => {
|
|
106
|
+
if (toolId) emit("agent:tool_executed", { turn_id: turnId, tool_category: category, outcome, duration_bucket: bucketMs(telemetryNow() - startedAt) }, toolId);
|
|
107
|
+
},
|
|
108
|
+
};
|
|
109
|
+
},
|
|
110
|
+
/** The first proof that the scene actually changed, once per turn. */
|
|
111
|
+
applied: () => advisory(() => { if (!applied) { applied = true; emit("agent:result_applied", { turn_id: turnId }); } }),
|
|
112
|
+
finished: (outcome, failureCode) => emit(`agent:turn_${outcome}`, {
|
|
113
|
+
turn_id: turnId,
|
|
114
|
+
duration_bucket: bucketMs(telemetryNow() - turnStartedAt),
|
|
115
|
+
...(outcome !== "succeeded" ? { failure_code: failureCode } : {}),
|
|
116
|
+
}),
|
|
117
|
+
};
|
|
118
|
+
}
|
|
119
|
+
|
|
29
120
|
/** Reject anything that is not a list of {role, name?, dataUrl} inline images. */
|
|
30
121
|
function validReferences(references) {
|
|
31
122
|
if (references === undefined) return true;
|
|
@@ -77,14 +168,43 @@ function quotaEvent(codex, headers) {
|
|
|
77
168
|
};
|
|
78
169
|
}
|
|
79
170
|
|
|
171
|
+
// Backend errors may echo credentials or image inputs, so their bodies are
|
|
172
|
+
// never forwarded. Only a structured message survives, redacted and clipped to
|
|
173
|
+
// one line — without it neither the log nor the panel can say why a turn died.
|
|
174
|
+
const UPSTREAM_DETAIL_MAX = 200;
|
|
175
|
+
export function sanitizeUpstreamDetail(body) {
|
|
176
|
+
let message = null;
|
|
177
|
+
try {
|
|
178
|
+
const parsed = JSON.parse(body);
|
|
179
|
+
const candidate = parsed?.error?.message ?? parsed?.error ?? parsed?.message ?? parsed?.detail;
|
|
180
|
+
message = typeof candidate === "string" ? candidate : null;
|
|
181
|
+
} catch { return null; }
|
|
182
|
+
if (!message) return null;
|
|
183
|
+
const cleaned = message
|
|
184
|
+
.replace(/data:[^\s"']+/gi, "[image]")
|
|
185
|
+
.replace(/\b(?:AIza[\w-]{20,}|(?:sk-or-|sk-ant-|sk-|Bearer\s+|eyJ)[\w.\-+/=]+)/gi, "[redacted]")
|
|
186
|
+
.replace(/\s+/g, " ")
|
|
187
|
+
.trim();
|
|
188
|
+
if (!cleaned) return null;
|
|
189
|
+
return cleaned.length > UPSTREAM_DETAIL_MAX ? `${cleaned.slice(0, UPSTREAM_DETAIL_MAX - 1)}\u2026` : cleaned;
|
|
190
|
+
}
|
|
191
|
+
|
|
80
192
|
function errorInfo(error, quota) {
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
193
|
+
const status = Number.isInteger(error?.status) ? error.status : null;
|
|
194
|
+
const detail = typeof error?.detail === "string" && error.detail ? error.detail : null;
|
|
195
|
+
// The status and the backend's own sanitized words are the only things that
|
|
196
|
+
// can tell an author WHY the turn died; a generic sentence cannot.
|
|
197
|
+
const explain = (message) => ({
|
|
198
|
+
message: status === null ? message : `${status} \u2014 ${detail || message}`,
|
|
199
|
+
...(status === null ? {} : { status }),
|
|
200
|
+
...(detail ? { detail } : {}),
|
|
201
|
+
});
|
|
202
|
+
if (error?.status === 401 || error?.code === "unauthorized") return { code: "auth", ...explain("Authentication required. Sign in again.") };
|
|
203
|
+
if (error?.status === 429) return { code: "rate_limit", ...explain("Rate limit exceeded."), resetAt: quota?.primary.resetAt ?? null };
|
|
204
|
+
if (error?.code === "entitlement") return { code: "entitlement", ...explain("This account cannot generate images.") };
|
|
205
|
+
if (error?.code === "overloaded") return { code: "overloaded", ...explain("The model service is overloaded right now. Try again in a moment.") };
|
|
206
|
+
if (error?.code === "server_error") return { code: "overloaded", ...explain("The model service hit an internal error. Try again in a moment.") };
|
|
207
|
+
return { code: "upstream", ...explain("The model or live editor could not complete this turn.") };
|
|
88
208
|
}
|
|
89
209
|
|
|
90
210
|
/** Use the existing client and its request queue, retaining failure headers that
|
|
@@ -98,18 +218,25 @@ function defaultClient(auth, requestContext) {
|
|
|
98
218
|
requestContext.getStore()?.(response.headers);
|
|
99
219
|
if (response.ok) return response;
|
|
100
220
|
const detail = await response.text();
|
|
101
|
-
const error = Object.assign(new Error("Codex backend request failed."), { status: response.status, headers: response.headers });
|
|
221
|
+
const error = Object.assign(new Error("Codex backend request failed."), { status: response.status, headers: response.headers, detail: sanitizeUpstreamDetail(detail) });
|
|
102
222
|
if (url.includes("/images/") && /entitlement|plan/i.test(detail)) error.code = "entitlement";
|
|
103
223
|
throw error;
|
|
104
224
|
},
|
|
105
225
|
});
|
|
106
226
|
}
|
|
107
227
|
|
|
228
|
+
// The launcher and the dev runner both own a hub of their own; the endpoint
|
|
229
|
+
// file names which one a controller has reached.
|
|
230
|
+
const liveHubOwner = () => process.env.COZYCLAY_LIVE_OWNER
|
|
231
|
+
|| (basename(process.argv[1] ?? "") === "dev-full.mjs" ? "dev-full" : "cozyclay");
|
|
232
|
+
|
|
108
233
|
/** Start the optional registry/live dependencies without making signed-out
|
|
109
234
|
* startup depend on an MCP dependency install. Failures remain visible on use. */
|
|
110
235
|
function liveToolsRuntime() {
|
|
111
236
|
return Promise.all([import("../../mcp/tool-handlers.mjs"), import("../../mcp/live-hub.mjs")]).then(async ([registry, { startLiveHub }]) => {
|
|
112
|
-
const
|
|
237
|
+
const owner = liveHubOwner();
|
|
238
|
+
const token = randomBytes(32).toString("hex");
|
|
239
|
+
const liveHub = await startLiveHub(Number(process.env.COZYCLAY_LIVE_PORT ?? 5184), { token, owner });
|
|
113
240
|
registry.setLiveHub(liveHub);
|
|
114
241
|
const handlers = registry.createToolHandlers().map((tool) => ({
|
|
115
242
|
...tool,
|
|
@@ -119,11 +246,22 @@ function liveToolsRuntime() {
|
|
|
119
246
|
return liveHub?.connected ? liveHub.runExclusive(tool.name, workspaceHandle, run) : run(workspaceHandle);
|
|
120
247
|
},
|
|
121
248
|
}));
|
|
249
|
+
if (liveHub) {
|
|
250
|
+
// A controller runs the same per-workspace wrapper the panel's own tools
|
|
251
|
+
// run, so registry state stays serialized across both surfaces.
|
|
252
|
+
liveHub.serveTool = (name, args, workspaceHandle) => {
|
|
253
|
+
const tool = handlers.find((entry) => entry.name === name);
|
|
254
|
+
if (!tool) throw Object.assign(new Error(`Unknown live tool "${name}".`), { code: "UNKNOWN_TOOL" });
|
|
255
|
+
return tool.handler(args ?? {}, { workspaceHandle });
|
|
256
|
+
};
|
|
257
|
+
publishLiveEndpoint({ port: liveHub.port, token, owner });
|
|
258
|
+
liveHub.server?.once("close", () => removeLiveEndpoint(liveHub.port));
|
|
259
|
+
}
|
|
122
260
|
return { liveHub, handlers };
|
|
123
261
|
}).catch((error) => ({ error }));
|
|
124
262
|
}
|
|
125
263
|
|
|
126
|
-
export function createAgentHandler({ auth = defaultAuth, codex, handlers, liveHub, port,
|
|
264
|
+
export function createAgentHandler({ auth = defaultAuth, codex, models, codexBaseUrl, cliproxyBaseUrl, env, fauxProvider, handlers, liveHub, port, getBridgeOrigin, studioRuntime, clock = Date.now, setIntervalImpl = setInterval, clearIntervalImpl = clearInterval, sessionStore: injectedSessionStore } = {}) {
|
|
127
265
|
const requestContext = new AsyncLocalStorage();
|
|
128
266
|
codex ||= defaultClient(auth, requestContext);
|
|
129
267
|
const runtime = handlers !== undefined || liveHub !== undefined ? Promise.resolve({ handlers: handlers ?? [], liveHub }) : liveToolsRuntime();
|
|
@@ -140,26 +278,380 @@ export function createAgentHandler({ auth = defaultAuth, codex, handlers, liveHu
|
|
|
140
278
|
} catch { return ""; }
|
|
141
279
|
};
|
|
142
280
|
const sessions = new Map();
|
|
143
|
-
const
|
|
281
|
+
const workflowRunners = new Map();
|
|
282
|
+
const studioRunners = new Map();
|
|
283
|
+
let workflowModels = models;
|
|
284
|
+
let workflowModelsPromise = models ? Promise.resolve(models) : null;
|
|
285
|
+
const ensureWorkflowModels = async () => {
|
|
286
|
+
const pending = workflowModelsPromise ??= import("./providers.mjs").then(({ createModels }) => createModels({ auth, codexBaseUrl, cliproxyBaseUrl, env }));
|
|
287
|
+
try { return await pending; }
|
|
288
|
+
catch (error) {
|
|
289
|
+
// A retired build must not clear a newer identity's initialization.
|
|
290
|
+
if (workflowModelsPromise === pending) workflowModelsPromise = null;
|
|
291
|
+
throw error;
|
|
292
|
+
}
|
|
293
|
+
};
|
|
294
|
+
const hasAnyCredential = async () => {
|
|
295
|
+
try { if (await auth.getAccessToken()) return true; } catch { /* an unavailable ChatGPT token is not a credential */ }
|
|
296
|
+
const registry = await ensureWorkflowModels();
|
|
297
|
+
if (typeof registry?.getAuth !== "function") return false;
|
|
298
|
+
const { PROVIDERS } = await import("./providers.mjs");
|
|
299
|
+
for (const provider of PROVIDERS) {
|
|
300
|
+
try { if (await registry.getAuth(provider.id)) return true; } catch { /* one broken provider does not break the gate */ }
|
|
301
|
+
}
|
|
302
|
+
return false;
|
|
303
|
+
};
|
|
304
|
+
// Tests hand in a store of their own; only the real sidecar writes the
|
|
305
|
+
// author's config dir (#375).
|
|
306
|
+
const sessionStore = injectedSessionStore ?? createSessionStore();
|
|
307
|
+
const studioSessions = new Map();
|
|
308
|
+
const studioEvents = new Map();
|
|
309
|
+
let ownedStudioRuntime = studioRuntime || null;
|
|
310
|
+
const studioOwnerTokens = new Map();
|
|
311
|
+
const parseCookies = req => Object.fromEntries(String(req.headers.cookie || "").split(";").map(part => part.trim().split("=")).filter(([key, value]) => key && value).map(([key, value]) => [key, decodeURIComponent(value)]));
|
|
312
|
+
const pruneStudioSessions = () => {
|
|
313
|
+
const now = clock();
|
|
314
|
+
for (const [id, session] of studioSessions) if (!session.activeJobId && now - session.updatedAt > 600_000) { studioSessions.delete(id); studioOwnerTokens.delete(id); for (const turn of session.turns.keys()) studioEvents.delete(turn); }
|
|
315
|
+
const retired = [...studioSessions.entries()].filter(([, session]) => !session.activeJobId).sort((a, b) => a[1].updatedAt - b[1].updatedAt);
|
|
316
|
+
while (studioSessions.size > 256 && retired.length) { const [id, session] = retired.shift(); studioSessions.delete(id); studioOwnerTokens.delete(id); for (const turn of session.turns.keys()) studioEvents.delete(turn); }
|
|
317
|
+
};
|
|
318
|
+
const studioOwner = (req, sessionId, create = false) => {
|
|
319
|
+
pruneStudioSessions();
|
|
320
|
+
const cookies = parseCookies(req), supplied = cookies.studio_owner;
|
|
321
|
+
let owner = studioOwnerTokens.get(sessionId);
|
|
322
|
+
if (!owner && create) { owner = randomBytes(24).toString("hex"); studioOwnerTokens.set(sessionId, owner); return owner; }
|
|
323
|
+
if (!owner || supplied !== owner) throw Object.assign(new Error("Studio session owner mismatch."), { code: "AUTH_REQUIRED" });
|
|
324
|
+
return owner;
|
|
325
|
+
};
|
|
326
|
+
const studioRuntimeFor = async hub => {
|
|
327
|
+
if (ownedStudioRuntime) return ownedStudioRuntime;
|
|
328
|
+
const { createStudioMotionRuntime } = await import("./motion-runtime.mjs");
|
|
329
|
+
if (!hub) return null;
|
|
330
|
+
ownedStudioRuntime = createStudioMotionRuntime({ liveHub: hub, getBridgeOrigin, clock });
|
|
331
|
+
return ownedStudioRuntime;
|
|
332
|
+
};
|
|
333
|
+
const emitStudioEvent = (turnId, event) => {
|
|
334
|
+
const record = studioEvents.get(turnId) || { next: 0, events: [], listeners: new Set(), terminal: false };
|
|
335
|
+
// Execution telemetry is advisory and is validated key-by-key in the
|
|
336
|
+
// browser: it carries no replay cursor, is never retained for a resume,
|
|
337
|
+
// and a reconnect simply misses the frames it was not there for.
|
|
338
|
+
if (["execution_telemetry", "execution_tool_started"].includes(event.type)) {
|
|
339
|
+
studioEvents.set(turnId, record);
|
|
340
|
+
for (const listener of [...record.listeners]) listener(event);
|
|
341
|
+
return;
|
|
342
|
+
}
|
|
343
|
+
const value = { ...event, eventSeq: ++record.next };
|
|
344
|
+
record.next = value.eventSeq;
|
|
345
|
+
const previous = record.events.at(-1);
|
|
346
|
+
if (value.type === "job.progress" && previous?.type === "job.progress" && previous.jobId === value.jobId) record.events[record.events.length - 1] = value;
|
|
347
|
+
else record.events.push(value);
|
|
348
|
+
if (record.events.length > 256) record.events.splice(0, record.events.length - 256);
|
|
349
|
+
if (["done", "error", "receipt"].includes(value.type)) record.terminal = true;
|
|
350
|
+
studioEvents.set(turnId, record); for (const listener of [...record.listeners]) listener(value);
|
|
351
|
+
};
|
|
352
|
+
const unsubscribe = auth.onAuthChange?.(({ kind }) => {
|
|
353
|
+
if (kind !== "signed_out" && kind !== "replaced") return;
|
|
144
354
|
for (const session of sessions.values()) session.controller?.abort();
|
|
145
|
-
|
|
355
|
+
for (const session of studioSessions.values()) session.controller?.abort();
|
|
356
|
+
void ownedStudioRuntime?.dispose?.(); ownedStudioRuntime = studioRuntime || null;
|
|
357
|
+
for (const runner of [...workflowRunners.values(), ...studioRunners.values()]) void runner.close?.();
|
|
358
|
+
workflowRunners.clear(); studioRunners.clear();
|
|
359
|
+
workflowModels = models;
|
|
360
|
+
workflowModelsPromise = models ? Promise.resolve(models) : null;
|
|
361
|
+
sessions.clear(); studioSessions.clear(); studioEvents.clear(); studioOwnerTokens.clear();
|
|
146
362
|
});
|
|
147
363
|
|
|
364
|
+
const studioIdentity = host => Object.fromEntries(["workspaceId", "documentEpoch", "sceneId", "sceneEpoch"].map(key => [key, host[key]]));
|
|
365
|
+
const writeStudioStream = (res, record, after = 0, req = null) => {
|
|
366
|
+
res.writeHead(200, { "content-type": "text/event-stream; charset=utf-8", "cache-control": "no-cache", connection: "keep-alive" }); res.flushHeaders?.();
|
|
367
|
+
let cursor = after; const send = event => {
|
|
368
|
+
if (res.destroyed) return;
|
|
369
|
+
if (event.eventSeq === undefined) { res.write(`data: ${JSON.stringify(event)}\n\n`); return; }
|
|
370
|
+
if (event.eventSeq > cursor) { cursor = event.eventSeq; res.write(`data: ${JSON.stringify(event)}\n\n`); }
|
|
371
|
+
};
|
|
372
|
+
const listener = event => { send(event); if (event.type === "done" && !res.writableEnded) res.end(); }; record.listeners.add(listener); for (const event of record.events) send(event);
|
|
373
|
+
if (record.terminal) { record.listeners.delete(listener); res.end(); return () => {}; }
|
|
374
|
+
const heartbeat = setIntervalImpl(() => { clock(); if (!res.writableEnded && !res.destroyed) res.write(": heartbeat\n\n"); }, 15_000);
|
|
375
|
+
const close = () => { clearIntervalImpl(heartbeat); record.listeners.delete(listener); };
|
|
376
|
+
res.once("close", close); return close;
|
|
377
|
+
};
|
|
378
|
+
const authoritativeStudioContext = async (value, hub) => {
|
|
379
|
+
if (!hub?.workspaceId || !hub?.command) throw Object.assign(new Error("A connected editor is required."), { code: "LIVE_HUB_UNAVAILABLE" });
|
|
380
|
+
let actualHandle;
|
|
381
|
+
try { actualHandle = hub.resolveWorkspace("studio context", value.context.host.workspaceHandle); }
|
|
382
|
+
catch { throw Object.assign(new Error("The submitted Studio handle is stale or unknown."), { code: "LIVE_HUB_UNAVAILABLE" }); }
|
|
383
|
+
if (hub.workspaceId(actualHandle) !== value.context.host.workspaceId) throw Object.assign(new Error("Studio handle belongs to a different workspace."), { code: "STALE_SCENE" });
|
|
384
|
+
const result = await hub.command("read_studio_context", { host: studioIdentity(value.context.host) }, actualHandle);
|
|
385
|
+
return result?.context ?? result;
|
|
386
|
+
};
|
|
387
|
+
const handleStudioTurn = async (req, res, value, path) => {
|
|
388
|
+
const { StudioProtocolError, validateStudioContextFreshness } = await import("../../src/studio-agent-protocol.js");
|
|
389
|
+
const [{ createStudioTools }, { encodeStudioContext }] = await Promise.all([
|
|
390
|
+
import("./studio-tools.mjs"), import("../../src/studio-agent-context.js"),
|
|
391
|
+
]);
|
|
392
|
+
const hubDeps = await runtime; const hub = hubDeps.liveHub || liveHub;
|
|
393
|
+
if (path === "/agent/stop") {
|
|
394
|
+
const session = studioSessions.get(value.sessionId);
|
|
395
|
+
if (!session || session.owner !== parseCookies(req).studio_owner || !session.turns.has(value.turnId)) throw new StudioProtocolError("AUTH_REQUIRED", "Studio stop is not owned by this session.");
|
|
396
|
+
const jobId = value.jobId ?? session.activeJobId;
|
|
397
|
+
// Explicit ids are checked against every motion job this session admitted.
|
|
398
|
+
// Retired ids remain in the set so a repeat Stop is idempotent, while an id
|
|
399
|
+
// admitted by another session is stale even when this session is idle.
|
|
400
|
+
if (value.jobId && !session.motionJobIds.has(value.jobId)) throw new StudioProtocolError("STALE_TARGET", "Stop does not own that motion job.");
|
|
401
|
+
// An owned retired id is still stale while another motion job is active:
|
|
402
|
+
// this Stop cannot acknowledge the current turn's different job.
|
|
403
|
+
if (value.jobId && session.activeJobId && value.jobId !== session.activeJobId) throw new StudioProtocolError("STALE_TARGET", "Stop does not target the active motion job.");
|
|
404
|
+
// #379 / 16r: a job's id is only "acknowledged" by THIS turn's held motion
|
|
405
|
+
// tool — session.activeJobId now stays set only while that job is genuinely
|
|
406
|
+
// still in flight (or pending an explicit accept), and activeJobTurnId ties
|
|
407
|
+
// it to the turn that admitted it. A retired job from an earlier turn must
|
|
408
|
+
// never make a later, unrelated turn's Stop go quiet.
|
|
409
|
+
const acknowledged = Boolean(session.activeJobId) && session.activeJobTurnId === value.turnId;
|
|
410
|
+
// The runtime is the only thing that knows whether the job was applied.
|
|
411
|
+
// Forwarding its outcome keeps the panel from turning "I could not find
|
|
412
|
+
// out" into "nothing was applied"; a discarded outcome reads as proof.
|
|
413
|
+
let outcome = null;
|
|
414
|
+
if (jobId && ownedStudioRuntime?.stop) outcome = await ownedStudioRuntime.stop(jobId);
|
|
415
|
+
if (jobId && session.activeJobId === jobId) { session.activeJobId = null; session.activeJobTurnId = null; }
|
|
416
|
+
// #379 / 16q: `session.controller.signal` is the SAME signal wired into the
|
|
417
|
+
// active turn's prompt context (`:520 signal: controller.signal`), which
|
|
418
|
+
// pi's `withAbortSignal`/`awaitWithContext` races against every awaited
|
|
419
|
+
// internal step — aborting it here interrupts the in-flight generate_motion
|
|
420
|
+
// tool call's OWN completion machinery before it can settle, well before
|
|
421
|
+
// pi's own graceful `lane.abort()` (below) ever runs. A job the runtime
|
|
422
|
+
// actually acknowledged already resolved the held tool call with its real
|
|
423
|
+
// outcome via `runtime.stop()` above; the only abort this stop still needs
|
|
424
|
+
// is the graceful one that stops the model from being prompted again, so
|
|
425
|
+
// `session.controller` must NOT be touched on that path (no other tool is
|
|
426
|
+
// concurrently in flight while generate_motion holds the turn). A stop
|
|
427
|
+
// for a job this turn never acknowledged (none active, or a retired one
|
|
428
|
+
// from an earlier turn) never resolved anything for THIS turn and stays a
|
|
429
|
+
// plain (loud) abort on both signals.
|
|
430
|
+
if (!acknowledged) session.controller?.abort();
|
|
431
|
+
await session.modelSession?.abort?.("studio stop", acknowledged ? { quiet: true } : undefined);
|
|
432
|
+
json(res, 200, { ok: true, status: jobId ? "stopped" : "detached", ...(outcome ? { outcome: { status: outcome.status ?? null, code: outcome.code ?? null, mutated: outcome.mutated ?? null } } : {}) }); return true;
|
|
433
|
+
}
|
|
434
|
+
if (!studioRuntime && (!hub?.command || !hub?.workspaceId)) throw new StudioProtocolError("CAPABILITY_MISSING", "Studio execution is not installed.");
|
|
435
|
+
if (!value.context.host.workspaceHandle) throw new StudioProtocolError("LIVE_HUB_UNAVAILABLE", "A connected editor handle is required.");
|
|
436
|
+
let session = studioSessions.get(value.sessionId);
|
|
437
|
+
const suppliedOwner = parseCookies(req).studio_owner;
|
|
438
|
+
if (session && suppliedOwner && session.owner !== suppliedOwner) throw new StudioProtocolError("AUTH_REQUIRED", "Studio session owner mismatch.");
|
|
439
|
+
if (!session) {
|
|
440
|
+
studioOwner(req, value.sessionId, true);
|
|
441
|
+
let persisted = null;
|
|
442
|
+
try { persisted = sessionStore.read(value.sessionId); } catch { persisted = null; }
|
|
443
|
+
session = { owner: studioOwnerTokens.get(value.sessionId), history: persisted?.history ?? [], persistedItems: persisted?.history?.length ?? 0, meta: persisted?.meta ?? null, turns: new Map(), controller: null, activeJobId: null, activeJobTurnId: null, motionJobIds: new Set(Array.isArray(persisted?.meta?.motionJobIds) ? persisted.meta.motionJobIds.filter(id => typeof id === "string") : []), generationPrompt: null, host: null, updatedAt: clock() };
|
|
444
|
+
studioSessions.set(value.sessionId, session);
|
|
445
|
+
}
|
|
446
|
+
session.updatedAt = clock();
|
|
447
|
+
const existing = session.turns.get(value.turnId);
|
|
448
|
+
if (existing) { writeStudioStream(res, existing, 0, req); return true; }
|
|
449
|
+
let current;
|
|
450
|
+
try { current = studioRuntime?.readContext ? await studioRuntime.readContext(value.context.host) : await authoritativeStudioContext(value, hub); }
|
|
451
|
+
catch (error) { if (error instanceof StudioProtocolError) throw error; if (error?.code) throw new StudioProtocolError(error.code, error.message); throw new StudioProtocolError("CAPABILITY_MISSING", "The connected editor does not expose authoritative Studio context."); }
|
|
452
|
+
validateStudioContextFreshness(value.context, current);
|
|
453
|
+
if (studioRuntime?.handleTurn) { await studioRuntime.handleTurn(value, req, res); return true; }
|
|
454
|
+
const record = { next: 0, events: [], listeners: new Set(), terminal: false }; studioEvents.set(value.turnId, record); session.turns.set(value.turnId, record); session.host = studioIdentity(value.context.host);
|
|
455
|
+
res.setHeader("set-cookie", `studio_owner=${encodeURIComponent(session.owner)}; Path=/agent; HttpOnly; SameSite=Strict`);
|
|
456
|
+
const close = writeStudioStream(res, record, 0, req);
|
|
457
|
+
const send = event => emitStudioEvent(value.turnId, event);
|
|
458
|
+
// The Studio turn is measured exactly like the Workflow turn, correlated by
|
|
459
|
+
// the turn id the host already owns (the frozen envelope carries no room
|
|
460
|
+
// for a second one).
|
|
461
|
+
const telemetry = createTurnTelemetry(send, value.turnId);
|
|
462
|
+
let turnOutcome = "succeeded", turnFailureCode = null, toolFailed = false;
|
|
463
|
+
const controller = new AbortController(); session.controller = controller;
|
|
464
|
+
// Admission is the document identity plus the exact scene revision each
|
|
465
|
+
// command expects. Per-entity tokens are deliberately absent: the revision
|
|
466
|
+
// already moves whenever any authored content changes, and a turn's second
|
|
467
|
+
// edit to the same entity would otherwise carry a token the first edit retired.
|
|
468
|
+
const admission = {
|
|
469
|
+
commandId: () => randomUUID(), host: studioIdentity(value.context.host), revision: value.context.revision.scene,
|
|
470
|
+
refresh: async () => {
|
|
471
|
+
const refreshed = studioRuntime?.readContext ? await studioRuntime.readContext(value.context.host) : await authoritativeStudioContext(value, hub);
|
|
472
|
+
admission.revision = refreshed.revision.scene;
|
|
473
|
+
},
|
|
474
|
+
};
|
|
475
|
+
const runtimeForJob = await studioRuntimeFor(hub);
|
|
476
|
+
const tools = createStudioTools({ liveHub: hub, workspaceHandle: value.context.host.workspaceHandle, session: { signal: controller.signal, admission }, resolveImage: async (id, correlation) => hub.command("resolve_studio_image", { imageId: id, ...correlation }, value.context.host.workspaceHandle) });
|
|
477
|
+
const motion = async args => {
|
|
478
|
+
if (session.generationPrompt === value.text) throw new StudioProtocolError("AUTH_REQUIRED", "This generation request already has a retained result; start a new explicit request.");
|
|
479
|
+
if (!runtimeForJob) throw new StudioProtocolError("CAPABILITY_MISSING", "Studio motion runtime is unavailable.");
|
|
480
|
+
const character = value.context.entities.find(entity => entity.id === args.characterId && entity.kind === "character");
|
|
481
|
+
if (!character) throw new StudioProtocolError("TARGET_NOT_READY", "The admitted character is unavailable.");
|
|
482
|
+
const commandId = randomUUID(); const host = { ...value.context.host, workspaceHandle: value.context.host.workspaceHandle };
|
|
483
|
+
const admissionResult = runtimeForJob.admit({ hostBinding: host, characterId: args.characterId, targetToken: character.token, turnId: value.turnId, commandId, authorization: { id: randomUUID(), generations: 1 }, source: args.source, repair: args.repair ?? "bounded" });
|
|
484
|
+
session.motionJobIds.add(admissionResult.jobId); persistenceMeta.motionJobIds = [...session.motionJobIds]; session.activeJobId = admissionResult.jobId; session.activeJobTurnId = value.turnId; session.generationPrompt = value.text;
|
|
485
|
+
const unsubscribe = runtimeForJob.subscribe(admissionResult.jobId, event => send({ ...event, sourceEventSeq: event.eventSeq }));
|
|
486
|
+
// Subscription precedes start, including replay of the queued admission event.
|
|
487
|
+
let outcome;
|
|
488
|
+
try { outcome = await runtimeForJob.start(admissionResult.jobId); if (outcome?.ok && outcome.status === "installed") send({ type: "receipt", receipt: outcome }); return outcome; }
|
|
489
|
+
finally {
|
|
490
|
+
unsubscribe();
|
|
491
|
+
// A job still awaiting an explicit accept (review_required) stays "active"
|
|
492
|
+
// for the accept route's ownership check (:678); every other outcome —
|
|
493
|
+
// installed, cancelled, proved-not-applied, failed, or a thrown error —
|
|
494
|
+
// is terminal for this session and must not leak into a later turn's Stop.
|
|
495
|
+
if (session.activeJobId === admissionResult.jobId && outcome?.status !== "review_required") { session.activeJobId = null; session.activeJobTurnId = null; }
|
|
496
|
+
}
|
|
497
|
+
};
|
|
498
|
+
const modelTools = tools.map(tool => ({
|
|
499
|
+
...tool,
|
|
500
|
+
handler: async args => {
|
|
501
|
+
const result = await (tool.name === "generate_motion" ? motion(args) : tool.handler(args));
|
|
502
|
+
if (!result || !Array.isArray(result.visualRefs) || !result.visualRefs.length) return result;
|
|
503
|
+
const ref = result.visualRefs.find(item => item?.imageId || item?.id);
|
|
504
|
+
if (!ref) return result;
|
|
505
|
+
const visual = await tools.resolveImage(ref.imageId || ref.id, { receiptId: result.receiptId, revision: result.revision });
|
|
506
|
+
return { ...result, visualStatus: visual.visualStatus, imageId: visual.imageId, revision: visual.revision, receiptId: visual.receiptId, ...(visual.dataUrl ? { dataUrl: visual.dataUrl } : {}) };
|
|
507
|
+
},
|
|
508
|
+
}));
|
|
509
|
+
let frameObservation;
|
|
510
|
+
if (value.attachFrame) {
|
|
511
|
+
const captured = await hub.command("capture_framing_png", {}, value.context.host.workspaceHandle);
|
|
512
|
+
if (!captured?.dataUrl?.startsWith("data:image/")) throw new StudioProtocolError("TARGET_NOT_READY", "The current frame has no image bytes.");
|
|
513
|
+
const match = /^data:([^;]+);base64,(.*)$/.exec(captured.dataUrl);
|
|
514
|
+
frameObservation = { data: match[2], mimeType: match[1], revision: captured.revision, receiptId: captured.receiptId };
|
|
515
|
+
}
|
|
516
|
+
const telemetryTools = new Map();
|
|
517
|
+
const emitFrame = frame => {
|
|
518
|
+
if (frame.type === "tool.start") {
|
|
519
|
+
telemetryTools.set(frame.callId, { execution: telemetry.toolStarted(agentToolCategory(frame.name)), name: frame.name });
|
|
520
|
+
return send(frame);
|
|
521
|
+
}
|
|
522
|
+
if (frame.type === "tool.done") {
|
|
523
|
+
const record = telemetryTools.get(frame.callId);
|
|
524
|
+
if (record && frame.ok && frame.result?.ok && frame.result?.authored) telemetry.applied();
|
|
525
|
+
if (!frame.ok) {
|
|
526
|
+
toolFailed = true;
|
|
527
|
+
if (typeof frame.error === "string" && frame.error.startsWith("Validation failed for tool ")) frame.error = "INVALID_ARGUMENT: Unexpected field.";
|
|
528
|
+
}
|
|
529
|
+
send(frame);
|
|
530
|
+
if (record) {
|
|
531
|
+
record.execution.executed(frame.ok ? "succeeded" : (controller.signal.aborted ? "cancelled" : "failed"));
|
|
532
|
+
record.finished = true;
|
|
533
|
+
}
|
|
534
|
+
return;
|
|
535
|
+
}
|
|
536
|
+
if (frame.type === "error") {
|
|
537
|
+
turnOutcome = controller.signal.aborted || frame.code === "aborted" ? "cancelled" : "failed";
|
|
538
|
+
turnFailureCode = controller.signal.aborted ? "aborted" : frame.code;
|
|
539
|
+
return send(frame);
|
|
540
|
+
}
|
|
541
|
+
if (frame.type === "done") {
|
|
542
|
+
if (turnOutcome === "succeeded" && !toolFailed) telemetry.finished("succeeded", null);
|
|
543
|
+
else telemetry.finished(turnOutcome, turnFailureCode || "tool_failed");
|
|
544
|
+
}
|
|
545
|
+
send(frame);
|
|
546
|
+
};
|
|
547
|
+
const persistenceMeta = { sceneName: value.context?.scene?.name ?? value.context?.sceneName ?? null, firstText: value.text, motionJobIds: [...session.motionJobIds] };
|
|
548
|
+
let runner = studioRunners.get(value.sessionId);
|
|
549
|
+
if (!runner) {
|
|
550
|
+
runner = createAgentRunner({ models: await ensureWorkflowModels(), fauxProvider, sessionStore, clock, codexBaseUrl, cliproxyBaseUrl, auth, env });
|
|
551
|
+
studioRunners.set(value.sessionId, runner);
|
|
552
|
+
}
|
|
553
|
+
try {
|
|
554
|
+
if (!session.modelSession) session.modelSession = await runner.openSession(value.sessionId, { surface: "studio" });
|
|
555
|
+
for await (const frame of session.modelSession.start({
|
|
556
|
+
surface: "studio", sessionId: value.sessionId, model: value.model || (fauxProvider ? `${fauxProvider.provider?.id || fauxProvider.provider || "faux"}/scripted` : "gpt-6-astra"), effort: value.effort,
|
|
557
|
+
text: value.text, attachments: value.attachments, contextText: encodeStudioContext(value.context), frameObservation,
|
|
558
|
+
context: value.context, tools: modelTools, signal: controller.signal, emit: send,
|
|
559
|
+
meta: persistenceMeta,
|
|
560
|
+
quotaEvent: headers => codex?.parseQuotaHeaders ? quotaEvent(codex, headers) : { type: "quota", plan: null, primary: { usedPercent: null, windowMinutes: null, resetAt: null }, credits: { has: null } },
|
|
561
|
+
})) emitFrame(frame);
|
|
562
|
+
} catch (error) {
|
|
563
|
+
const info = errorInfo(error);
|
|
564
|
+
turnOutcome = controller.signal.aborted ? "cancelled" : "failed";
|
|
565
|
+
turnFailureCode = controller.signal.aborted ? "aborted" : agentFailureCode(error, controller.signal, toolFailed);
|
|
566
|
+
send({ type: "error", code: controller.signal.aborted ? "aborted" : error.code || "upstream", message: info.status === undefined ? error.message : info.message, ...(info.status === undefined ? {} : { status: info.status }) });
|
|
567
|
+
emitFrame({ type: "done" });
|
|
568
|
+
}
|
|
569
|
+
if (controller.signal.aborted) { turnOutcome = "cancelled"; turnFailureCode = "aborted"; }
|
|
570
|
+
if (!record.terminal) emitFrame({ type: "done" });
|
|
571
|
+
close(); if (!res.writableEnded) res.end(); session.controller = null; return true;
|
|
572
|
+
};
|
|
148
573
|
const handle = async (req, res, path = new URL(req.url, "http://127.0.0.1").pathname) => {
|
|
149
574
|
if (!path.startsWith("/agent/")) return false;
|
|
150
575
|
if (port !== undefined && !allowAgentOrigin(req, typeof port === "function" ? port() : port)) {
|
|
151
576
|
json(res, 403, { error: "forbidden origin" }); return true;
|
|
152
577
|
}
|
|
578
|
+
if (path.startsWith("/agent/turn/") && path.endsWith("/events") && req.method === "GET") {
|
|
579
|
+
if (!await hasAnyCredential()) { json(res, 401, { error: { code: "AUTH_REQUIRED", message: "Sign in or configure a provider key." } }); return true; }
|
|
580
|
+
const turnId = decodeURIComponent(path.slice("/agent/turn/".length, -"/events".length));
|
|
581
|
+
const session = [...studioSessions.values()].find(candidate => candidate.turns.has(turnId)); const record = studioEvents.get(turnId);
|
|
582
|
+
if (!record || !session || parseCookies(req).studio_owner !== session.owner) { json(res, 403, { error: { code: "AUTH_REQUIRED", message: "Studio event stream is not owned by this session." } }); return true; }
|
|
583
|
+
const after = Number(new URL(req.url, "http://127.0.0.1").searchParams.get("after") || 0);
|
|
584
|
+
if (!Number.isSafeInteger(after) || after < 0) { json(res, 400, { error: "invalid cursor" }); return true; }
|
|
585
|
+
writeStudioStream(res, record, after, req); return true;
|
|
586
|
+
}
|
|
587
|
+
if (path.startsWith("/agent/turn/") && path.endsWith("/steer") && req.method === "POST") {
|
|
588
|
+
const turnId = decodeURIComponent(path.slice("/agent/turn/".length, -"/steer".length));
|
|
589
|
+
let value;
|
|
590
|
+
try {
|
|
591
|
+
value = await readBody(req, TURN_BODY_LIMIT);
|
|
592
|
+
if (!value || typeof value.text !== "string" || !value.text.trim() || !validAttachments(value.attachments)) throw new Error("Invalid request.");
|
|
593
|
+
} catch { json(res, 400, { error: "invalid request" }); return true; }
|
|
594
|
+
// Studio turnIds are frozen envelopes; their events are keyed in studioSessions,
|
|
595
|
+
// never in the Workflow `sessions` map, so that lookup alone tells them apart.
|
|
596
|
+
if ([...studioSessions.values()].some(candidate => candidate.turns.has(turnId))) { json(res, 409, { error: { code: "STEER_UNSUPPORTED", message: "Steering a Studio turn is not supported." } }); return true; }
|
|
597
|
+
const session = [...sessions.values()].find(candidate => candidate.turnId === turnId);
|
|
598
|
+
if (!session) { json(res, 404, { error: "turn not found" }); return true; }
|
|
599
|
+
if (!session.running || !session.modelSession) { json(res, 409, { error: { code: "NO_ACTIVE_TURN", message: "This turn already ended." } }); return true; }
|
|
600
|
+
const images = (Array.isArray(value.attachments) ? value.attachments : []).map(attachment => {
|
|
601
|
+
const match = /^data:([^;]+);base64,(.*)$/.exec(attachment?.dataUrl || "");
|
|
602
|
+
return match ? { type: "image", data: match[2], mimeType: match[1] } : null;
|
|
603
|
+
}).filter(Boolean);
|
|
604
|
+
await session.modelSession.steer(value.text, images);
|
|
605
|
+
json(res, 200, { ok: true, queued: true }); return true;
|
|
606
|
+
}
|
|
607
|
+
if (path === "/agent/providers" && req.method === "GET") {
|
|
608
|
+
const { providerStatus } = await import("./providers.mjs");
|
|
609
|
+
try { json(res, 200, { providers: providerStatus({ auth }) }); }
|
|
610
|
+
catch { json(res, 500, { error: "provider credentials unavailable" }); }
|
|
611
|
+
return true;
|
|
612
|
+
}
|
|
613
|
+
if (/^\/agent\/providers\/[^/]+$/.test(path) && ["PUT", "DELETE"].includes(req.method)) {
|
|
614
|
+
const { PROVIDERS } = await import("./providers.mjs");
|
|
615
|
+
const keys = await import("./provider-keys.mjs");
|
|
616
|
+
const { createCredentialStore } = await import("./credential-store.mjs");
|
|
617
|
+
const id = path.slice("/agent/providers/".length);
|
|
618
|
+
if (!PROVIDERS.some((provider) => provider.id === id)) { json(res, 404, { error: "provider not found" }); return true; }
|
|
619
|
+
const credentials = createCredentialStore({ auth, keys });
|
|
620
|
+
let key;
|
|
621
|
+
if (req.method === "PUT") {
|
|
622
|
+
if (id === "openai-codex") { json(res, 400, { error: "Use ChatGPT sign-in for OpenAI Codex." }); return true; }
|
|
623
|
+
try {
|
|
624
|
+
const value = await readBody(req);
|
|
625
|
+
if (typeof value?.key !== "string" || !value.key.trim()) throw new Error("Invalid key.");
|
|
626
|
+
key = value.key.trim();
|
|
627
|
+
} catch { json(res, 400, { error: "a non-empty API key is required" }); return true; }
|
|
628
|
+
}
|
|
629
|
+
try {
|
|
630
|
+
if (req.method === "PUT") await credentials.modify(id, async () => ({ type: "api_key", key }));
|
|
631
|
+
else await credentials.delete(id);
|
|
632
|
+
json(res, 200, { ok: true });
|
|
633
|
+
} catch { json(res, 500, { error: "provider credentials could not be saved" }); }
|
|
634
|
+
return true;
|
|
635
|
+
}
|
|
636
|
+
if (path === "/agent/sessions" && req.method === "GET") {
|
|
637
|
+
const surface = new URL(req.url, "http://127.0.0.1").searchParams.get("surface") || undefined;
|
|
638
|
+
try { json(res, 200, { sessions: sessionStore.list({ surface }) }); }
|
|
639
|
+
catch { json(res, 500, { error: "session history unavailable" }); }
|
|
640
|
+
return true;
|
|
641
|
+
}
|
|
642
|
+
if (path.startsWith("/agent/sessions/") && req.method === "GET") {
|
|
643
|
+
const sessionId = decodeURIComponent(path.slice("/agent/sessions/".length));
|
|
644
|
+
try {
|
|
645
|
+
const persisted = sessionStore.read(sessionId);
|
|
646
|
+
if (!persisted) { json(res, 404, { error: "session not found" }); return true; }
|
|
647
|
+
json(res, 200, { sessionId, transcript: transcriptFromHistory(persisted.history), meta: persisted.meta });
|
|
648
|
+
} catch { json(res, 404, { error: "session not found" }); }
|
|
649
|
+
return true;
|
|
650
|
+
}
|
|
153
651
|
if (path === "/agent/models" && req.method === "GET") {
|
|
154
652
|
try {
|
|
155
|
-
const
|
|
156
|
-
|
|
157
|
-
const id = typeof model === "string" ? model : model.slug || model.id;
|
|
158
|
-
const efforts = Array.isArray(model.supported_reasoning_levels) ? model.supported_reasoning_levels.map((level) => (typeof level === "string" ? level : level.effort)).filter(Boolean) : [];
|
|
159
|
-
return { id, label: id, efforts, defaultEffort: typeof model.default_reasoning_level === "string" ? model.default_reasoning_level : efforts[0] ?? null };
|
|
160
|
-
});
|
|
161
|
-
models.sort((a, b) => Number(b.id === "gpt-6-astra") - Number(a.id === "gpt-6-astra"));
|
|
162
|
-
json(res, 200, { models });
|
|
653
|
+
const { listAgentModels } = await import("./providers.mjs");
|
|
654
|
+
json(res, 200, await listAgentModels({ auth, codex, models: await ensureWorkflowModels(), env }));
|
|
163
655
|
} catch (error) { json(res, error.status === 401 ? 401 : 502, { error: errorInfo(error) }); }
|
|
164
656
|
return true;
|
|
165
657
|
}
|
|
@@ -183,7 +675,7 @@ export function createAgentHandler({ auth = defaultAuth, codex, handlers, liveHu
|
|
|
183
675
|
return true;
|
|
184
676
|
}
|
|
185
677
|
if (path === "/agent/video/providers" && req.method === "GET") {
|
|
186
|
-
json(res, 200, { providers: createVideoAdapters().map((adapter) => ({ id: adapter.id, name: adapter.name, configured: adapter.configured() })) });
|
|
678
|
+
json(res, 200, { providers: createVideoAdapters().map((adapter) => ({ id: adapter.id, name: adapter.name, configured: adapter.configured(), ...(adapter.model ? { model: adapter.model, resolution: adapter.resolution } : {}) })) });
|
|
187
679
|
return true;
|
|
188
680
|
}
|
|
189
681
|
if (path === "/agent/video" && req.method === "POST") {
|
|
@@ -191,33 +683,55 @@ export function createAgentHandler({ auth = defaultAuth, codex, handlers, liveHu
|
|
|
191
683
|
try {
|
|
192
684
|
value = await readBody(req, IMAGE_BODY_LIMIT);
|
|
193
685
|
if (!value || typeof value.provider !== "string" || typeof value.prompt !== "string" || !value.prompt.trim() || typeof value.imageDataUrl !== "string" || !value.imageDataUrl.startsWith("data:image/") || (value.lastFrameDataUrl !== undefined && (typeof value.lastFrameDataUrl !== "string" || !value.lastFrameDataUrl.startsWith("data:image/"))) || !Number.isFinite(Number(value.durationSeconds)) || Number(value.durationSeconds) < 1 || Number(value.durationSeconds) > 15 || typeof value.aspect !== "string" || (value.model !== undefined && typeof value.model !== "string")) throw new Error("Invalid request.");
|
|
194
|
-
|
|
686
|
+
if (value.provider === "fal") validateFalVideoRequest({ model: process.env.FAL_MODEL || undefined, durationSeconds: Number(value.durationSeconds), aspect: value.aspect });
|
|
687
|
+
} catch (error) { json(res, error?.code === "fal-invalid-request" ? 422 : 400, { error: error?.message || "invalid request" }); return true; }
|
|
195
688
|
const adapter = createVideoAdapters().find((entry) => entry.id === value.provider);
|
|
196
689
|
if (!adapter || !adapter.configured()) { json(res, 409, { error: "video provider is not configured" }); return true; }
|
|
197
690
|
try {
|
|
198
691
|
const result = await adapter.generate({ ...value, durationSeconds: Number(value.durationSeconds) });
|
|
199
|
-
json(res, 200, { ...(result.mp4Base64 ? { dataUrl: `data:video/mp4;base64,${result.mp4Base64}` } : { url: result.url }), width: result.width, height: result.height, seconds: result.seconds, ...(result.preservation ? { preservation: result.preservation } : {}) });
|
|
692
|
+
json(res, 200, { ...(result.mp4Base64 ? { dataUrl: `data:video/mp4;base64,${result.mp4Base64}` } : { url: result.url }), width: result.width, height: result.height, fps: result.fps, seconds: result.seconds, ...(result.metadataMeasured !== undefined ? { metadataMeasured: result.metadataMeasured } : {}), ...(result.preservation ? { preservation: result.preservation } : {}) });
|
|
200
693
|
} catch (error) {
|
|
201
694
|
// A generated H3 take that fails the plate check is unsafe to show as
|
|
202
695
|
// a locked shot. Keep the distinction visible to the client so it can
|
|
203
696
|
// ask for a retry instead of silently accepting a drifting set.
|
|
204
|
-
const status =
|
|
697
|
+
const status = ["h3-preservation-failed", "fal-invalid-request"].includes(error?.code) ? 422 : 502;
|
|
205
698
|
json(res, status, { error: error?.message || "video provider failed", ...(error?.preservation ? { preservation: error.preservation } : {}) });
|
|
206
699
|
}
|
|
207
700
|
return true;
|
|
208
701
|
}
|
|
702
|
+
if (path.startsWith("/agent/jobs/") && path.endsWith("/accept") && req.method === "POST") {
|
|
703
|
+
if (!await hasAnyCredential()) { json(res, 401, { error: { code: "AUTH_REQUIRED", message: "Sign in or configure a provider key." } }); return true; }
|
|
704
|
+
let value; try { value = await readBody(req); } catch { json(res, 400, { error: "invalid request" }); return true; }
|
|
705
|
+
const jobId = decodeURIComponent(path.slice("/agent/jobs/".length, -"/accept".length)); const session = studioSessions.get(value?.sessionId);
|
|
706
|
+
if (!session || session.owner !== parseCookies(req).studio_owner || value.surface !== "studio" || value.explicitUnverifiedAcceptance !== true || !session.turns.has(value.turnId) || session.activeJobId !== jobId) { json(res, 403, { error: { code: "AUTH_REQUIRED", message: "Only the owning Studio UI may accept this candidate." } }); return true; }
|
|
707
|
+
try { const receipt = await ownedStudioRuntime.accept(jobId); const record = studioEvents.get(value.turnId); if (record) { emitStudioEvent(value.turnId, { type: "receipt", receipt }); emitStudioEvent(value.turnId, { type: "done" }); record.terminal = true; } session.activeJobId = null; session.activeJobTurnId = null; json(res, 200, { receipt }); } catch (error) { json(res, 409, { error: { code: error.code || "VERIFICATION_FAILED", message: error.message } }); }
|
|
708
|
+
return true;
|
|
709
|
+
}
|
|
209
710
|
if (req.method !== "POST" || !["/agent/turn", "/agent/stop"].includes(path)) {
|
|
210
711
|
json(res, 404, { error: "not found" }); return true;
|
|
211
712
|
}
|
|
212
713
|
let value;
|
|
213
714
|
try {
|
|
214
|
-
value = await readBody(req);
|
|
215
|
-
if (
|
|
715
|
+
value = await readBody(req, TURN_BODY_LIMIT);
|
|
716
|
+
if (value?.surface === "studio") {
|
|
717
|
+
// Lazy only for the minimal legacy-sidecar fixture, which omits src/.
|
|
718
|
+
// Actual npm packages include src; there is exactly one validator.
|
|
719
|
+
const protocol = await import("../../src/studio-agent-protocol.js");
|
|
720
|
+
value = path === "/agent/stop" ? protocol.validateStudioStopEnvelope(value) : protocol.validateStudioTurnEnvelope(value);
|
|
721
|
+
} else if (!value || (value.surface !== undefined && value.surface !== "workflow")
|
|
722
|
+
|| value.context !== undefined || value.turnId !== undefined || typeof value.sessionId !== "string" || !value.sessionId
|
|
216
723
|
|| (path === "/agent/turn" && (typeof value.text !== "string"
|
|
217
724
|
|| (value.attachFrame !== undefined && typeof value.attachFrame !== "boolean")
|
|
725
|
+
|| !validAttachments(value.attachments)
|
|
218
726
|
|| (value.model !== undefined && typeof value.model !== "string")
|
|
219
727
|
|| (value.effort !== undefined && !REASONING_EFFORTS.includes(value.effort))))) throw new Error("Invalid request.");
|
|
220
|
-
} catch
|
|
728
|
+
} catch (error) {
|
|
729
|
+
json(res, 400, { error: error?.name === "StudioProtocolError" ? error.toJSON() : "invalid request" }); return true;
|
|
730
|
+
}
|
|
731
|
+
if (value.surface === "studio") {
|
|
732
|
+
try { return await handleStudioTurn(req, res, value, path); }
|
|
733
|
+
catch (error) { if (error instanceof (await import("../../src/studio-agent-protocol.js")).StudioProtocolError) json(res, 409, { error: error.toJSON() }); else throw error; return true; }
|
|
734
|
+
}
|
|
221
735
|
if (path === "/agent/stop") {
|
|
222
736
|
sessions.get(value.sessionId)?.controller?.abort();
|
|
223
737
|
json(res, 200, { ok: true }); return true;
|
|
@@ -234,115 +748,101 @@ export function createAgentHandler({ auth = defaultAuth, codex, handlers, liveHu
|
|
|
234
748
|
sessions.set(value.sessionId, session);
|
|
235
749
|
const controller = new AbortController();
|
|
236
750
|
const { signal } = controller;
|
|
237
|
-
Object.assign(session, { running: true, controller, signal });
|
|
751
|
+
Object.assign(session, { running: true, controller, signal, emit: send });
|
|
238
752
|
const disconnect = () => controller.abort();
|
|
239
753
|
res.once("close", disconnect);
|
|
754
|
+
const turnId = typeof value.turn_id === "string" && /^[a-f0-9]{32}$/.test(value.turn_id) ? value.turn_id : null;
|
|
755
|
+
session.turnId = turnId;
|
|
756
|
+
const telemetry = createTurnTelemetry(send, turnId);
|
|
757
|
+
const telemetryTools = new Map();
|
|
758
|
+
let turnOutcome = "succeeded";
|
|
759
|
+
let turnFailureCode = null;
|
|
760
|
+
let toolFailed = false;
|
|
240
761
|
let quota;
|
|
241
|
-
const
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
762
|
+
const quotaForResponse = (headers, model) => {
|
|
763
|
+
if (quota) return quota;
|
|
764
|
+
quota = model?.provider === "openai-codex" ? quotaEvent(codex, headers) : {
|
|
765
|
+
type: "quota", plan: null,
|
|
766
|
+
primary: { usedPercent: null, windowMinutes: null, resetAt: null },
|
|
767
|
+
credits: { has: null },
|
|
768
|
+
};
|
|
769
|
+
return quota;
|
|
245
770
|
};
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
771
|
+
const emitFrame = (frame) => {
|
|
772
|
+
if (frame.type === "tool.start") {
|
|
773
|
+
telemetryTools.set(frame.callId, { execution: telemetry.toolStarted(agentToolCategory(frame.name)), name: frame.name });
|
|
774
|
+
return send(frame);
|
|
775
|
+
}
|
|
776
|
+
if (frame.type === "tool.done") {
|
|
777
|
+
send(frame);
|
|
778
|
+
const record = telemetryTools.get(frame.callId);
|
|
779
|
+
if (record) {
|
|
780
|
+
if (frame.ok && appliedCanvasResult(record.name, frame.result)) telemetry.applied();
|
|
781
|
+
record.execution.executed(frame.ok ? "succeeded" : (signal.aborted ? "cancelled" : "failed"));
|
|
782
|
+
record.finished = true;
|
|
254
783
|
}
|
|
784
|
+
if (!frame.ok) toolFailed = true;
|
|
785
|
+
return;
|
|
255
786
|
}
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
787
|
+
if (frame.type === "error") {
|
|
788
|
+
if (frame.code === "truncated") {
|
|
789
|
+
turnOutcome = "unresolved";
|
|
790
|
+
return;
|
|
791
|
+
}
|
|
792
|
+
turnOutcome = signal.aborted || frame.code === "aborted" ? "cancelled" : "failed";
|
|
793
|
+
turnFailureCode = signal.aborted ? "aborted" : frame.code;
|
|
794
|
+
return send(frame);
|
|
795
|
+
}
|
|
796
|
+
if (frame.type === "done") {
|
|
797
|
+
for (const record of telemetryTools.values()) if (!record.finished) {
|
|
798
|
+
record.execution.executed(signal.aborted ? "cancelled" : "failed");
|
|
799
|
+
record.finished = true;
|
|
800
|
+
}
|
|
801
|
+
if (turnOutcome === "unresolved") return send(frame);
|
|
802
|
+
if (turnOutcome === "succeeded" && !toolFailed) telemetry.finished("succeeded", null);
|
|
803
|
+
else telemetry.finished(toolFailed ? "failed" : turnOutcome, toolFailed ? "tool_failed" : turnFailureCode);
|
|
265
804
|
}
|
|
805
|
+
send(frame);
|
|
266
806
|
};
|
|
267
807
|
const turn = async () => {
|
|
268
|
-
if (!await
|
|
808
|
+
if (!await hasAnyCredential()) throw Object.assign(new Error("Authentication required."), { status: 401 });
|
|
809
|
+
workflowModels = await ensureWorkflowModels();
|
|
269
810
|
const dependencies = await runtime;
|
|
270
|
-
session.codex = { editImage: (args) =>
|
|
271
|
-
const
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
const args = typeof item.arguments === "string" ? JSON.parse(item.arguments) : item.arguments;
|
|
277
|
-
send({ type: "tool.start", callId: item.call_id, name: item.name, label: item.name.replaceAll("_", " "), args });
|
|
278
|
-
const started = performance.now();
|
|
279
|
-
try {
|
|
280
|
-
if (dependencies.error) throw dependencies.error;
|
|
281
|
-
const result = await tool.handler(args);
|
|
282
|
-
signal.throwIfAborted();
|
|
283
|
-
send({ type: "tool.done", callId: item.call_id, ok: true, elapsedMs: Math.round(performance.now() - started), result });
|
|
284
|
-
return result;
|
|
285
|
-
} catch (error) {
|
|
286
|
-
if (process.env.COZYCLAY_AGENT_DEBUG) console.error("[agent] tool", item.name, "failed:", error?.message);
|
|
287
|
-
send({ type: "tool.done", callId: item.call_id, ok: false, elapsedMs: Math.round(performance.now() - started), error: errorInfo(error).message });
|
|
288
|
-
throw error;
|
|
289
|
-
}
|
|
290
|
-
};
|
|
291
|
-
let text = value.text;
|
|
292
|
-
if (value.attachFrame) {
|
|
293
|
-
const captured = await executeTool({ call_id: "attached-frame", name: "capture_blocking_frame", arguments: {} }, tools.internal.capture);
|
|
294
|
-
text += `\nAttached frame imageId: ${captured.imageId}`;
|
|
811
|
+
session.codex = { editImage: (args) => codex.editImage(args) };
|
|
812
|
+
const workflowTools = createAgentTools({ ...dependencies, session, emit: send });
|
|
813
|
+
let workflowRunner = workflowRunners.get(value.sessionId);
|
|
814
|
+
if (!workflowRunner) {
|
|
815
|
+
workflowRunner = createAgentRunner({ models: workflowModels, tools: workflowTools, systemPrompt: SYSTEM_PROMPT, clock, fauxProvider, sessionStore, codexBaseUrl, cliproxyBaseUrl, auth, env, onQuota: quotaForResponse });
|
|
816
|
+
workflowRunners.set(value.sessionId, workflowRunner);
|
|
295
817
|
}
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
// the failed request rather than replaying already-executed scene tools.
|
|
300
|
-
while (true) {
|
|
301
|
-
const output = await retryOverloaded(() => retryAuth(async () => {
|
|
302
|
-
const stream = codex.streamResponses({ input: history, tools: agentToolSchemas(tools), instructions: SYSTEM_PROMPT, model: value.model, effort: value.effort, signal });
|
|
303
|
-
const headers = stream.headers.then(observeHeaders, () => {});
|
|
304
|
-
const items = [];
|
|
305
|
-
try {
|
|
306
|
-
for await (const event of stream) {
|
|
307
|
-
signal.throwIfAborted();
|
|
308
|
-
if (event.type === "response.output_text.delta") send({ type: "text.delta", text: event.delta });
|
|
309
|
-
if (event.type === "response.output_item.done") items.push(event.item);
|
|
310
|
-
if (event.type === "error" || event.type === "response.failed") {
|
|
311
|
-
if (process.env.COZYCLAY_AGENT_DEBUG) console.error("[agent] model event:", JSON.stringify(event).slice(0, 600));
|
|
312
|
-
const code = event.error?.code ?? event.response?.error?.code;
|
|
313
|
-
throw Object.assign(new Error("Model response failed."), code === "server_is_overloaded" ? { code: "overloaded" } : code === "server_error" ? { code: "server_error" } : {});
|
|
314
|
-
}
|
|
315
|
-
}
|
|
316
|
-
await headers;
|
|
317
|
-
return items;
|
|
318
|
-
} finally { await headers; }
|
|
319
|
-
}));
|
|
320
|
-
for (const item of output) {
|
|
321
|
-
history.push(item); // Preserve reasoning items verbatim.
|
|
322
|
-
if (item.type === "function_call") {
|
|
323
|
-
const result = await executeTool(item);
|
|
324
|
-
history.push({ type: "function_call_output", call_id: item.call_id, output: JSON.stringify(result) });
|
|
325
|
-
}
|
|
326
|
-
}
|
|
327
|
-
if (!output.some((item) => item.type === "function_call")) break;
|
|
328
|
-
}
|
|
329
|
-
session.history = history;
|
|
818
|
+
if (!session.modelSession) session.modelSession = await workflowRunner.openSession(value.sessionId, { surface: "workflow" });
|
|
819
|
+
const turnInput = { surface: "workflow", sessionId: value.sessionId, text: value.text, model: value.model || "gpt-6-astra", effort: value.effort, attachments: value.attachments, attachFrame: value.attachFrame, signal, emit: send, quotaEvent: quotaForResponse };
|
|
820
|
+
for await (const frame of session.modelSession.start(turnInput)) emitFrame(frame);
|
|
330
821
|
};
|
|
331
|
-
try { await
|
|
822
|
+
try { await turn(); }
|
|
332
823
|
catch (error) {
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
824
|
+
turnOutcome = signal.aborted ? "cancelled" : "failed";
|
|
825
|
+
turnFailureCode = signal.aborted ? "aborted" : agentFailureCode(error, signal, toolFailed);
|
|
826
|
+
if (!signal.aborted) emitFrame({ type: "error", ...errorInfo(error, quota) });
|
|
827
|
+
if (!res.writableEnded) emitFrame({ type: "done" });
|
|
828
|
+
}
|
|
829
|
+
finally {
|
|
830
|
+
if (!res.writableEnded) res.end();
|
|
831
|
+
session.running = false;
|
|
832
|
+
res.off("close", disconnect);
|
|
340
833
|
}
|
|
341
834
|
return true;
|
|
342
835
|
};
|
|
836
|
+
|
|
343
837
|
handle.close = async () => {
|
|
344
838
|
unsubscribe?.();
|
|
345
839
|
for (const session of sessions.values()) session.controller?.abort();
|
|
840
|
+
for (const session of studioSessions.values()) session.controller?.abort();
|
|
841
|
+
for (const runner of workflowRunners.values()) await runner.close?.();
|
|
842
|
+
for (const runner of studioRunners.values()) await runner.close?.();
|
|
843
|
+
workflowRunners.clear(); studioRunners.clear();
|
|
844
|
+
if (ownedStudioRuntime?.dispose) await ownedStudioRuntime.dispose();
|
|
845
|
+
studioSessions.clear(); studioEvents.clear(); studioOwnerTokens.clear();
|
|
346
846
|
sessions.clear();
|
|
347
847
|
const { liveHub: hub } = await runtime;
|
|
348
848
|
if (hub?.server) {
|