@sema-agent/server 7.137.0 → 7.138.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/MIGRATION.md +328 -139
- package/README.md +1 -0
- package/README.zh-CN.md +1 -0
- package/USAGE.md +67 -20
- package/dist/approval-card.d.ts +21 -14
- package/dist/approval-card.js +1 -0
- package/dist/audit.js +1 -0
- package/dist/background-agent-name.d.ts +5 -0
- package/dist/background-agent-name.js +5 -0
- package/dist/boot/budget-tracing.d.ts +1 -12
- package/dist/boot/deployment-mods.d.ts +46 -16
- package/dist/boot/deployment-mods.js +82 -6
- package/dist/boot/governance-seams.d.ts +10 -17
- package/dist/boot/governance-seams.js +5 -17
- package/dist/boot/leader.d.ts +3 -0
- package/dist/boot/leader.js +1 -0
- package/dist/boot/org-delivery.js +15 -2
- package/dist/boot/resolve-spec.d.ts +3 -0
- package/dist/boot/resolve-spec.js +23 -12
- package/dist/boot/retention-lane.d.ts +11 -5
- package/dist/boot/retention-lane.js +2 -2
- package/dist/boot/runner-deps.d.ts +7 -3
- package/dist/boot/runner-deps.js +1 -0
- package/dist/boot/stage-04-stores.d.ts +1 -12
- package/dist/boot/stage-05-execution-env.d.ts +3 -12
- package/dist/boot/stage-05-execution-env.js +7 -4
- package/dist/boot/stage-06-runners.d.ts +3 -12
- package/dist/boot/stage-06-runners.js +5 -2
- package/dist/boot/stage-07-capability-layer.d.ts +3 -12
- package/dist/boot/stage-07-capability-layer.js +2 -2
- package/dist/boot/stage-08-reapers.d.ts +3 -12
- package/dist/boot/stage-09-leader.d.ts +3 -12
- package/dist/boot/stage-09-leader.js +2 -2
- package/dist/boot/stage-10-http-server.d.ts +3 -12
- package/dist/boot/stage-10-http-server.js +4 -2
- package/dist/boot/web-admin.d.ts +5 -2
- package/dist/boot/web-admin.js +3 -3
- package/dist/brain.js +5 -5
- package/dist/budget.d.ts +17 -15
- package/dist/budget.js +2 -2
- package/dist/capabilities/mcp-probe.d.ts +36 -11
- package/dist/capabilities/mcp-probe.js +44 -5
- package/dist/capability-facts.d.ts +6 -0
- package/dist/capability-facts.js +5 -0
- package/dist/checkpoint-identity.d.ts +13 -6
- package/dist/config-catalog.js +7 -1
- package/dist/config-types.d.ts +15 -7
- package/dist/config.js +8 -5
- package/dist/deployment-shape.d.ts +6 -6
- package/dist/gate-settlement.d.ts +71 -0
- package/dist/gate-settlement.js +40 -0
- package/dist/hooks/classic-stations.d.ts +1 -0
- package/dist/hooks/classic-stations.js +1 -0
- package/dist/hooks/command-spawn.d.ts +2 -2
- package/dist/hooks/hook-runner.js +14 -14
- package/dist/http/admission.js +4 -2
- package/dist/http/dispatch.d.ts +4 -0
- package/dist/http/dispatch.js +3 -0
- package/dist/http/durable-run-leg.js +4 -2
- package/dist/http/memory-compliance-failure-reply.js +3 -0
- package/dist/http/resume-legs.d.ts +5 -11
- package/dist/http/resume-legs.js +136 -91
- package/dist/http/route-ctx.d.ts +30 -51
- package/dist/http/route-ctx.js +2 -5
- package/dist/http/routes/a2a-serve.js +10 -4
- package/dist/http/routes/agents-roster.d.ts +1 -1
- package/dist/http/routes/agents-roster.js +5 -1
- package/dist/http/routes/approvals-assistant.js +6 -6
- package/dist/http/routes/capabilities.js +12 -10
- package/dist/http/routes/mcp-reconnect.d.ts +3 -2
- package/dist/http/routes/mcp-reconnect.js +7 -5
- package/dist/http/routes/memory-consolidation.d.ts +1 -1
- package/dist/http/routes/memory-origin.d.ts +1 -1
- package/dist/http/routes/rules.js +3 -3
- package/dist/http/routes/runs.js +11 -7
- package/dist/http/routes/sessions.js +6 -4
- package/dist/http/routes/side-query.js +2 -3
- package/dist/http/routes/tasks.js +792 -780
- package/dist/http/server.d.ts +31 -25
- package/dist/http/wire-types.d.ts +14 -6
- package/dist/identity-source.d.ts +1 -1
- package/dist/index.d.ts +2 -0
- package/dist/index.js +1 -0
- package/dist/leader/wire.d.ts +7 -0
- package/dist/leader/wire.js +5 -4
- package/dist/mcp-assertion-header.d.ts +51 -40
- package/dist/mcp-assertion-header.js +10 -11
- package/dist/mcp-authorization-face.d.ts +27 -0
- package/dist/mcp-authorization-face.js +15 -0
- package/dist/memory-operator-faces.d.ts +13 -4
- package/dist/memory-operator-faces.js +1 -17
- package/dist/memory-sync-client.js +72 -57
- package/dist/memory-sync.d.ts +1 -1
- package/dist/model-route-gap.d.ts +6 -22
- package/dist/model-route-gap.js +7 -5
- package/dist/mods-declaration.d.ts +110 -0
- package/dist/mods-declaration.js +109 -0
- package/dist/observability/err-text.d.ts +5 -0
- package/dist/observability/err-text.js +9 -0
- package/dist/observability/fail-open.d.ts +5 -1
- package/dist/observability/fail-open.js +5 -1
- package/dist/observability/secret-env-scrub.d.ts +1 -1
- package/dist/observability/secret-env-scrub.js +1 -1
- package/dist/parked-decide.d.ts +15 -2
- package/dist/parked-decide.js +14 -8
- package/dist/peer-seat.d.ts +150 -0
- package/dist/peer-seat.js +239 -0
- package/dist/permission-rule-vocab.d.ts +7 -7
- package/dist/permission-rule-vocab.js +3 -3
- package/dist/plain-string-record.d.ts +38 -0
- package/dist/plain-string-record.js +38 -0
- package/dist/plugins/background-agent-store-sql.js +6 -3
- package/dist/plugins/caching-session-store.d.ts +5 -0
- package/dist/plugins/caching-session-store.js +6 -0
- package/dist/plugins/checkpoint-store-sql.d.ts +15 -20
- package/dist/plugins/checkpoint-store-sql.js +8 -8
- package/dist/plugins/file-history-store-sql.js +3 -2
- package/dist/plugins/k8s-pod-reaper.d.ts +2 -8
- package/dist/plugins/k8s-pod-reaper.js +3 -3
- package/dist/plugins/local-checkpoint-store.d.ts +1 -11
- package/dist/plugins/local-checkpoint-store.js +0 -1
- package/dist/plugins/local-session-store.d.ts +66 -27
- package/dist/plugins/local-session-store.js +60 -28
- package/dist/plugins/mailbox-store-sql.d.ts +11 -3
- package/dist/plugins/mailbox-store-sql.js +103 -15
- package/dist/plugins/memory-sync-store-pg.d.ts +16 -5
- package/dist/plugins/memory-sync-store-tidb.d.ts +3 -3
- package/dist/plugins/one-time-migrations.d.ts +6 -3
- package/dist/plugins/one-time-migrations.js +3 -4
- package/dist/plugins/permission-rule-store-sql.js +21 -13
- package/dist/plugins/pg-session-storage.d.ts +10 -2
- package/dist/plugins/pg-session-storage.js +18 -5
- package/dist/plugins/remote-env-adb.d.ts +3 -1
- package/dist/plugins/remote-env-adb.js +1 -1
- package/dist/plugins/remote-env-device.d.ts +3 -1
- package/dist/plugins/remote-env-device.js +1 -1
- package/dist/plugins/remote-env-e2b.d.ts +2 -0
- package/dist/plugins/remote-env-e2b.js +1 -1
- package/dist/plugins/remote-env-host.d.ts +3 -1
- package/dist/plugins/remote-env-host.js +1 -1
- package/dist/plugins/remote-env-k8s.d.ts +3 -1
- package/dist/plugins/remote-env-k8s.js +1 -1
- package/dist/plugins/remote-env-local-docker.d.ts +2 -0
- package/dist/plugins/remote-env-local-docker.js +1 -1
- package/dist/plugins/remote-env-ssh.d.ts +3 -1
- package/dist/plugins/remote-env-ssh.js +1 -1
- package/dist/plugins/retention-store-sql.d.ts +14 -2
- package/dist/plugins/retention-store-sql.js +6 -0
- package/dist/plugins/session-fork.d.ts +10 -13
- package/dist/plugins/session-fork.js +5 -25
- package/dist/plugins/session-ownership.d.ts +7 -0
- package/dist/plugins/session-ownership.js +5 -0
- package/dist/plugins/session-workspace-cell.d.ts +17 -0
- package/dist/plugins/session-workspace-cell.js +38 -0
- package/dist/plugins/store-backend.js +7 -7
- package/dist/plugins/tidb-pool.js +6 -0
- package/dist/plugins/tidb-session-store.d.ts +17 -2
- package/dist/plugins/tidb-session-store.js +16 -5
- package/dist/plugins/tool-result-store-sql.d.ts +13 -6
- package/dist/plugins/tool-result-store-sql.js +8 -7
- package/dist/plugins/worktree-isolation.d.ts +3 -2
- package/dist/plugins/worktree-isolation.js +1 -7
- package/dist/project-identity.d.ts +1 -14
- package/dist/project-identity.js +0 -4
- package/dist/rules-consent.js +6 -4
- package/dist/run-local.js +12 -6
- package/dist/runs.d.ts +7 -1
- package/dist/runs.js +6 -1
- package/dist/sandbox-git-plane.d.ts +2 -2
- package/dist/server-secret-env.d.ts +6 -3
- package/dist/server-secret-env.js +4 -2
- package/dist/session-incarnation.d.ts +4 -5
- package/dist/session-incarnation.js +2 -2
- package/dist/session-workspace.d.ts +130 -0
- package/dist/session-workspace.js +83 -0
- package/dist/spec-fields.d.ts +1 -1
- package/dist/standing-answer-grant.d.ts +2 -13
- package/dist/standing-answer-grant.js +0 -7
- package/dist/task-a2a.d.ts +44 -14
- package/dist/task-a2a.js +46 -20
- package/dist/task-mcp.d.ts +33 -20
- package/dist/task-mcp.js +46 -22
- package/dist/tool-approval.d.ts +32 -8
- package/dist/tool-approval.js +49 -12
- package/dist/trace/core-keyset-guard.d.ts +33 -18
- package/dist/trace/engine-notice-wire.d.ts +1 -1
- package/dist/trace/engine-notice-wire.js +2 -0
- package/dist/trace/exit-seat.d.ts +20 -1
- package/dist/trace/exit-seat.js +3 -0
- package/dist/trace/extension-call-face.d.ts +33 -0
- package/dist/trace/extension-call-face.js +39 -0
- package/dist/trace/project.d.ts +25 -7
- package/dist/trace/project.js +29 -3
- package/dist/trace/redact.js +42 -13
- package/dist/trace/self-minted-wire.d.ts +129 -10
- package/dist/trace/self-minted-wire.js +62 -3
- package/dist/trace/wire-projection-faces.d.ts +6 -6
- package/dist/trace/wire-projection-faces.js +6 -5
- package/dist/user-authored-text.d.ts +2 -2
- package/dist/user-authored-text.js +16 -1
- package/dist/web-admin-client/center-request.d.ts +3 -2
- package/dist/web-admin-client/center-request.js +2 -3
- package/dist/web-admin-client/skills-mcp.d.ts +8 -4
- package/dist/web-admin-client/skills-mcp.js +31 -18
- package/dist/web-admin-client/types.d.ts +5 -0
- package/package.json +3 -3
- package/wire-changes/server@7.138.0.json +446 -0
- package/dist/http/mcp-remote-text.d.ts +0 -5
- package/dist/http/mcp-remote-text.js +0 -7
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { redactErrorMessage, redactHead, redactTerminalResult } from "../../observability/run-terminal.js";
|
|
2
2
|
import { runWithVerification, runCascade, taskNotificationDedupKey, uuidv7 } from "@sema-agent/core";
|
|
3
|
+
import { heldFor } from "../../peer-seat.js";
|
|
3
4
|
import { graftParkToolCallId, markChildrenStoppedByUserOnAbort, syncRefusalHttpStatus, zeroWorkRefusalFacts, releaseKeyIfZeroWorkRefusal, createPromptCommitReader, HEARTBEAT_MS } from "../../runs.js";
|
|
4
5
|
import { sealResultForExit, outwardPlane } from "../../terminal.js";
|
|
5
6
|
import { withPrincipal } from "../../observability/principal-context.js";
|
|
@@ -38,6 +39,7 @@ import { createInstructionSelectionLeg } from "../../instruction-selection.js";
|
|
|
38
39
|
import { fieldConflictFacts } from "../../field-conflict.js";
|
|
39
40
|
import { refusalBody } from "../refusal-body.js";
|
|
40
41
|
import { heldWorkflowParkDecisions } from "../../parked-decide.js";
|
|
42
|
+
import { createSessionWorkspaceRecorder, legSessionWitnessOf, sessionWorkspaceStoreOf } from "../../session-workspace.js";
|
|
41
43
|
const DISCONNECT_PARK_GRACE_MS = 10_000;
|
|
42
44
|
function sseData(res, payload) {
|
|
43
45
|
const anchor = payload.eventId;
|
|
@@ -151,666 +153,616 @@ export async function handleTasks(req, res, match, ctx) {
|
|
|
151
153
|
const prepared = await prepareSpec(req, res, keyedBody, undefined, { liveStream });
|
|
152
154
|
if (!prepared)
|
|
153
155
|
return;
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
const
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
156
|
+
try {
|
|
157
|
+
const declaredFamilies = declaredApprovalFrameFamilies(prepared.body.settings);
|
|
158
|
+
const approvalLegInput = (streamApprovalOn) => {
|
|
159
|
+
const legWalltimeMs = typeof prepared.spec.limits?.maxWalltimeMs === "number" ? prepared.spec.limits.maxWalltimeMs : undefined;
|
|
160
|
+
return {
|
|
161
|
+
streamApprovalOn,
|
|
162
|
+
windowMs: deps.config.streamApproval?.windowMs,
|
|
163
|
+
windowMarginMs: deps.config.streamAskWindowMarginMs,
|
|
164
|
+
...(legWalltimeMs !== undefined ? { legWalltimeMs } : {}),
|
|
165
|
+
declaredFamilies,
|
|
166
|
+
};
|
|
163
167
|
};
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
sendError(res, 400, "request.precondition_unmet", "x-detach-on-disconnect requires a durable run (a run store + sessionId) — without one the detached result would be unqueryable");
|
|
182
|
-
return;
|
|
183
|
-
}
|
|
184
|
-
reqState.streamTaskId = earlyDurableTid;
|
|
185
|
-
let usageKey;
|
|
186
|
-
reqState.streamDetached = detachOnDisconnect;
|
|
187
|
-
sseHeaders(res, earlyDurableTid ? { "x-task-id": earlyDurableTid } : undefined);
|
|
188
|
-
if (earlyDurableTid)
|
|
189
|
-
writeMetaFrame(res, earlyDurableTid, prepared.spec.sessionId, prepared.warnings);
|
|
190
|
-
const ac = createRunAbort();
|
|
191
|
-
let closed = false;
|
|
192
|
-
let detachLogged = false;
|
|
193
|
-
let disconnectParkGrace;
|
|
194
|
-
const onDisconnect = () => {
|
|
195
|
-
if (detachOnDisconnect) {
|
|
196
|
-
if (res.writableEnded)
|
|
197
|
-
return;
|
|
198
|
-
if (!detachLogged) {
|
|
199
|
-
detachLogged = true;
|
|
200
|
-
res.on("error", () => undefined);
|
|
201
|
-
deps.logger?.info("stream_client_detached", { taskId: earlyDurableTid, sessionId: prepared.spec.sessionId, detached: true });
|
|
202
|
-
}
|
|
168
|
+
const assembleApprovalLeg = (streamApprovalOn) => resolveApprovalLeg({ ...approvalLegInput(streamApprovalOn), nowMonotonicMs: performance.now() });
|
|
169
|
+
const principal = prepared.auth?.principal;
|
|
170
|
+
const wiringOperatorFace = explicitOperatorOk(gatedPrincipal(req, deps.config), deps.config);
|
|
171
|
+
let wiringReceipt;
|
|
172
|
+
const instructionLeg = createInstructionSelectionLeg();
|
|
173
|
+
const legWiringSections = () => ({ instructions: instructionLeg.segment(), skills: runSkillsOf(prepared.spec) });
|
|
174
|
+
const withRunReceipt = (out) => wiringReceipt !== undefined ? { ...out, wiringManifest: wiringReceipt } : out;
|
|
175
|
+
if (liveStream) {
|
|
176
|
+
if (prepared.verify || prepared.cascade) {
|
|
177
|
+
sendError(res, 400, "request.field_conflict", "verify / cascade are not supported on /v1/tasks/stream (both are multi-attempt, not a single stream) — use /v1/tasks or /v1/runs", fieldConflictFacts("cascade_verify", prepared.verify ? (prepared.cascade ? ["verify", "cascade"] : ["verify"]) : ["cascade"]));
|
|
178
|
+
return;
|
|
179
|
+
}
|
|
180
|
+
const earlyDurableTid = facts.durableRuns(deps).available && prepared.spec.sessionId ? uuidv7() : undefined;
|
|
181
|
+
const streamLegStartedAt = Date.now();
|
|
182
|
+
const detachOnDisconnect = headerStr(req.headers["x-detach-on-disconnect"]) === "true";
|
|
183
|
+
if (detachOnDisconnect && earlyDurableTid === undefined) {
|
|
184
|
+
sendError(res, 400, "request.precondition_unmet", "x-detach-on-disconnect requires a durable run (a run store + sessionId) — without one the detached result would be unqueryable");
|
|
203
185
|
return;
|
|
204
186
|
}
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
187
|
+
reqState.streamTaskId = earlyDurableTid;
|
|
188
|
+
let usageKey;
|
|
189
|
+
reqState.streamDetached = detachOnDisconnect;
|
|
190
|
+
sseHeaders(res, earlyDurableTid ? { "x-task-id": earlyDurableTid } : undefined);
|
|
191
|
+
if (earlyDurableTid)
|
|
192
|
+
writeMetaFrame(res, earlyDurableTid, prepared.spec.sessionId, prepared.warnings);
|
|
193
|
+
const ac = createRunAbort();
|
|
194
|
+
let closed = false;
|
|
195
|
+
let detachLogged = false;
|
|
196
|
+
let disconnectParkGrace;
|
|
197
|
+
const onDisconnect = () => {
|
|
198
|
+
if (detachOnDisconnect) {
|
|
199
|
+
if (res.writableEnded)
|
|
200
|
+
return;
|
|
201
|
+
if (!detachLogged) {
|
|
202
|
+
detachLogged = true;
|
|
203
|
+
res.on("error", () => undefined);
|
|
204
|
+
deps.logger?.info("stream_client_detached", { taskId: earlyDurableTid, sessionId: prepared.spec.sessionId, detached: true });
|
|
205
|
+
}
|
|
215
206
|
return;
|
|
216
207
|
}
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
208
|
+
if (deps.checkpointStore && earlyDurableTid && disconnectParkGrace === undefined) {
|
|
209
|
+
const parked = deps.toolApproval?.parkOpenAsksForTask(earlyDurableTid) ?? 0;
|
|
210
|
+
if (parked > 0) {
|
|
211
|
+
res.on("error", () => undefined);
|
|
212
|
+
deps.logger?.info("stream_disconnect_ask_parked", { taskId: earlyDurableTid, sessionId: prepared.spec.sessionId, asks: parked, graceMs: DISCONNECT_PARK_GRACE_MS });
|
|
213
|
+
disconnectParkGrace = setTimeout(() => {
|
|
214
|
+
closed = true;
|
|
215
|
+
ac.abort("ask_park_grace_exhausted");
|
|
216
|
+
}, DISCONNECT_PARK_GRACE_MS);
|
|
217
|
+
disconnectParkGrace.unref?.();
|
|
218
|
+
return;
|
|
219
|
+
}
|
|
220
|
+
}
|
|
221
|
+
closed = true;
|
|
222
|
+
ac.abort("client_disconnected");
|
|
223
|
+
};
|
|
224
|
+
req.on("close", onDisconnect);
|
|
225
|
+
res.on("close", () => {
|
|
226
|
+
if (!res.writableEnded)
|
|
227
|
+
onDisconnect();
|
|
228
|
+
});
|
|
229
|
+
if ((res.destroyed || res.closed) && !res.writableEnded)
|
|
224
230
|
onDisconnect();
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
231
|
+
const replayGate = resolveStreamApprovalGate({
|
|
232
|
+
toolApprovalEnabled: Boolean(deps.toolApproval),
|
|
233
|
+
streamApprovalEnabled: deps.config.streamApproval?.enabled === true,
|
|
234
|
+
backend: deps.backend,
|
|
235
|
+
checkpointStore: deps.checkpointStore,
|
|
236
|
+
});
|
|
237
|
+
const replaySessionId = approvalPreambleReplays(approvalLegInput(replayGate.active)) ? prepared.spec.sessionId : undefined;
|
|
238
|
+
const presentedOnConnection = new Set();
|
|
239
|
+
const presentedCompleted = new Set();
|
|
240
|
+
const admitPresentedComplete = (askId) => {
|
|
241
|
+
if (!presentedOnConnection.has(askId))
|
|
242
|
+
return true;
|
|
243
|
+
if (presentedCompleted.has(askId))
|
|
244
|
+
return false;
|
|
245
|
+
presentedCompleted.add(askId);
|
|
239
246
|
return true;
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
if (
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
res.write(`event: ${frame.type}\ndata: ${JSON.stringify(frame)}\n\n`);
|
|
271
|
-
});
|
|
272
|
-
res.once("close", unwatch);
|
|
247
|
+
};
|
|
248
|
+
if (replaySessionId) {
|
|
249
|
+
const replayOwner = gatedPrincipal(req, deps.config) ?? null;
|
|
250
|
+
const askStore = replayGate.active ? replayGate.askStore : undefined;
|
|
251
|
+
let presentedAskIds = [];
|
|
252
|
+
const preambleFrames = await collectSsePreamble(async (signal) => {
|
|
253
|
+
if (!askStore)
|
|
254
|
+
return [];
|
|
255
|
+
const rows = await askStore.listPendingBySession(replaySessionId, replayOwner, signal);
|
|
256
|
+
const { frames, skipped, dropped } = buildApprovalPreamble(rows, Date.now(), deps.config.streamApproval.replayMax);
|
|
257
|
+
presentedAskIds = frames.map((f) => f.askId);
|
|
258
|
+
if (skipped > 0)
|
|
259
|
+
deps.logger?.warn?.("approval_preamble_row_skipped", { sessionId: replaySessionId, skipped, face: "tasks.stream" });
|
|
260
|
+
if (dropped > 0)
|
|
261
|
+
deps.logger?.warn?.("approval_preamble_capped", { sessionId: replaySessionId, dropped, replayMax: deps.config.streamApproval.replayMax, face: "tasks.stream" });
|
|
262
|
+
return buildApprovalPreambleSseFrames(frames);
|
|
263
|
+
}, (reason, err) => deps.logger?.warn?.("approval_preamble_failed", { sessionId: replaySessionId, reason, face: "tasks.stream", error: err instanceof Error ? err.message : String(err ?? "") }));
|
|
264
|
+
if (!res.writableEnded && !res.destroyed) {
|
|
265
|
+
for (const f of preambleFrames)
|
|
266
|
+
res.write(`event: ${f.event}\ndata: ${JSON.stringify(f.data)}\n\n`);
|
|
267
|
+
if (deps.toolApproval && preambleFrames.length > 0) {
|
|
268
|
+
for (const askId of presentedAskIds)
|
|
269
|
+
presentedOnConnection.add(askId);
|
|
270
|
+
const unwatch = deps.toolApproval.watchPresentedCards(presentedAskIds, (frame) => {
|
|
271
|
+
if (res.writableEnded || res.destroyed || !admitPresentedComplete(frame.askId))
|
|
272
|
+
return;
|
|
273
|
+
res.write(`event: ${frame.type}\ndata: ${JSON.stringify(frame)}\n\n`);
|
|
274
|
+
});
|
|
275
|
+
res.once("close", unwatch);
|
|
276
|
+
}
|
|
273
277
|
}
|
|
274
278
|
}
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
279
|
+
const hb = setInterval(() => {
|
|
280
|
+
if (!res.writableEnded && !res.destroyed)
|
|
281
|
+
res.write(`event: heartbeat\ndata: {}\n\n`);
|
|
282
|
+
}, 15_000);
|
|
283
|
+
hb.unref?.();
|
|
284
|
+
let ranLive = false;
|
|
285
|
+
let resp;
|
|
286
|
+
const runStreamSubmitLeg = async () => {
|
|
287
|
+
ranLive = true;
|
|
288
|
+
let finalResult;
|
|
289
|
+
let runRowSettled = false;
|
|
290
|
+
let durableTaskId;
|
|
291
|
+
let durableHeartbeat;
|
|
292
|
+
const cancelledResult = (errorMessage, cause) => mintCancelledResult({ taskId: durableTaskId ?? "", sessionId: prepared.spec.sessionId ?? "", startedAtMs: streamLegStartedAt, message: errorMessage, cause });
|
|
293
|
+
let userMsgEntryId;
|
|
294
|
+
const promptCommit = createPromptCommitReader();
|
|
295
|
+
let anchorPut = false;
|
|
296
|
+
const putRewindAnchor = () => {
|
|
297
|
+
if (anchorPut || !userMsgEntryId || !durableTaskId || !prepared.spec.sessionId || !deps.resumeAnchorStore)
|
|
298
|
+
return;
|
|
299
|
+
anchorPut = true;
|
|
300
|
+
void deps.resumeAnchorStore
|
|
301
|
+
.put(prepared.spec.sessionId, durableTaskId, userMsgEntryId, principal ?? null)
|
|
302
|
+
.catch(() => deps.metrics?.inc("resume_anchor_capture_failed"));
|
|
303
|
+
};
|
|
304
|
+
if (deps.runStore && prepared.spec.sessionId) {
|
|
305
|
+
const tid = earlyDurableTid ?? uuidv7();
|
|
306
|
+
const created = await deps.runStore.createRun(tid, prepared.spec.sessionId, principal ?? null, deps.instanceId ?? "default", { ...runMeta(prepared, source), identity: runIdentityOf(req, deps.config, principal ?? null), idem: idem ?? null });
|
|
307
|
+
if (created.ok)
|
|
308
|
+
clearTurnActivity(tid);
|
|
309
|
+
if (created.ok)
|
|
310
|
+
recordResourceWindow(tid, prepared.spec.resourceSuspend);
|
|
311
|
+
if (created.existing) {
|
|
312
|
+
const replay = await replayOfExistingRun(created.existing, { principal: submitter, idem }, "sync", deps, deps.runStore);
|
|
313
|
+
sseData(res, { type: "done", result: doneResultOfOutcome(replay), replay: true });
|
|
314
|
+
return replay;
|
|
315
|
+
}
|
|
316
|
+
if (!created.ok) {
|
|
317
|
+
const conflict = await buildActiveRunConflict({ runStore: deps.runStore, checkpointStore: deps.checkpointStore, governance: deps.config, runStaleSec: deps.config.runStaleSec, turnActivity: readTurnActivityMs }, prepared.spec.sessionId, created.activeTaskId);
|
|
318
|
+
const refused = { status: 409, body: conflict };
|
|
319
|
+
sseData(res, { type: "done", result: doneResultOfOutcome(refused) });
|
|
320
|
+
return refused;
|
|
321
|
+
}
|
|
322
|
+
const placementReject = await commitPlacement(deps, deps.runStore, tid, prepared.bindingIntent);
|
|
323
|
+
if (placementReject) {
|
|
324
|
+
const body = devicePlacementErrorBody(placementReject);
|
|
325
|
+
const refused = { status: placementReject.status, body };
|
|
326
|
+
sseData(res, { type: "done", result: doneResultOfOutcome(refused) });
|
|
327
|
+
return refused;
|
|
328
|
+
}
|
|
329
|
+
durableTaskId = tid;
|
|
312
330
|
}
|
|
313
|
-
if (
|
|
314
|
-
const
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
331
|
+
if (deps.checkpointStore && prepared.spec.sessionId) {
|
|
332
|
+
const ctxRefusal = await persistSubmitCtx(deps.checkpointStore, deps, durableTaskId !== undefined && deps.runStore ? { runStore: deps.runStore, taskId: durableTaskId } : undefined, prepared.spec.sessionId, { body: prepared.body, memoryScope: prepared.auth?.memoryScope, writerVersion: serviceVersion() });
|
|
333
|
+
if (ctxRefusal) {
|
|
334
|
+
sseData(res, { type: "done", result: doneResultOfOutcome(ctxRefusal) });
|
|
335
|
+
return ctxRefusal;
|
|
336
|
+
}
|
|
318
337
|
}
|
|
319
|
-
|
|
320
|
-
if (
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
338
|
+
commitSessionInheritEnv(deps, prepared.spec.sessionId, prepared.body);
|
|
339
|
+
if (prepared.spec.sessionId)
|
|
340
|
+
markChildrenStoppedByUserOnAbort(ac.signal, durableTaskId ?? prepared.spec.sessionId, prepared.auth?.principal);
|
|
341
|
+
let ledgerSink;
|
|
342
|
+
let ledgerTerminal = false;
|
|
343
|
+
if (durableTaskId && deps.runStore) {
|
|
344
|
+
const rs = deps.runStore;
|
|
345
|
+
const tid = durableTaskId;
|
|
346
|
+
ledgerSink = createLedgerSink({ appendEvent: (seq, type, data) => rs.appendEvent(tid, seq, type, data), persistThinking: deps.config.traceThinking, onActivity: (kind, statusData) => recordTurnActivity(tid, kind, statusData), legSections: legWiringSections });
|
|
347
|
+
usageKey = tid;
|
|
348
|
+
deps.modelUsage?.register(tid);
|
|
349
|
+
const owner = principal ?? null;
|
|
350
|
+
const abortFromVerb = () => ac.abort("cancel_verb");
|
|
351
|
+
durableHeartbeat = setInterval(() => {
|
|
352
|
+
void rs.heartbeat(tid, owner).catch(() => undefined);
|
|
353
|
+
if (!ac.signal.aborted) {
|
|
354
|
+
void Promise.resolve(rs.isCancelRequested?.(tid, owner)).then((c) => { if (c)
|
|
355
|
+
abortFromVerb(); }).catch(() => undefined);
|
|
356
|
+
}
|
|
357
|
+
}, HEARTBEAT_MS);
|
|
358
|
+
durableHeartbeat.unref?.();
|
|
359
|
+
inflightRuns.set(tid, ac);
|
|
360
|
+
void Promise.resolve(rs.isCancelRequested?.(tid, owner)).then((c) => { if (c && !ac.signal.aborted)
|
|
361
|
+
abortFromVerb(); }).catch(() => undefined);
|
|
325
362
|
}
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
sseData(res, { type: "done", result: doneResultOfOutcome(ctxRefusal) });
|
|
332
|
-
return ctxRefusal;
|
|
363
|
+
let fleetPub;
|
|
364
|
+
try {
|
|
365
|
+
fleetPub = durableTaskId
|
|
366
|
+
? fleetRunPublisher(deps.fleetBus, { runId: durableTaskId, scope: gatedPrincipal(req, deps.config) ?? "default", rootTaskId: durableTaskId, sessionId: prepared.spec.sessionId, ...fleetRunLabels(prepared.spec.objective) })
|
|
367
|
+
: undefined;
|
|
333
368
|
}
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
const
|
|
343
|
-
|
|
344
|
-
usageKey = tid;
|
|
345
|
-
deps.modelUsage?.register(tid);
|
|
346
|
-
const owner = principal ?? null;
|
|
347
|
-
const abortFromVerb = () => ac.abort("cancel_verb");
|
|
348
|
-
durableHeartbeat = setInterval(() => {
|
|
349
|
-
void rs.heartbeat(tid, owner).catch(() => undefined);
|
|
350
|
-
if (!ac.signal.aborted) {
|
|
351
|
-
void Promise.resolve(rs.isCancelRequested?.(tid, owner)).then((c) => { if (c)
|
|
352
|
-
abortFromVerb(); }).catch(() => undefined);
|
|
353
|
-
}
|
|
354
|
-
}, HEARTBEAT_MS);
|
|
355
|
-
durableHeartbeat.unref?.();
|
|
356
|
-
inflightRuns.set(tid, ac);
|
|
357
|
-
void Promise.resolve(rs.isCancelRequested?.(tid, owner)).then((c) => { if (c && !ac.signal.aborted)
|
|
358
|
-
abortFromVerb(); }).catch(() => undefined);
|
|
359
|
-
}
|
|
360
|
-
let fleetPub;
|
|
361
|
-
try {
|
|
362
|
-
fleetPub = durableTaskId
|
|
363
|
-
? fleetRunPublisher(deps.fleetBus, { runId: durableTaskId, scope: gatedPrincipal(req, deps.config) ?? "default", rootTaskId: durableTaskId, sessionId: prepared.spec.sessionId, ...fleetRunLabels(prepared.spec.objective) })
|
|
364
|
-
: undefined;
|
|
365
|
-
}
|
|
366
|
-
catch (e) {
|
|
367
|
-
if (durableHeartbeat)
|
|
368
|
-
clearInterval(durableHeartbeat);
|
|
369
|
-
if (durableTaskId && inflightRuns.get(durableTaskId) === ac)
|
|
370
|
-
inflightRuns.delete(durableTaskId);
|
|
371
|
-
throw e;
|
|
372
|
-
}
|
|
373
|
-
let fleetSettled = false;
|
|
374
|
-
const settleFleet = (status, residuals) => {
|
|
375
|
-
if (fleetSettled)
|
|
376
|
-
return;
|
|
377
|
-
fleetSettled = true;
|
|
378
|
-
fleetPub?.onTerminal(status, residuals);
|
|
379
|
-
};
|
|
380
|
-
let liveStreamRef;
|
|
381
|
-
const subagentHandleEvictions = [];
|
|
382
|
-
let syncLegLive = true;
|
|
383
|
-
const syncNotifiedKeys = new NotifiedKeys();
|
|
384
|
-
try {
|
|
385
|
-
fleetPub?.onStart();
|
|
386
|
-
}
|
|
387
|
-
catch (e) {
|
|
388
|
-
if (durableHeartbeat)
|
|
389
|
-
clearInterval(durableHeartbeat);
|
|
390
|
-
if (durableTaskId && inflightRuns.get(durableTaskId) === ac)
|
|
391
|
-
inflightRuns.delete(durableTaskId);
|
|
392
|
-
settleFleet("failed");
|
|
393
|
-
throw e;
|
|
394
|
-
}
|
|
395
|
-
const noticeLedger = ledgerSink;
|
|
396
|
-
const unregisterEngineNotice = registerEngineNoticeLeg({
|
|
397
|
-
sessionId: prepared.spec.sessionId,
|
|
398
|
-
...(prepared.engineNotices !== undefined ? { pending: prepared.engineNotices } : {}),
|
|
399
|
-
live: (row) => {
|
|
400
|
-
if (res.writableEnded || res.destroyed) {
|
|
401
|
-
recordFailOpen(failOpenTagForDroppedFrame("engine_notice"), "frame=engine_notice");
|
|
369
|
+
catch (e) {
|
|
370
|
+
if (durableHeartbeat)
|
|
371
|
+
clearInterval(durableHeartbeat);
|
|
372
|
+
if (durableTaskId && inflightRuns.get(durableTaskId) === ac)
|
|
373
|
+
inflightRuns.delete(durableTaskId);
|
|
374
|
+
throw e;
|
|
375
|
+
}
|
|
376
|
+
let fleetSettled = false;
|
|
377
|
+
const settleFleet = (status, residuals) => {
|
|
378
|
+
if (fleetSettled)
|
|
402
379
|
return;
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
}
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
380
|
+
fleetSettled = true;
|
|
381
|
+
fleetPub?.onTerminal(status, residuals);
|
|
382
|
+
};
|
|
383
|
+
let liveStreamRef;
|
|
384
|
+
const subagentHandleEvictions = [];
|
|
385
|
+
let syncLegLive = true;
|
|
386
|
+
const syncNotifiedKeys = new NotifiedKeys();
|
|
387
|
+
try {
|
|
388
|
+
fleetPub?.onStart();
|
|
389
|
+
}
|
|
390
|
+
catch (e) {
|
|
391
|
+
if (durableHeartbeat)
|
|
392
|
+
clearInterval(durableHeartbeat);
|
|
393
|
+
if (durableTaskId && inflightRuns.get(durableTaskId) === ac)
|
|
394
|
+
inflightRuns.delete(durableTaskId);
|
|
395
|
+
settleFleet("failed");
|
|
396
|
+
throw e;
|
|
397
|
+
}
|
|
398
|
+
const noticeLedger = ledgerSink;
|
|
399
|
+
const unregisterEngineNotice = registerEngineNoticeLeg({
|
|
400
|
+
sessionId: prepared.spec.sessionId,
|
|
401
|
+
...(prepared.engineNotices !== undefined ? { pending: prepared.engineNotices } : {}),
|
|
402
|
+
live: (row) => {
|
|
403
|
+
if (res.writableEnded || res.destroyed) {
|
|
404
|
+
recordFailOpen(failOpenTagForDroppedFrame("engine_notice"), "frame=engine_notice");
|
|
405
|
+
return;
|
|
406
|
+
}
|
|
407
|
+
sseData(res, { type: "engine_notice", ...row });
|
|
408
|
+
},
|
|
409
|
+
...(noticeLedger
|
|
410
|
+
? {
|
|
411
|
+
durable: (row) => noticeLedger.append("engine_notice", row).then(() => true, () => (recordFailOpen("server.engine-notice.durable-append-failed", `leg=sync code=${row.code} task=${durableTaskId ?? ""}`), false)),
|
|
412
|
+
}
|
|
413
|
+
: {}),
|
|
414
|
+
});
|
|
415
|
+
if (deps.userInputLedger && durableTaskId && prepared.spec.sessionId)
|
|
416
|
+
await deps.userInputLedger.announceUndrained({ sessionId: prepared.spec.sessionId, taskId: durableTaskId });
|
|
417
|
+
try {
|
|
418
|
+
const streamBody = async () => {
|
|
419
|
+
await withPrincipal(principal, async () => {
|
|
420
|
+
const tailContentMode = prepared.spec.forwardSubagentEvents === true ? "on" : "progress_only";
|
|
421
|
+
const childReasoning = createReasoningSegments();
|
|
422
|
+
const topReasoning = createReasoningSegments();
|
|
423
|
+
const heldDecisions = prepared.spec.sessionId !== undefined ? await heldWorkflowParkDecisions(deps, prepared.spec.sessionId) : {};
|
|
424
|
+
const fwdInternals = {
|
|
425
|
+
...heldDecisions,
|
|
426
|
+
onForwardEvent: (e) => {
|
|
427
|
+
fleetPub?.onForwardEvent(e);
|
|
428
|
+
ledgerSink?.onForwardEvent(e);
|
|
429
|
+
{
|
|
430
|
+
const bg = e.bgAgentId;
|
|
431
|
+
if (bg !== undefined)
|
|
432
|
+
defaultSubagentTailBus.onForwardFrame(bg, tailContentMode, e);
|
|
433
|
+
}
|
|
434
|
+
if (res.writableEnded)
|
|
435
|
+
return;
|
|
436
|
+
for (const end of childReasoning.onEvent(e))
|
|
437
|
+
sseData(res, end);
|
|
438
|
+
const t = e.type;
|
|
439
|
+
if (t === "task_progress") {
|
|
440
|
+
sseData(res, { type: "task_progress", ...taskProgressEventData(e) });
|
|
441
|
+
}
|
|
442
|
+
else if (t === "task_started") {
|
|
443
|
+
sseData(res, { type: "task_started", ...taskStartedEventData(e) });
|
|
444
|
+
}
|
|
445
|
+
else if (t === "tool_start") {
|
|
446
|
+
const tsv = e;
|
|
447
|
+
sseData(res, { type: "tool_start", ...toolStartEventData(tsv) });
|
|
448
|
+
}
|
|
449
|
+
else if (t === "tool_end") {
|
|
450
|
+
const te = e;
|
|
451
|
+
sseData(res, { type: "tool_end", ...toolEndEventData(te) });
|
|
452
|
+
}
|
|
453
|
+
else if (t === "text_delta") {
|
|
454
|
+
const td = e;
|
|
455
|
+
sseData(res, { type: "text_delta", delta: td.delta, ...(td.eventId ? { eventId: td.eventId } : {}), ...(td.parentToolCallId ? { parentToolCallId: td.parentToolCallId } : {}) });
|
|
456
|
+
}
|
|
457
|
+
else if (t === "text_end") {
|
|
458
|
+
const tend = textEndEventData(e);
|
|
459
|
+
if (tend !== null)
|
|
460
|
+
sseData(res, { type: "text_end", ...tend });
|
|
461
|
+
}
|
|
462
|
+
else if (t === "reasoning_delta") {
|
|
463
|
+
const rd = e;
|
|
464
|
+
sseData(res, { type: "reasoning_delta", delta: redactSecrets(rd.delta), ...(rd.eventId ? { eventId: rd.eventId } : {}), ...(rd.parentToolCallId ? { parentToolCallId: rd.parentToolCallId } : {}) });
|
|
465
|
+
}
|
|
466
|
+
},
|
|
467
|
+
onTaskNotification: (n, opts) => {
|
|
468
|
+
if (n.task_type === "workflow")
|
|
469
|
+
return;
|
|
470
|
+
if (!res.writableEnded)
|
|
471
|
+
for (const end of childReasoning.cut(reasoningStreamOfTask(n.task_id, n.sessionId)))
|
|
472
|
+
sseData(res, end);
|
|
473
|
+
defaultSubagentTailBus.forgetHandle(n.task_id);
|
|
474
|
+
if (defaultSubagentTailBus.hasSubscribers(n.task_id)) {
|
|
475
|
+
defaultSubagentTailBus.publish(n.task_id, { type: "task_settled", taskId: n.task_id, status: n.status, ...(typeof n.seq === "number" ? { seq: n.seq } : {}), ...(n.summary ? { summary: redactSecrets(n.summary) } : {}) });
|
|
476
|
+
}
|
|
477
|
+
const hadRow = isFleetAgentTerminalNotification(n) ? (fleetPub?.onChildTerminal(n.sessionId ?? n.task_id, n.status, n.task_id, n.toolUseId) ?? false) : false;
|
|
478
|
+
const deliverable = syncLegLive && !res.writableEnded && !res.destroyed;
|
|
479
|
+
const parked = !deliverable && Boolean(deps.workflowCompletionInbox && prepared.spec.sessionId);
|
|
480
|
+
deps.logger?.info?.("task_notification_observed", { route: "sync-stream", taskId: n.task_id, taskType: n.task_type, status: n.status, hadFleetRow: hadRow, legLive: syncLegLive, parkedDurable: parked });
|
|
481
|
+
if (parked) {
|
|
482
|
+
void deps.workflowCompletionInbox.enqueue(taskNotificationInboxEntry(prepared.spec.sessionId, askOwner, n, Date.now(), durableTaskId, opts?.priority)).catch((err) => deps.logger?.warn?.("park_enqueue_failed", { route: "sync-stream", taskId: n.task_id, err: err instanceof Error ? err.message : String(err) }));
|
|
483
|
+
}
|
|
484
|
+
else if (deliverable) {
|
|
485
|
+
const key = taskNotificationDedupKey(n);
|
|
486
|
+
if (syncNotifiedKeys.get(key) === undefined) {
|
|
487
|
+
syncNotifiedKeys.set(key, Promise.resolve(true));
|
|
488
|
+
sseData(res, { type: "task_notification", ...taskNotificationEventData({ notification: n, ...(opts?.priority !== undefined ? { priority: opts.priority } : {}) }) });
|
|
489
|
+
}
|
|
490
|
+
}
|
|
491
|
+
},
|
|
492
|
+
...(deps.subagentSteerRegistry && durableTaskId
|
|
493
|
+
? {
|
|
494
|
+
onSubagentSpawn: (handle) => {
|
|
495
|
+
subagentHandleEvictions.push(deps.subagentSteerRegistry.register(durableTaskId, handle));
|
|
496
|
+
},
|
|
497
|
+
}
|
|
498
|
+
: {}),
|
|
499
|
+
};
|
|
500
|
+
const liveStream = prepared.runner.runTaskStream({ ...prepared.spec, signal: ac.signal, ...(durableTaskId ? { taskId: durableTaskId } : {}) }, undefined, fwdInternals);
|
|
501
|
+
liveStreamRef = liveStream;
|
|
502
|
+
void heldFor(prepared.peerLease, () => Promise.allSettled([Promise.resolve().then(() => liveStream.result())]));
|
|
503
|
+
if (durableTaskId)
|
|
504
|
+
steerableRuns.set(durableTaskId, liveStream);
|
|
505
|
+
prepared.acceptance.attachStream({ reachable: () => !closed && !res.writableEnded && !res.destroyed, write: (frame) => sseData(res, { ...frame }) });
|
|
506
|
+
const liveWorkspace = createSessionWorkspaceRecorder(sessionWorkspaceStoreOf(deps.sessionStorage), legSessionWitnessOf(prepared.spec));
|
|
507
|
+
for await (const ev of liveStream) {
|
|
508
|
+
if (closed)
|
|
509
|
+
break;
|
|
510
|
+
for (const end of ev.type === "done" ? [...childReasoning.cut(), ...topReasoning.cut()] : topReasoning.onEvent(ev))
|
|
434
511
|
sseData(res, end);
|
|
435
|
-
const
|
|
436
|
-
if (
|
|
437
|
-
|
|
512
|
+
const promptEntryId = promptCommit(ev);
|
|
513
|
+
if (promptEntryId !== undefined)
|
|
514
|
+
userMsgEntryId = promptEntryId;
|
|
515
|
+
const causeAtDone = ev.type === "done" ? ac.cause() : undefined;
|
|
516
|
+
await deps.userInputLedger?.observeLegEvent(liveStream, ev);
|
|
517
|
+
prepared.acceptance.observe(ev);
|
|
518
|
+
await liveWorkspace.observe(ev);
|
|
519
|
+
if (ev.type === "done") {
|
|
520
|
+
finalResult = await mintSubmitExitResult(ev.result, deps.checkpointStore);
|
|
521
|
+
if (durableTaskId && finalResult)
|
|
522
|
+
finalResult = { ...finalResult, taskId: durableTaskId };
|
|
523
|
+
if (causeAtDone !== undefined && finalResult !== undefined) {
|
|
524
|
+
finalResult = mintCancelledResult({ taskId: durableTaskId ?? "", sessionId: prepared.spec.sessionId ?? "", startedAtMs: streamLegStartedAt, message: "cancelled by user", cause: causeAtDone, engineResult: finalResult });
|
|
525
|
+
}
|
|
526
|
+
finalizeTaskResult(ev.result, principal, prepared.spec.objective);
|
|
527
|
+
noteRunOutcome(finalResult);
|
|
528
|
+
putRewindAnchor();
|
|
529
|
+
settleFleet(outwardPlane(finalResult.terminal).status, fleetRunResiduals(finalResult));
|
|
530
|
+
if (ledgerSink && finalResult) {
|
|
531
|
+
if (usageKey && deps.modelUsage && deps.runStore && durableTaskId) {
|
|
532
|
+
const rs2 = deps.runStore;
|
|
533
|
+
const tid2 = durableTaskId;
|
|
534
|
+
const fr = finalResult;
|
|
535
|
+
finalResult = await attachModelUsage(fr, { append: ledgerSink.append, getEvents: (_id, after) => rs2.getEvents(tid2, after), modelUsage: deps.modelUsage, taskId: usageKey }).catch(() => fr);
|
|
536
|
+
}
|
|
537
|
+
await ledgerSink.onDone(finalResult);
|
|
538
|
+
ledgerTerminal = true;
|
|
539
|
+
}
|
|
540
|
+
sseData(res, { ...ev, result: finalResult });
|
|
541
|
+
continue;
|
|
438
542
|
}
|
|
439
|
-
|
|
440
|
-
|
|
543
|
+
if (ledgerSink) {
|
|
544
|
+
await ledgerSink.onEvent(ev);
|
|
545
|
+
if (ev.type === "turn_end" && usageKey)
|
|
546
|
+
await appendModelUsageDelta(ledgerSink.append, deps.modelUsage, usageKey);
|
|
441
547
|
}
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
548
|
+
fleetPub?.onEvent(ev);
|
|
549
|
+
if (ev.type === "status")
|
|
550
|
+
deps.metrics?.inc("brain_retry_total", { phase: String(ev.phase) });
|
|
551
|
+
if (ev.type === "status") {
|
|
552
|
+
sseData(res, { type: "status", ...brainStatusEventData(ev) });
|
|
445
553
|
}
|
|
446
|
-
else if (
|
|
447
|
-
const te =
|
|
554
|
+
else if (ev.type === "tool_end") {
|
|
555
|
+
const te = ev;
|
|
448
556
|
sseData(res, { type: "tool_end", ...toolEndEventData(te) });
|
|
449
557
|
}
|
|
450
|
-
else if (
|
|
451
|
-
const
|
|
452
|
-
sseData(res, { type: "
|
|
558
|
+
else if (ev.type === "tool_start") {
|
|
559
|
+
const tsv = ev;
|
|
560
|
+
sseData(res, { type: "tool_start", ...toolStartEventData(tsv) });
|
|
453
561
|
}
|
|
454
|
-
else if (
|
|
455
|
-
const
|
|
456
|
-
|
|
457
|
-
|
|
562
|
+
else if (ev.type === "task_notification") {
|
|
563
|
+
const tnv = ev.notification;
|
|
564
|
+
const tnvKey = taskNotificationStreamKeyOf(tnv);
|
|
565
|
+
const tnvPrior = tnvKey !== undefined ? syncNotifiedKeys.get(tnvKey) : undefined;
|
|
566
|
+
if (tnvPrior === undefined || !(await tnvPrior)) {
|
|
567
|
+
if (tnvKey !== undefined)
|
|
568
|
+
syncNotifiedKeys.set(tnvKey, Promise.resolve(true));
|
|
569
|
+
sseData(res, { type: "task_notification", ...taskNotificationEventData(ev) });
|
|
570
|
+
}
|
|
458
571
|
}
|
|
459
|
-
else if (
|
|
460
|
-
|
|
461
|
-
sseData(res, { type: "reasoning_delta", delta: redactSecrets(rd.delta), ...(rd.eventId ? { eventId: rd.eventId } : {}), ...(rd.parentToolCallId ? { parentToolCallId: rd.parentToolCallId } : {}) });
|
|
572
|
+
else if (ev.type === "reasoning_delta" && typeof ev.delta === "string") {
|
|
573
|
+
sseData(res, { type: "reasoning_delta", delta: redactSecrets(ev.delta), ...(ev.eventId ? { eventId: ev.eventId } : {}), ...(ev.parentToolCallId ? { parentToolCallId: ev.parentToolCallId } : {}) });
|
|
462
574
|
}
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
if (n.task_type === "workflow")
|
|
466
|
-
return;
|
|
467
|
-
if (!res.writableEnded)
|
|
468
|
-
for (const end of childReasoning.cut(reasoningStreamOfTask(n.task_id, n.sessionId)))
|
|
469
|
-
sseData(res, end);
|
|
470
|
-
defaultSubagentTailBus.forgetHandle(n.task_id);
|
|
471
|
-
if (defaultSubagentTailBus.hasSubscribers(n.task_id)) {
|
|
472
|
-
defaultSubagentTailBus.publish(n.task_id, { type: "task_settled", taskId: n.task_id, status: n.status, ...(typeof n.seq === "number" ? { seq: n.seq } : {}), ...(n.summary ? { summary: redactSecrets(n.summary) } : {}) });
|
|
575
|
+
else if (ev.type === "task_progress") {
|
|
576
|
+
sseData(res, { type: "task_progress", ...taskProgressEventData(ev) });
|
|
473
577
|
}
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
const parked = !deliverable && Boolean(deps.workflowCompletionInbox && prepared.spec.sessionId);
|
|
477
|
-
deps.logger?.info?.("task_notification_observed", { route: "sync-stream", taskId: n.task_id, taskType: n.task_type, status: n.status, hadFleetRow: hadRow, legLive: syncLegLive, parkedDurable: parked });
|
|
478
|
-
if (parked) {
|
|
479
|
-
void deps.workflowCompletionInbox.enqueue(taskNotificationInboxEntry(prepared.spec.sessionId, askOwner, n, Date.now(), durableTaskId, opts?.priority)).catch((err) => deps.logger?.warn?.("park_enqueue_failed", { route: "sync-stream", taskId: n.task_id, err: err instanceof Error ? err.message : String(err) }));
|
|
578
|
+
else if (ev.type === "task_started") {
|
|
579
|
+
sseData(res, { type: "task_started", ...taskStartedEventData(ev) });
|
|
480
580
|
}
|
|
481
|
-
else if (
|
|
482
|
-
|
|
483
|
-
if (syncNotifiedKeys.get(key) === undefined) {
|
|
484
|
-
syncNotifiedKeys.set(key, Promise.resolve(true));
|
|
485
|
-
sseData(res, { type: "task_notification", ...taskNotificationEventData({ notification: n, ...(opts?.priority !== undefined ? { priority: opts.priority } : {}) }) });
|
|
486
|
-
}
|
|
581
|
+
else if (ev.type === "compacted") {
|
|
582
|
+
sseData(res, { type: "compacted", ...compactedEventData(ev) });
|
|
487
583
|
}
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
? {
|
|
491
|
-
onSubagentSpawn: (handle) => {
|
|
492
|
-
subagentHandleEvictions.push(deps.subagentSteerRegistry.register(durableTaskId, handle));
|
|
493
|
-
},
|
|
584
|
+
else if (ev.type === "diagnostics") {
|
|
585
|
+
sseData(res, { type: "diagnostics", ...diagnosticsEventData(ev) });
|
|
494
586
|
}
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
const liveStream = prepared.runner.runTaskStream({ ...prepared.spec, signal: ac.signal, ...(durableTaskId ? { taskId: durableTaskId } : {}) }, undefined, fwdInternals);
|
|
498
|
-
liveStreamRef = liveStream;
|
|
499
|
-
if (durableTaskId)
|
|
500
|
-
steerableRuns.set(durableTaskId, liveStream);
|
|
501
|
-
prepared.acceptance.attachStream({ reachable: () => !closed && !res.writableEnded && !res.destroyed, write: (frame) => sseData(res, { ...frame }) });
|
|
502
|
-
for await (const ev of liveStream) {
|
|
503
|
-
if (closed)
|
|
504
|
-
break;
|
|
505
|
-
for (const end of ev.type === "done" ? [...childReasoning.cut(), ...topReasoning.cut()] : topReasoning.onEvent(ev))
|
|
506
|
-
sseData(res, end);
|
|
507
|
-
const promptEntryId = promptCommit(ev);
|
|
508
|
-
if (promptEntryId !== undefined)
|
|
509
|
-
userMsgEntryId = promptEntryId;
|
|
510
|
-
const causeAtDone = ev.type === "done" ? ac.cause() : undefined;
|
|
511
|
-
await deps.userInputLedger?.observeLegEvent(liveStream, ev);
|
|
512
|
-
prepared.acceptance.observe(ev);
|
|
513
|
-
if (ev.type === "done") {
|
|
514
|
-
finalResult = await mintSubmitExitResult(ev.result, deps.checkpointStore);
|
|
515
|
-
if (durableTaskId && finalResult)
|
|
516
|
-
finalResult = { ...finalResult, taskId: durableTaskId };
|
|
517
|
-
if (causeAtDone !== undefined && finalResult !== undefined) {
|
|
518
|
-
finalResult = mintCancelledResult({ taskId: durableTaskId ?? "", sessionId: prepared.spec.sessionId ?? "", startedAtMs: streamLegStartedAt, message: "cancelled by user", cause: causeAtDone, engineResult: finalResult });
|
|
587
|
+
else if (ev.type === "steering_injected") {
|
|
588
|
+
sseData(res, { type: "steering_injected", ...steeringInjectedEventData(ev) });
|
|
519
589
|
}
|
|
520
|
-
|
|
521
|
-
|
|
522
|
-
putRewindAnchor();
|
|
523
|
-
settleFleet(outwardPlane(finalResult.terminal).status, fleetRunResiduals(finalResult));
|
|
524
|
-
if (ledgerSink && finalResult) {
|
|
525
|
-
if (usageKey && deps.modelUsage && deps.runStore && durableTaskId) {
|
|
526
|
-
const rs2 = deps.runStore;
|
|
527
|
-
const tid2 = durableTaskId;
|
|
528
|
-
const fr = finalResult;
|
|
529
|
-
finalResult = await attachModelUsage(fr, { append: ledgerSink.append, getEvents: (_id, after) => rs2.getEvents(tid2, after), modelUsage: deps.modelUsage, taskId: usageKey }).catch(() => fr);
|
|
530
|
-
}
|
|
531
|
-
await ledgerSink.onDone(finalResult);
|
|
532
|
-
ledgerTerminal = true;
|
|
590
|
+
else if (ev.type === "workspace_changed") {
|
|
591
|
+
sseData(res, { type: "workspace_changed", ...workspaceChangedEventData(ev) });
|
|
533
592
|
}
|
|
534
|
-
|
|
535
|
-
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
|
|
544
|
-
|
|
545
|
-
|
|
546
|
-
|
|
547
|
-
|
|
548
|
-
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
|
|
552
|
-
|
|
553
|
-
|
|
554
|
-
|
|
555
|
-
|
|
556
|
-
|
|
557
|
-
|
|
558
|
-
|
|
559
|
-
|
|
560
|
-
if (
|
|
561
|
-
|
|
562
|
-
|
|
563
|
-
|
|
593
|
+
else if (ev.type === "compaction_outcome") {
|
|
594
|
+
sseData(res, { type: "compaction_outcome", ...compactionOutcomeEventData(ev) });
|
|
595
|
+
}
|
|
596
|
+
else if (ev.type === "compaction_started") {
|
|
597
|
+
sseData(res, { type: "compaction_started", ...compactionStartedEventData(ev) });
|
|
598
|
+
}
|
|
599
|
+
else if (ev.type === "wiring_manifest") {
|
|
600
|
+
const wiringFace = wiringManifestFaceEventData(ev, wiringOperatorFace, legWiringSections());
|
|
601
|
+
wiringReceipt ??= wiringFace;
|
|
602
|
+
sseData(res, { type: "wiring_manifest", ...wiringFace });
|
|
603
|
+
}
|
|
604
|
+
else if (ev.type === "tool_roster_delta") {
|
|
605
|
+
const rosterDelta = toolRosterDeltaEventData(ev);
|
|
606
|
+
if (rosterDelta !== undefined)
|
|
607
|
+
sseData(res, { type: "tool_roster_delta", ...rosterDelta });
|
|
608
|
+
}
|
|
609
|
+
else if (ev.type === "tool_progress") {
|
|
610
|
+
const progress = toolProgressEventData(ev);
|
|
611
|
+
if (progress !== undefined)
|
|
612
|
+
sseData(res, { type: "tool_progress", ...progress });
|
|
613
|
+
}
|
|
614
|
+
else if (ev.type === "tool_disclosure") {
|
|
615
|
+
const disclosure = toolDisclosureEventData(ev);
|
|
616
|
+
if (disclosure !== undefined)
|
|
617
|
+
sseData(res, { type: "tool_disclosure", ...disclosure });
|
|
618
|
+
}
|
|
619
|
+
else if (ev.type === "human_input") {
|
|
620
|
+
sseData(res, { type: "human_input", ...humanInputEventData(ev) });
|
|
621
|
+
}
|
|
622
|
+
else if (ev.type === "text_delta" || ev.type === "text_end" || ev.type === "turn_end" || ev.type === "message_committed" || ev.type === "context_usage") {
|
|
623
|
+
const e = ev;
|
|
624
|
+
const ident = liveIdentityFields(e);
|
|
625
|
+
const textEnd = ev.type === "text_end" ? textEndEventData(e) : undefined;
|
|
626
|
+
const arm = ev.type === "text_delta"
|
|
627
|
+
? { type: "text_delta", delta: e.delta, ...ident }
|
|
628
|
+
: ev.type === "text_end"
|
|
629
|
+
?
|
|
630
|
+
textEnd === null || textEnd === undefined ? null : { type: "text_end", ...textEnd }
|
|
631
|
+
: ev.type === "turn_end"
|
|
632
|
+
? { type: "turn_end", ...turnEndEventData(ev), ...ident }
|
|
633
|
+
: ev.type === "message_committed"
|
|
634
|
+
? { type: "message_committed", entryId: e.entryId, role: e.role, ...(e.toolCallId !== undefined ? { toolCallId: e.toolCallId } : {}), ...ident }
|
|
635
|
+
: { type: "context_usage", ...contextUsageEventData(ev), ...ident };
|
|
636
|
+
if (arm !== null)
|
|
637
|
+
sseData(res, arm);
|
|
638
|
+
}
|
|
639
|
+
else {
|
|
640
|
+
sseData(res, { ...ev });
|
|
564
641
|
}
|
|
565
642
|
}
|
|
566
|
-
|
|
567
|
-
|
|
568
|
-
|
|
569
|
-
|
|
570
|
-
|
|
571
|
-
|
|
572
|
-
|
|
573
|
-
sseData(res, { type: "task_started", ...taskStartedEventData(ev) });
|
|
574
|
-
}
|
|
575
|
-
else if (ev.type === "compacted") {
|
|
576
|
-
sseData(res, { type: "compacted", ...compactedEventData(ev) });
|
|
577
|
-
}
|
|
578
|
-
else if (ev.type === "diagnostics") {
|
|
579
|
-
sseData(res, { type: "diagnostics", ...diagnosticsEventData(ev) });
|
|
580
|
-
}
|
|
581
|
-
else if (ev.type === "steering_injected") {
|
|
582
|
-
sseData(res, { type: "steering_injected", ...steeringInjectedEventData(ev) });
|
|
583
|
-
}
|
|
584
|
-
else if (ev.type === "workspace_changed") {
|
|
585
|
-
sseData(res, { type: "workspace_changed", ...workspaceChangedEventData(ev) });
|
|
586
|
-
}
|
|
587
|
-
else if (ev.type === "compaction_outcome") {
|
|
588
|
-
sseData(res, { type: "compaction_outcome", ...compactionOutcomeEventData(ev) });
|
|
589
|
-
}
|
|
590
|
-
else if (ev.type === "compaction_started") {
|
|
591
|
-
sseData(res, { type: "compaction_started", ...compactionStartedEventData(ev) });
|
|
592
|
-
}
|
|
593
|
-
else if (ev.type === "wiring_manifest") {
|
|
594
|
-
const wiringFace = wiringManifestFaceEventData(ev, wiringOperatorFace, legWiringSections());
|
|
595
|
-
wiringReceipt ??= wiringFace;
|
|
596
|
-
sseData(res, { type: "wiring_manifest", ...wiringFace });
|
|
597
|
-
}
|
|
598
|
-
else if (ev.type === "tool_roster_delta") {
|
|
599
|
-
const rosterDelta = toolRosterDeltaEventData(ev);
|
|
600
|
-
if (rosterDelta !== undefined)
|
|
601
|
-
sseData(res, { type: "tool_roster_delta", ...rosterDelta });
|
|
602
|
-
}
|
|
603
|
-
else if (ev.type === "tool_progress") {
|
|
604
|
-
const progress = toolProgressEventData(ev);
|
|
605
|
-
if (progress !== undefined)
|
|
606
|
-
sseData(res, { type: "tool_progress", ...progress });
|
|
607
|
-
}
|
|
608
|
-
else if (ev.type === "tool_disclosure") {
|
|
609
|
-
const disclosure = toolDisclosureEventData(ev);
|
|
610
|
-
if (disclosure !== undefined)
|
|
611
|
-
sseData(res, { type: "tool_disclosure", ...disclosure });
|
|
612
|
-
}
|
|
613
|
-
else if (ev.type === "human_input") {
|
|
614
|
-
sseData(res, { type: "human_input", ...humanInputEventData(ev) });
|
|
615
|
-
}
|
|
616
|
-
else if (ev.type === "text_delta" || ev.type === "text_end" || ev.type === "turn_end" || ev.type === "message_committed" || ev.type === "context_usage") {
|
|
617
|
-
const e = ev;
|
|
618
|
-
const ident = liveIdentityFields(e);
|
|
619
|
-
const textEnd = ev.type === "text_end" ? textEndEventData(e) : undefined;
|
|
620
|
-
const arm = ev.type === "text_delta"
|
|
621
|
-
? { type: "text_delta", delta: e.delta, ...ident }
|
|
622
|
-
: ev.type === "text_end"
|
|
623
|
-
?
|
|
624
|
-
textEnd === null || textEnd === undefined ? null : { type: "text_end", ...textEnd }
|
|
625
|
-
: ev.type === "turn_end"
|
|
626
|
-
? { type: "turn_end", ...turnEndEventData(ev), ...ident }
|
|
627
|
-
: ev.type === "message_committed"
|
|
628
|
-
? { type: "message_committed", entryId: e.entryId, role: e.role, ...(e.toolCallId !== undefined ? { toolCallId: e.toolCallId } : {}), ...ident }
|
|
629
|
-
: { type: "context_usage", ...contextUsageEventData(ev), ...ident };
|
|
630
|
-
if (arm !== null)
|
|
631
|
-
sseData(res, arm);
|
|
632
|
-
}
|
|
633
|
-
else {
|
|
634
|
-
sseData(res, { ...ev });
|
|
635
|
-
}
|
|
636
|
-
}
|
|
637
|
-
});
|
|
638
|
-
};
|
|
639
|
-
const askTaskId = durableTaskId ?? uuidv7();
|
|
640
|
-
const askOwner = gatedPrincipal(req, deps.config) ?? null;
|
|
641
|
-
const emitAsk = (frame) => {
|
|
642
|
-
if (res.writableEnded || res.destroyed) {
|
|
643
|
-
recordFailOpen(failOpenTagForDroppedFrame(frame.type), `frame=${frame.type}`);
|
|
644
|
-
return;
|
|
645
|
-
}
|
|
646
|
-
res.write(`event: ${frame.type}\ndata: ${JSON.stringify(frame)}\n\n`);
|
|
647
|
-
};
|
|
648
|
-
const faceAlive = () => !res.writableEnded && !res.destroyed;
|
|
649
|
-
const emitElicitation = createLedgerTwinEmit({ sink: ledgerSink, writeLive: emitAsk, liveReachable: faceAlive });
|
|
650
|
-
const writeQuestionLive = (frame) => {
|
|
651
|
-
if (!faceAlive())
|
|
652
|
-
throw new Error("live stream ended — question frame undeliverable");
|
|
653
|
-
emitAsk(frame);
|
|
654
|
-
};
|
|
655
|
-
const emitQuestion = createLedgerTwinEmit({ sink: ledgerSink, writeLive: writeQuestionLive, liveReachable: faceAlive });
|
|
656
|
-
await emitPendingWorkflowCompletions(deps.workflowCompletionInbox, prepared.spec.sessionId, askOwner, (frame) => {
|
|
657
|
-
if (closed || res.writableEnded || res.destroyed)
|
|
658
|
-
throw new Error("stream ended before the completion frame was written");
|
|
659
|
-
const f = frame;
|
|
660
|
-
const fKey = f.type === "task_notification" ? taskNotificationStreamKeyOf(f) : undefined;
|
|
661
|
-
if (fKey !== undefined)
|
|
662
|
-
syncNotifiedKeys.set(fKey, Promise.resolve(true));
|
|
663
|
-
sseData(res, frame);
|
|
664
|
-
}, { route: "sync-stream-open", connection: "live-sse", log: (m, x) => deps.logger?.info?.(m, x) });
|
|
665
|
-
const withElicit = deps.elicitation
|
|
666
|
-
? () => deps.elicitation.runWithContext({ taskId: askTaskId, owner: askOwner, emit: emitElicitation, abortSignal: ac.signal }, streamBody)
|
|
667
|
-
: streamBody;
|
|
668
|
-
const withQuestion = () => deps.question
|
|
669
|
-
? deps.question.runWithContext({ taskId: askTaskId, owner: askOwner, emit: emitQuestion, abortSignal: ac.signal, deliverable: faceAlive }, withElicit)
|
|
670
|
-
: withElicit();
|
|
671
|
-
const emitApproval = (frame) => {
|
|
672
|
-
if (res.writableEnded || res.destroyed)
|
|
673
|
-
throw new Error("live stream ended — approval card undeliverable");
|
|
674
|
-
emitAsk(frame);
|
|
675
|
-
};
|
|
676
|
-
const streamApprovalOn = resolveStreamApprovalGate({
|
|
677
|
-
toolApprovalEnabled: Boolean(deps.toolApproval),
|
|
678
|
-
streamApprovalEnabled: deps.config.streamApproval?.enabled === true,
|
|
679
|
-
backend: deps.backend,
|
|
680
|
-
checkpointStore: deps.checkpointStore,
|
|
681
|
-
}).active;
|
|
682
|
-
const approvalLeg = assembleApprovalLeg(streamApprovalOn);
|
|
683
|
-
const legDeadlineMonotonic = approvalLeg.legDeadlineMonotonic;
|
|
684
|
-
const unattended = approvalLeg.unattended;
|
|
685
|
-
const emitCard = createApprovalCardEmitter({
|
|
686
|
-
...(ledgerSink
|
|
687
|
-
? {
|
|
688
|
-
appendDurable: async (frame) => {
|
|
689
|
-
const { type, ...rest } = frame;
|
|
690
|
-
await ledgerSink.append(type, rest);
|
|
691
|
-
},
|
|
692
|
-
}
|
|
693
|
-
: {}),
|
|
694
|
-
writeLive: (frame) => {
|
|
695
|
-
if (res.writableEnded || res.destroyed)
|
|
696
|
-
throw new Error("live stream ended");
|
|
697
|
-
if (frame.type === "approval_complete" && !admitPresentedComplete(frame.askId))
|
|
643
|
+
});
|
|
644
|
+
};
|
|
645
|
+
const askTaskId = durableTaskId ?? uuidv7();
|
|
646
|
+
const askOwner = gatedPrincipal(req, deps.config) ?? null;
|
|
647
|
+
const emitAsk = (frame) => {
|
|
648
|
+
if (res.writableEnded || res.destroyed) {
|
|
649
|
+
recordFailOpen(failOpenTagForDroppedFrame(frame.type), `frame=${frame.type}`);
|
|
698
650
|
return;
|
|
651
|
+
}
|
|
652
|
+
res.write(`event: ${frame.type}\ndata: ${JSON.stringify(frame)}\n\n`);
|
|
653
|
+
};
|
|
654
|
+
const faceAlive = () => !res.writableEnded && !res.destroyed;
|
|
655
|
+
const emitElicitation = createLedgerTwinEmit({ sink: ledgerSink, writeLive: emitAsk, liveReachable: faceAlive });
|
|
656
|
+
const writeQuestionLive = (frame) => {
|
|
657
|
+
if (!faceAlive())
|
|
658
|
+
throw new Error("live stream ended — question frame undeliverable");
|
|
699
659
|
emitAsk(frame);
|
|
700
|
-
}
|
|
701
|
-
|
|
702
|
-
|
|
703
|
-
|
|
660
|
+
};
|
|
661
|
+
const emitQuestion = createLedgerTwinEmit({ sink: ledgerSink, writeLive: writeQuestionLive, liveReachable: faceAlive });
|
|
662
|
+
await emitPendingWorkflowCompletions(deps.workflowCompletionInbox, prepared.spec.sessionId, askOwner, (frame) => {
|
|
663
|
+
if (closed || res.writableEnded || res.destroyed)
|
|
664
|
+
throw new Error("stream ended before the completion frame was written");
|
|
665
|
+
const f = frame;
|
|
666
|
+
const fKey = f.type === "task_notification" ? taskNotificationStreamKeyOf(f) : undefined;
|
|
667
|
+
if (fKey !== undefined)
|
|
668
|
+
syncNotifiedKeys.set(fKey, Promise.resolve(true));
|
|
669
|
+
sseData(res, frame);
|
|
670
|
+
}, { route: "sync-stream-open", connection: "live-sse", log: (m, x) => deps.logger?.info?.(m, x) });
|
|
671
|
+
const withElicit = deps.elicitation
|
|
672
|
+
? () => deps.elicitation.runWithContext({ taskId: askTaskId, owner: askOwner, emit: emitElicitation, abortSignal: ac.signal }, streamBody)
|
|
673
|
+
: streamBody;
|
|
674
|
+
const withQuestion = () => deps.question
|
|
675
|
+
? deps.question.runWithContext({ taskId: askTaskId, owner: askOwner, emit: emitQuestion, abortSignal: ac.signal, deliverable: faceAlive }, withElicit)
|
|
676
|
+
: withElicit();
|
|
677
|
+
const emitApproval = (frame) => {
|
|
678
|
+
if (res.writableEnded || res.destroyed)
|
|
679
|
+
throw new Error("live stream ended — approval card undeliverable");
|
|
704
680
|
emitAsk(frame);
|
|
705
|
-
|
|
706
|
-
|
|
707
|
-
|
|
708
|
-
|
|
709
|
-
|
|
710
|
-
|
|
711
|
-
|
|
712
|
-
|
|
681
|
+
};
|
|
682
|
+
const streamApprovalOn = resolveStreamApprovalGate({
|
|
683
|
+
toolApprovalEnabled: Boolean(deps.toolApproval),
|
|
684
|
+
streamApprovalEnabled: deps.config.streamApproval?.enabled === true,
|
|
685
|
+
backend: deps.backend,
|
|
686
|
+
checkpointStore: deps.checkpointStore,
|
|
687
|
+
}).active;
|
|
688
|
+
const approvalLeg = assembleApprovalLeg(streamApprovalOn);
|
|
689
|
+
const legDeadlineMonotonic = approvalLeg.legDeadlineMonotonic;
|
|
690
|
+
const unattended = approvalLeg.unattended;
|
|
691
|
+
const emitCard = createApprovalCardEmitter({
|
|
692
|
+
...(ledgerSink
|
|
713
693
|
? {
|
|
714
|
-
|
|
715
|
-
|
|
716
|
-
|
|
717
|
-
|
|
694
|
+
appendDurable: async (frame) => {
|
|
695
|
+
const { type, ...rest } = frame;
|
|
696
|
+
await ledgerSink.append(type, rest);
|
|
697
|
+
},
|
|
718
698
|
}
|
|
719
|
-
: {
|
|
720
|
-
|
|
721
|
-
|
|
722
|
-
|
|
723
|
-
|
|
724
|
-
|
|
725
|
-
|
|
726
|
-
|
|
727
|
-
|
|
699
|
+
: {}),
|
|
700
|
+
writeLive: (frame) => {
|
|
701
|
+
if (res.writableEnded || res.destroyed)
|
|
702
|
+
throw new Error("live stream ended");
|
|
703
|
+
if (frame.type === "approval_complete" && !admitPresentedComplete(frame.askId))
|
|
704
|
+
return;
|
|
705
|
+
emitAsk(frame);
|
|
706
|
+
},
|
|
707
|
+
});
|
|
708
|
+
const emitRevokeLive = (frame) => {
|
|
709
|
+
if (!res.writableEnded && !res.destroyed)
|
|
710
|
+
emitAsk(frame);
|
|
711
|
+
};
|
|
712
|
+
const approvalCtx = deps.toolApproval && unattended === undefined
|
|
713
|
+
? {
|
|
728
714
|
taskId: askTaskId,
|
|
715
|
+
owner: askOwner,
|
|
716
|
+
abortSignal: ac.signal,
|
|
729
717
|
...(prepared.spec.sessionId ? { sessionId: prepared.spec.sessionId } : {}),
|
|
730
|
-
...(streamApprovalOn
|
|
731
|
-
|
|
732
|
-
|
|
733
|
-
|
|
734
|
-
|
|
735
|
-
|
|
736
|
-
|
|
737
|
-
|
|
738
|
-
void (await emitAsk(f));
|
|
739
|
-
if (ledgerSink) {
|
|
740
|
-
const { type, ...rest } = f;
|
|
741
|
-
await ledgerSink.append(type, rest).catch(() => undefined);
|
|
742
|
-
}
|
|
743
|
-
},
|
|
744
|
-
}, withApproval)
|
|
745
|
-
: withApproval()));
|
|
746
|
-
if (durableTaskId && deps.runStore) {
|
|
747
|
-
const finalPlane = finalResult ? outwardPlane(finalResult.terminal) : undefined;
|
|
748
|
-
if (finalResult && finalPlane?.status === "suspended" && deps.checkpointStore)
|
|
749
|
-
await deps.runStore.setSuspended(durableTaskId, finalResult);
|
|
750
|
-
else if (finalResult && finalPlane?.status === "needs_review" && deps.checkpointStore)
|
|
751
|
-
await deps.runStore.setNeedsReview(durableTaskId, finalResult);
|
|
752
|
-
else if (finalResult && finalPlane) {
|
|
753
|
-
await deps.runStore.setTerminal(durableTaskId, finalPlane.status, finalResult, finalPlane.errorMessage ?? null);
|
|
754
|
-
await releaseKeyIfZeroWorkRefusal(deps.runStore, durableTaskId, finalResult);
|
|
755
|
-
}
|
|
756
|
-
else {
|
|
757
|
-
putRewindAnchor();
|
|
758
|
-
const pending = await deps.checkpointStore?.findPendingTokenBySession(prepared.spec.sessionId);
|
|
759
|
-
const dcCause = ac.cause();
|
|
760
|
-
if (pending) {
|
|
761
|
-
if (ledgerSink && !ledgerTerminal) {
|
|
762
|
-
ledgerTerminal = true;
|
|
763
|
-
await ledgerSink.flush().catch(() => undefined);
|
|
764
|
-
await ledgerSink.appendParked("suspended", {}).catch(() => undefined);
|
|
765
|
-
}
|
|
766
|
-
await deps.runStore.setSuspended(durableTaskId, null);
|
|
767
|
-
settleFleet("suspended");
|
|
718
|
+
...(streamApprovalOn
|
|
719
|
+
? {
|
|
720
|
+
emitCard,
|
|
721
|
+
emitRevoke: emitRevokeLive,
|
|
722
|
+
legKey: "",
|
|
723
|
+
...(legDeadlineMonotonic !== undefined ? { legDeadlineMonotonic } : {}),
|
|
724
|
+
}
|
|
725
|
+
: { emit: emitApproval }),
|
|
768
726
|
}
|
|
769
|
-
|
|
770
|
-
|
|
771
|
-
|
|
772
|
-
|
|
773
|
-
|
|
774
|
-
|
|
775
|
-
|
|
776
|
-
|
|
777
|
-
|
|
778
|
-
|
|
779
|
-
|
|
727
|
+
: undefined;
|
|
728
|
+
if (deps.toolApproval) {
|
|
729
|
+
const toolApproval = deps.toolApproval;
|
|
730
|
+
prepared.spec.onAsk = unattended
|
|
731
|
+
? toolApproval.boundUnattendedAsk({ owner: askOwner, taskId: askTaskId, ...(prepared.spec.sessionId ? { sessionId: prepared.spec.sessionId } : {}) }, unattended)
|
|
732
|
+
: toolApproval.boundAsk({
|
|
733
|
+
owner: askOwner,
|
|
734
|
+
taskId: askTaskId,
|
|
735
|
+
...(prepared.spec.sessionId ? { sessionId: prepared.spec.sessionId } : {}),
|
|
736
|
+
...(streamApprovalOn ? { legKey: "", ...(legDeadlineMonotonic !== undefined ? { legDeadlineMonotonic } : {}) } : {}),
|
|
737
|
+
});
|
|
738
|
+
}
|
|
739
|
+
const withApproval = () => approvalCtx && deps.toolApproval ? deps.toolApproval.runWithContext(approvalCtx, withQuestion) : withQuestion();
|
|
740
|
+
await instructionLeg.run(prepared.spec.sessionId, () => withModelReachScope({ sessionId: prepared.spec.sessionId, taskId: durableTaskId }, () => deps.sendUserFile
|
|
741
|
+
? deps.sendUserFile.runWithContext({
|
|
742
|
+
taskId: askTaskId,
|
|
743
|
+
emit: async (f) => {
|
|
744
|
+
void (await emitAsk(f));
|
|
745
|
+
if (ledgerSink) {
|
|
746
|
+
const { type, ...rest } = f;
|
|
747
|
+
await ledgerSink.append(type, rest).catch(() => undefined);
|
|
748
|
+
}
|
|
749
|
+
},
|
|
750
|
+
}, withApproval)
|
|
751
|
+
: withApproval()));
|
|
752
|
+
if (durableTaskId && deps.runStore) {
|
|
753
|
+
const finalPlane = finalResult ? outwardPlane(finalResult.terminal) : undefined;
|
|
754
|
+
if (finalResult && finalPlane?.status === "suspended" && deps.checkpointStore)
|
|
755
|
+
await deps.runStore.setSuspended(durableTaskId, finalResult);
|
|
756
|
+
else if (finalResult && finalPlane?.status === "needs_review" && deps.checkpointStore)
|
|
757
|
+
await deps.runStore.setNeedsReview(durableTaskId, finalResult);
|
|
758
|
+
else if (finalResult && finalPlane) {
|
|
759
|
+
await deps.runStore.setTerminal(durableTaskId, finalPlane.status, finalResult, finalPlane.errorMessage ?? null);
|
|
760
|
+
await releaseKeyIfZeroWorkRefusal(deps.runStore, durableTaskId, finalResult);
|
|
780
761
|
}
|
|
781
762
|
else {
|
|
782
|
-
|
|
783
|
-
|
|
784
|
-
|
|
785
|
-
await ledgerSink.flush().catch(() => undefined);
|
|
786
|
-
await ledgerSink.append("failed", { errorMessage: err }).catch(() => undefined);
|
|
787
|
-
}
|
|
788
|
-
await deps.runStore.setTerminal(durableTaskId, "failed", null, err);
|
|
789
|
-
settleFleet("failed");
|
|
790
|
-
}
|
|
791
|
-
}
|
|
792
|
-
runRowSettled = true;
|
|
793
|
-
}
|
|
794
|
-
}
|
|
795
|
-
finally {
|
|
796
|
-
syncLegLive = false;
|
|
797
|
-
unregisterEngineNotice();
|
|
798
|
-
if (durableHeartbeat)
|
|
799
|
-
clearInterval(durableHeartbeat);
|
|
800
|
-
if (durableTaskId && steerableRuns.get(durableTaskId) === liveStreamRef)
|
|
801
|
-
steerableRuns.delete(durableTaskId);
|
|
802
|
-
if (liveStreamRef !== undefined)
|
|
803
|
-
await deps.userInputLedger?.settleLeg(liveStreamRef);
|
|
804
|
-
if (durableTaskId && inflightRuns.get(durableTaskId) === ac)
|
|
805
|
-
inflightRuns.delete(durableTaskId);
|
|
806
|
-
for (const evict of subagentHandleEvictions)
|
|
807
|
-
evict();
|
|
808
|
-
if (!runRowSettled && durableTaskId && deps.runStore) {
|
|
809
|
-
try {
|
|
810
|
-
const row = await deps.runStore.getRun(durableTaskId);
|
|
811
|
-
if (row?.status === "running") {
|
|
812
|
-
const pending = prepared.spec.sessionId ? await deps.checkpointStore?.findPendingTokenBySession(prepared.spec.sessionId) : undefined;
|
|
813
|
-
const backstopCause = ac.cause();
|
|
763
|
+
putRewindAnchor();
|
|
764
|
+
const pending = await deps.checkpointStore?.findPendingTokenBySession(prepared.spec.sessionId);
|
|
765
|
+
const dcCause = ac.cause();
|
|
814
766
|
if (pending) {
|
|
815
767
|
if (ledgerSink && !ledgerTerminal) {
|
|
816
768
|
ledgerTerminal = true;
|
|
@@ -820,195 +772,255 @@ export async function handleTasks(req, res, match, ctx) {
|
|
|
820
772
|
await deps.runStore.setSuspended(durableTaskId, null);
|
|
821
773
|
settleFleet("suspended");
|
|
822
774
|
}
|
|
823
|
-
else if (
|
|
824
|
-
|
|
825
|
-
|
|
826
|
-
await ledgerSink.onDone(finalResult).catch(() => undefined);
|
|
827
|
-
}
|
|
828
|
-
const retryPlane = outwardPlane(finalResult.terminal);
|
|
829
|
-
await deps.runStore.setTerminal(durableTaskId, retryPlane.status, finalResult, retryPlane.errorMessage ?? null);
|
|
830
|
-
await releaseKeyIfZeroWorkRefusal(deps.runStore, durableTaskId, finalResult);
|
|
831
|
-
settleFleet(retryPlane.status, fleetRunResiduals(finalResult));
|
|
832
|
-
}
|
|
833
|
-
else if (backstopCause !== undefined) {
|
|
834
|
-
const err = "stream aborted before completion (run cancelled)";
|
|
835
|
-
const c = cancelledResult(err, backstopCause);
|
|
775
|
+
else if (dcCause !== undefined) {
|
|
776
|
+
const dcErr = "stream client disconnected before completion (run aborted)";
|
|
777
|
+
const dc = cancelledResult(dcErr, dcCause);
|
|
836
778
|
if (ledgerSink && !ledgerTerminal) {
|
|
837
779
|
ledgerTerminal = true;
|
|
838
780
|
await ledgerSink.flush().catch(() => undefined);
|
|
839
|
-
await ledgerSink.append("failed", buildCancelledLedgerRow(
|
|
781
|
+
await ledgerSink.append("failed", buildCancelledLedgerRow(dc)).catch(() => undefined);
|
|
840
782
|
}
|
|
841
|
-
await deps.runStore.setTerminal(durableTaskId, "failed",
|
|
783
|
+
await deps.runStore.setTerminal(durableTaskId, "failed", dc, dcErr);
|
|
784
|
+
noteRunOutcome(dc);
|
|
785
|
+
settleFleet("failed");
|
|
842
786
|
}
|
|
843
787
|
else {
|
|
844
|
-
const err = "stream
|
|
788
|
+
const err = "stream ended before a terminal event";
|
|
845
789
|
if (ledgerSink && !ledgerTerminal) {
|
|
846
790
|
ledgerTerminal = true;
|
|
847
791
|
await ledgerSink.flush().catch(() => undefined);
|
|
848
792
|
await ledgerSink.append("failed", { errorMessage: err }).catch(() => undefined);
|
|
849
793
|
}
|
|
850
794
|
await deps.runStore.setTerminal(durableTaskId, "failed", null, err);
|
|
795
|
+
settleFleet("failed");
|
|
851
796
|
}
|
|
852
797
|
}
|
|
853
|
-
|
|
854
|
-
catch {
|
|
798
|
+
runRowSettled = true;
|
|
855
799
|
}
|
|
856
800
|
}
|
|
857
|
-
|
|
858
|
-
|
|
859
|
-
|
|
860
|
-
|
|
861
|
-
|
|
862
|
-
|
|
863
|
-
|
|
864
|
-
|
|
865
|
-
|
|
866
|
-
|
|
867
|
-
|
|
868
|
-
|
|
869
|
-
|
|
870
|
-
|
|
871
|
-
|
|
872
|
-
|
|
873
|
-
|
|
874
|
-
|
|
875
|
-
|
|
876
|
-
|
|
877
|
-
|
|
878
|
-
|
|
879
|
-
|
|
880
|
-
|
|
881
|
-
|
|
882
|
-
|
|
883
|
-
|
|
884
|
-
|
|
885
|
-
|
|
886
|
-
|
|
887
|
-
|
|
888
|
-
|
|
889
|
-
|
|
890
|
-
|
|
891
|
-
|
|
892
|
-
|
|
893
|
-
|
|
894
|
-
|
|
895
|
-
|
|
896
|
-
|
|
897
|
-
|
|
898
|
-
|
|
899
|
-
|
|
900
|
-
|
|
901
|
-
|
|
902
|
-
|
|
903
|
-
|
|
904
|
-
|
|
905
|
-
|
|
906
|
-
|
|
907
|
-
|
|
908
|
-
|
|
909
|
-
|
|
910
|
-
|
|
911
|
-
|
|
912
|
-
|
|
913
|
-
|
|
914
|
-
|
|
915
|
-
|
|
916
|
-
|
|
917
|
-
|
|
918
|
-
if (durableTaskId && deps.runStore) {
|
|
919
|
-
const rs = deps.runStore;
|
|
920
|
-
const tid = durableTaskId;
|
|
921
|
-
const owner = principal ?? null;
|
|
922
|
-
const abortFromVerb = () => syncCancelCtrl.abort("cancel_verb");
|
|
923
|
-
durableHeartbeat = setInterval(() => {
|
|
924
|
-
void rs.heartbeat(tid, owner).catch(() => undefined);
|
|
925
|
-
if (!syncCancelCtrl.signal.aborted) {
|
|
926
|
-
void Promise.resolve(rs.isCancelRequested?.(tid, owner)).then((c) => { if (c)
|
|
927
|
-
abortFromVerb(); }).catch(() => undefined);
|
|
801
|
+
finally {
|
|
802
|
+
syncLegLive = false;
|
|
803
|
+
unregisterEngineNotice();
|
|
804
|
+
if (durableHeartbeat)
|
|
805
|
+
clearInterval(durableHeartbeat);
|
|
806
|
+
if (durableTaskId && steerableRuns.get(durableTaskId) === liveStreamRef)
|
|
807
|
+
steerableRuns.delete(durableTaskId);
|
|
808
|
+
if (liveStreamRef !== undefined)
|
|
809
|
+
await deps.userInputLedger?.settleLeg(liveStreamRef);
|
|
810
|
+
if (durableTaskId && inflightRuns.get(durableTaskId) === ac)
|
|
811
|
+
inflightRuns.delete(durableTaskId);
|
|
812
|
+
for (const evict of subagentHandleEvictions)
|
|
813
|
+
evict();
|
|
814
|
+
if (!runRowSettled && durableTaskId && deps.runStore) {
|
|
815
|
+
try {
|
|
816
|
+
const row = await deps.runStore.getRun(durableTaskId);
|
|
817
|
+
if (row?.status === "running") {
|
|
818
|
+
const pending = prepared.spec.sessionId ? await deps.checkpointStore?.findPendingTokenBySession(prepared.spec.sessionId) : undefined;
|
|
819
|
+
const backstopCause = ac.cause();
|
|
820
|
+
if (pending) {
|
|
821
|
+
if (ledgerSink && !ledgerTerminal) {
|
|
822
|
+
ledgerTerminal = true;
|
|
823
|
+
await ledgerSink.flush().catch(() => undefined);
|
|
824
|
+
await ledgerSink.appendParked("suspended", {}).catch(() => undefined);
|
|
825
|
+
}
|
|
826
|
+
await deps.runStore.setSuspended(durableTaskId, null);
|
|
827
|
+
settleFleet("suspended");
|
|
828
|
+
}
|
|
829
|
+
else if (finalResult) {
|
|
830
|
+
if (ledgerSink && !ledgerTerminal) {
|
|
831
|
+
ledgerTerminal = true;
|
|
832
|
+
await ledgerSink.onDone(finalResult).catch(() => undefined);
|
|
833
|
+
}
|
|
834
|
+
const retryPlane = outwardPlane(finalResult.terminal);
|
|
835
|
+
await deps.runStore.setTerminal(durableTaskId, retryPlane.status, finalResult, retryPlane.errorMessage ?? null);
|
|
836
|
+
await releaseKeyIfZeroWorkRefusal(deps.runStore, durableTaskId, finalResult);
|
|
837
|
+
settleFleet(retryPlane.status, fleetRunResiduals(finalResult));
|
|
838
|
+
}
|
|
839
|
+
else if (backstopCause !== undefined) {
|
|
840
|
+
const err = "stream aborted before completion (run cancelled)";
|
|
841
|
+
const c = cancelledResult(err, backstopCause);
|
|
842
|
+
if (ledgerSink && !ledgerTerminal) {
|
|
843
|
+
ledgerTerminal = true;
|
|
844
|
+
await ledgerSink.flush().catch(() => undefined);
|
|
845
|
+
await ledgerSink.append("failed", buildCancelledLedgerRow(c)).catch(() => undefined);
|
|
846
|
+
}
|
|
847
|
+
await deps.runStore.setTerminal(durableTaskId, "failed", c, err);
|
|
848
|
+
}
|
|
849
|
+
else {
|
|
850
|
+
const err = "stream leg threw before a terminal event";
|
|
851
|
+
if (ledgerSink && !ledgerTerminal) {
|
|
852
|
+
ledgerTerminal = true;
|
|
853
|
+
await ledgerSink.flush().catch(() => undefined);
|
|
854
|
+
await ledgerSink.append("failed", { errorMessage: err }).catch(() => undefined);
|
|
855
|
+
}
|
|
856
|
+
await deps.runStore.setTerminal(durableTaskId, "failed", null, err);
|
|
857
|
+
}
|
|
858
|
+
}
|
|
859
|
+
}
|
|
860
|
+
catch {
|
|
861
|
+
}
|
|
928
862
|
}
|
|
929
|
-
|
|
930
|
-
|
|
931
|
-
|
|
932
|
-
void Promise.resolve(rs.isCancelRequested?.(tid, owner)).then((c) => { if (c && !syncCancelCtrl.signal.aborted)
|
|
933
|
-
abortFromVerb(); }).catch(() => undefined);
|
|
934
|
-
}
|
|
935
|
-
else if (durableTaskId) {
|
|
936
|
-
inflightRuns.set(durableTaskId, syncCancelCtrl);
|
|
937
|
-
}
|
|
938
|
-
const specWithSignal = { ...prepared.spec, signal: syncCancelCtrl.signal, ...(durableTaskId ? { taskId: durableTaskId } : {}) };
|
|
939
|
-
const observeSyncLegEvent = (ev) => {
|
|
940
|
-
if (ev.type !== "wiring_manifest" || wiringReceipt !== undefined)
|
|
941
|
-
return;
|
|
942
|
-
wiringReceipt = wiringManifestFaceEventData(ev, wiringOperatorFace, legWiringSections());
|
|
863
|
+
settleFleet("failed");
|
|
864
|
+
}
|
|
865
|
+
return { status: 200, body: finalResult };
|
|
943
866
|
};
|
|
944
|
-
let result;
|
|
945
867
|
counters.uncountedBillableInflight++;
|
|
946
868
|
try {
|
|
947
|
-
|
|
948
|
-
|
|
949
|
-
|
|
950
|
-
|
|
951
|
-
|
|
952
|
-
|
|
953
|
-
observeSyncLegEvent(ev);
|
|
954
|
-
prepared.acceptance.observe(ev);
|
|
955
|
-
}
|
|
956
|
-
return stream.result();
|
|
957
|
-
};
|
|
958
|
-
result = await instructionLeg.run(prepared.spec.sessionId, () => withPrincipal(principal, () => prepared.verify ? verifyLeg(prepared.verify) : prepared.cascade ? cascadeLeg() : plainLeg()));
|
|
869
|
+
resp = await idemCache.run(inflightKey, async () => withRunReceipt(await runStreamSubmitLeg()));
|
|
870
|
+
}
|
|
871
|
+
catch (e) {
|
|
872
|
+
deps.logger?.error("stream_error", { closed, err: redactHead(e instanceof Error ? e.message : String(e), 2000) });
|
|
873
|
+
if (!closed)
|
|
874
|
+
throw e;
|
|
959
875
|
}
|
|
960
876
|
finally {
|
|
961
877
|
counters.uncountedBillableInflight--;
|
|
962
|
-
|
|
963
|
-
|
|
964
|
-
|
|
965
|
-
inflightRuns.delete(durableTaskId);
|
|
878
|
+
clearInterval(hb);
|
|
879
|
+
if (usageKey)
|
|
880
|
+
deps.modelUsage?.clear(usageKey);
|
|
966
881
|
}
|
|
967
|
-
|
|
968
|
-
|
|
969
|
-
result = mintCancelledResult({
|
|
970
|
-
taskId: durableTaskId ?? "",
|
|
971
|
-
sessionId: prepared.spec.sessionId ?? "",
|
|
972
|
-
startedAtMs: syncLegStartedAt,
|
|
973
|
-
message: "cancelled by user",
|
|
974
|
-
cause: syncAbortCause,
|
|
975
|
-
engineResult: result,
|
|
976
|
-
});
|
|
882
|
+
if (!ranLive && resp && !res.writableEnded) {
|
|
883
|
+
sseData(res, { type: "done", result: doneResultOfOutcome(resp), replay: true });
|
|
977
884
|
}
|
|
978
|
-
|
|
979
|
-
|
|
980
|
-
|
|
885
|
+
res.end();
|
|
886
|
+
}
|
|
887
|
+
else {
|
|
888
|
+
const withSyncSubmitKeys = (body) => {
|
|
889
|
+
const extra = {
|
|
890
|
+
...(prepared.verify || prepared.cascade ? { notice: MULTI_ATTEMPT_NO_EVENT_CHANNEL_NOTICE } : {}),
|
|
891
|
+
...(prepared.warnings !== undefined ? { warnings: prepared.warnings } : {}),
|
|
892
|
+
};
|
|
893
|
+
return Object.keys(extra).length > 0 ? { ...body, ...extra } : body;
|
|
894
|
+
};
|
|
895
|
+
const runSyncSubmitLeg = async () => {
|
|
896
|
+
const syncLegStartedAt = Date.now();
|
|
897
|
+
let durableTaskId;
|
|
898
|
+
let durableHeartbeat;
|
|
899
|
+
if (deps.runStore && prepared.spec.sessionId) {
|
|
900
|
+
const tid = uuidv7();
|
|
901
|
+
const created = await deps.runStore.createRun(tid, prepared.spec.sessionId, principal ?? null, deps.instanceId ?? "default", { ...runMeta(prepared, source), identity: runIdentityOf(req, deps.config, principal ?? null), idem: idem ?? null });
|
|
902
|
+
if (created.ok)
|
|
903
|
+
clearTurnActivity(tid);
|
|
904
|
+
if (created.ok)
|
|
905
|
+
recordResourceWindow(tid, prepared.spec.resourceSuspend);
|
|
906
|
+
if (created.existing)
|
|
907
|
+
return await replayOfExistingRun(created.existing, { principal: submitter, idem }, "sync", deps, deps.runStore);
|
|
908
|
+
if (!created.ok)
|
|
909
|
+
return { status: 409, body: await buildActiveRunConflict({ runStore: deps.runStore, checkpointStore: deps.checkpointStore, governance: deps.config, runStaleSec: deps.config.runStaleSec, turnActivity: readTurnActivityMs }, prepared.spec.sessionId, created.activeTaskId) };
|
|
910
|
+
const placementReject = await commitPlacement(deps, deps.runStore, tid, prepared.bindingIntent);
|
|
911
|
+
if (placementReject)
|
|
912
|
+
return { status: placementReject.status, body: devicePlacementErrorBody(placementReject) };
|
|
913
|
+
durableTaskId = tid;
|
|
914
|
+
}
|
|
915
|
+
if (deps.checkpointStore && prepared.spec.sessionId) {
|
|
916
|
+
const ctxRefusal = await persistSubmitCtx(deps.checkpointStore, deps, durableTaskId !== undefined && deps.runStore ? { runStore: deps.runStore, taskId: durableTaskId } : undefined, prepared.spec.sessionId, { body: prepared.body, memoryScope: prepared.auth?.memoryScope, writerVersion: serviceVersion() });
|
|
917
|
+
if (ctxRefusal)
|
|
918
|
+
return ctxRefusal;
|
|
919
|
+
}
|
|
920
|
+
commitSessionInheritEnv(deps, prepared.spec.sessionId, prepared.body);
|
|
921
|
+
const syncCancelCtrl = createRunAbort();
|
|
922
|
+
if (prepared.spec.sessionId)
|
|
923
|
+
markChildrenStoppedByUserOnAbort(syncCancelCtrl.signal, durableTaskId ?? prepared.spec.sessionId, prepared.auth?.principal);
|
|
981
924
|
if (durableTaskId && deps.runStore) {
|
|
982
|
-
|
|
983
|
-
|
|
925
|
+
const rs = deps.runStore;
|
|
926
|
+
const tid = durableTaskId;
|
|
927
|
+
const owner = principal ?? null;
|
|
928
|
+
const abortFromVerb = () => syncCancelCtrl.abort("cancel_verb");
|
|
929
|
+
durableHeartbeat = setInterval(() => {
|
|
930
|
+
void rs.heartbeat(tid, owner).catch(() => undefined);
|
|
931
|
+
if (!syncCancelCtrl.signal.aborted) {
|
|
932
|
+
void Promise.resolve(rs.isCancelRequested?.(tid, owner)).then((c) => { if (c)
|
|
933
|
+
abortFromVerb(); }).catch(() => undefined);
|
|
934
|
+
}
|
|
935
|
+
}, HEARTBEAT_MS);
|
|
936
|
+
durableHeartbeat.unref?.();
|
|
937
|
+
inflightRuns.set(tid, syncCancelCtrl);
|
|
938
|
+
void Promise.resolve(rs.isCancelRequested?.(tid, owner)).then((c) => { if (c && !syncCancelCtrl.signal.aborted)
|
|
939
|
+
abortFromVerb(); }).catch(() => undefined);
|
|
984
940
|
}
|
|
985
|
-
|
|
986
|
-
|
|
987
|
-
|
|
988
|
-
const
|
|
989
|
-
|
|
990
|
-
|
|
991
|
-
|
|
992
|
-
|
|
993
|
-
|
|
994
|
-
|
|
995
|
-
|
|
996
|
-
|
|
997
|
-
|
|
998
|
-
|
|
999
|
-
|
|
1000
|
-
|
|
1001
|
-
|
|
1002
|
-
|
|
1003
|
-
|
|
1004
|
-
|
|
1005
|
-
|
|
1006
|
-
|
|
1007
|
-
|
|
1008
|
-
|
|
1009
|
-
|
|
941
|
+
else if (durableTaskId) {
|
|
942
|
+
inflightRuns.set(durableTaskId, syncCancelCtrl);
|
|
943
|
+
}
|
|
944
|
+
const specWithSignal = { ...prepared.spec, signal: syncCancelCtrl.signal, ...(durableTaskId ? { taskId: durableTaskId } : {}) };
|
|
945
|
+
const observeSyncLegEvent = (ev) => {
|
|
946
|
+
if (ev.type !== "wiring_manifest" || wiringReceipt !== undefined)
|
|
947
|
+
return;
|
|
948
|
+
wiringReceipt = wiringManifestFaceEventData(ev, wiringOperatorFace, legWiringSections());
|
|
949
|
+
};
|
|
950
|
+
let result;
|
|
951
|
+
counters.uncountedBillableInflight++;
|
|
952
|
+
try {
|
|
953
|
+
const heldDecisions = prepared.spec.sessionId !== undefined ? await heldWorkflowParkDecisions(deps, prepared.spec.sessionId) : {};
|
|
954
|
+
const verifyLeg = (v) => runWithVerification(prepared.runner, specWithSignal, v, heldDecisions);
|
|
955
|
+
const cascadeLeg = () => runCascade(prepared.runner, specWithSignal, cascadeConfig(deps.config.cascadeLadder, prepared.spec.limits?.maxCostUsd), heldDecisions);
|
|
956
|
+
const plainLeg = async () => {
|
|
957
|
+
const stream = prepared.runner.runTaskStream(specWithSignal, undefined, heldDecisions);
|
|
958
|
+
const syncWorkspace = createSessionWorkspaceRecorder(sessionWorkspaceStoreOf(deps.sessionStorage), legSessionWitnessOf(prepared.spec));
|
|
959
|
+
for await (const ev of stream) {
|
|
960
|
+
observeSyncLegEvent(ev);
|
|
961
|
+
prepared.acceptance.observe(ev);
|
|
962
|
+
await syncWorkspace.observe(ev);
|
|
963
|
+
}
|
|
964
|
+
return stream.result();
|
|
965
|
+
};
|
|
966
|
+
result = await instructionLeg.run(prepared.spec.sessionId, () => withPrincipal(principal, () => prepared.verify ? verifyLeg(prepared.verify) : prepared.cascade ? cascadeLeg() : plainLeg()));
|
|
967
|
+
}
|
|
968
|
+
finally {
|
|
969
|
+
counters.uncountedBillableInflight--;
|
|
970
|
+
if (durableHeartbeat)
|
|
971
|
+
clearInterval(durableHeartbeat);
|
|
972
|
+
if (durableTaskId && inflightRuns.get(durableTaskId) === syncCancelCtrl)
|
|
973
|
+
inflightRuns.delete(durableTaskId);
|
|
974
|
+
}
|
|
975
|
+
const syncAbortCause = syncCancelCtrl.cause();
|
|
976
|
+
if (syncAbortCause !== undefined) {
|
|
977
|
+
result = mintCancelledResult({
|
|
978
|
+
taskId: durableTaskId ?? "",
|
|
979
|
+
sessionId: prepared.spec.sessionId ?? "",
|
|
980
|
+
startedAtMs: syncLegStartedAt,
|
|
981
|
+
message: "cancelled by user",
|
|
982
|
+
cause: syncAbortCause,
|
|
983
|
+
engineResult: result,
|
|
984
|
+
});
|
|
985
|
+
}
|
|
986
|
+
const resumeAtStatus = syncRefusalHttpStatus(result);
|
|
987
|
+
if (resumeAtStatus) {
|
|
988
|
+
const resumeAtPlane = outwardPlane(result.terminal);
|
|
989
|
+
if (durableTaskId && deps.runStore) {
|
|
990
|
+
await deps.runStore.setTerminal(durableTaskId, "failed", sealResultForExit(result), resumeAtPlane.errorMessage ?? null);
|
|
991
|
+
await releaseKeyIfZeroWorkRefusal(deps.runStore, durableTaskId, result);
|
|
992
|
+
}
|
|
993
|
+
const retryAfterSec = resumeAtPlane.errorCode === "memory.admission_required" && result.retryAfterMs !== undefined
|
|
994
|
+
? Math.max(1, Math.ceil(result.retryAfterMs / 1000))
|
|
995
|
+
: undefined;
|
|
996
|
+
const refusalFacts = zeroWorkRefusalFacts(redactTerminalResult(sealResultForExit(result)).terminal, prepared.warnings);
|
|
997
|
+
return { status: resumeAtStatus, body: refusalBody({ errorCode: httpErrorCode(resumeAtStatus, resumeAtPlane.errorCode), message: redactErrorMessage(resumeAtPlane.errorMessage) ?? "resume-at failed", facts: { ...(retryAfterSec !== undefined ? { retryAfterSec } : {}), ...refusalFacts } }) };
|
|
998
|
+
}
|
|
999
|
+
const exitResult = await mintSubmitExitResult(result, deps.checkpointStore);
|
|
1000
|
+
const syncPlane = outwardPlane(result.terminal);
|
|
1001
|
+
if (syncPlane.status === "suspended" && deps.checkpointStore && deps.runStore && durableTaskId && prepared.spec.sessionId) {
|
|
1002
|
+
await deps.runStore.setSuspended(durableTaskId, exitResult);
|
|
1003
|
+
return { status: 200, body: withSyncSubmitKeys(buildSyncSubmitBody(exitResult, { taskId: durableTaskId, sessionId: prepared.spec.sessionId, status: "suspended" })) };
|
|
1004
|
+
}
|
|
1005
|
+
if (syncPlane.status === "needs_review" && deps.checkpointStore && deps.runStore && durableTaskId && prepared.spec.sessionId) {
|
|
1006
|
+
await deps.runStore.setNeedsReview(durableTaskId, exitResult);
|
|
1007
|
+
return { status: 200, body: withSyncSubmitKeys(buildSyncSubmitBody(exitResult, { taskId: durableTaskId, sessionId: prepared.spec.sessionId, status: "needs_review" })) };
|
|
1008
|
+
}
|
|
1009
|
+
if (durableTaskId && deps.runStore)
|
|
1010
|
+
await deps.runStore.setTerminal(durableTaskId, syncPlane.status, exitResult, syncPlane.errorMessage ?? null);
|
|
1011
|
+
prepared.acceptance.settle(result);
|
|
1012
|
+
finalizeTaskResult(result, principal, prepared.spec.objective);
|
|
1013
|
+
noteRunOutcome(result);
|
|
1014
|
+
return { status: 200, body: withSyncSubmitKeys(buildSyncSubmitBody(exitResult)) };
|
|
1015
|
+
};
|
|
1016
|
+
const resp = await idemCache.run(inflightKey, async () => withRunReceipt(await runSyncSubmitLeg()));
|
|
1017
|
+
sendSubmitResult(res, resp);
|
|
1018
|
+
}
|
|
1019
|
+
return;
|
|
1020
|
+
}
|
|
1021
|
+
finally {
|
|
1022
|
+
prepared.peerLease.release();
|
|
1010
1023
|
}
|
|
1011
|
-
return;
|
|
1012
1024
|
}
|
|
1013
1025
|
}
|
|
1014
1026
|
export const TASKS_ROUTES = [
|