@sema-agent/server 1.316.0 → 1.317.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (50) hide show
  1. package/README.md +9 -0
  2. package/README.zh-CN.md +7 -0
  3. package/dist/config-types.d.ts +1 -1
  4. package/dist/config.d.ts +14 -0
  5. package/dist/config.js +148 -45
  6. package/dist/elicitation.js +2 -2
  7. package/dist/http/route-ctx.d.ts +51 -2
  8. package/dist/http/routes/approvals-assistant.d.ts +11 -0
  9. package/dist/http/routes/approvals-assistant.js +530 -0
  10. package/dist/http/routes/attachments.js +7 -7
  11. package/dist/http/routes/fleet.d.ts +4 -0
  12. package/dist/http/routes/fleet.js +147 -0
  13. package/dist/http/routes/images.js +29 -29
  14. package/dist/http/routes/leader.d.ts +4 -0
  15. package/dist/http/routes/leader.js +48 -0
  16. package/dist/http/routes/memory-policy.js +10 -10
  17. package/dist/http/routes/notify-wake.d.ts +4 -0
  18. package/dist/http/routes/notify-wake.js +133 -0
  19. package/dist/http/routes/observability.js +5 -5
  20. package/dist/http/routes/runs.d.ts +19 -0
  21. package/dist/http/routes/runs.js +967 -0
  22. package/dist/http/routes/session-sync.js +28 -28
  23. package/dist/http/routes/sessions-list.js +8 -8
  24. package/dist/http/routes/sessions.js +47 -47
  25. package/dist/http/routes/side-query.d.ts +4 -0
  26. package/dist/http/routes/side-query.js +88 -0
  27. package/dist/http/routes/tasks.d.ts +4 -0
  28. package/dist/http/routes/tasks.js +632 -0
  29. package/dist/http/routes/trace-usage.d.ts +4 -0
  30. package/dist/http/routes/trace-usage.js +239 -0
  31. package/dist/http/routes/workflows.d.ts +5 -0
  32. package/dist/http/routes/workflows.js +337 -0
  33. package/dist/http/run-meta.d.ts +11 -0
  34. package/dist/http/run-meta.js +16 -0
  35. package/dist/http/send.d.ts +1 -0
  36. package/dist/http/send.js +15 -0
  37. package/dist/http/server.d.ts +6 -5
  38. package/dist/http/server.js +241 -3166
  39. package/dist/http/sse-log.js +2 -2
  40. package/dist/http/wire-types.d.ts +6 -0
  41. package/dist/leader/endpoint.js +4 -4
  42. package/dist/main.js +5 -5
  43. package/dist/plugins/remote-env-host.js +4 -3
  44. package/dist/question.js +2 -2
  45. package/dist/run-local.js +1 -1
  46. package/dist/tool-approval.js +2 -2
  47. package/dist/trace/ledger-sink.js +1 -1
  48. package/dist/trace/project.d.ts +1 -0
  49. package/dist/trace/project.js +3 -0
  50. package/package.json +1 -1
@@ -1,41 +1,30 @@
1
1
  import http from "node:http";
2
2
  import { once } from "node:events";
3
- import { timingSafeEqual, createHash, randomBytes } from "node:crypto";
4
- import { sessionLogDigest, sessionLogDigestsComparable, uuidv7, sanitizePathComponent, callKeyOrdinal, isThinkingLevel, DEFAULT_EFFORT_LEVELS, expandTiers, materializeMcpTools, runWithVerification, resumeWithVerification, runCascade, CheckpointError, mintCheckpointToken, SessionPolicyError, SessionError, HAND_TOOL_EFFECTS, canonicalToolName, formatUserScope, defaultTaskRegistry, validatePendingSteer, StreamingImportValidator, getWorkflowRun, subscribeWorkflow, deriveAgentDisplayStatus } from "@sema-agent/core";
3
+ import { createHash } from "node:crypto";
4
+ import { uuidv7, isThinkingLevel, expandTiers, resumeWithVerification, CheckpointError, HAND_TOOL_EFFECTS, canonicalToolName, defaultTaskRegistry, validatePendingSteer, subscribeWorkflow } from "@sema-agent/core";
5
5
  import { decideParkedAgent, findParkedAgentForCheckpoint } from "../parked-decide.js";
6
- import { tarHeader, tarPadding, splitTarPath, TAR_END } from "./tar.js";
7
6
  import { matchCatalogModel } from "../model-select.js";
8
- import { mcpForScenario } from "../sema-registry.js";
9
- import { HttpError, principalFrom, verifiedPrincipal, setSsoPrincipal, ssoVerifiedPrincipal, setSsoScope, isUuidV7, isUuidShape, verifyDirectDoorProof, MAX_APPROVAL_REASON_CHARS } from "../security.js";
10
- import { exportSession, exportSessionManifest, importSession, classifySyncRelationshipByIds, SyncConflictError, stagingIdFor } from "../session-sync.js";
11
- import { windowMessages, truncateMessageBlobs } from "../audit.js";
12
- import { parseMemorySyncRequest } from "../memory-sync.js";
13
- import { StringDecoder } from "node:string_decoder";
7
+ import {} from "../sema-registry.js";
8
+ import { HttpError, principalFrom, verifiedPrincipal, setSsoPrincipal, ssoVerifiedPrincipal, setSsoScope, isUuidV7, verifyDirectDoorProof } from "../security.js";
9
+ import { exportSession, importSession } from "../session-sync.js";
10
+ import {} from "../memory-sync.js";
14
11
  import { MAX_SETTINGS_OUTPUT_STYLE_CHARS, MAX_SETTINGS_PERMISSION_RULES, MAX_SETTINGS_ENV_VARS, MAX_SETTINGS_ENV_KEY_CHARS, MAX_SETTINGS_ENV_VALUE_CHARS } from "../task-settings.js";
15
12
  import { parseHooksConfig } from "../hooks/hook-runner.js";
16
13
  import { validateTaskAgents } from "../spec-fields.js";
17
- import { isValidCwd, cwdHonored, MAX_ADDITIONAL_DIRS } from "../task-cwd.js";
18
- import { mcpInjectionHonored } from "../task-mcp.js";
19
- import { validateBake, buildBakeArgv } from "../images/bake-validate.js";
20
- import { manifestToIndexEntry } from "../images/manifest.js";
21
- import { runInBackground, evictIfConflict, stripCheckpointToken, resumeAtHttpStatus, TurnAnchorCapture, HEARTBEAT_MS, markChildrenStoppedByUserOnAbort } from "../runs.js";
14
+ import { isValidCwd, MAX_ADDITIONAL_DIRS } from "../task-cwd.js";
15
+ import { runInBackground, evictIfConflict, stripCheckpointToken, TurnAnchorCapture, HEARTBEAT_MS, markChildrenStoppedByUserOnAbort } from "../runs.js";
22
16
  import { looksLikeJwt } from "../auth-bridge.js";
23
- import { redactSteerIn, STEER_IN_MAX_CHARS, STEER_IN_MAX_REQUEST_CHARS } from "../orchestration/workflow-agent-steer.js";
24
- import { emitPendingWorkflowCompletions, taskNotificationInboxEntry, taskNotificationFoldKey, taskNotificationStreamKey, NotifiedKeys } from "../orchestration/workflow-completion-inbox.js";
17
+ import {} from "../orchestration/workflow-agent-steer.js";
18
+ import { emitPendingWorkflowCompletions, taskNotificationInboxEntry, taskNotificationStreamKey, NotifiedKeys } from "../orchestration/workflow-completion-inbox.js";
25
19
  import { fleetRunPublisher, fleetRunLabels, fleetRunResiduals } from "../fleet/fleet-bus.js";
26
20
  import { defaultSubagentTailBus, projectTailFrame } from "../fleet/subagent-tail-bus.js";
27
- import { RateLimiter } from "../observability/rate-limit.js";
21
+ import {} from "../observability/rate-limit.js";
28
22
  import { withPrincipal } from "../observability/principal-context.js";
29
- import { projectEvents, runSummary, mapTraceEvent, turnEndEventData, contextUsageEventData, toolStartEventData, toolEndEventData, taskProgressEventData, taskNotificationEventData, compactedEventData, diagnosticsEventData, brainStatusEventData, steeringInjectedEventData, workspaceChangedEventData, appendModelUsageDelta, appendPromptManifest, attachModelUsage } from "../trace/project.js";
30
- import { createLedgerSink } from "../trace/ledger-sink.js";
31
- import { cacheFamilyOfMirror } from "../budget.js";
32
- import { projectArtifacts } from "../trace/artifacts.js";
33
- import { composeSupervisorCost, infraCost, infraUsageFromEvents, hasInfraPricing } from "../observability/cost-taxonomy.js";
34
- import { redactSecrets, redactDeep, redactedPreview } from "../trace/redact.js";
35
- import { usageSummary, usageSeries, usageBreakdown } from "../usage-analytics.js";
23
+ import { turnEndEventData, contextUsageEventData, toolStartEventData, toolEndEventData, taskProgressEventData, taskNotificationEventData, compactedEventData, diagnosticsEventData, brainStatusEventData, steeringInjectedEventData, workspaceChangedEventData, appendModelUsageDelta, appendPromptManifest, attachModelUsage } from "../trace/project.js";
24
+ import { redactSecrets } from "../trace/redact.js";
36
25
  import { IdempotencyCache, scopedIdempotencyKey } from "./idempotency.js";
37
26
  export { scopedIdempotencyKey };
38
- import { streamSseLog, sleep } from "./sse-log.js";
27
+ import { streamSseLog } from "./sse-log.js";
39
28
  import { handleCapabilities } from "./routes/capabilities.js";
40
29
  import { handleObservability } from "./routes/observability.js";
41
30
  import { handleMemoryPolicy } from "./routes/memory-policy.js";
@@ -43,12 +32,23 @@ import { handleSessionsList } from "./routes/sessions-list.js";
43
32
  import { handleSessions, createSessionsLocal } from "./routes/sessions.js";
44
33
  import { handleSessionSync, createSessionSyncLocal } from "./routes/session-sync.js";
45
34
  import { handleAttachments } from "./routes/attachments.js";
46
- import { handleImages, createImagesLocal, coarseStatusForState, errorCodeForExit, IMAGE_DIGEST_RE, IMAGE_PROFILE_RE, BAKE_ID_RE, BAKE_EVENTS_RE, BAKE_CANCEL_RE, BAKE_CLAIM_RE, BAKE_INGEST_RE } from "./routes/images.js";
35
+ import { handleFleet } from "./routes/fleet.js";
36
+ import { handleTraceUsage } from "./routes/trace-usage.js";
37
+ import { handleWorkflows, handleWorkflowAgentSteer } from "./routes/workflows.js";
38
+ import { handleNotifyWake } from "./routes/notify-wake.js";
39
+ import { handleApprovalsAssistant, streamApprovals, isQuestionAnswer, ASSISTANT_PREEMPT_RE, ASSISTANT_RESUME_RE, ASSISTANT_PLAN_REVIEW_RE } from "./routes/approvals-assistant.js";
40
+ export { streamApprovals, isQuestionAnswer };
41
+ import { handleRuns, handleRunVerbs, RUN_CANCEL_RE, RUN_STEER_RE, RUN_DETACH_RE, RUN_SUBAGENT_RESUME_RE } from "./routes/runs.js";
42
+ import { handleSideQuery } from "./routes/side-query.js";
43
+ import { handleTasks } from "./routes/tasks.js";
44
+ import { handleLeader } from "./routes/leader.js";
45
+ import { cascadeConfig } from "./run-meta.js";
46
+ export { cascadeConfig };
47
+ import { handleImages, createImagesLocal, coarseStatusForState, errorCodeForExit, BAKE_ID_RE, BAKE_EVENTS_RE, BAKE_CANCEL_RE, BAKE_CLAIM_RE, BAKE_INGEST_RE } from "./routes/images.js";
47
48
  export { coarseStatusForState, errorCodeForExit };
48
- import { sendJson, sendError, msg, sseHeaders } from "./send.js";
49
- import { headerStr, safeEqual, authorized, systemFor, bearerPresentedButUnverified, gatedPrincipal, explicitOperatorOk, isOperator, explicitOperator } from "./principal-gate.js";
49
+ import { sendJson, sendError, httpErrorCode, msg } from "./send.js";
50
+ import { authorized, systemFor, gatedPrincipal, explicitOperatorOk, isOperator, explicitOperator } from "./principal-gate.js";
50
51
  export { explicitOperatorOk, isOperator, explicitOperator };
51
- import { pgHasUnstorable } from "../plugins/pg-safe-json.js";
52
52
  export function flattenServiceDeps(deps) {
53
53
  const { stores, coordinators, seams, observability, governance, deployment, knobs, ...flat } = deps;
54
54
  if (!(stores || coordinators || seams || observability || governance || deployment || knobs))
@@ -59,14 +59,6 @@ export const MAX_USER_SKILLS = 10;
59
59
  export const MAX_SKILL_CONTENT_CHARS = 1_048_576;
60
60
  export const MAX_SYSTEM_PROMPT_CHARS = 16_384;
61
61
  export const MAX_OUTPUT_SCHEMA_CHARS = 32_768;
62
- function runMeta(prepared, source) {
63
- const obj = prepared.spec.objective ?? "";
64
- return {
65
- jobId: prepared.jobId ?? null,
66
- source,
67
- objectivePreview: obj ? redactSecrets(obj).slice(0, 120) : null,
68
- };
69
- }
70
62
  export function validateUserSkills(skills) {
71
63
  if (skills === undefined)
72
64
  return null;
@@ -86,77 +78,37 @@ export function validateUserSkills(skills) {
86
78
  }
87
79
  return null;
88
80
  }
89
- export function cascadeConfig(ladder, maxCostUsd) {
90
- return {
91
- ladder: ladder.map((model) => ({ model })),
92
- ...(maxCostUsd && maxCostUsd > 0 ? { costCeilingMicroUsd: Math.round(maxCostUsd * 1e6) } : {}),
93
- };
94
- }
95
81
  export function clampVerifyRounds(v) {
96
82
  const n = typeof v === "number" && Number.isFinite(v) ? Math.floor(v) : 2;
97
83
  return Math.min(5, Math.max(1, n));
98
84
  }
99
- export function isQuestionAnswer(v) {
100
- if (!v || typeof v !== "object" || Array.isArray(v))
101
- return false;
102
- const a = v.answers;
103
- if (!Array.isArray(a) || a.length === 0)
104
- return false;
105
- return a.every((it) => {
106
- if (!it || typeof it !== "object" || Array.isArray(it))
107
- return false;
108
- const { header, selected, note } = it;
109
- return (typeof header === "string" &&
110
- Array.isArray(selected) &&
111
- selected.length > 0 &&
112
- selected.every((s) => typeof s === "string") &&
113
- (note === undefined || typeof note === "string"));
114
- });
115
- }
116
85
  const MAX_BODY = 8 * 1024 * 1024;
117
86
  const MAX_IMAGES_PER_REQUEST = 20;
118
87
  const MAX_IMAGE_BASE64_BYTES = 6 * 1024 * 1024;
119
- const RUN_ID_RE = /^\/v1\/runs\/([^/]+)(\/events)?$/;
120
- const RUN_CANCEL_RE = /^\/v1\/runs\/([^/]+)\/cancel$/;
121
- const RUN_STEER_RE = /^\/v1\/runs\/([^/]+)\/steer$/;
122
- const RUN_DETACH_RE = /^\/v1\/runs\/([^/]+)\/detach$/;
123
- const RUN_COMPACT_RE = /^\/v1\/runs\/([^/]+)\/compact$/;
124
- function runNotFoundBody(id) {
125
- const aStar = /^(?:a|wa)[0-9a-f]{4,}$/i.test(id);
126
- return {
127
- error: aStar
128
- ? `run not found — "${id}" looks like an agent handle (a*/wa* domain), which is never in the run store; use the TaskOutput tool (or the workflow journal / subagents surface) to read it, and the run's durable taskId (X-Task-Id) for this endpoint`
129
- : "run not found — the id belongs to no run in this deployment's run store (a run from another server process, or an in-memory store that did not survive a restart, is not visible here)",
130
- };
131
- }
132
- const ELICIT_RESPOND_RE = /^\/v1\/elicitations\/([^/]+)\/respond$/;
133
- const QUESTION_RESPOND_RE = /^\/v1\/questions\/([^/]+)\/respond$/;
134
- const TOOL_APPROVAL_RESPOND_RE = /^\/v1\/tool-approvals\/([^/]+)\/respond$/;
135
- const ASSISTANT_PREEMPT_RE = /^\/v1\/assistant\/tasks\/([^/]+)\/preempt$/;
136
- const ASSISTANT_RESUME_RE = /^\/v1\/assistant\/tasks\/([^/]+)\/resume$/;
137
- const ASSISTANT_PLAN_REVIEW_RE = /^\/v1\/assistant\/tasks\/([^/]+)\/plan_review$/;
138
- const TRACE_RE = /^\/v1\/tasks(?:\/([^/]+)(\/turns|\/stream|\/artifacts))?$/;
139
- const WORKFLOWS_RE = /^\/v1\/workflows(?:\/([^/]+)(\/stream|\/journal)?)?$/;
140
- const WORKFLOW_AGENT_STEER_RE = /^\/v1\/workflows\/([^/]+)\/agents\/([^/]+)\/steer$/;
141
- const RUN_SUBAGENT_STEER_RE = /^\/v1\/runs\/([^/]+)\/subagents\/([^/]+)\/steer$/;
142
- const RUN_SUBAGENT_RESUME_RE = /^\/v1\/runs\/([^/]+)\/subagents\/([^/]+)\/resume$/;
143
- const RUN_SUBAGENT_OUTPUT_RE = /^\/v1\/runs\/([^/]+)\/subagents\/([^/]+)\/output$/;
144
- const RUN_SUBAGENT_STREAM_RE = /^\/v1\/runs\/([^/]+)\/subagents\/([^/]+)\/stream$/;
145
- const RUN_TASK_OUTPUT_RE = /^\/v1\/runs\/([^/]+)\/tasks\/([^/]+)\/output$/;
146
- const RUN_TASK_STOP_RE = /^\/v1\/runs\/([^/]+)\/tasks\/([^/]+)\/stop$/;
147
88
  const HAND_TOOL_NAMES = Object.keys(HAND_TOOL_EFFECTS);
148
89
  function isCheckpointReopenedFailure(r) {
149
90
  return r.status === "failed" && (r.errorCode === "resume.env_failed" || r.errorCode === "resume.tool_unavailable");
150
91
  }
151
92
  const ROUTE_DOMAINS = [
93
+ handleTraceUsage,
94
+ handleWorkflows,
95
+ handleFleet,
152
96
  handleCapabilities,
97
+ handleSideQuery,
98
+ handleTasks,
99
+ handleRuns,
100
+ handleWorkflowAgentSteer,
101
+ handleRunVerbs,
102
+ handleLeader,
153
103
  handleImages,
104
+ handleApprovalsAssistant,
154
105
  handleObservability,
155
106
  handleMemoryPolicy,
156
107
  handleSessionsList,
157
108
  handleSessions,
158
109
  handleSessionSync,
159
110
  handleAttachments,
111
+ handleNotifyWake,
160
112
  ];
161
113
  void ROUTE_DOMAINS;
162
114
  export function createHttpServer(rawDeps) {
@@ -219,14 +171,14 @@ export function createHttpServer(rawDeps) {
219
171
  void handle(req, res).catch((err) => {
220
172
  if (err instanceof HttpError) {
221
173
  if (!res.headersSent)
222
- sendJson(res, err.status, { error: err.message, ...(err.code ? { code: err.code, errorCode: err.code } : {}), ...(err.extra ?? {}) });
174
+ sendError(res, err.status, httpErrorCode(err.status, err.code), err.message, { ...(err.code ? { code: err.code } : {}), ...(err.extra ?? {}) });
223
175
  else
224
176
  res.end();
225
177
  return;
226
178
  }
227
179
  deps.logger?.error("request_error", { method: req.method, url: req.url, err: msg(err) });
228
180
  if (!res.headersSent)
229
- sendJson(res, 500, { error: "internal error" });
181
+ sendError(res, 500, "internal.error", "internal error");
230
182
  else
231
183
  res.end();
232
184
  });
@@ -236,6 +188,7 @@ export function createHttpServer(rawDeps) {
236
188
  registry: { idemCache, inflightRuns, preemptableRuns, cancelledViaVerb, steerableRuns, wakeParkMints, counters },
237
189
  helpers: { readJson, readRawBody, rateLimited, quotaExceeded, leaseDenied, safeDecode, isFleetWide, runOwnerOk, sessionOwnerScope, sessionOwnerScopeForWrite, runSessionAcceptOk },
238
190
  local: { images: createImagesLocal(deps), sessions: createSessionsLocal(), sessionSync: createSessionSyncLocal() },
191
+ legs: { prepareSpec, finalizeTaskResult, resumeCheckpoint, driveResumeIntoRunLog, resumePreempted, resumeWake, resumePlanReview },
239
192
  };
240
193
  async function handle(req, res) {
241
194
  const startedAt = Date.now();
@@ -247,7 +200,7 @@ export function createHttpServer(rawDeps) {
247
200
  counters.admittedInflight++;
248
201
  let logged = false;
249
202
  const reqState = { streamTaskId: undefined, streamDetached: false, source: null };
250
- const ctx = { deps: routeCtxBase.deps, registry: routeCtxBase.registry, helpers: routeCtxBase.helpers, local: routeCtxBase.local, req: reqState };
203
+ const ctx = { deps: routeCtxBase.deps, registry: routeCtxBase.registry, helpers: routeCtxBase.helpers, local: routeCtxBase.local, legs: routeCtxBase.legs, req: reqState };
251
204
  const record = (viaClose) => {
252
205
  if (logged)
253
206
  return;
@@ -293,2438 +246,117 @@ export function createHttpServer(rawDeps) {
293
246
  port: deps.config.port,
294
247
  ...(deps.config.directDoorActive ? { directDoorActive: true } : {}),
295
248
  ...(restart ? { restartRequired: true, restart: { reasons: restart.reasons, version: restart.version, since: restart.since } } : {}),
296
- ...(() => {
297
- const stuck = deps.planeDeferredState?.();
298
- return stuck ? { modelPlaneDeferred: { version: stuck.version, since: stuck.since, noHandoff: true, ...(stuck.blocked ? { blockedReasons: stuck.blocked } : {}) } } : {};
299
- })(),
300
- });
301
- return;
302
- }
303
- if (req.method === "GET" && (url === "/metrics" || url === "/metrics/summary" || url === "/metrics/plan-cache")) {
304
- if (!deps.metrics) {
305
- sendJson(res, 404, { error: "metrics disabled" });
306
- return;
307
- }
308
- const { authToken: t, metricsToken: mt } = deps.config;
309
- const anyT = Boolean(t) || Object.keys(deps.config.authTokens ?? {}).length > 0;
310
- const ok = (!anyT && !mt) || (anyT && systemFor(req, deps.config) !== undefined) || (!!mt && authorized(req, mt));
311
- if (!ok) {
312
- sendJson(res, 401, { error: "unauthorized" });
313
- return;
314
- }
315
- if (url === "/metrics/summary") {
316
- sendJson(res, 200, { model: deps.config.model.id, ...deps.metrics.summarize() });
317
- return;
318
- }
319
- if (url === "/metrics/plan-cache") {
320
- sendJson(res, 200, { scopes: deps.planCacheProbe?.dump() ?? {} });
321
- return;
322
- }
323
- res.writeHead(200, { "content-type": "text/plain; version=0.0.4" });
324
- res.end(deps.metrics.render());
325
- return;
326
- }
327
- if (deps.registryJwtVerifier) {
328
- const authz = req.headers.authorization ?? "";
329
- const rawBearer = /^Bearer\s/i.test(authz) ? authz.replace(/^Bearer\s+/i, "") : "";
330
- if (rawBearer && looksLikeJwt(rawBearer)) {
331
- const v = await deps.registryJwtVerifier.verify(rawBearer);
332
- if (v.ok) {
333
- setSsoPrincipal(req, v.identity.principal);
334
- req.headers[deps.config.principalHeader.toLowerCase()] = v.identity.principal;
335
- if (v.identity.scope)
336
- setSsoScope(req, v.identity.scope);
337
- deps.logger?.info?.("auth_bridge_principal_accepted", { principal: v.identity.principal, ...(v.identity.scope ? { scope: v.identity.scope } : {}) });
338
- }
339
- }
340
- }
341
- if (req.method === "GET" && url === "/v1/tasks/source-summary") {
342
- if (!isFleetWide(req)) {
343
- sendJson(res, 401, { error: "unauthorized" });
344
- return;
345
- }
346
- if (!deps.runStore) {
347
- sendJson(res, 501, { error: "trace API requires the TiDB run store (SESSION_BACKEND=tidb)" });
348
- return;
349
- }
350
- const q = new URL(req.url ?? "", "http://x").searchParams;
351
- const sinceSec = Math.min(30 * 86_400, Math.max(60, Number(q.get("sinceSec") ?? 86_400) || 86_400));
352
- sendJson(res, 200, { sinceSec, rows: await deps.runStore.sourceSummary(Date.now() - sinceSec * 1000) });
353
- return;
354
- }
355
- const traceMatch = req.method === "GET" ? TRACE_RE.exec(url) : null;
356
- if (traceMatch) {
357
- const fleetWide = isFleetWide(req);
358
- const gateOwner = fleetWide ? null : (gatedPrincipal(req, deps.config) || null);
359
- if (!fleetWide && gateOwner === null) {
360
- sendJson(res, 401, { error: "unauthorized" });
361
- return;
362
- }
363
- if (!deps.runStore) {
364
- sendJson(res, 501, { error: "trace API requires the TiDB run store (SESSION_BACKEND=tidb)" });
365
- return;
366
- }
367
- const query = new URL(req.url ?? "", "http://x").searchParams;
368
- const taskId = traceMatch[1];
369
- const sub = traceMatch[2];
370
- if (taskId && gateOwner !== null) {
371
- const run = await deps.runStore.getRun(taskId);
372
- if (!run || (run.owner !== null && run.owner !== gateOwner)) {
373
- sendJson(res, 404, { error: "task not found" });
374
- return;
375
- }
376
- }
377
- if (!taskId)
378
- await handleTaskList(res, deps.runStore, query, gateOwner);
379
- else if (sub === "/turns")
380
- await handleTaskTurns(res, deps.runStore, taskId, query);
381
- else if (sub === "/stream")
382
- await streamTaskTrace(req, res, deps.runStore, taskId, deps.config.runStaleSec * 1000);
383
- else if (sub === "/artifacts")
384
- await handleTaskArtifacts(res, deps.runStore, taskId);
385
- else
386
- sendJson(res, 404, { error: "not found" });
387
- return;
388
- }
389
- if (url.startsWith("/v1/usage/") && req.method === "GET") {
390
- const sub = url.slice("/v1/usage/".length).split("?")[0];
391
- if (sub === "summary" || sub === "series" || sub === "breakdown") {
392
- if (!deps.runStore) {
393
- sendJson(res, 501, { error: "usage analytics requires a run store" });
394
- return;
395
- }
396
- const fleetWide = isFleetWide(req);
397
- const jwtOwner = fleetWide ? undefined : gatedPrincipal(req, deps.config);
398
- if (!fleetWide && !jwtOwner) {
399
- sendJson(res, 401, { error: "unauthorized" });
400
- return;
401
- }
402
- const q = new URL(req.url ?? "", "http://x").searchParams;
403
- const parseT = (v, dflt) => {
404
- if (!v)
405
- return dflt;
406
- const n = /^\d+$/.test(v) ? Number(v) : Date.parse(v);
407
- return Number.isFinite(n) ? n : NaN;
408
- };
409
- const now = Date.now();
410
- const from = parseT(q.get("from"), now - 7 * 86_400_000);
411
- const to = parseT(q.get("to"), now);
412
- if (Number.isNaN(from) || Number.isNaN(to) || from >= to) {
413
- sendJson(res, 400, { error: "invalid from/to (ISO-8601 or epoch-ms; from < to)" });
414
- return;
415
- }
416
- const owner = jwtOwner ?? (q.get("principal") ?? undefined);
417
- try {
418
- const scan = await deps.runStore.usageScan(from, to, owner !== undefined ? { owner } : undefined);
419
- const base = { from: new Date(from).toISOString(), to: new Date(to).toISOString(), ...(scan.truncated ? { truncated: true } : {}) };
420
- if (sub === "summary") {
421
- sendJson(res, 200, { ...base, ...usageSummary(scan.rows) });
422
- }
423
- else if (sub === "series") {
424
- const metric = (q.get("metric") ?? "costUsd");
425
- const granularity = (q.get("granularity") ?? "day");
426
- if (!["tasks", "tokensIn", "tokensOut", "costUsd"].includes(metric) || !["hour", "day"].includes(granularity)) {
427
- sendJson(res, 400, { error: "metric=tasks|tokensIn|tokensOut|costUsd, granularity=hour|day" });
428
- return;
429
- }
430
- sendJson(res, 200, { ...base, metric, granularity, series: usageSeries(scan.rows, metric, granularity) });
431
- }
432
- else {
433
- const dimension = (q.get("dimension") ?? "principal");
434
- if (!["principal", "model"].includes(dimension)) {
435
- sendJson(res, 400, { error: "dimension=principal|model" });
436
- return;
437
- }
438
- sendJson(res, 200, { ...base, dimension, breakdown: usageBreakdown(scan.rows, dimension) });
439
- }
440
- }
441
- catch (err) {
442
- deps.logger?.warn?.("usage_analytics_failed", { err: String(err) });
443
- sendJson(res, 500, { error: "usage analytics failed" });
444
- }
445
- return;
446
- }
447
- }
448
- const wfMatch = req.method === "GET" ? WORKFLOWS_RE.exec(url) : null;
449
- if (wfMatch) {
450
- const principal = gatedPrincipal(req, deps.config);
451
- if (deps.config.requirePrincipal && !principal) {
452
- sendJson(res, 401, { error: `missing principal header '${deps.config.principalHeader}'` });
453
- return;
454
- }
455
- if (!deps.workflowRunStore) {
456
- sendJson(res, 501, { error: "workflow runs require self-orchestration (SELF_ORCHESTRATION_ENABLED)" });
457
- return;
458
- }
459
- const scope = principal ?? "default";
460
- const wfId = wfMatch[1];
461
- const isStream = wfMatch[2] === "/stream";
462
- if (!wfId) {
463
- const q = new URL(req.url ?? "", "http://x").searchParams;
464
- const statusFilter = q.get("status");
465
- const limit = Math.min(100, Math.max(1, Number(q.get("limit") ?? 50) || 50));
466
- const sessionFilter = q.get("session");
467
- const sessionRuns = await deps.workflowRunStore.listByScope(scope, {
468
- ...(statusFilter ? { status: statusFilter } : {}),
469
- limit,
470
- ...(sessionFilter !== null ? { session: sessionFilter } : {}),
471
- });
472
- const redacted = sessionRuns.map((r) => ({
473
- ...r,
474
- ...(r.name !== undefined ? { name: redactSecrets(r.name) } : {}),
475
- ...(r.description !== undefined ? { description: redactSecrets(r.description) } : {}),
476
- ...(r.currentPhase !== undefined ? { currentPhase: redactSecrets(r.currentPhase) } : {}),
477
- }));
478
- sendJson(res, 200, { workflows: redacted });
479
- return;
480
- }
481
- const run = await getWorkflowRun(deps.workflowRunStore, wfId, scope);
482
- if (!run || run.scope !== scope) {
483
- sendJson(res, 404, { error: "workflow not found" });
484
- return;
485
- }
486
- if (!runSessionAcceptOk(req, res, { sessionId: run.originatingSessionId ?? null }, wfMatch[2] === "/journal" ? "workflow.journal" : isStream ? "workflow.stream" : "workflow.detail", "workflow not found"))
487
- return;
488
- if (wfMatch[2] === "/journal") {
489
- if (!deps.workflowJournalStore) {
490
- sendJson(res, 501, { error: "workflow journal requires the journal store (SELF_ORCHESTRATION_ENABLED + a store backend)" });
491
- return;
492
- }
493
- const jq = new URL(req.url ?? "", "http://x").searchParams;
494
- const jLimit = Math.min(50, Math.max(1, Number(jq.get("limit") ?? 20) || 20));
495
- const jOffset = Math.max(0, Number(jq.get("offset") ?? 0) || 0);
496
- const MAX_ROW_BYTES = 64 * 1024;
497
- const firstLine = (t) => (t ? redactSecrets(t.split("\n")[0].slice(0, 300)) : undefined);
498
- const projectResult = (callKey, r) => ({
499
- callKey,
500
- ordinal: callKeyOrdinal(callKey),
501
- status: r.status,
502
- ...(r.errorMessage ? { error: firstLine(r.errorMessage) } : {}),
503
- ...(r.result ? { result: redactSecrets(r.result.slice(0, 2000)) } : {}),
504
- ...(r.stats ? { tokens: r.stats.tokens, turns: r.stats.turns } : {}),
505
- });
506
- const js = deps.workflowJournalStore;
507
- let projected;
508
- if (js.loadPage) {
509
- const page = await js.loadPage(wfId, scope, { offset: jOffset, limit: jLimit, maxResultBytes: MAX_ROW_BYTES });
510
- projected = page.map((row) => {
511
- if (row.resultJson === null) {
512
- return { callKey: row.callKey, ordinal: callKeyOrdinal(row.callKey), truncated: true, resultBytes: row.resultBytes };
513
- }
514
- try {
515
- return projectResult(row.callKey, JSON.parse(row.resultJson));
516
- }
517
- catch {
518
- return { callKey: row.callKey, ordinal: callKeyOrdinal(row.callKey), truncated: true, resultBytes: row.resultBytes };
519
- }
520
- });
521
- }
522
- else {
523
- const entries = await deps.workflowJournalStore.load(wfId, scope);
524
- projected = entries
525
- .sort((a, b) => callKeyOrdinal(a.callKey) - callKeyOrdinal(b.callKey))
526
- .slice(jOffset, jOffset + jLimit)
527
- .map((en) => {
528
- const bytes = Buffer.byteLength(JSON.stringify(en.result));
529
- return bytes > MAX_ROW_BYTES
530
- ? { callKey: en.callKey, ordinal: callKeyOrdinal(en.callKey), truncated: true, resultBytes: bytes }
531
- : projectResult(en.callKey, en.result);
532
- });
533
- }
534
- sendJson(res, 200, { runId: wfId, entries: projected, ...(projected.length === jLimit ? { nextOffset: jOffset + jLimit } : {}) });
535
- return;
536
- }
537
- if (isStream)
538
- await streamWorkflowRun(req, res, wfId, scope);
539
- else
540
- sendJson(res, 200, summarizeWorkflowDetail(run));
541
- return;
542
- }
543
- if (req.method === "GET" && new URL(req.url ?? "", "http://x").pathname === "/v1/fleet/stream") {
544
- if (!deps.fleetBus) {
545
- sendJson(res, 501, { error: "fleet stream requires self-orchestration / fleet wiring" });
546
- return;
547
- }
548
- const { fleetWide } = sessionOwnerScope(req);
549
- const principal = gatedPrincipal(req, deps.config);
550
- if (deps.config.requirePrincipal && !principal && !fleetWide) {
551
- sendJson(res, 401, { error: `missing principal header '${deps.config.principalHeader}'` });
552
- return;
553
- }
554
- const fleetParams = new URL(req.url ?? "", "http://x").searchParams;
555
- const callerSession = fleetParams.get("session");
556
- const observeOnly = fleetParams.get("observe") === "1";
557
- await streamFleet(req, res, deps.fleetBus, fleetWide ? null : (principal ?? "default"), callerSession || null, observeOnly ? undefined : deps.workflowCompletionInbox);
558
- return;
559
- }
560
- let source = null;
561
- const anyServiceAuth = Boolean(deps.config.authToken) || Object.keys(deps.config.authTokens ?? {}).length > 0;
562
- if (anyServiceAuth) {
563
- const sys = systemFor(req, deps.config);
564
- if (sys === undefined) {
565
- if (ssoVerifiedPrincipal(req)) {
566
- source = "sso";
567
- }
568
- else {
569
- sendJson(res, 401, { error: "unauthorized" });
570
- return;
571
- }
572
- }
573
- else {
574
- source = sys;
575
- }
576
- }
577
- reqState.source = source;
578
- if (req.method === "POST" &&
579
- !anyServiceAuth &&
580
- !deps.config.allowUnauthedWrites &&
581
- (isBillableSubmitPath(url) || url.startsWith("/v1/images/bakes"))) {
582
- sendJson(res, 503, {
583
- error: "this worker requires a service auth token (set SERVICE_AUTH_TOKEN) before accepting task submissions",
584
- });
585
- return;
586
- }
587
- if (deps.drainState?.draining && req.method === "POST" && isBillableSubmitPath(url)) {
588
- res.setHeader("retry-after", "15");
589
- sendError(res, 503, "draining", "draining", { message: "this instance is draining for shutdown/upgrade — retry against the replacement instance" });
590
- return;
591
- }
592
- if (deps.modelReady && !deps.modelReady() && req.method === "POST" && isBillableSubmitPath(url)) {
593
- res.setHeader("retry-after", "5");
594
- sendJson(res, 503, { error: "model_roster_pending", message: "this worker has no model yet (waiting for the first effective-config pull to land the roster) — retry shortly" });
595
- return;
596
- }
597
- if (req.method === "PUT" &&
598
- !anyServiceAuth &&
599
- !deps.config.allowUnauthedWrites &&
600
- /^\/v1\/sessions\/[^/]+\/policy$/.test(url)) {
601
- sendJson(res, 503, {
602
- error: "this worker requires a service auth token (set SERVICE_AUTH_TOKEN) before accepting session-policy writes",
603
- });
604
- return;
605
- }
606
- if (await handleCapabilities(req, res, url, ctx))
607
- return;
608
- if (req.method === "POST" && url === "/v1/side-query") {
609
- const ac = new AbortController();
610
- const onClose = () => ac.abort();
611
- res.on("close", onClose);
612
- try {
613
- if (rateLimited(req, res) || quotaExceeded(req, res) || (await leaseDenied(req, res)))
614
- return;
615
- const principal = gatedPrincipal(req, deps.config);
616
- if (deps.config.requirePrincipal && !principal) {
617
- sendJson(res, 401, { error: `missing principal header '${deps.config.principalHeader}'` });
618
- return;
619
- }
620
- let body;
621
- try {
622
- const raw = await readJson(req);
623
- if (raw === null || typeof raw !== "object" || Array.isArray(raw))
624
- throw new HttpError(400, "body must be a JSON object (SideQuerySpec shape)");
625
- body = raw;
626
- }
627
- catch (e) {
628
- const he = e instanceof HttpError ? e : new HttpError(400, msg(e));
629
- sendJson(res, he.status, { error: he.message });
630
- return;
631
- }
632
- if (!Array.isArray(body.messages) || body.messages.length === 0) {
633
- sendJson(res, 400, { error: "messages must be a non-empty array (SideQueryMessage[])" });
634
- return;
635
- }
636
- if (body.thinking !== undefined && !isThinkingLevel(body.thinking)) {
637
- sendJson(res, 400, { error: "invalid thinking level" });
638
- return;
639
- }
640
- if (ac.signal.aborted)
641
- return;
642
- const spec = {
643
- messages: body.messages,
644
- ...(typeof body.model === "string" ? { model: body.model } : {}),
645
- ...(typeof body.modelRole === "string" ? { modelRole: body.modelRole } : {}),
646
- ...(isThinkingLevel(body.thinking) ? { thinking: body.thinking } : {}),
647
- ...(typeof body.systemPrompt === "string" ? { systemPrompt: body.systemPrompt } : {}),
648
- ...(Array.isArray(body.tools) ? { tools: body.tools } : {}),
649
- ...(typeof body.maxOutputTokens === "number" && Number.isFinite(body.maxOutputTokens) && body.maxOutputTokens > 0 ? { maxOutputTokens: Math.floor(body.maxOutputTokens) } : {}),
650
- signal: ac.signal,
651
- };
652
- try {
653
- const catalogSnapshot = Object.values(deps.runner.agentCatalog?.models ?? {}).map((m) => ({ id: m?.id, api: m?.api, params: m?.params?.promptCacheFamily ? { promptCacheFamily: m.params.promptCacheFamily } : undefined }));
654
- const result = await deps.runner.sideQuery(spec);
655
- const rr = result;
656
- const modelId = typeof rr.model === "string" ? rr.model : "unknown";
657
- const byId = catalogSnapshot.filter((m) => m.id === modelId);
658
- const families = new Set(byId.map((m) => cacheFamilyOfMirror(m)));
659
- const family = families.size === 1 ? [...families][0] : undefined;
660
- deps.sideQueryAccounting?.(principal, { model: modelId, ...(family ? { family } : {}), usage: rr.usage });
661
- deps.metrics?.inc?.("side_query_total", { result: "ok" });
662
- if (!res.writableEnded)
663
- sendJson(res, 200, result);
664
- }
665
- catch (e) {
666
- deps.metrics?.inc?.("side_query_total", { result: "error" });
667
- if (!res.writableEnded && !ac.signal.aborted) {
668
- const m = msg(e);
669
- const isValidation = /^Unknown model ref |^No model for role |requires a non-empty messages array/.test(m);
670
- sendJson(res, isValidation ? 400 : 500, { error: redactSecrets(m) });
671
- }
672
- }
673
- }
674
- finally {
675
- res.off("close", onClose);
676
- }
677
- return;
678
- }
679
- if (req.method === "POST" && (url === "/v1/tasks" || url === "/v1/tasks/stream")) {
680
- const rawIdem = headerStr(req.headers["idempotency-key"]);
681
- const idemKey = rawIdem ? scopedIdempotencyKey(rawIdem, source, gatedPrincipal(req, deps.config)) : undefined;
682
- const cached = idemKey ? idemCache.peek(idemKey) : undefined;
683
- if (cached) {
684
- if (url === "/v1/tasks/stream") {
685
- sseHeaders(res);
686
- const hb = setInterval(() => {
687
- if (!res.writableEnded)
688
- res.write(`event: heartbeat\ndata: {}\n\n`);
689
- }, 15_000);
690
- try {
691
- const resp = await cached;
692
- if (!res.writableEnded)
693
- res.write(`data: ${JSON.stringify({ type: "done", result: resp.body, replay: true })}\n\n`);
694
- }
695
- finally {
696
- clearInterval(hb);
697
- }
698
- res.end();
699
- }
700
- else {
701
- const resp = await cached;
702
- sendJson(res, resp.status, resp.body);
703
- }
704
- return;
705
- }
706
- if (rateLimited(req, res) || quotaExceeded(req, res) || (await leaseDenied(req, res)))
707
- return;
708
- const prepared = await prepareSpec(req, res);
709
- if (!prepared)
710
- return;
711
- const principal = prepared.auth?.principal;
712
- if (url === "/v1/tasks/stream") {
713
- if (prepared.verify || prepared.cascade) {
714
- sendJson(res, 400, { error: "verify / cascade are not supported on /v1/tasks/stream (both are multi-attempt, not a single stream) — use /v1/tasks or /v1/runs" });
715
- return;
716
- }
717
- const earlyDurableTid = deps.runStore && prepared.spec.sessionId ? uuidv7() : undefined;
718
- const detachOnDisconnect = headerStr(req.headers["x-detach-on-disconnect"]) === "true";
719
- if (detachOnDisconnect && earlyDurableTid === undefined) {
720
- sendJson(res, 400, { error: "x-detach-on-disconnect requires a durable run (a run store + sessionId) — without one the detached result would be unqueryable" });
721
- return;
722
- }
723
- reqState.streamTaskId = earlyDurableTid;
724
- let usageKey;
725
- reqState.streamDetached = detachOnDisconnect;
726
- sseHeaders(res, earlyDurableTid ? { "x-task-id": earlyDurableTid } : undefined);
727
- if (earlyDurableTid) {
728
- res.write(`event: meta\ndata: ${JSON.stringify({ type: "meta", taskId: earlyDurableTid, ...(prepared.spec.sessionId ? { sessionId: prepared.spec.sessionId } : {}) })}\n\n`);
729
- }
730
- const ac = new AbortController();
731
- if (prepared.spec.sessionId)
732
- markChildrenStoppedByUserOnAbort(ac.signal, prepared.spec.sessionId, prepared.auth?.principal);
733
- let closed = false;
734
- let detachLogged = false;
735
- const onDisconnect = () => {
736
- if (detachOnDisconnect) {
737
- if (res.writableEnded)
738
- return;
739
- if (!detachLogged) {
740
- detachLogged = true;
741
- res.on("error", () => undefined);
742
- deps.logger?.info("stream_client_detached", { taskId: earlyDurableTid, sessionId: prepared.spec.sessionId, detached: true });
743
- }
744
- return;
745
- }
746
- closed = true;
747
- ac.abort();
748
- };
749
- req.on("close", onDisconnect);
750
- res.on("close", () => {
751
- if (!res.writableEnded)
752
- onDisconnect();
753
- });
754
- const hb = setInterval(() => {
755
- if (!res.writableEnded && !res.destroyed)
756
- res.write(`event: heartbeat\ndata: {}\n\n`);
757
- }, 15_000);
758
- let ranLive = false;
759
- let resp;
760
- counters.uncountedBillableInflight++;
761
- try {
762
- resp = await idemCache.run(idemKey, async () => {
763
- ranLive = true;
764
- let finalResult;
765
- let runRowSettled = false;
766
- let durableTaskId;
767
- let durableHeartbeat;
768
- let userMsgEntryId;
769
- let anchorPut = false;
770
- const putRewindAnchor = () => {
771
- if (anchorPut || !userMsgEntryId || !durableTaskId || !prepared.spec.sessionId || !deps.resumeAnchorStore)
772
- return;
773
- anchorPut = true;
774
- void deps.resumeAnchorStore
775
- .put(prepared.spec.sessionId, durableTaskId, userMsgEntryId, principal ?? null)
776
- .catch(() => deps.metrics?.inc("resume_anchor_capture_failed"));
777
- };
778
- if (deps.runStore && prepared.spec.sessionId) {
779
- const tid = earlyDurableTid ?? uuidv7();
780
- const created = await deps.runStore.createRun(tid, prepared.spec.sessionId, principal ?? null, deps.instanceId ?? "default", runMeta(prepared, source));
781
- if (created.ok)
782
- deps.sessionTitler?.maybeTitle(prepared.spec.sessionId, prepared.spec.objective);
783
- if (!created.ok) {
784
- const conflict = { error: "session already has an active run — POST /v1/runs/{activeTaskId}/cancel stops it (same-instance interactive runs abort immediately)", activeTaskId: created.activeTaskId };
785
- res.write(`data: ${JSON.stringify({ type: "done", result: { status: "failed", errorMessage: conflict.error, activeTaskId: conflict.activeTaskId } })}\n\n`);
786
- return { status: 409, body: conflict };
787
- }
788
- durableTaskId = tid;
789
- }
790
- let ledgerSink;
791
- let ledgerTerminal = false;
792
- if (detachOnDisconnect && durableTaskId && deps.runStore) {
793
- const rs = deps.runStore;
794
- const tid = durableTaskId;
795
- ledgerSink = createLedgerSink({ appendEvent: (seq, type, data) => rs.appendEvent(tid, seq, type, data), persistThinking: deps.config.traceThinking });
796
- usageKey = prepared.spec.sessionId;
797
- if (usageKey)
798
- deps.modelUsage?.register(usageKey);
799
- }
800
- if (deps.checkpointStore && prepared.spec.sessionId) {
801
- await deps.checkpointStore.putCtx(prepared.spec.sessionId, { body: prepared.body, memoryScope: prepared.auth?.memoryScope });
802
- }
803
- if (durableTaskId && deps.runStore) {
804
- const rs = deps.runStore;
805
- const tid = durableTaskId;
806
- const owner = principal ?? null;
807
- const abortFromVerb = () => {
808
- cancelledViaVerb.add(tid);
809
- ac.abort();
810
- };
811
- durableHeartbeat = setInterval(() => {
812
- void rs.heartbeat(tid, owner).catch(() => undefined);
813
- if (!ac.signal.aborted) {
814
- void Promise.resolve(rs.isCancelRequested?.(tid, owner)).then((c) => { if (c)
815
- abortFromVerb(); }).catch(() => undefined);
816
- }
817
- }, HEARTBEAT_MS);
818
- durableHeartbeat.unref?.();
819
- inflightRuns.set(tid, ac);
820
- void Promise.resolve(rs.isCancelRequested?.(tid, owner)).then((c) => { if (c && !ac.signal.aborted)
821
- abortFromVerb(); }).catch(() => undefined);
822
- }
823
- let fleetPub;
824
- try {
825
- fleetPub = durableTaskId
826
- ? fleetRunPublisher(deps.fleetBus, { runId: durableTaskId, scope: gatedPrincipal(req, deps.config) ?? "default", rootTaskId: prepared.spec.sessionId, ...fleetRunLabels(prepared.spec.objective) })
827
- : undefined;
828
- }
829
- catch (e) {
830
- if (durableHeartbeat)
831
- clearInterval(durableHeartbeat);
832
- if (durableTaskId && inflightRuns.get(durableTaskId) === ac) {
833
- inflightRuns.delete(durableTaskId);
834
- cancelledViaVerb.delete(durableTaskId);
835
- }
836
- throw e;
837
- }
838
- let fleetSettled = false;
839
- const settleFleet = (status, residuals) => {
840
- if (fleetSettled)
841
- return;
842
- fleetSettled = true;
843
- fleetPub?.onTerminal(status, residuals);
844
- };
845
- let liveStreamRef;
846
- const subagentHandleEvictions = [];
847
- let syncLegLive = true;
848
- const syncNotifiedKeys = new NotifiedKeys();
849
- try {
850
- fleetPub?.onStart();
851
- }
852
- catch (e) {
853
- if (durableHeartbeat)
854
- clearInterval(durableHeartbeat);
855
- if (durableTaskId && inflightRuns.get(durableTaskId) === ac) {
856
- inflightRuns.delete(durableTaskId);
857
- cancelledViaVerb.delete(durableTaskId);
858
- }
859
- settleFleet("failed");
860
- throw e;
861
- }
862
- try {
863
- const streamBody = async () => {
864
- await withPrincipal(principal, async () => {
865
- const fwdInternals = {
866
- onForwardEvent: (e) => {
867
- fleetPub?.onForwardEvent(e);
868
- ledgerSink?.onForwardEvent(e);
869
- {
870
- const bg = e.bgAgentId;
871
- if (bg !== undefined && defaultSubagentTailBus.hasSubscribers(bg)) {
872
- const f = projectTailFrame(e);
873
- if (f)
874
- defaultSubagentTailBus.publish(bg, f);
875
- }
876
- }
877
- if (res.writableEnded)
878
- return;
879
- const t = e.type;
880
- if (t === "task_progress") {
881
- res.write(`data: ${JSON.stringify({ type: "task_progress", ...taskProgressEventData(e) })}\n\n`);
882
- }
883
- else if (t === "tool_start") {
884
- const tsv = e;
885
- res.write(`data: ${JSON.stringify({ type: "tool_start", ...toolStartEventData(tsv) })}\n\n`);
886
- }
887
- else if (t === "tool_end") {
888
- const te = e;
889
- res.write(`data: ${JSON.stringify({ type: "tool_end", ...toolEndEventData(te) })}\n\n`);
890
- }
891
- else if (t === "text_delta") {
892
- const td = e;
893
- res.write(`data: ${JSON.stringify({ type: "text_delta", delta: td.delta, ...(td.eventId ? { eventId: td.eventId } : {}), ...(td.parentToolCallId ? { parentToolCallId: td.parentToolCallId } : {}) })}\n\n`);
894
- }
895
- else if (t === "reasoning_delta") {
896
- const rd = e;
897
- res.write(`data: ${JSON.stringify({ type: "reasoning_delta", delta: redactSecrets(rd.delta), ...(rd.eventId ? { eventId: rd.eventId } : {}), ...(rd.parentToolCallId ? { parentToolCallId: rd.parentToolCallId } : {}) })}\n\n`);
898
- }
899
- },
900
- onTaskNotification: (n) => {
901
- if (n.task_type === "workflow")
902
- return;
903
- if (defaultSubagentTailBus.hasSubscribers(n.task_id)) {
904
- defaultSubagentTailBus.publish(n.task_id, { type: "task_settled", taskId: n.task_id, status: n.status, ...(typeof n.seq === "number" ? { seq: n.seq } : {}), ...(n.summary ? { summary: redactSecrets(n.summary) } : {}) });
905
- }
906
- const hadRow = fleetPub?.onChildTerminal(n.sessionId ?? n.task_id, n.status, n.task_id, n.toolUseId) ?? false;
907
- const deliverable = syncLegLive && !res.writableEnded && !res.destroyed;
908
- const parked = !deliverable && Boolean(deps.workflowCompletionInbox && prepared.spec.sessionId);
909
- deps.logger?.info?.("task_notification_observed", { route: "sync-stream", taskId: n.task_id, taskType: n.task_type, status: n.status, hadFleetRow: hadRow, legLive: syncLegLive, parkedDurable: parked });
910
- if (parked) {
911
- void deps.workflowCompletionInbox.enqueue(taskNotificationInboxEntry(prepared.spec.sessionId, askOwner, n, Date.now(), durableTaskId)).catch((err) => deps.logger?.warn?.("park_enqueue_failed", { route: "sync-stream", taskId: n.task_id, err: err instanceof Error ? err.message : String(err) }));
912
- }
913
- else if (deliverable) {
914
- const key = taskNotificationStreamKey(n);
915
- if (syncNotifiedKeys.get(key) === undefined) {
916
- syncNotifiedKeys.set(key, Promise.resolve(true));
917
- res.write(`data: ${JSON.stringify({ type: "task_notification", ...taskNotificationEventData({ notification: n }) })}\n\n`);
918
- }
919
- }
920
- },
921
- ...(deps.subagentSteerRegistry && durableTaskId
922
- ? {
923
- onSubagentSpawn: (handle) => {
924
- subagentHandleEvictions.push(deps.subagentSteerRegistry.register(durableTaskId, handle));
925
- },
926
- }
927
- : {}),
928
- };
929
- const liveStream = deps.runner.runTaskStream({ ...prepared.spec, signal: ac.signal }, undefined, fwdInternals);
930
- liveStreamRef = liveStream;
931
- if (durableTaskId)
932
- steerableRuns.set(durableTaskId, liveStream);
933
- for await (const ev of liveStream) {
934
- if (closed)
935
- break;
936
- if (ev.type === "message_committed" && ev.role === "user")
937
- userMsgEntryId = ev.entryId;
938
- if (ev.type === "done") {
939
- finalResult = stripCheckpointToken(ev.result);
940
- if (durableTaskId && finalResult)
941
- finalResult = { ...finalResult, taskId: durableTaskId };
942
- if (durableTaskId && cancelledViaVerb.has(durableTaskId) && finalResult?.status === "failed" && !finalResult.errorCode) {
943
- finalResult = { ...finalResult, errorCode: "cancelled" };
944
- }
945
- finalizeTaskResult(ev.result, principal, prepared.spec.objective, prepared.spec.sessionId);
946
- putRewindAnchor();
947
- settleFleet(finalResult?.status ?? "completed", fleetRunResiduals(finalResult));
948
- if (ledgerSink && finalResult) {
949
- if (usageKey && deps.modelUsage && deps.runStore && durableTaskId) {
950
- const rs2 = deps.runStore;
951
- const tid2 = durableTaskId;
952
- const fr = finalResult;
953
- finalResult = await attachModelUsage(fr, { append: ledgerSink.append, getEvents: (_id, after) => rs2.getEvents(tid2, after), modelUsage: deps.modelUsage, taskId: usageKey }).catch(() => fr);
954
- }
955
- await ledgerSink.onDone(finalResult);
956
- ledgerTerminal = true;
957
- }
958
- res.write(`data: ${JSON.stringify({ ...ev, result: finalResult })}\n\n`);
959
- continue;
960
- }
961
- if (ledgerSink) {
962
- await ledgerSink.onEvent(ev);
963
- if (ev.type === "turn_end" && usageKey)
964
- await appendModelUsageDelta(ledgerSink.append, deps.modelUsage, usageKey);
965
- }
966
- fleetPub?.onEvent(ev);
967
- if (ev.type === "status")
968
- deps.metrics?.inc("brain_retry_total", { phase: String(ev.phase) });
969
- if (ev.type === "status") {
970
- res.write(`data: ${JSON.stringify({ type: "status", ...brainStatusEventData(ev) })}\n\n`);
971
- }
972
- else if (ev.type === "tool_end") {
973
- const te = ev;
974
- res.write(`data: ${JSON.stringify({ type: "tool_end", ...toolEndEventData(te) })}\n\n`);
975
- }
976
- else if (ev.type === "tool_start") {
977
- const tsv = ev;
978
- res.write(`data: ${JSON.stringify({ type: "tool_start", ...toolStartEventData(tsv) })}\n\n`);
979
- }
980
- else if (ev.type === "task_notification") {
981
- const tnv = ev.notification;
982
- const tnvKey = tnv?.task_id ? taskNotificationStreamKey({ task_type: tnv.task_type, task_id: tnv.task_id, status: String(tnv.status), seq: tnv.seq }) : undefined;
983
- const tnvPrior = tnvKey !== undefined ? syncNotifiedKeys.get(tnvKey) : undefined;
984
- if (tnvPrior === undefined || !(await tnvPrior)) {
985
- if (tnvKey !== undefined)
986
- syncNotifiedKeys.set(tnvKey, Promise.resolve(true));
987
- res.write(`data: ${JSON.stringify({ type: "task_notification", ...taskNotificationEventData(ev) })}\n\n`);
988
- }
989
- }
990
- else if (ev.type === "reasoning_delta" && typeof ev.delta === "string") {
991
- res.write(`data: ${JSON.stringify({ type: "reasoning_delta", delta: redactSecrets(ev.delta), ...(ev.eventId ? { eventId: ev.eventId } : {}), ...(ev.parentToolCallId ? { parentToolCallId: ev.parentToolCallId } : {}) })}\n\n`);
992
- }
993
- else if (ev.type === "task_progress") {
994
- res.write(`data: ${JSON.stringify({ type: "task_progress", ...taskProgressEventData(ev) })}\n\n`);
995
- }
996
- else if (ev.type === "compacted") {
997
- res.write(`data: ${JSON.stringify({ type: "compacted", ...compactedEventData(ev) })}\n\n`);
998
- }
999
- else if (ev.type === "diagnostics") {
1000
- res.write(`data: ${JSON.stringify({ type: "diagnostics", ...diagnosticsEventData(ev) })}\n\n`);
1001
- }
1002
- else if (ev.type === "steering_injected") {
1003
- res.write(`data: ${JSON.stringify({ type: "steering_injected", ...steeringInjectedEventData(ev) })}\n\n`);
1004
- }
1005
- else if (ev.type === "workspace_changed") {
1006
- res.write(`data: ${JSON.stringify({ type: "workspace_changed", ...workspaceChangedEventData(ev) })}\n\n`);
1007
- }
1008
- else if (ev.type === "text_delta" || ev.type === "turn_end" || ev.type === "message_committed" || ev.type === "context_usage") {
1009
- const e = ev;
1010
- const ident = {
1011
- ...(e.eventId !== undefined ? { eventId: e.eventId } : {}),
1012
- ...(e.parentToolCallId !== undefined ? { parentToolCallId: e.parentToolCallId } : {}),
1013
- ...(e.sourceTaskId !== undefined ? { sourceTaskId: e.sourceTaskId } : {}),
1014
- ...(e.bgAgentId !== undefined ? { bgAgentId: e.bgAgentId } : {}),
1015
- };
1016
- const arm = ev.type === "text_delta"
1017
- ? { type: "text_delta", delta: e.delta, ...ident }
1018
- : ev.type === "turn_end"
1019
- ? { type: "turn_end", ...(e.usage !== undefined ? { usage: e.usage } : {}), ...(e.usageMissing !== undefined ? { usageMissing: e.usageMissing } : {}), ...(e.stopReason !== undefined ? { stopReason: e.stopReason } : {}), ...ident }
1020
- : ev.type === "message_committed"
1021
- ? { type: "message_committed", entryId: e.entryId, role: e.role, ...(e.toolCallId !== undefined ? { toolCallId: e.toolCallId } : {}), ...ident }
1022
- : { type: "context_usage", ...contextUsageEventData(ev), ...ident };
1023
- res.write(`data: ${JSON.stringify(arm)}\n\n`);
1024
- }
1025
- else {
1026
- res.write(`data: ${JSON.stringify(ev)}\n\n`);
1027
- }
1028
- }
1029
- });
1030
- };
1031
- const askTaskId = durableTaskId ?? uuidv7();
1032
- const askOwner = gatedPrincipal(req, deps.config) ?? null;
1033
- const emitAsk = (frame) => {
1034
- if (!res.writableEnded)
1035
- res.write(`event: ${frame.type}\ndata: ${JSON.stringify(frame)}\n\n`);
1036
- };
1037
- await emitPendingWorkflowCompletions(deps.workflowCompletionInbox, prepared.spec.sessionId, askOwner, (frame) => {
1038
- if (closed || res.writableEnded || res.destroyed)
1039
- throw new Error("stream ended before the completion frame was written");
1040
- const f = frame;
1041
- if (f.type === "task_notification" && f.task_id)
1042
- syncNotifiedKeys.set(taskNotificationStreamKey({ task_type: f.task_type, task_id: f.task_id, status: String(f.status), seq: f.seq }), Promise.resolve(true));
1043
- res.write(`data: ${JSON.stringify(frame)}\n\n`);
1044
- }, { route: "sync-stream-open", connection: "live-sse", log: (m, x) => deps.logger?.info?.(m, x) });
1045
- const withElicit = deps.elicitation
1046
- ? () => deps.elicitation.runWithContext({ taskId: askTaskId, owner: askOwner, emit: emitAsk, abortSignal: ac.signal }, streamBody)
1047
- : streamBody;
1048
- const withQuestion = () => deps.question
1049
- ? deps.question.runWithContext({ taskId: askTaskId, owner: askOwner, emit: emitAsk, abortSignal: ac.signal }, withElicit)
1050
- : withElicit();
1051
- const emitApproval = (frame) => {
1052
- if (res.writableEnded || res.destroyed)
1053
- throw new Error("live stream ended — approval card undeliverable");
1054
- emitAsk(frame);
1055
- };
1056
- const approvalCtx = deps.toolApproval
1057
- ? { taskId: askTaskId, owner: askOwner, emit: emitApproval, abortSignal: ac.signal, ...(prepared.spec.sessionId ? { sessionId: prepared.spec.sessionId } : {}) }
1058
- : undefined;
1059
- if (approvalCtx && deps.toolApproval) {
1060
- prepared.spec.onAsk = deps.toolApproval.boundAsk({
1061
- owner: askOwner,
1062
- taskId: askTaskId,
1063
- ...(prepared.spec.sessionId ? { sessionId: prepared.spec.sessionId } : {}),
1064
- });
1065
- }
1066
- const withApproval = () => approvalCtx && deps.toolApproval ? deps.toolApproval.runWithContext(approvalCtx, withQuestion) : withQuestion();
1067
- await (deps.sendUserFile
1068
- ? deps.sendUserFile.runWithContext({
1069
- taskId: askTaskId,
1070
- emit: async (f) => {
1071
- void (await emitAsk(f));
1072
- if (ledgerSink) {
1073
- const { type, ...rest } = f;
1074
- await ledgerSink.append(type, rest).catch(() => undefined);
1075
- }
1076
- },
1077
- }, withApproval)
1078
- : withApproval());
1079
- if (durableTaskId && deps.runStore) {
1080
- if (finalResult?.status === "suspended" && deps.checkpointStore)
1081
- await deps.runStore.setSuspended(durableTaskId);
1082
- else if (finalResult?.status === "needs_review" && deps.checkpointStore)
1083
- await deps.runStore.setNeedsReview(durableTaskId);
1084
- else if (finalResult)
1085
- await deps.runStore.setTerminal(durableTaskId, finalResult.status, finalResult, finalResult.errorMessage ?? null);
1086
- else {
1087
- putRewindAnchor();
1088
- const pending = await deps.checkpointStore?.findPendingTokenBySession(prepared.spec.sessionId);
1089
- if (pending) {
1090
- if (ledgerSink && !ledgerTerminal) {
1091
- ledgerTerminal = true;
1092
- await ledgerSink.flush().catch(() => undefined);
1093
- await ledgerSink.appendParked("suspended", {}).catch(() => undefined);
1094
- }
1095
- await deps.runStore.setSuspended(durableTaskId);
1096
- settleFleet("suspended");
1097
- }
1098
- else {
1099
- const dcErr = "stream client disconnected before completion (run aborted)";
1100
- const dc = { taskId: durableTaskId, sessionId: prepared.spec.sessionId ?? "", status: "failed", errorCode: "cancelled", errorMessage: dcErr, stats: { turns: 0, tokens: 0 } };
1101
- if (ledgerSink && !ledgerTerminal) {
1102
- ledgerTerminal = true;
1103
- await ledgerSink.flush().catch(() => undefined);
1104
- await ledgerSink.append("failed", { errorMessage: dcErr, errorCode: "cancelled" }).catch(() => undefined);
1105
- }
1106
- await deps.runStore.setTerminal(durableTaskId, "failed", dc, dcErr);
1107
- settleFleet("failed");
1108
- }
1109
- }
1110
- runRowSettled = true;
1111
- }
1112
- }
1113
- finally {
1114
- syncLegLive = false;
1115
- if (durableHeartbeat)
1116
- clearInterval(durableHeartbeat);
1117
- if (durableTaskId && steerableRuns.get(durableTaskId) === liveStreamRef)
1118
- steerableRuns.delete(durableTaskId);
1119
- if (durableTaskId && inflightRuns.get(durableTaskId) === ac) {
1120
- inflightRuns.delete(durableTaskId);
1121
- cancelledViaVerb.delete(durableTaskId);
1122
- }
1123
- for (const evict of subagentHandleEvictions)
1124
- evict();
1125
- if (!runRowSettled && durableTaskId && deps.runStore) {
1126
- try {
1127
- const row = await deps.runStore.getRun(durableTaskId);
1128
- if (row?.status === "running") {
1129
- const pending = prepared.spec.sessionId ? await deps.checkpointStore?.findPendingTokenBySession(prepared.spec.sessionId) : undefined;
1130
- if (pending) {
1131
- if (ledgerSink && !ledgerTerminal) {
1132
- ledgerTerminal = true;
1133
- await ledgerSink.flush().catch(() => undefined);
1134
- await ledgerSink.appendParked("suspended", {}).catch(() => undefined);
1135
- }
1136
- await deps.runStore.setSuspended(durableTaskId);
1137
- settleFleet("suspended");
1138
- }
1139
- else if (finalResult) {
1140
- if (ledgerSink && !ledgerTerminal) {
1141
- ledgerTerminal = true;
1142
- await ledgerSink.onDone(finalResult).catch(() => undefined);
1143
- }
1144
- await deps.runStore.setTerminal(durableTaskId, finalResult.status, finalResult, finalResult.errorMessage ?? null);
1145
- settleFleet(finalResult.status, fleetRunResiduals(finalResult));
1146
- }
1147
- else if (ac.signal.aborted) {
1148
- const err = "stream aborted before completion (run cancelled)";
1149
- const c = { taskId: durableTaskId, sessionId: prepared.spec.sessionId ?? "", status: "failed", errorCode: "cancelled", errorMessage: err, stats: { turns: 0, tokens: 0 } };
1150
- if (ledgerSink && !ledgerTerminal) {
1151
- ledgerTerminal = true;
1152
- await ledgerSink.flush().catch(() => undefined);
1153
- await ledgerSink.append("failed", { errorMessage: err, errorCode: "cancelled" }).catch(() => undefined);
1154
- }
1155
- await deps.runStore.setTerminal(durableTaskId, "failed", c, err);
1156
- }
1157
- else {
1158
- const err = "stream leg threw before a terminal event";
1159
- if (ledgerSink && !ledgerTerminal) {
1160
- ledgerTerminal = true;
1161
- await ledgerSink.flush().catch(() => undefined);
1162
- await ledgerSink.append("failed", { errorMessage: err }).catch(() => undefined);
1163
- }
1164
- await deps.runStore.setTerminal(durableTaskId, "failed", null, err);
1165
- }
1166
- }
1167
- }
1168
- catch {
1169
- }
1170
- }
1171
- settleFleet("failed");
1172
- }
1173
- return { status: 200, body: finalResult };
1174
- }, (r) => r.body?.status === "completed");
1175
- }
1176
- catch (e) {
1177
- deps.logger?.error("stream_error", { closed, err: e instanceof Error ? e.message : String(e) });
1178
- if (!closed)
1179
- throw e;
1180
- }
1181
- finally {
1182
- counters.uncountedBillableInflight--;
1183
- clearInterval(hb);
1184
- if (usageKey)
1185
- deps.modelUsage?.clear(usageKey);
1186
- }
1187
- if (!ranLive && resp && !res.writableEnded) {
1188
- res.write(`data: ${JSON.stringify({ type: "done", result: resp.body, replay: true })}\n\n`);
1189
- }
1190
- res.end();
1191
- }
1192
- else {
1193
- const resp = await idemCache.run(idemKey, async () => {
1194
- let durableTaskId;
1195
- let durableHeartbeat;
1196
- if (deps.runStore && prepared.spec.sessionId) {
1197
- const tid = uuidv7();
1198
- const created = await deps.runStore.createRun(tid, prepared.spec.sessionId, principal ?? null, deps.instanceId ?? "default", runMeta(prepared, source));
1199
- if (!created.ok)
1200
- return { status: 409, body: { error: "session already has an active run — POST /v1/runs/{activeTaskId}/cancel stops it (same-instance interactive runs abort immediately)", activeTaskId: created.activeTaskId } };
1201
- deps.sessionTitler?.maybeTitle(prepared.spec.sessionId, prepared.spec.objective);
1202
- durableTaskId = tid;
1203
- }
1204
- if (deps.checkpointStore && prepared.spec.sessionId) {
1205
- await deps.checkpointStore.putCtx(prepared.spec.sessionId, { body: prepared.body, memoryScope: prepared.auth?.memoryScope });
1206
- }
1207
- const syncCancelCtrl = new AbortController();
1208
- if (prepared.spec.sessionId)
1209
- markChildrenStoppedByUserOnAbort(syncCancelCtrl.signal, prepared.spec.sessionId, prepared.auth?.principal);
1210
- if (durableTaskId && deps.runStore) {
1211
- const rs = deps.runStore;
1212
- const tid = durableTaskId;
1213
- const owner = principal ?? null;
1214
- const abortFromVerb = () => {
1215
- cancelledViaVerb.add(tid);
1216
- syncCancelCtrl.abort();
1217
- };
1218
- durableHeartbeat = setInterval(() => {
1219
- void rs.heartbeat(tid, owner).catch(() => undefined);
1220
- if (!syncCancelCtrl.signal.aborted) {
1221
- void Promise.resolve(rs.isCancelRequested?.(tid, owner)).then((c) => { if (c)
1222
- abortFromVerb(); }).catch(() => undefined);
1223
- }
1224
- }, HEARTBEAT_MS);
1225
- durableHeartbeat.unref?.();
1226
- inflightRuns.set(tid, syncCancelCtrl);
1227
- void Promise.resolve(rs.isCancelRequested?.(tid, owner)).then((c) => { if (c && !syncCancelCtrl.signal.aborted)
1228
- abortFromVerb(); }).catch(() => undefined);
1229
- }
1230
- else if (durableTaskId) {
1231
- inflightRuns.set(durableTaskId, syncCancelCtrl);
1232
- }
1233
- const specWithSignal = { ...prepared.spec, signal: syncCancelCtrl.signal };
1234
- let result;
1235
- let cancelVerbLabel = false;
1236
- counters.uncountedBillableInflight++;
1237
- try {
1238
- result = await withPrincipal(principal, () => prepared.verify
1239
- ? runWithVerification(deps.runner, specWithSignal, prepared.verify)
1240
- : prepared.cascade
1241
- ? runCascade(deps.runner, specWithSignal, cascadeConfig(deps.config.cascadeLadder, prepared.spec.maxCostUsd))
1242
- : deps.runner.runTask(specWithSignal));
1243
- }
1244
- finally {
1245
- counters.uncountedBillableInflight--;
1246
- if (durableHeartbeat)
1247
- clearInterval(durableHeartbeat);
1248
- if (durableTaskId && inflightRuns.get(durableTaskId) === syncCancelCtrl) {
1249
- inflightRuns.delete(durableTaskId);
1250
- cancelVerbLabel = cancelledViaVerb.has(durableTaskId);
1251
- cancelledViaVerb.delete(durableTaskId);
1252
- }
1253
- }
1254
- if (cancelVerbLabel && result.status === "failed" && !result.errorCode) {
1255
- result = { ...result, errorCode: "cancelled" };
1256
- }
1257
- const resumeAtStatus = resumeAtHttpStatus(result);
1258
- if (resumeAtStatus) {
1259
- if (durableTaskId && deps.runStore)
1260
- await deps.runStore.setTerminal(durableTaskId, "failed", stripCheckpointToken(result), result.errorMessage ?? null);
1261
- return { status: resumeAtStatus, body: { errorCode: result.errorCode, error: result.errorMessage ?? "resume-at failed" } };
1262
- }
1263
- if (result.status === "suspended" && deps.checkpointStore && deps.runStore && prepared.spec.sessionId) {
1264
- if (durableTaskId)
1265
- await deps.runStore.setSuspended(durableTaskId);
1266
- return { status: 200, body: { taskId: durableTaskId, sessionId: prepared.spec.sessionId, status: "suspended" } };
1267
- }
1268
- if (result.status === "needs_review" && deps.checkpointStore && deps.runStore && prepared.spec.sessionId) {
1269
- if (durableTaskId)
1270
- await deps.runStore.setNeedsReview(durableTaskId);
1271
- return { status: 200, body: { taskId: durableTaskId, sessionId: prepared.spec.sessionId, status: "needs_review" } };
1272
- }
1273
- if (durableTaskId && deps.runStore)
1274
- await deps.runStore.setTerminal(durableTaskId, result.status, stripCheckpointToken(result), result.errorMessage ?? null);
1275
- finalizeTaskResult(result, principal, prepared.spec.objective, prepared.spec.sessionId);
1276
- return { status: 200, body: stripCheckpointToken(result) };
1277
- }, (r) => r.status < 400);
1278
- sendJson(res, resp.status, resp.body);
1279
- }
1280
- return;
1281
- }
1282
- if (req.method === "POST" && url === "/v1/runs") {
1283
- if (!deps.runStore) {
1284
- sendJson(res, 501, { error: "async runs require the TiDB run store (SESSION_BACKEND=tidb)" });
1285
- return;
1286
- }
1287
- const rawIdem = headerStr(req.headers["idempotency-key"]);
1288
- const idemKey = rawIdem ? scopedIdempotencyKey(rawIdem, source, gatedPrincipal(req, deps.config)) : undefined;
1289
- const cached = idemKey ? idemCache.peek(idemKey) : undefined;
1290
- if (cached) {
1291
- const resp = await cached;
1292
- sendJson(res, resp.status, resp.body);
1293
- return;
1294
- }
1295
- if (rateLimited(req, res) || quotaExceeded(req, res))
1296
- return;
1297
- const prepared = await prepareSpec(req, res);
1298
- if (!prepared)
1299
- return;
1300
- const runStore = deps.runStore;
1301
- const rawClientTaskId = prepared.body.taskId;
1302
- if (rawClientTaskId !== undefined && (typeof rawClientTaskId !== "string" || !isUuidV7(rawClientTaskId))) {
1303
- sendJson(res, 400, { error: "body.taskId must be a uuidv7 string (caller-minted idempotency key)" });
1304
- return;
1305
- }
1306
- const clientTaskId = rawClientTaskId;
1307
- if (clientTaskId) {
1308
- const existing = await runStore.getRun(clientTaskId);
1309
- if (existing) {
1310
- const verified = verifiedPrincipal(req, deps.config);
1311
- if (existing.owner !== null && existing.owner !== verified) {
1312
- sendJson(res, 409, { error: "taskId already exists" });
1313
- return;
1314
- }
1315
- sendJson(res, 202, { taskId: existing.taskId, sessionId: existing.sessionId, status: existing.status });
1316
- return;
1317
- }
1318
- }
1319
- if (await leaseDenied(req, res))
1320
- return;
1321
- const resp = await idemCache.run(idemKey, async () => {
1322
- const sessionId = prepared.spec.sessionId ?? uuidv7();
1323
- const taskId = clientTaskId ?? uuidv7();
1324
- const created = await runStore.createRun(taskId, sessionId, prepared.auth?.principal ?? null, deps.instanceId ?? "default", runMeta(prepared, source));
1325
- if (created.ok)
1326
- deps.sessionTitler?.maybeTitle(sessionId, prepared.spec.objective);
1327
- if (!created.ok) {
1328
- if (clientTaskId && created.activeTaskId === clientTaskId) {
1329
- return { status: 202, body: { taskId: clientTaskId, sessionId, status: "running" } };
1330
- }
1331
- return { status: 409, body: { error: "session already has an active run — POST /v1/runs/{activeTaskId}/cancel stops it (same-instance interactive runs abort immediately)", activeTaskId: created.activeTaskId } };
1332
- }
1333
- if (deps.checkpointStore) {
1334
- await deps.checkpointStore.putCtx(sessionId, { body: prepared.body, memoryScope: prepared.auth?.memoryScope });
1335
- }
1336
- const anchorOwner = prepared.auth?.principal ?? null;
1337
- const getLeafId = deps.sessionStorage?.getLeafId?.bind(deps.sessionStorage);
1338
- const captureTurnAnchor = deps.resumeAnchorStore && getLeafId
1339
- ? async (eventId) => {
1340
- const leaf = await getLeafId(sessionId);
1341
- if (leaf)
1342
- await deps.resumeAnchorStore.put(sessionId, eventId, leaf, anchorOwner);
1343
- }
1344
- : undefined;
1345
- const captureUserMessageAnchor = deps.resumeAnchorStore
1346
- ? async (entryId) => { await deps.resumeAnchorStore.put(sessionId, taskId, entryId, anchorOwner); }
1347
- : undefined;
1348
- const fleetPub = fleetRunPublisher(deps.fleetBus, {
1349
- runId: taskId,
1350
- scope: gatedPrincipal(req, deps.config) ?? "default",
1351
- rootTaskId: taskId,
1352
- ...fleetRunLabels(prepared.spec.objective),
1353
- });
1354
- void runInBackground(deps.runner, { ...prepared.spec, sessionId }, runStore, taskId, deps.metrics, prepared.auth?.principal, prepared.verify, prepared.cascade ? cascadeConfig(deps.config.cascadeLadder, prepared.spec.maxCostUsd) : undefined, deps.instrumentDegenerate, deps.planCacheProbe, deps.config.traceThinking, inflightRuns, preemptableRuns, steerableRuns, deps.modelUsage, deps.elicitation, gatedPrincipal(req, deps.config) ?? null, captureTurnAnchor, fleetPub, captureUserMessageAnchor, deps.question, deps.workflowCompletionInbox, deps.subagentSteerRegistry, (m, x) => deps.logger?.info?.(m, x), deps.sendUserFile, deps.promptManifests);
1355
- return { status: 202, body: { taskId, sessionId, status: "running" } };
1356
- }, (r) => r.status === 202);
1357
- sendJson(res, resp.status, resp.body);
1358
- return;
1359
- }
1360
- const runMatch = req.method === "GET" ? RUN_ID_RE.exec(url) : null;
1361
- if (runMatch) {
1362
- if (!deps.runStore) {
1363
- sendJson(res, 501, { error: "async runs require the TiDB run store" });
1364
- return;
1365
- }
1366
- const taskId = runMatch[1];
1367
- if (deps.config.requirePrincipal && !gatedPrincipal(req, deps.config)) {
1368
- sendJson(res, 401, { error: `missing principal header '${deps.config.principalHeader}'` });
1369
- return;
1370
- }
1371
- const run = await deps.runStore.getRun(taskId);
1372
- if (!run) {
1373
- sendJson(res, 404, runNotFoundBody(taskId));
1374
- return;
1375
- }
1376
- if (!runOwnerOk(req, res, run.owner))
1377
- return;
1378
- if (!runSessionAcceptOk(req, res, run, runMatch[2] ? "run.events" : "run.poll"))
1379
- return;
1380
- if (runMatch[2]) {
1381
- await streamRunEvents(req, res, deps.runStore, taskId, deps.config.runStaleSec * 1000);
1382
- }
1383
- else {
1384
- const stale = run.status === "running" && Date.now() - new Date(run.updatedAt).getTime() > deps.config.runStaleSec * 1000;
1385
- const infraRates = deps.config.infraCostRates;
1386
- const needCost = Boolean(run.result?.stats && !stale && infraRates && hasInfraPricing(infraRates));
1387
- const needSuggestions = !stale && run.status === "completed";
1388
- const events = (needCost || needSuggestions) && deps.runStore.getEvents ? await deps.runStore.getEvents(taskId, 0).catch(() => []) : undefined;
1389
- let supervisorCost;
1390
- if (needCost && events && run.result?.stats) {
1391
- const durMs = new Date(run.updatedAt).getTime() - new Date(run.createdAt).getTime();
1392
- const usage = infraUsageFromEvents(events, Number.isFinite(durMs) ? durMs : 0);
1393
- const st = run.result.stats;
1394
- supervisorCost = composeSupervisorCost(st.costBreakdown ?? null, st.costMicroUsd ?? 0, infraCost(usage, infraRates));
1395
- }
1396
- let suggestions;
1397
- if (needSuggestions && events) {
1398
- const sug = [...events].reverse().find((e) => e.type === "suggestions");
1399
- const arr = sug?.data?.suggestions;
1400
- if (Array.isArray(arr))
1401
- suggestions = arr.map((s) => String(s));
1402
- }
1403
- sendJson(res, 200, {
1404
- taskId: run.taskId,
1405
- sessionId: run.sessionId,
1406
- status: stale ? "failed" : run.status,
1407
- result: run.result ?? undefined,
1408
- supervisorCost,
1409
- suggestions,
1410
- errorCode: stale ? undefined : (run.result?.errorCode ?? undefined),
1411
- error: stale ? "run stalled (instance lost?)" : run.error ?? undefined,
1412
- jobId: run.jobId ?? undefined,
1413
- source: run.source ?? undefined,
1414
- });
1415
- }
1416
- return;
1417
- }
1418
- const cancelMatch = req.method === "POST" ? RUN_CANCEL_RE.exec(url) : null;
1419
- if (cancelMatch) {
1420
- if (!deps.runStore) {
1421
- sendJson(res, 501, { error: "async runs require the TiDB run store" });
1422
- return;
1423
- }
1424
- if (deps.config.requirePrincipal && !gatedPrincipal(req, deps.config)) {
1425
- sendJson(res, 401, { error: `missing principal header '${deps.config.principalHeader}'` });
1426
- return;
1427
- }
1428
- const taskId = cancelMatch[1];
1429
- const run = await deps.runStore.getRun(taskId);
1430
- if (!run) {
1431
- sendJson(res, 404, { error: "run not found — the id belongs to no run in this deployment's run store (a run from another server process, or an in-memory store that did not survive a restart, is not visible here)" });
1432
- return;
1433
- }
1434
- if (!runOwnerOk(req, res, run.owner))
1435
- return;
1436
- if (!runSessionAcceptOk(req, res, run, "run.cancel"))
1437
- return;
1438
- const rs = deps.runStore;
1439
- const cancelSuspended = async () => {
1440
- const cs = deps.checkpointStore;
1441
- let note = "cancelled while suspended (pending approval settled)";
1442
- if (cs) {
1443
- const token = await cs.findPendingTokenBySession(run.sessionId);
1444
- if (token) {
1445
- const cp = await cs.get(token);
1446
- const won = cp ? await cs.expire(token, cp.scope) : false;
1447
- if (!won) {
1448
- const now = await rs.getRun(taskId);
1449
- const st = now?.status;
1450
- if (st === "suspended" || st === "needs_review" || st === "running") {
1451
- sendJson(res, 409, { taskId, status: st, error: "pending approval was settled concurrently (decided or expired) — re-check the run and retry cancel if it is still active" });
1452
- }
1453
- else {
1454
- sendJson(res, 202, { taskId, status: st ?? "failed", note: "run already terminal — cancel is a no-op" });
1455
- }
1456
- return;
1457
- }
1458
- }
1459
- else {
1460
- const claimed = await rs.markResuming(taskId);
1461
- if (!claimed) {
1462
- const now = await rs.getRun(taskId);
1463
- const st = now?.status;
1464
- if (st === "running" || st === "suspended" || st === "needs_review") {
1465
- sendJson(res, 409, { taskId, status: st, error: "pending approval was settled concurrently (decided or expired) — re-check the run and retry cancel if it is still active" });
1466
- }
1467
- else {
1468
- sendJson(res, 202, { taskId, status: st ?? "failed", note: "run already terminal — cancel is a no-op" });
1469
- }
1470
- return;
1471
- }
1472
- note = "cancelled while suspended (no pending approval found — stale park released)";
1473
- }
1474
- }
1475
- else {
1476
- note = "cancelled while suspended (no checkpoint store on this deployment — run row terminalized only)";
1477
- }
1478
- const err = note;
1479
- try {
1480
- const result = { taskId, sessionId: run.sessionId, status: "failed", errorCode: "cancelled", errorMessage: err, stats: { turns: 0, tokens: 0 } };
1481
- let lastErr;
1482
- for (let attempt = 0;; attempt++) {
1483
- try {
1484
- await rs.setTerminal(taskId, "failed", result, err);
1485
- lastErr = undefined;
1486
- break;
1487
- }
1488
- catch (e) {
1489
- lastErr = e;
1490
- if (attempt >= 2)
1491
- break;
1492
- await new Promise((r) => setTimeout(r, 100 * (attempt + 1)));
1493
- }
1494
- }
1495
- if (lastErr !== undefined)
1496
- throw lastErr;
1497
- }
1498
- catch (e) {
1499
- sendJson(res, 500, { taskId, error: `cancel progressed (${note}) but could not terminalize the run row (retried) — retry cancel; a stuck row is reaped after the stale window (${e instanceof Error ? e.message : String(e)})` });
1500
- return;
1501
- }
1502
- const finalRow = await rs.getRun(taskId).catch(() => undefined);
1503
- if (finalRow && finalRow.errorCode !== "cancelled" && (finalRow.status === "failed" || finalRow.status === "completed" || finalRow.status === "blocked" || finalRow.status === "timeout")) {
1504
- sendJson(res, 202, { taskId, status: finalRow.status, errorCode: finalRow.errorCode ?? null, note: `session unlocked; the run was terminalized concurrently (${finalRow.errorCode ?? finalRow.status}) before this cancel's write — reporting the actual ledger state` });
1505
- return;
1506
- }
1507
- sendJson(res, 202, { taskId, status: "failed", errorCode: "cancelled", note });
1508
- };
1509
- if (run.status === "running") {
1510
- const flagged = await deps.runStore.requestCancel(taskId, run.owner);
1511
- if (!flagged) {
1512
- const now = await deps.runStore.getRun(taskId);
1513
- if (now?.status === "suspended") {
1514
- await cancelSuspended();
1515
- }
1516
- else {
1517
- sendJson(res, 202, { taskId, status: now?.status ?? "failed", note: "run already terminal — cancel is a no-op" });
1518
- }
1519
- return;
1520
- }
1521
- if (inflightRuns.has(taskId)) {
1522
- cancelledViaVerb.add(taskId);
1523
- inflightRuns.get(taskId).abort();
1524
- }
1525
- sendJson(res, 202, { taskId, status: "cancelling" });
1526
- }
1527
- else if (run.status === "suspended" || run.status === "needs_review") {
1528
- await cancelSuspended();
1529
- }
1530
- else {
1531
- sendJson(res, 202, { taskId, status: run.status, note: "run already terminal — cancel is a no-op" });
1532
- }
1533
- return;
1534
- }
1535
- const steerMatch = req.method === "POST" ? RUN_STEER_RE.exec(url) : null;
1536
- if (steerMatch) {
1537
- if (rateLimited(req, res) || quotaExceeded(req, res) || (await leaseDenied(req, res)))
1538
- return;
1539
- if (!deps.runStore) {
1540
- sendJson(res, 501, { error: "async runs require the TiDB run store" });
1541
- return;
1542
- }
1543
- const principal = gatedPrincipal(req, deps.config);
1544
- if (deps.config.requirePrincipal && principal === undefined) {
1545
- sendJson(res, 401, { error: `missing principal header '${deps.config.principalHeader}'` });
1546
- return;
1547
- }
1548
- const taskId = steerMatch[1];
1549
- let body;
1550
- try {
1551
- body = (await readJson(req));
1552
- }
1553
- catch {
1554
- sendJson(res, 400, { error: "invalid JSON body" });
1555
- return;
1556
- }
1557
- if (typeof body.text !== "string" || body.text.length === 0) {
1558
- sendJson(res, 400, { error: "body must be { text: string (non-empty), mode?: 'all' | 'one-at-a-time', priority?: 'now' | 'next' | 'later' }" });
1559
- return;
1560
- }
1561
- if (body.mode !== undefined && body.mode !== "all" && body.mode !== "one-at-a-time") {
1562
- sendJson(res, 400, { error: "mode must be 'all' or 'one-at-a-time' when present" });
1563
- return;
1564
- }
1565
- if (body.priority !== undefined && body.priority !== "now" && body.priority !== "next" && body.priority !== "later") {
1566
- sendJson(res, 400, { error: "priority must be 'now', 'next', or 'later' when present" });
1567
- return;
1568
- }
1569
- const text = body.text;
1570
- const priority = body.priority;
1571
- const messageId = uuidv7();
1572
- const trusted = explicitOperatorOk(principal, deps.config.operatorPrincipals);
1573
- try {
1574
- validatePendingSteer({ text, trusted });
1575
- }
1576
- catch (e) {
1577
- if (e instanceof CheckpointError && e.code === "steering.invalid_content") {
1578
- sendError(res, 422, "steering.invalid_content", e.message);
1579
- return;
1580
- }
1581
- throw e;
1582
- }
1583
- const run = await deps.runStore.getRun(taskId);
1584
- if (!run) {
1585
- sendJson(res, 404, runNotFoundBody(taskId));
1586
- return;
1587
- }
1588
- if (!trusted && run.owner !== null && run.owner !== principal) {
1589
- sendJson(res, 404, { error: "run not found" });
1590
- return;
1591
- }
1592
- if (!runSessionAcceptOk(req, res, run, "run.steer"))
1593
- return;
1594
- const tryPark = async () => {
1595
- const cs = deps.checkpointStore;
1596
- if (!cs)
1597
- return "no-store";
1598
- const scope = await cs.peekPendingScope(run.sessionId);
1599
- if (scope === null || scope === undefined)
1600
- return "no-checkpoint";
1601
- const token = await cs.findPendingTokenBySession(run.sessionId, scope);
1602
- if (!token)
1603
- return "no-checkpoint";
1604
- return (await cs.setPendingSteer(token, scope, { text, trusted })) ? "parked" : "no-checkpoint";
1605
- };
1606
- const sendParked = () => sendJson(res, 202, { taskId, status: "suspended", delivery: "queued", messageId, ...(priority ? { priority } : {}), note: "steer parked on the checkpoint — injected when the run resumes" });
1607
- const sendNotRunning = (error) => sendError(res, 409, "steering.not_running", error);
1608
- const live = steerableRuns.get(taskId);
1609
- if (live) {
1610
- try {
1611
- await live.steer(text, { trusted });
1612
- sendJson(res, 200, { taskId, status: "running", delivery: "applied", messageId, ...(priority ? { priority } : {}) });
1613
- return;
1614
- }
1615
- catch (e) {
1616
- const code = e.code;
1617
- if (code === "steering.invalid_content") {
1618
- sendError(res, 422, "steering.invalid_content", e instanceof Error ? e.message : "invalid steering content");
1619
- return;
1620
- }
1621
- if (code !== "steering.not_running")
1622
- throw e;
1623
- if (await tryPark() === "parked") {
1624
- sendParked();
1625
- return;
1626
- }
1627
- sendNotRunning("run just finished — no longer accepting steers");
1628
- return;
1629
- }
1630
- }
1631
- if (run.status === "suspended") {
1632
- const outcome = await tryPark();
1633
- if (outcome === "no-store") {
1634
- sendJson(res, 501, { error: "steering a suspended run requires the checkpoint store" });
1635
- return;
1636
- }
1637
- if (outcome === "parked") {
1638
- sendParked();
1639
- return;
1640
- }
1641
- sendNotRunning("run is no longer suspended (resolved or expired)");
1642
- return;
1643
- }
1644
- if (run.status === "running") {
1645
- sendNotRunning("run is active on another replica — cross-replica live-steer is not yet supported");
1646
- return;
1647
- }
1648
- if (deps.checkpointStore && deps.sessionStorage?.getLeafId) {
1649
- const activeTaskId = await deps.runStore.getActiveTaskId?.(run.sessionId).catch(() => undefined);
1650
- if (activeTaskId !== undefined && activeTaskId !== null && activeTaskId !== taskId) {
1651
- sendNotRunning(`run is ${run.status} and its session has an ACTIVE run (${activeTaskId}) — steer that run instead`);
1652
- return;
1653
- }
1654
- const prior = wakeParkMints.get(run.sessionId) ?? Promise.resolve();
1655
- const mintResult = { parked: false };
1656
- const job = prior.then(async () => {
1657
- const cs = deps.checkpointStore;
1658
- if (await tryPark() === "parked") {
1659
- mintResult.parked = true;
1660
- return;
1661
- }
1662
- const leafId = await Promise.resolve(deps.sessionStorage.getLeafId(run.sessionId)).catch(() => undefined);
1663
- if (leafId === undefined || leafId === null)
1664
- return;
1665
- const cpScope = run.owner ?? "_";
1666
- const wakeToken = mintCheckpointToken();
1667
- await cs.put(wakeToken, {
1668
- token: wakeToken,
1669
- scope: cpScope,
1670
- sessionId: run.sessionId,
1671
- leafId,
1672
- gate: { kind: "task_done" },
1673
- pendingAction: { kind: "task_done" },
1674
- state: { activeTools: [], nestedStats: { tokens: 0, turns: 0, tasks: 0, costUsd: 0, costMicroUsd: 0 } },
1675
- status: "pending",
1676
- createdAt: Date.now(),
1677
- sourceTaskId: taskId,
1678
- });
1679
- const winner = await cs.findPendingTokenBySession(run.sessionId).catch(() => null);
1680
- if (winner !== null && winner !== wakeToken) {
1681
- await cs.expire(wakeToken, cpScope).catch(() => undefined);
1682
- if (await cs.setPendingSteer(winner, cpScope, { text, trusted }))
1683
- mintResult.parked = true;
1684
- return;
1685
- }
1686
- if (await cs.setPendingSteer(wakeToken, cpScope, { text, trusted }))
1687
- mintResult.parked = true;
1688
- });
1689
- const wrapped = job.catch(() => undefined);
1690
- wakeParkMints.set(run.sessionId, wrapped);
1691
- try {
1692
- await job;
1693
- }
1694
- finally {
1695
- if (wakeParkMints.get(run.sessionId) === wrapped)
1696
- wakeParkMints.delete(run.sessionId);
1697
- }
1698
- if (mintResult.parked) {
1699
- sendJson(res, 202, { taskId, status: run.status, delivery: "parked_for_wake", messageId, ...(priority ? { priority } : {}), note: "run already ended — steer parked on a task_done checkpoint; deliver it with POST /v1/sessions/:id/wake" });
1700
- return;
1701
- }
1702
- }
1703
- sendNotRunning(`run is ${run.status} — not accepting steers`);
1704
- return;
1705
- }
1706
- const compactMatch = req.method === "POST" ? RUN_COMPACT_RE.exec(url) : null;
1707
- if (compactMatch) {
1708
- if (rateLimited(req, res) || quotaExceeded(req, res) || (await leaseDenied(req, res)))
1709
- return;
1710
- if (!deps.runStore) {
1711
- sendJson(res, 501, { error: "async runs require the TiDB run store" });
1712
- return;
1713
- }
1714
- const principal = gatedPrincipal(req, deps.config);
1715
- if (deps.config.requirePrincipal && principal === undefined) {
1716
- sendJson(res, 401, { error: `missing principal header '${deps.config.principalHeader}'` });
1717
- return;
1718
- }
1719
- const taskId = compactMatch[1];
1720
- const run = await deps.runStore.getRun(taskId);
1721
- if (!run) {
1722
- sendJson(res, 404, { error: "run not found" });
1723
- return;
1724
- }
1725
- const operator = explicitOperatorOk(principal, deps.config.operatorPrincipals);
1726
- if (!operator && run.owner !== null && run.owner !== principal) {
1727
- sendJson(res, 404, { error: "run not found" });
1728
- return;
1729
- }
1730
- if (!runSessionAcceptOk(req, res, run, "run.compact"))
1731
- return;
1732
- let compactBody;
1733
- try {
1734
- const parsed = await readJson(req);
1735
- if (parsed === null || typeof parsed !== "object") {
1736
- sendJson(res, 400, { error: "body must be a JSON object when present" });
1737
- return;
1738
- }
1739
- compactBody = parsed;
1740
- }
1741
- catch (e) {
1742
- if (e instanceof HttpError) {
1743
- sendJson(res, e.status, { error: e.message });
1744
- return;
1745
- }
1746
- sendJson(res, 400, { error: "invalid JSON body" });
1747
- return;
1748
- }
1749
- if (compactBody.instructions !== undefined && (typeof compactBody.instructions !== "string" || compactBody.instructions.length === 0 || [...compactBody.instructions].length > 2_048)) {
1750
- sendJson(res, 400, { error: "instructions must be a non-empty string of at most 2048 characters (code points) when present — the engine caps compaction instructions there" });
1751
- return;
1752
- }
1753
- const compactInstructions = compactBody.instructions;
1754
- const live = steerableRuns.get(taskId);
1755
- if (!live) {
1756
- sendError(res, 409, "compact.not_running", run.status === "running" ? "run is active on another replica — manual compact is replica-local" : `run is ${run.status} — not accepting compaction`);
1757
- return;
1758
- }
1759
- void live.compact(compactInstructions !== undefined ? { instructions: compactInstructions } : undefined).then((outcome) => {
1760
- const level = outcome === "failed" ? "warn" : "info";
1761
- deps.logger?.[level]?.("manual_compact_outcome", { taskId, outcome, note: outcome === "compacted" ? "a compacted{trigger:'manual'} event rode the run stream" : `no event (see compaction.${outcome === "mooted" ? "mooted" : outcome} trace / task terminal events)` });
1762
- }, (e) => {
1763
- if (e.code !== "steering.not_running") {
1764
- deps.logger?.warn?.("manual_compact_failed", { taskId, err: e instanceof Error ? e.message : String(e) });
1765
- }
1766
- else {
1767
- deps.logger?.info?.("manual_compact_not_running", { taskId, note: "stream settled between lookup and compact() — benign unless systematic" });
1768
- }
1769
- });
1770
- sendJson(res, 202, { taskId, status: "running", delivery: "accepted", note: "compaction will run at the next turn boundary; a compacted{trigger:'manual'} event rides the run stream if anything is summarized" });
1771
- return;
1772
- }
1773
- const detachMatch = req.method === "POST" ? RUN_DETACH_RE.exec(url) : null;
1774
- if (detachMatch) {
1775
- if (rateLimited(req, res))
1776
- return;
1777
- if (!deps.runStore) {
1778
- sendJson(res, 501, { error: "async runs require the TiDB run store" });
1779
- return;
1780
- }
1781
- const principal = gatedPrincipal(req, deps.config);
1782
- if (deps.config.requirePrincipal && principal === undefined) {
1783
- sendJson(res, 401, { error: `missing principal header '${deps.config.principalHeader}'` });
1784
- return;
1785
- }
1786
- const taskId = detachMatch[1];
1787
- let body;
1788
- try {
1789
- body = (await readJson(req));
1790
- }
1791
- catch {
1792
- sendJson(res, 400, { error: "invalid JSON body" });
1793
- return;
1794
- }
1795
- if (typeof body.toolCallId !== "string" || body.toolCallId.length === 0 || body.toolCallId.length > 256) {
1796
- sendJson(res, 400, { error: "body must be { toolCallId: string (non-empty, ≤256 chars) }" });
1797
- return;
1798
- }
1799
- const run = await deps.runStore.getRun(taskId);
1800
- if (!run) {
1801
- sendJson(res, 404, { error: "run not found" });
1802
- return;
1803
- }
1804
- const operator = explicitOperatorOk(principal, deps.config.operatorPrincipals);
1805
- if (!operator && run.owner !== null && run.owner !== principal) {
1806
- sendJson(res, 404, { error: "run not found" });
1807
- return;
1808
- }
1809
- if (!runSessionAcceptOk(req, res, run, "run.detach"))
1810
- return;
1811
- const live = steerableRuns.get(taskId);
1812
- if (!live) {
1813
- sendError(res, 409, "detach.not_running", run.status === "running" ? "run is active on another replica — detach is replica-local" : `run is ${run.status} — no running tool call to detach`);
1814
- return;
1815
- }
1816
- live.detach(body.toolCallId);
1817
- sendJson(res, 202, { taskId, toolCallId: body.toolCallId, delivery: "requested", note: "if the tool call is running in a detach-capable env it settles early with 'moved to background; task_id=b*'; otherwise the request is a no-op (fail-safe)" });
1818
- return;
1819
- }
1820
- const wfSteerMatch = req.method === "POST" ? WORKFLOW_AGENT_STEER_RE.exec(url) : null;
1821
- if (wfSteerMatch) {
1822
- if (rateLimited(req, res) || quotaExceeded(req, res) || (await leaseDenied(req, res)))
1823
- return;
1824
- if (!deps.workflowRunStore) {
1825
- sendJson(res, 501, { error: "workflow runs require self-orchestration (SELF_ORCHESTRATION_ENABLED)" });
1826
- return;
1827
- }
1828
- const principal = gatedPrincipal(req, deps.config);
1829
- if (deps.config.requirePrincipal && principal === undefined) {
1830
- sendJson(res, 401, { error: `missing principal header '${deps.config.principalHeader}'` });
1831
- return;
1832
- }
1833
- const wfRunId = safeDecode(wfSteerMatch[1]);
1834
- const label = safeDecode(wfSteerMatch[2]);
1835
- if (wfRunId === null || label === null) {
1836
- sendJson(res, 400, { error: "malformed workflow path (invalid percent-encoding)" });
1837
- return;
1838
- }
1839
- let body;
1840
- try {
1841
- body = (await readJson(req));
1842
- }
1843
- catch {
1844
- sendJson(res, 400, { error: "invalid JSON body" });
1845
- return;
1846
- }
1847
- if (typeof body.content !== "string" || body.content.length === 0) {
1848
- sendJson(res, 400, { error: "body must be { content: string (non-empty) }" });
1849
- return;
1850
- }
1851
- if (body.content.length > STEER_IN_MAX_REQUEST_CHARS) {
1852
- sendError(res, 413, "steer.content_too_large", `content exceeds ${STEER_IN_MAX_REQUEST_CHARS} characters (got ${body.content.length}); note only the first ${STEER_IN_MAX_CHARS} are delivered anyway`);
1853
- return;
1854
- }
1855
- const trusted = explicitOperatorOk(principal, deps.config.operatorPrincipals);
1856
- const ownerScope = principal ?? "default";
1857
- const wfRun = await getWorkflowRun(deps.workflowRunStore, wfRunId, ownerScope);
1858
- if (!trusted && !wfRun) {
1859
- sendJson(res, 404, { error: "workflow not found" });
1860
- return;
1861
- }
1862
- if (wfRun && !runSessionAcceptOk(req, res, { sessionId: wfRun.originatingSessionId ?? null }, "workflow.agent-steer", "workflow not found"))
1863
- return;
1864
- const sendNotRunningWf = (error) => sendError(res, 409, "steering.not_running", error);
1865
- const redacted = redactSteerIn(body.content, label);
1866
- const resolution = deps.workflowAgentRegistry?.resolve(wfRunId, label);
1867
- if (resolution && resolution.count > 1) {
1868
- sendError(res, 409, "steering.ambiguous_label", `${resolution.count} live agents share label '${label}' in this run — steer target is ambiguous`);
1869
- return;
1870
- }
1871
- const handle = resolution?.handle;
1872
- if (handle) {
1873
- try {
1874
- const marker = await handle.steer(redacted);
1875
- sendJson(res, 200, { runId: wfRunId, label, status: "running", delivery: "applied", marker });
1876
- return;
1877
- }
1878
- catch (e) {
1879
- const code = e.code;
1880
- if (code === "steering.not_running") {
1881
- sendNotRunningWf("agent just finished — no longer accepting steers");
1882
- return;
1883
- }
1884
- throw e;
1885
- }
1886
- }
1887
- sendNotRunningWf(!wfRun
1888
- ? "workflow agent is not running on this replica (no live handle)"
1889
- : wfRun.status === "running"
1890
- ? "workflow agent is active on another replica — cross-replica live-steer is not yet supported"
1891
- : `workflow is ${wfRun.status} — agent is not running`);
1892
- return;
1893
- }
1894
- const subOutputMatch = req.method === "GET" ? RUN_SUBAGENT_OUTPUT_RE.exec(url) : null;
1895
- if (subOutputMatch) {
1896
- if (!deps.runStore || !deps.subagentTaskOutput) {
1897
- sendJson(res, 501, { error: "subagent output reads require the run store" });
1898
- return;
1899
- }
1900
- const principal = gatedPrincipal(req, deps.config);
1901
- if (deps.config.requirePrincipal && principal === undefined) {
1902
- sendJson(res, 401, { error: `missing principal header '${deps.config.principalHeader}'` });
1903
- return;
1904
- }
1905
- const runId = safeDecode(subOutputMatch[1]);
1906
- const target = safeDecode(subOutputMatch[2]);
1907
- if (runId === null || target === null) {
1908
- sendJson(res, 400, { error: "malformed subagent path (invalid percent-encoding)" });
1909
- return;
1910
- }
1911
- const trusted = explicitOperatorOk(principal, deps.config.operatorPrincipals);
1912
- const run = await deps.runStore.getRun(runId);
1913
- if (!run || (!trusted && run.owner !== null && run.owner !== principal)) {
1914
- sendJson(res, 404, { error: "run not found" });
1915
- return;
1916
- }
1917
- const callerSession = new URL(req.url ?? "", "http://x").searchParams.get("session");
1918
- if (!trusted && run.sessionId && callerSession !== run.sessionId) {
1919
- sendJson(res, 404, { error: "run not found" });
1920
- return;
1921
- }
1922
- const out = await deps.subagentTaskOutput(target, { owner: runId, scope: run.owner ?? "default", ...(run.sessionId ? { sessionId: run.sessionId } : {}) });
1923
- const details = out.details;
1924
- if (details?.error === "not_found" || details?.type !== "background_agent") {
1925
- sendJson(res, 404, { error: `no background agent "${target}" under this run (unknown handle, not this run's child, or already reaped — bg children live in the replica-local registry for the parent's lifetime; wa… workflow-agent rows are read via the workflow journal, not this verb)` });
1926
- return;
1927
- }
1928
- sendJson(res, 200, { taskId: runId, target, content: out.content, output: out.details });
1929
- return;
1930
- }
1931
- const subStreamMatch = req.method === "GET" ? RUN_SUBAGENT_STREAM_RE.exec(url) : null;
1932
- if (subStreamMatch) {
1933
- if (!deps.runStore || !deps.subagentTaskOutput) {
1934
- sendJson(res, 501, { error: "subagent streams require the run store" });
1935
- return;
1936
- }
1937
- const principal = gatedPrincipal(req, deps.config);
1938
- if (deps.config.requirePrincipal && principal === undefined) {
1939
- sendJson(res, 401, { error: `missing principal header '${deps.config.principalHeader}'` });
1940
- return;
1941
- }
1942
- const runId = safeDecode(subStreamMatch[1]);
1943
- const target = safeDecode(subStreamMatch[2]);
1944
- if (runId === null || target === null) {
1945
- sendJson(res, 400, { error: "malformed subagent path (invalid percent-encoding)" });
1946
- return;
1947
- }
1948
- const trusted = explicitOperatorOk(principal, deps.config.operatorPrincipals);
1949
- const run = await deps.runStore.getRun(runId);
1950
- if (!run || (!trusted && run.owner !== null && run.owner !== principal)) {
1951
- sendJson(res, 404, { error: "run not found" });
1952
- return;
1953
- }
1954
- const callerSession = new URL(req.url ?? "", "http://x").searchParams.get("session");
1955
- if (!trusted && run.sessionId && callerSession !== run.sessionId) {
1956
- sendJson(res, 404, { error: "run not found" });
1957
- return;
1958
- }
1959
- const it = defaultSubagentTailBus.subscribe(target);
1960
- const probe = await deps.subagentTaskOutput(target, { owner: runId, scope: run.owner ?? "default", ...(run.sessionId ? { sessionId: run.sessionId } : {}) });
1961
- const probeDetails = probe.details;
1962
- if (probeDetails?.error === "not_found" || probeDetails?.type !== "background_agent") {
1963
- void it.return?.();
1964
- sendJson(res, 404, { error: `no background agent "${target}" under this run (unknown handle, not this run's child, or already reaped — bg children live in the replica-local registry for the parent's lifetime; wa… workflow-agent rows are read via the workflow journal, not this verb)` });
1965
- return;
1966
- }
1967
- sseHeaders(res);
1968
- res.write(`event: meta\ndata: ${JSON.stringify({ version: 1, runId, target, status: probeDetails.status ?? "running", ...(typeof probeDetails.seq === "number" ? { seq: probeDetails.seq } : {}), live: "replica-local", replayFace: "GET /v1/runs/:id/subagents/:handle/output" })}\n\n`);
1969
- if (probeDetails.status !== "running" && probeDetails.status !== "pending" && probeDetails.status !== "parked") {
1970
- void it.return?.();
1971
- res.end();
1972
- return;
1973
- }
1974
- let closed = false;
1975
- const hb = setInterval(() => {
1976
- if (!res.writableEnded)
1977
- res.write(`event: heartbeat\ndata: {}\n\n`);
1978
- }, 15_000);
1979
- if (typeof hb.unref === "function")
1980
- hb.unref();
1981
- req.on("close", () => {
1982
- closed = true;
1983
- void it.return?.();
1984
- });
1985
- try {
1986
- for (;;) {
1987
- const n = await it.next();
1988
- if (n.done || closed || res.writableEnded)
1989
- break;
1990
- res.write(`event: forward\ndata: ${JSON.stringify(n.value)}\n\n`);
1991
- if (n.value.type === "task_settled")
1992
- break;
1993
- }
1994
- }
1995
- finally {
1996
- clearInterval(hb);
1997
- void it.return?.();
1998
- if (!res.writableEnded)
1999
- res.end();
2000
- }
2001
- return;
2002
- }
2003
- const taskVerbMatch = req.method === "GET" ? RUN_TASK_OUTPUT_RE.exec(url) : req.method === "POST" ? RUN_TASK_STOP_RE.exec(url) : null;
2004
- if (taskVerbMatch) {
2005
- const stopVerb = req.method === "POST";
2006
- if (!deps.runStore || !deps.taskHandleOutput || !deps.taskHandleStop) {
2007
- sendJson(res, 501, { error: "task-handle verbs require the run store + task-registry seams" });
2008
- return;
2009
- }
2010
- if (stopVerb && rateLimited(req, res))
2011
- return;
2012
- const principal = gatedPrincipal(req, deps.config);
2013
- if (deps.config.requirePrincipal && principal === undefined) {
2014
- sendJson(res, 401, { error: `missing principal header '${deps.config.principalHeader}'` });
2015
- return;
2016
- }
2017
- const runId = safeDecode(taskVerbMatch[1]);
2018
- const target = safeDecode(taskVerbMatch[2]);
2019
- if (runId === null || target === null) {
2020
- sendJson(res, 400, { error: "malformed task path (invalid percent-encoding)" });
2021
- return;
2022
- }
2023
- const trusted = explicitOperatorOk(principal, deps.config.operatorPrincipals);
2024
- const run = await deps.runStore.getRun(runId);
2025
- if (!run || (!trusted && run.owner !== null && run.owner !== principal)) {
2026
- sendJson(res, 404, { error: "run not found" });
2027
- return;
2028
- }
2029
- const q = new URL(req.url ?? "", "http://x").searchParams;
2030
- if (!trusted && run.sessionId && q.get("session") !== run.sessionId) {
2031
- sendJson(res, 404, { error: "run not found" });
2032
- return;
2033
- }
2034
- const access = { owner: runId, scope: run.owner ?? "default", ...(run.sessionId ? { sessionId: run.sessionId } : {}) };
2035
- if (q.get("filter") !== null) {
2036
- sendJson(res, 400, { error: "filter is not accepted on the wire — fetch the output and filter client-side (note: the wire serves the clipped projection)" });
2037
- return;
2038
- }
2039
- const out = stopVerb
2040
- ? await deps.taskHandleStop(target, access)
2041
- : await deps.taskHandleOutput(target, access);
2042
- const details = out.details;
2043
- if (details?.error === "not_found") {
2044
- sendJson(res, 404, { error: `no background task "${target}" under this run (unknown handle, not this run's task, already reaped — handles live in the replica-local registry for the parent's lifetime — or a workflow handle: workflow rows read via GET /v1/workflows/:id/journal; stopping a workflow is not on this wire)` });
2045
- return;
2046
- }
2047
- const parkArbiterUnreachable = details?.error === "park_arbiter_unreachable";
2048
- const parkResumeWon = details?.error === "park_resume_won";
2049
- const stillParked = details?.error === "parked_pending_approval" || details?.status === "parked";
2050
- if (stopVerb && details?.error !== undefined && (details.status === "running" || stillParked || parkResumeWon || parkArbiterUnreachable)) {
2051
- const notLocal = details.error === "not_local";
2052
- sendError(res, 409, notLocal
2053
- ? "stop.not_local"
2054
- : parkArbiterUnreachable
2055
- ? "stop.park_arbiter_unreachable"
2056
- : parkResumeWon
2057
- ? "stop.park_resume_won"
2058
- : stillParked
2059
- ? "stop.parked"
2060
- : "stop.not_landed", notLocal
2061
- ? "this task is not attached to this instance — cannot stop it here (its durable record is still marked running; if its host process is gone, the retention sweep settles it as interrupted)"
2062
- : parkArbiterUnreachable
2063
- ? "the approval arbitration store was unreachable — this task's row stays parked (not stopped); retry the stop"
2064
- : parkResumeWon
2065
- ? "the pending tool-approval was resolved (resumed) concurrently with this stop attempt — the resume won the race, so nothing was stopped; re-check the task's current status"
2066
- : stillParked
2067
- ? "this task is durably parked awaiting a tool-approval decision, not actively running — there is no live process to interrupt; resolve (or let expire) the pending approval instead"
2068
- : `stop did not land (${details.error}) — the process may still be running`, { taskId: runId, target, content: out.content, output: out.details });
2069
- return;
2070
- }
2071
- const g14 = (() => {
2072
- const d = out.details;
2073
- if (!d || d.error !== undefined || d.retrieval_status === "not_ready")
2074
- return {};
2075
- if (d.type === "background_bash")
2076
- return { cursorSemantics: d.details && "bytesDroppedBeforeCursor" in d.details ? "cursor" : "full" };
2077
- if (d.type === "monitor" || d.type === "background_agent")
2078
- return { cursorSemantics: "full" };
2079
- return {};
2080
- })();
2081
- sendJson(res, 200, { taskId: runId, target, content: out.content, output: out.details, ...g14 });
2082
- return;
2083
- }
2084
- const subVerbMatch = req.method === "POST" ? (RUN_SUBAGENT_STEER_RE.exec(url) ?? RUN_SUBAGENT_RESUME_RE.exec(url)) : null;
2085
- if (subVerbMatch) {
2086
- const verb = url.endsWith("/resume") ? "resume" : "steer";
2087
- if (rateLimited(req, res) || quotaExceeded(req, res) || (await leaseDenied(req, res)))
2088
- return;
2089
- if (!deps.runStore) {
2090
- sendJson(res, 501, { error: "async runs require the TiDB run store" });
2091
- return;
2092
- }
2093
- const principal = gatedPrincipal(req, deps.config);
2094
- if (deps.config.requirePrincipal && principal === undefined) {
2095
- sendJson(res, 401, { error: `missing principal header '${deps.config.principalHeader}'` });
2096
- return;
2097
- }
2098
- const runId = safeDecode(subVerbMatch[1]);
2099
- const target = safeDecode(subVerbMatch[2]);
2100
- if (runId === null || target === null) {
2101
- sendJson(res, 400, { error: "malformed subagent path (invalid percent-encoding)" });
2102
- return;
2103
- }
2104
- let body;
2105
- try {
2106
- body = (await readJson(req));
2107
- }
2108
- catch {
2109
- sendJson(res, 400, { error: "invalid JSON body" });
2110
- return;
2111
- }
2112
- if (typeof body.content !== "string" || body.content.length === 0) {
2113
- sendJson(res, 400, { error: "body must be { content: string (non-empty) }" });
2114
- return;
2115
- }
2116
- if (body.content.length > STEER_IN_MAX_REQUEST_CHARS) {
2117
- sendError(res, 413, "steer.content_too_large", `content exceeds ${STEER_IN_MAX_REQUEST_CHARS} characters (got ${body.content.length}); note only the first ${STEER_IN_MAX_CHARS} are delivered anyway`);
2118
- return;
2119
- }
2120
- const trusted = explicitOperatorOk(principal, deps.config.operatorPrincipals);
2121
- const run = await deps.runStore.getRun(runId);
2122
- if (!run || (!trusted && run.owner !== null && run.owner !== principal)) {
2123
- sendJson(res, 404, { error: "run not found" });
2124
- return;
2125
- }
2126
- if (!runSessionAcceptOk(req, res, run, `subagent.${verb}`))
2127
- return;
2128
- const send409 = (errorCode, error) => sendError(res, 409, errorCode, error);
2129
- const redacted = redactSteerIn(body.content, target);
2130
- const resolution = deps.subagentSteerRegistry?.resolve(runId, target);
2131
- if (resolution && resolution.count > 1) {
2132
- send409("steering.ambiguous_target", `${resolution.count} sub-agents match '${target}' in this run — address by parentToolCallId`);
2133
- return;
2134
- }
2135
- const handle = resolution?.handle;
2136
- if (handle) {
2137
- try {
2138
- if (verb === "resume") {
2139
- if (!handle.resume) {
2140
- send409("resume.retain_off", "the parent run did not retain sub-agent sessions (set retainSubagentSessions on the run to enable revival)");
2141
- return;
2142
- }
2143
- const marker = await handle.resume(redacted);
2144
- sendJson(res, 200, { taskId: runId, target, status: "running", delivery: "applied", marker, note: `Message queued for delivery to ${handle.agentName ?? target}; it will continue in the background.` });
2145
- return;
2146
- }
2147
- const marker = await handle.steer(redacted);
2148
- sendJson(res, 200, { taskId: runId, target, status: "running", delivery: "applied", marker, note: `Message queued for delivery to ${handle.agentName ?? target} at its next tool round.` });
2149
- return;
2150
- }
2151
- catch (e) {
2152
- const code = e.code;
2153
- if (code === "steering.not_running") {
2154
- send409(code, "sub-agent just finished — no longer accepting steers");
2155
- return;
2156
- }
2157
- if (code === "steering.still_running") {
2158
- send409(code, "sub-agent (or a prior resume) is still in flight — steer it instead, or wait for it to settle");
2159
- return;
2160
- }
2161
- if (code === "resume.retain_off" || code === "resume.evicted" || code === "resume.cap" || code === "resume.session_not_found") {
2162
- send409(code, e instanceof Error ? e.message : "resume rejected");
2163
- return;
2164
- }
2165
- throw e;
2166
- }
2167
- }
2168
- send409("steering.not_running", run.status === "running"
2169
- ? "no sub-agent matches on this replica (it runs backgrounded, or the run is on another replica)"
2170
- : `run is ${run.status} — its sub-agents are no longer addressable`);
2171
- return;
2172
- }
2173
- const elicitMatch = req.method === "POST" ? ELICIT_RESPOND_RE.exec(url) : null;
2174
- if (elicitMatch) {
2175
- if (rateLimited(req, res))
2176
- return;
2177
- if (!deps.elicitation) {
2178
- sendJson(res, 501, { error: "inbound MCP elicitation is not enabled on this worker (MCP_ELICITATION_ENABLED)" });
2179
- return;
2180
- }
2181
- const principal = gatedPrincipal(req, deps.config);
2182
- if (deps.config.requirePrincipal && principal === undefined) {
2183
- sendJson(res, 401, { error: `missing principal header '${deps.config.principalHeader}'` });
2184
- return;
2185
- }
2186
- const id = elicitMatch[1];
2187
- let body;
2188
- try {
2189
- body = await readJson(req);
2190
- }
2191
- catch {
2192
- sendJson(res, 400, { error: "invalid JSON body" });
2193
- return;
2194
- }
2195
- const { status, body: respBody } = deps.elicitation.respond(id, principal, body);
2196
- sendJson(res, status, respBody);
2197
- return;
2198
- }
2199
- const questionMatch = req.method === "POST" ? QUESTION_RESPOND_RE.exec(url) : null;
2200
- if (questionMatch) {
2201
- if (rateLimited(req, res))
2202
- return;
2203
- if (!deps.question) {
2204
- sendJson(res, 501, { error: "AskUserQuestion live HITL is not enabled on this worker (ASK_QUESTION_ENABLED)" });
2205
- return;
2206
- }
2207
- const principal = gatedPrincipal(req, deps.config);
2208
- if (deps.config.requirePrincipal && principal === undefined) {
2209
- sendJson(res, 401, { error: `missing principal header '${deps.config.principalHeader}'` });
2210
- return;
2211
- }
2212
- const id = questionMatch[1];
2213
- let body;
2214
- try {
2215
- body = await readJson(req);
2216
- }
2217
- catch {
2218
- sendJson(res, 400, { error: "invalid JSON body" });
2219
- return;
2220
- }
2221
- const { status, body: respBody } = deps.question.respond(id, principal, body);
2222
- sendJson(res, status, respBody);
2223
- return;
2224
- }
2225
- const approvalMatch = req.method === "POST" ? TOOL_APPROVAL_RESPOND_RE.exec(url) : null;
2226
- if (approvalMatch) {
2227
- if (rateLimited(req, res))
2228
- return;
2229
- if (!deps.toolApproval) {
2230
- sendJson(res, 501, { error: "live tool-approval HITL is not enabled on this worker (TOOL_APPROVAL_ENABLED)" });
2231
- return;
2232
- }
2233
- const principal = gatedPrincipal(req, deps.config);
2234
- if (deps.config.requirePrincipal && principal === undefined) {
2235
- sendJson(res, 401, { error: `missing principal header '${deps.config.principalHeader}'` });
2236
- return;
2237
- }
2238
- const id = approvalMatch[1];
2239
- let body;
2240
- try {
2241
- body = await readJson(req);
2242
- }
2243
- catch {
2244
- sendJson(res, 400, { error: "invalid JSON body" });
2245
- return;
2246
- }
2247
- const { status, body: respBody } = deps.toolApproval.respond(id, principal, body);
2248
- sendJson(res, status, respBody);
2249
- return;
2250
- }
2251
- if (deps.leaderEndpoint) {
2252
- const isLeaderPost = req.method === "POST" && url === "/v1/leader";
2253
- const isLeaderGet = req.method === "GET" && /^\/v1\/leader\/[^/]+$/.test(url);
2254
- if (isLeaderPost || isLeaderGet) {
2255
- if (deps.config.requirePrincipal && !gatedPrincipal(req, deps.config)) {
2256
- sendJson(res, 401, { error: `missing principal header '${deps.config.principalHeader}'` });
2257
- return;
2258
- }
2259
- const requester = gatedPrincipal(req, deps.config) ?? null;
2260
- if (isLeaderPost) {
2261
- if (rateLimited(req, res) || quotaExceeded(req, res) || (await leaseDenied(req, res)))
2262
- return;
2263
- let body;
2264
- try {
2265
- body = await readJson(req);
2266
- }
2267
- catch {
2268
- sendJson(res, 400, { error: "invalid JSON body" });
2269
- return;
2270
- }
2271
- const r = deps.leaderEndpoint.handle("POST", url, body, requester);
2272
- if (r) {
2273
- sendJson(res, r.status, r.body);
2274
- return;
2275
- }
2276
- }
2277
- else {
2278
- const r = deps.leaderEndpoint.handle("GET", url, undefined, requester);
2279
- if (r) {
2280
- sendJson(res, r.status, r.body);
2281
- return;
2282
- }
2283
- }
2284
- }
2285
- }
2286
- if (await handleImages(req, res, url, ctx))
2287
- return;
2288
- if (deps.checkpointStore && (url.startsWith("/v1/approvals") || url.startsWith("/v1/assistant"))) {
2289
- const cs = deps.checkpointStore;
2290
- const principal = gatedPrincipal(req, deps.config);
2291
- const operator = isOperator(principal, deps.config.operatorPrincipals);
2292
- if (req.method === "GET" && url === "/v1/approvals/stream") {
2293
- const scope = operator
2294
- ? (new URL(req.url ?? "", "http://x").searchParams.get("owner") ?? undefined)
2295
- : (principal ?? "__none__");
2296
- await streamApprovals(req, res, cs, scope);
2297
- return;
2298
- }
2299
- if (req.method === "GET" && url === "/v1/approvals") {
2300
- const scope = operator
2301
- ? (new URL(req.url ?? "", "http://x").searchParams.get("owner") ?? undefined)
2302
- : (principal ?? "__none__");
2303
- sendJson(res, 200, { pending: await cs.listPending(scope) });
2304
- return;
2305
- }
2306
- {
2307
- const em = /^\/v1\/approvals\/([^/]+)\/exemptions(?:\/([^/]+))?$/.exec(url);
2308
- if (em && (req.method === "GET" || req.method === "DELETE")) {
2309
- if (!deps.approvalExemptionStore) {
2310
- sendJson(res, 501, { error: "approval exemptions need a store backend" });
2311
- return;
2312
- }
2313
- let exSessionId;
2314
- let exToolName;
2315
- try {
2316
- exSessionId = decodeURIComponent(em[1]);
2317
- exToolName = em[2] !== undefined ? decodeURIComponent(em[2]) : undefined;
2318
- }
2319
- catch {
2320
- sendJson(res, 400, { error: "malformed id (invalid percent-encoding)" });
2321
- return;
2322
- }
2323
- const operators = deps.config.operatorPrincipals;
2324
- const isExplicitOperator = operators.length > 0 && principal !== undefined && operators.includes(principal);
2325
- if (!isExplicitOperator) {
2326
- const ownerOf = deps.sessionStorage?.ownerOf?.bind(deps.sessionStorage);
2327
- if (!ownerOf) {
2328
- sendJson(res, 501, { error: "approval exemptions require a session-store backend" });
2329
- return;
2330
- }
2331
- const owner = await ownerOf(exSessionId);
2332
- if (owner === undefined || (owner !== null && owner !== principal)) {
2333
- sendJson(res, 404, { error: "not found" });
2334
- return;
2335
- }
2336
- }
2337
- if (req.method === "GET") {
2338
- if (exToolName !== undefined) {
2339
- sendJson(res, 404, { error: "not found" });
2340
- return;
2341
- }
2342
- sendJson(res, 200, { exemptions: await deps.approvalExemptionStore.list(exSessionId) });
2343
- return;
2344
- }
2345
- if (exToolName === undefined) {
2346
- sendJson(res, 400, { error: "DELETE needs /exemptions/:toolName" });
2347
- return;
2348
- }
2349
- const revoked = await deps.approvalExemptionStore.revoke(exSessionId, canonicalToolName(exToolName));
2350
- sendJson(res, revoked ? 200 : 404, revoked ? { revoked: true } : { error: "not found" });
2351
- return;
2352
- }
2353
- }
2354
- if (req.method === "GET" && url === "/v1/assistant/inbox") {
2355
- const ownerParam = new URL(req.url ?? "", "http://x").searchParams.get("owner") || undefined;
2356
- const scope = operator && ownerParam ? ownerParam : (principal ?? "__none__");
2357
- const summaries = await cs.listByScope(scope);
2358
- const inbox = (await Promise.all(summaries.map(async (s) => {
2359
- const ctx = await cs.getCtx(s.sessionId).catch(() => null);
2360
- const { token: _token, toolInput: _rawToolInput, ...safe } = s;
2361
- const input = s.toolInput != null ? redactedPreview(s.toolInput) : null;
2362
- return { ...safe, objective: ctx?.body?.objective ?? null, input };
2363
- }))).sort((a, b) => (b.severity ?? 0) - (a.severity ?? 0));
2364
- sendJson(res, 200, { inbox });
2365
- return;
2366
- }
2367
- if (req.method === "GET" && url === "/v1/assistant/tasks") {
2368
- const ownerParam = new URL(req.url ?? "", "http://x").searchParams.get("owner") || undefined;
2369
- const scope = operator && ownerParam ? ownerParam : (principal ?? "__none__");
2370
- if (!deps.runStore) {
2371
- sendJson(res, 200, { tasks: [] });
2372
- return;
2373
- }
2374
- const summaries = await cs.listByScope(scope);
2375
- const byGate = new Map(summaries.map((s) => [s.sessionId, s]));
2376
- const runs = await deps.runStore.listRuns({ owner: scope, limit: 100 });
2377
- const tasks = runs
2378
- .filter((r) => r.status === "running" || r.status === "suspended" || r.status === "needs_review")
2379
- .map((r) => {
2380
- const s = byGate.get(r.sessionId);
2381
- const gate = s ? { kind: s.gateKind, severity: s.severity ?? null, spentMicroUsd: s.spentMicroUsd ?? null, deadline: s.deadline ?? null } : null;
2382
- return { taskId: r.taskId, sessionId: r.sessionId, status: r.status, needsAttention: !!gate, gate, createdAt: r.createdAt, updatedAt: r.updatedAt };
2383
- })
2384
- .sort((a, b) => Number(b.needsAttention) - Number(a.needsAttention) || (b.gate?.severity ?? 0) - (a.gate?.severity ?? 0) || (b.gate?.spentMicroUsd ?? 0) - (a.gate?.spentMicroUsd ?? 0));
2385
- sendJson(res, 200, { tasks });
2386
- return;
2387
- }
2388
- const preemptMatch = req.method === "POST" ? ASSISTANT_PREEMPT_RE.exec(url) : null;
2389
- if (preemptMatch) {
2390
- if (rateLimited(req, res) || quotaExceeded(req, res))
2391
- return;
2392
- if (!deps.runStore) {
2393
- sendJson(res, 501, { error: "preemption requires the TiDB run store" });
2394
- return;
2395
- }
2396
- if (deps.config.requirePrincipal && principal === undefined) {
2397
- sendJson(res, 401, { error: `missing principal header '${deps.config.principalHeader}'` });
2398
- return;
2399
- }
2400
- if (!deps.config.resourceSuspend || !deps.checkpointStore) {
2401
- sendJson(res, 501, { error: "preemption not enabled on this worker (needs RESOURCE_SUSPEND=true + a durable checkpoint store)" });
2402
- return;
2403
- }
2404
- const taskId = preemptMatch[1];
2405
- const run = await deps.runStore.getRun(taskId);
2406
- if (!run) {
2407
- sendJson(res, 404, { error: "task not found" });
2408
- return;
2409
- }
2410
- const operators = deps.config.operatorPrincipals;
2411
- const explicitOperator = operators.length > 0 && principal !== undefined && operators.includes(principal);
2412
- if (!explicitOperator && run.owner !== null && run.owner !== principal) {
2413
- sendJson(res, 404, { error: "task not found" });
2414
- return;
2415
- }
2416
- if (run.status === "running") {
2417
- const ownedHere = run.instanceId != null && run.instanceId === deps.instanceId;
2418
- if (ownedHere && !preemptableRuns.has(taskId)) {
2419
- sendJson(res, 202, { taskId, status: run.status, note: "task is not preempt-eligible (verify/cascade leg) — preempt is a no-op" });
2420
- return;
2421
- }
2422
- const flagged = await deps.runStore.requestPreempt(taskId, run.owner);
2423
- if (!flagged) {
2424
- const now = await deps.runStore.getRun(taskId);
2425
- sendJson(res, 202, { taskId, status: now?.status ?? "failed", note: "task no longer running — preempt is a no-op" });
2426
- return;
2427
- }
2428
- preemptableRuns.get(taskId)?.abort();
2429
- sendJson(res, 202, { taskId, status: "preempting", note: "graceful durable yield — the task suspends at the next clean turn boundary if preempt-eligible; resume via POST /v1/assistant/tasks/:id/resume" });
2430
- }
2431
- else if (run.status === "suspended") {
2432
- sendJson(res, 409, { taskId, status: "suspended", error: "task is already suspended" });
2433
- }
2434
- else {
2435
- sendJson(res, 202, { taskId, status: run.status, note: "task already terminal — preempt is a no-op" });
2436
- }
2437
- return;
2438
- }
2439
- const resumeMatch = req.method === "POST" ? ASSISTANT_RESUME_RE.exec(url) : null;
2440
- if (resumeMatch) {
2441
- if (rateLimited(req, res) || quotaExceeded(req, res))
2442
- return;
2443
- if (!deps.runStore) {
2444
- sendJson(res, 501, { error: "resume requires the TiDB run store" });
2445
- return;
2446
- }
2447
- if (deps.config.requirePrincipal && principal === undefined) {
2448
- sendJson(res, 401, { error: `missing principal header '${deps.config.principalHeader}'` });
2449
- return;
2450
- }
2451
- const taskId = resumeMatch[1];
2452
- const run = await deps.runStore.getRun(taskId);
2453
- if (!run) {
2454
- sendJson(res, 404, { error: "task not found" });
2455
- return;
2456
- }
2457
- const operators = deps.config.operatorPrincipals;
2458
- const explicitOperator = operators.length > 0 && principal !== undefined && operators.includes(principal);
2459
- if (!explicitOperator && run.owner !== null && run.owner !== principal) {
2460
- sendJson(res, 404, { error: "task not found" });
2461
- return;
2462
- }
2463
- const out = await resumePreempted(run.sessionId, req);
2464
- sendJson(res, out.status, out.body);
2465
- return;
2466
- }
2467
- const planReviewMatch = req.method === "POST" ? ASSISTANT_PLAN_REVIEW_RE.exec(url) : null;
2468
- if (planReviewMatch) {
2469
- if (rateLimited(req, res) || quotaExceeded(req, res))
2470
- return;
2471
- if (!deps.runStore) {
2472
- sendJson(res, 501, { error: "plan_review requires the TiDB run store" });
2473
- return;
2474
- }
2475
- if (deps.config.requirePrincipal && principal === undefined) {
2476
- sendJson(res, 401, { error: `missing principal header '${deps.config.principalHeader}'` });
2477
- return;
2478
- }
2479
- const taskId = planReviewMatch[1];
2480
- const body = (await readJson(req));
2481
- const decision = body?.decision === "approve" ? "approve" : body?.decision === "edit" ? "edit" : body?.decision === "reject" ? "reject" : undefined;
2482
- if (!decision) {
2483
- sendJson(res, 400, { error: "body must be { decision: 'approve' | 'edit' | 'reject', editedPlan?, reason? }" });
2484
- return;
2485
- }
2486
- if (decision === "edit" && (typeof body.editedPlan !== "string" || body.editedPlan.length === 0)) {
2487
- sendJson(res, 400, { error: "decision 'edit' requires a non-empty editedPlan (the operator's revised plan)" });
2488
- return;
2489
- }
2490
- if (decision !== "edit" && body.editedPlan !== undefined) {
2491
- sendJson(res, 400, { error: `editedPlan is only valid with decision 'edit' (got '${decision}')` });
2492
- return;
2493
- }
2494
- if (body.reason !== undefined && typeof body.reason !== "string") {
2495
- sendJson(res, 400, { error: "reason must be a string" });
2496
- return;
2497
- }
2498
- if (typeof body.reason === "string" && body.reason.length > MAX_APPROVAL_REASON_CHARS) {
2499
- sendError(res, 413, "reason_too_large", `reason too large (max ${MAX_APPROVAL_REASON_CHARS} chars)`);
2500
- return;
2501
- }
2502
- const run = await deps.runStore.getRun(taskId);
2503
- if (!run) {
2504
- if (deps.config.directDoorActive) {
2505
- sendError(res, 401, "principal_unverified", "principal proof required");
2506
- return;
2507
- }
2508
- sendJson(res, 404, { error: "task not found" });
2509
- return;
2510
- }
2511
- let deciderPrincipal = principal;
2512
- if (deps.config.directDoorActive) {
2513
- const hdr = (n) => { const h = req.headers[n]; return Array.isArray(h) ? h[0] : h; };
2514
- const proof = verifyDirectDoorProof({ jwt: hdr("x-approval-principal-token"), mac: hdr("x-approval-mac"), kid: hdr("x-approval-mac-kid") }, { sessionId: run.sessionId, decision, reason: typeof body.reason === "string" ? body.reason : null }, deps.config, { actionBinding: false });
2515
- if (!proof.ok) {
2516
- sendError(res, proof.status, proof.errorCode, proof.error);
2517
- return;
2518
- }
2519
- deciderPrincipal = proof.principal;
2520
- }
2521
- const operators = deps.config.operatorPrincipals;
2522
- const explicitOperator = operators.length > 0 && deciderPrincipal !== undefined && operators.includes(deciderPrincipal);
2523
- if (!explicitOperator && run.owner !== null && run.owner !== deciderPrincipal) {
2524
- sendJson(res, 404, { error: "task not found" });
2525
- return;
2526
- }
2527
- const out = await resumePlanReview(run.sessionId, decision, decision === "edit" ? body.editedPlan : undefined, typeof body.reason === "string" ? body.reason : undefined, req);
2528
- sendJson(res, out.status, out.body);
249
+ ...(() => {
250
+ const stuck = deps.planeDeferredState?.();
251
+ return stuck ? { modelPlaneDeferred: { version: stuck.version, since: stuck.since, noHandoff: true, ...(stuck.blocked ? { blockedReasons: stuck.blocked } : {}) } } : {};
252
+ })(),
253
+ });
254
+ return;
255
+ }
256
+ if (req.method === "GET" && (url === "/metrics" || url === "/metrics/summary" || url === "/metrics/plan-cache")) {
257
+ if (!deps.metrics) {
258
+ sendError(res, 404, "feature.metrics_disabled", "metrics disabled");
2529
259
  return;
2530
260
  }
2531
- const m = /^\/v1\/approvals\/([^/]+?)(?:\/decide)?$/.exec(url);
2532
- if (m && req.method === "POST") {
2533
- if (rateLimited(req, res) || quotaExceeded(req, res))
2534
- return;
2535
- let sessionId;
2536
- try {
2537
- sessionId = decodeURIComponent(m[1]);
2538
- }
2539
- catch {
2540
- sendJson(res, 400, { error: "malformed approval id (invalid percent-encoding)" });
2541
- return;
2542
- }
2543
- const body = (await readJson(req));
2544
- const decision = body?.decision === "approve" ? "approve" : body?.decision === "deny" ? "deny" : undefined;
2545
- if (!decision) {
2546
- sendJson(res, 400, { error: "body must be { decision: 'approve' | 'deny', reason?, answer?, checkpointToken?, boundCallId?, boundInputHash?, updatedInput?, remember? }" });
2547
- return;
2548
- }
2549
- if (body.reason !== undefined && typeof body.reason !== "string") {
2550
- sendJson(res, 400, { error: "reason must be a string" });
2551
- return;
2552
- }
2553
- if (typeof body.reason === "string" && body.reason.length > MAX_APPROVAL_REASON_CHARS) {
2554
- sendError(res, 413, "reason_too_large", `reason too large (max ${MAX_APPROVAL_REASON_CHARS} chars)`);
2555
- return;
2556
- }
2557
- let remember = false;
2558
- if (body.remember !== undefined) {
2559
- if (body.remember !== "session") {
2560
- sendJson(res, 400, { error: `remember must be "session"` });
2561
- return;
2562
- }
2563
- if (decision !== "approve") {
2564
- sendJson(res, 400, { error: "remember requires decision 'approve'" });
2565
- return;
2566
- }
2567
- if (deps.config.directDoorActive) {
2568
- sendError(res, 400, "remember_not_in_proof", "remember is not supported on a direct-door worker (not covered by the decision proof)");
2569
- return;
2570
- }
2571
- if (!deps.approvalExemptionStore) {
2572
- sendJson(res, 501, { error: "approval exemptions need a store backend (DB_BACKEND / LOCAL lane)" });
2573
- return;
2574
- }
2575
- if (!body.checkpointToken && !body.boundCallId) {
2576
- sendError(res, 400, "remember_requires_binding", "remember requires the decision binding (echo checkpointToken and/or boundCallId from the pending approval)");
2577
- return;
2578
- }
2579
- remember = true;
2580
- }
2581
- let answer;
2582
- if (body.answer !== undefined) {
2583
- if (!isQuestionAnswer(body.answer)) {
2584
- sendJson(res, 400, { error: "malformed answer — expected { answers: [{ header, selected: string[], note? }] }" });
2585
- return;
2586
- }
2587
- answer = body.answer;
2588
- }
2589
- for (const f of ["checkpointToken", "boundCallId", "boundInputHash"]) {
2590
- if (body[f] !== undefined && typeof body[f] !== "string") {
2591
- sendJson(res, 400, { error: `${f} must be a string (the value surfaced on the pending approval record)` });
2592
- return;
2593
- }
2594
- }
2595
- if (deps.config.directDoorActive && body.updatedInput !== undefined) {
2596
- sendError(res, 400, "updated_input_not_in_proof", "updatedInput is not supported on a direct-door worker (not covered by the decision proof)");
2597
- return;
2598
- }
2599
- const binding = {
2600
- checkpointToken: body.checkpointToken,
2601
- boundCallId: body.boundCallId,
2602
- boundInputHash: body.boundInputHash,
2603
- updatedInput: body.updatedInput,
2604
- };
2605
- let deciderPrincipal = principal;
2606
- if (deps.config.directDoorActive) {
2607
- const hdr = (n) => {
2608
- const h = req.headers[n];
2609
- return Array.isArray(h) ? h[0] : h;
2610
- };
2611
- const proof = verifyDirectDoorProof({ jwt: hdr("x-approval-principal-token"), mac: hdr("x-approval-mac"), kid: hdr("x-approval-mac-kid") }, { sessionId, boundCallId: binding.boundCallId, boundInputHash: binding.boundInputHash, decision, reason: typeof body.reason === "string" ? body.reason : null }, deps.config);
2612
- if (!proof.ok) {
2613
- sendError(res, proof.status, proof.errorCode, proof.error);
2614
- return;
2615
- }
2616
- deciderPrincipal = proof.principal;
2617
- }
2618
- const operators = deps.config.operatorPrincipals;
2619
- const explicitOperator = operators.length > 0 && deciderPrincipal !== undefined && operators.includes(deciderPrincipal);
2620
- if (!explicitOperator) {
2621
- const cpScope = await cs.peekPendingScope(sessionId);
2622
- const ownsIt = cpScope == null || cpScope === "_" || cpScope === deciderPrincipal;
2623
- if (!ownsIt) {
2624
- sendJson(res, 404, { error: "approval not found" });
2625
- return;
2626
- }
2627
- }
2628
- let rememberToolName = null;
2629
- if (remember) {
2630
- const tok = await cs.findPendingTokenBySession(sessionId);
2631
- const cp0 = tok ? await cs.get(tok) : null;
2632
- const pa0 = cp0?.pendingAction;
2633
- const tn = pa0 && pa0.kind === "tool_approval" ? pa0.toolName : null;
2634
- rememberToolName = tn && tn !== "AskUserQuestion" ? tn : null;
2635
- }
2636
- let rememberApplied = false;
2637
- const grantOnCommit = remember && deps.approvalExemptionStore && rememberToolName
2638
- ? async (overrideSessionId) => {
2639
- const grantSessionId = overrideSessionId ?? sessionId;
2640
- try {
2641
- const canonical = canonicalToolName(rememberToolName);
2642
- await deps.approvalExemptionStore.grant(grantSessionId, canonical, deciderPrincipal ?? null);
2643
- rememberApplied = true;
2644
- deps.logger?.info?.("approval_exemption_granted", { sessionId: grantSessionId, toolName: canonical, grantedBy: deciderPrincipal ?? null });
2645
- }
2646
- catch (e) {
2647
- deps.logger?.warn?.("approval_exemption_grant_failed", { sessionId: grantSessionId, toolName: rememberToolName, error: String(e).slice(0, 200) });
2648
- }
2649
- }
2650
- : undefined;
2651
- const out = await resumeCheckpoint(sessionId, decision, body.reason ?? undefined, req, answer, binding, grantOnCommit);
2652
- if (remember && deps.approvalExemptionStore) {
2653
- sendJson(res, out.status, { ...out.body, rememberApplied });
2654
- return;
2655
- }
2656
- sendJson(res, out.status, out.body);
261
+ const { authToken: t, metricsToken: mt } = deps.config;
262
+ const anyT = Boolean(t) || Object.keys(deps.config.authTokens ?? {}).length > 0;
263
+ const ok = (!anyT && !mt) || (anyT && systemFor(req, deps.config) !== undefined) || (!!mt && authorized(req, mt));
264
+ if (!ok) {
265
+ sendError(res, 401, "auth.unauthorized", "unauthorized");
2657
266
  return;
2658
267
  }
2659
- sendJson(res, 404, { error: "not found" });
2660
- return;
2661
- }
2662
- if (deps.approvalStore && url.startsWith("/v1/approvals")) {
2663
- const store = deps.approvalStore;
2664
- if (deps.config.directApprovalDoor) {
2665
- sendError(res, 503, "approval_door_unconfigured", "approvals are served by the direct door on this worker");
268
+ if (url === "/metrics/summary") {
269
+ sendJson(res, 200, { model: deps.config.model.id, ...deps.metrics.summarize() });
2666
270
  return;
2667
271
  }
2668
- if (req.method === "GET" && url === "/v1/approvals") {
2669
- const principal = principalFrom(req, deps.config);
2670
- if (isOperator(principal, deps.config.operatorPrincipals)) {
2671
- const owner = new URL(req.url ?? "", "http://x").searchParams.get("owner") ?? undefined;
2672
- sendJson(res, 200, { pending: owner ? await store.listPending(owner, owner) : await store.listPendingAll() });
2673
- }
2674
- else {
2675
- sendJson(res, 200, { pending: principal ? await store.listPending(principal, principal) : [] });
2676
- }
272
+ if (url === "/metrics/plan-cache") {
273
+ sendJson(res, 200, { scopes: deps.planCacheProbe?.dump() ?? {} });
2677
274
  return;
2678
275
  }
2679
- const m = /^\/v1\/approvals\/([^/]+)$/.exec(url);
2680
- if (m) {
2681
- const id = m[1];
2682
- if (req.method === "GET") {
2683
- const principal = principalFrom(req, deps.config);
2684
- if (deps.config.requirePrincipal && !principal) {
2685
- sendJson(res, 401, { error: `missing principal header '${deps.config.principalHeader}'` });
2686
- return;
2687
- }
2688
- const operator = isOperator(principal, deps.config.operatorPrincipals);
2689
- const row = operator ? await store.getById(id) : principal != null ? await store.get(id, principal) : undefined;
2690
- if (!row || (!operator && row.owner !== null && row.owner !== principal)) {
2691
- sendJson(res, 404, { error: "approval not found" });
2692
- return;
2693
- }
2694
- sendJson(res, 200, row);
2695
- return;
276
+ res.writeHead(200, { "content-type": "text/plain; version=0.0.4" });
277
+ res.end(deps.metrics.render());
278
+ return;
279
+ }
280
+ if (deps.registryJwtVerifier) {
281
+ const authz = req.headers.authorization ?? "";
282
+ const rawBearer = /^Bearer\s/i.test(authz) ? authz.replace(/^Bearer\s+/i, "") : "";
283
+ if (rawBearer && looksLikeJwt(rawBearer)) {
284
+ const v = await deps.registryJwtVerifier.verify(rawBearer);
285
+ if (v.ok) {
286
+ setSsoPrincipal(req, v.identity.principal);
287
+ req.headers[deps.config.principalHeader.toLowerCase()] = v.identity.principal;
288
+ if (v.identity.scope)
289
+ setSsoScope(req, v.identity.scope);
290
+ deps.logger?.info?.("auth_bridge_principal_accepted", { principal: v.identity.principal, ...(v.identity.scope ? { scope: v.identity.scope } : {}) });
2696
291
  }
2697
- if (req.method === "POST") {
2698
- const principal = principalFrom(req, deps.config);
2699
- const operatorOk = deps.config.requirePrincipal
2700
- ? explicitOperatorOk(gatedPrincipal(req, deps.config), deps.config.operatorPrincipals)
2701
- : isOperator(principal, deps.config.operatorPrincipals);
2702
- if (!operatorOk) {
2703
- sendJson(res, 403, { error: "only an operator may decide an approval (set OPERATOR_PRINCIPALS)" });
2704
- return;
2705
- }
2706
- const body = (await readJson(req));
2707
- const decision = body?.decision === "approve" ? "approved" : body?.decision === "deny" ? "denied" : undefined;
2708
- if (!decision) {
2709
- sendJson(res, 400, { error: "body must be { decision: 'approve' | 'deny', reason? }" });
2710
- return;
2711
- }
2712
- const target = await store.getById(id);
2713
- if (!target) {
2714
- sendJson(res, 409, { error: "approval already decided or not found" });
2715
- return;
2716
- }
2717
- const applied = await store.decide(id, target.scope, decision, body.reason ?? null, principal ?? null);
2718
- if (applied)
2719
- sendJson(res, 200, { id, status: decision });
2720
- else
2721
- sendJson(res, 409, { error: "approval already decided or not found" });
292
+ }
293
+ }
294
+ if (await handleTraceUsage(req, res, url, ctx))
295
+ return;
296
+ if (await handleWorkflows(req, res, url, ctx))
297
+ return;
298
+ if (await handleFleet(req, res, url, ctx))
299
+ return;
300
+ let source = null;
301
+ const anyServiceAuth = Boolean(deps.config.authToken) || Object.keys(deps.config.authTokens ?? {}).length > 0;
302
+ if (anyServiceAuth) {
303
+ const sys = systemFor(req, deps.config);
304
+ if (sys === undefined) {
305
+ if (ssoVerifiedPrincipal(req)) {
306
+ source = "sso";
307
+ }
308
+ else {
309
+ sendError(res, 401, "auth.unauthorized", "unauthorized");
2722
310
  return;
2723
311
  }
2724
312
  }
2725
- sendJson(res, 404, { error: "not found" });
313
+ else {
314
+ source = sys;
315
+ }
316
+ }
317
+ reqState.source = source;
318
+ if (req.method === "POST" &&
319
+ !anyServiceAuth &&
320
+ !deps.config.allowUnauthedWrites &&
321
+ (isBillableSubmitPath(url) || url.startsWith("/v1/images/bakes"))) {
322
+ sendError(res, 503, "auth.service_token_required", "this worker requires a service auth token (set SERVICE_AUTH_TOKEN) before accepting task submissions");
323
+ return;
324
+ }
325
+ if (deps.drainState?.draining && req.method === "POST" && isBillableSubmitPath(url)) {
326
+ res.setHeader("retry-after", "15");
327
+ sendError(res, 503, "draining", "draining", { message: "this instance is draining for shutdown/upgrade — retry against the replacement instance" });
328
+ return;
329
+ }
330
+ if (deps.modelReady && !deps.modelReady() && req.method === "POST" && isBillableSubmitPath(url)) {
331
+ res.setHeader("retry-after", "5");
332
+ sendError(res, 503, "state.model_roster_pending", "model_roster_pending", { message: "this worker has no model yet (waiting for the first effective-config pull to land the roster) — retry shortly" });
333
+ return;
334
+ }
335
+ if (req.method === "PUT" &&
336
+ !anyServiceAuth &&
337
+ !deps.config.allowUnauthedWrites &&
338
+ /^\/v1\/sessions\/[^/]+\/policy$/.test(url)) {
339
+ sendError(res, 503, "auth.service_token_required", "this worker requires a service auth token (set SERVICE_AUTH_TOKEN) before accepting session-policy writes");
2726
340
  return;
2727
341
  }
342
+ if (await handleCapabilities(req, res, url, ctx))
343
+ return;
344
+ if (await handleSideQuery(req, res, url, ctx))
345
+ return;
346
+ if (await handleTasks(req, res, url, ctx))
347
+ return;
348
+ if (await handleRuns(req, res, url, ctx))
349
+ return;
350
+ if (await handleWorkflowAgentSteer(req, res, url, ctx))
351
+ return;
352
+ if (await handleRunVerbs(req, res, url, ctx))
353
+ return;
354
+ if (await handleLeader(req, res, url, ctx))
355
+ return;
356
+ if (await handleImages(req, res, url, ctx))
357
+ return;
358
+ if (await handleApprovalsAssistant(req, res, url, ctx))
359
+ return;
2728
360
  if (await handleObservability(req, res, url, ctx))
2729
361
  return;
2730
362
  if (await handleMemoryPolicy(req, res, url, ctx))
@@ -2737,193 +369,78 @@ export function createHttpServer(rawDeps) {
2737
369
  return;
2738
370
  if (await handleAttachments(req, res, url, ctx))
2739
371
  return;
2740
- if (req.method === "POST" && /^\/v1\/sessions\/[^/]+\/notify$/.test(url)) {
2741
- if (rateLimited(req, res) || quotaExceeded(req, res))
2742
- return;
2743
- const principal = gatedPrincipal(req, deps.config);
2744
- if (deps.config.requirePrincipal && principal === undefined) {
2745
- sendJson(res, 401, { error: `missing principal header '${deps.config.principalHeader}'` });
2746
- return;
2747
- }
2748
- const notifySession = decodeURIComponent(url.split("/")[3]);
2749
- let nb;
2750
- try {
2751
- nb = (await readJson(req));
2752
- }
2753
- catch {
2754
- sendJson(res, 400, { error: "body must be JSON: { task_id, status, summary, result?, seq?, source? }" });
2755
- return;
2756
- }
2757
- if (typeof nb.task_id !== "string" || nb.task_id.trim().length === 0 || nb.task_id.length > 190) {
2758
- sendJson(res, 400, { error: "task_id must be a non-empty string of at most 190 characters" });
2759
- return;
2760
- }
2761
- const NOTIFY_STATUSES = ["completed", "failed", "killed", "cancelled", "event"];
2762
- if (typeof nb.status !== "string" || !NOTIFY_STATUSES.includes(nb.status)) {
2763
- sendJson(res, 400, { error: `status must be one of ${NOTIFY_STATUSES.join("/")}` });
2764
- return;
2765
- }
2766
- if (typeof nb.summary !== "string" || nb.summary.trim().length === 0) {
2767
- sendJson(res, 400, { error: "summary must be a non-empty string" });
2768
- return;
2769
- }
2770
- if (nb.result !== undefined && typeof nb.result !== "string") {
2771
- sendJson(res, 400, { error: "result must be a string when present" });
2772
- return;
2773
- }
2774
- if (nb.seq !== undefined && (typeof nb.seq !== "number" || !Number.isInteger(nb.seq) || nb.seq < 1)) {
2775
- sendJson(res, 400, { error: "seq must be a positive integer when present" });
2776
- return;
2777
- }
2778
- if (nb.source !== undefined && (typeof nb.source !== "string" || nb.source.length > 190)) {
2779
- sendJson(res, 400, { error: "source must be a string of at most 190 characters when present" });
2780
- return;
2781
- }
2782
- if (!deps.sessionStorage?.ownerOf) {
2783
- sendJson(res, 501, { error: "external notify requires the session-ownership face (sessionStorage.ownerOf)" });
2784
- return;
2785
- }
2786
- const notifyOwner = await deps.sessionStorage.ownerOf(notifySession);
2787
- if (notifyOwner === undefined) {
2788
- sendJson(res, 404, { error: "session not found" });
2789
- return;
2790
- }
2791
- const notifyOperator = explicitOperatorOk(principal, deps.config.operatorPrincipals);
2792
- if (!notifyOperator && notifyOwner !== null && notifyOwner !== principal) {
2793
- sendJson(res, 404, { error: "session not found" });
2794
- return;
2795
- }
2796
- const payload = {
2797
- task_id: nb.task_id,
2798
- status: nb.status,
2799
- summary: nb.summary,
2800
- ...(nb.result !== undefined ? { result: nb.result } : {}),
2801
- ...(nb.seq !== undefined ? { seq: nb.seq } : {}),
2802
- ...(nb.source !== undefined ? { source: nb.source } : {}),
2803
- };
2804
- const liveTaskId = await deps.runStore?.getActiveTaskId?.(notifySession).catch(() => undefined);
2805
- const liveStream = liveTaskId !== undefined && liveTaskId !== null ? steerableRuns.get(liveTaskId) : undefined;
2806
- if (liveStream) {
2807
- try {
2808
- await liveStream.notify(payload);
2809
- sendJson(res, 200, { sessionId: notifySession, delivery: "live" });
2810
- return;
2811
- }
2812
- catch (e) {
2813
- const code = e.code;
2814
- if (typeof code === "string" && code.startsWith("notify.")) {
2815
- sendError(res, 400, code, e instanceof Error ? e.message : "invalid notification");
2816
- return;
2817
- }
2818
- }
2819
- }
2820
- if (!deps.workflowCompletionInbox) {
2821
- sendJson(res, 501, { error: "external notify park requires the workflow completion inbox (WORKFLOW_RUN_STORE)" });
2822
- return;
2823
- }
2824
- await deps.workflowCompletionInbox.enqueue(taskNotificationInboxEntry(notifySession, notifyOwner ?? principal ?? null, { task_id: payload.task_id, task_type: "external", status: payload.status, summary: payload.summary, ...(payload.result !== undefined ? { result: payload.result } : {}), ...(payload.seq !== undefined ? { seq: payload.seq } : {}), ...(payload.source !== undefined ? { source: payload.source } : {}) }, Date.now()));
2825
- sendJson(res, 202, { sessionId: notifySession, delivery: "parked", note: "no live stream — parked in the session inbox; drained as a task_notification on the next stream open" });
2826
- return;
2827
- }
2828
- if (req.method === "POST" && /^\/v1\/sessions\/[^/]+\/wake$/.test(url)) {
2829
- if (!deps.checkpointStore) {
2830
- sendJson(res, 501, { error: "wake requires the checkpoint store" });
2831
- return;
2832
- }
2833
- if (rateLimited(req, res) || quotaExceeded(req, res) || (await leaseDenied(req, res)))
2834
- return;
2835
- const principal = gatedPrincipal(req, deps.config);
2836
- if (deps.config.requirePrincipal && principal === undefined) {
2837
- sendJson(res, 401, { error: `missing principal header '${deps.config.principalHeader}'` });
2838
- return;
2839
- }
2840
- const wakeSession = decodeURIComponent(url.split("/")[3]);
2841
- let wakeBody;
2842
- try {
2843
- wakeBody = (await readJson(req));
2844
- }
2845
- catch {
2846
- sendJson(res, 400, { error: "body must be JSON: { message?: string }" });
2847
- return;
2848
- }
2849
- if (wakeBody.message !== undefined && (typeof wakeBody.message !== "string" || wakeBody.message.length === 0)) {
2850
- sendJson(res, 400, { error: "message must be a non-empty string when present" });
2851
- return;
2852
- }
2853
- const out = await resumeWake(wakeSession, wakeBody.message, principal, req);
2854
- sendJson(res, out.status, out.body);
372
+ if (await handleNotifyWake(req, res, url, ctx))
2855
373
  return;
2856
- }
2857
- sendJson(res, 404, { error: "not found" });
374
+ sendError(res, 404, "not_found.route", "not found");
2858
375
  }
2859
376
  async function prepareSpec(req, res) {
2860
377
  const body = (await readJson(req));
2861
378
  if (!body || typeof body.objective !== "string") {
2862
- sendJson(res, 400, { error: "missing 'objective' string" });
379
+ sendError(res, 400, "request.field_invalid", "missing 'objective' string");
2863
380
  return null;
2864
381
  }
2865
382
  if (body.objective.trim().length === 0) {
2866
- sendJson(res, 400, { error: "objective must not be empty or whitespace-only (an empty user message poisons the session history on strict providers)" });
383
+ sendError(res, 400, "request.field_invalid", "objective must not be empty or whitespace-only (an empty user message poisons the session history on strict providers)");
2867
384
  return null;
2868
385
  }
2869
386
  if (body.jobId !== undefined && (typeof body.jobId !== "string" || body.jobId.length === 0 || body.jobId.length > 64)) {
2870
- sendJson(res, 400, { error: "jobId must be a non-empty string of at most 64 characters" });
387
+ sendError(res, 400, "request.field_invalid", "jobId must be a non-empty string of at most 64 characters");
2871
388
  return null;
2872
389
  }
2873
390
  if (typeof body.sessionId === "string" && body.sessionId.length > 64) {
2874
- sendJson(res, 400, { error: "sessionId must be at most 64 characters" });
391
+ sendError(res, 400, "request.field_invalid", "sessionId must be at most 64 characters");
2875
392
  return null;
2876
393
  }
2877
394
  if (typeof body.systemPrompt === "string" && body.systemPrompt.length > MAX_SYSTEM_PROMPT_CHARS) {
2878
- sendJson(res, 400, { error: `systemPrompt must be at most ${MAX_SYSTEM_PROMPT_CHARS} characters` });
395
+ sendError(res, 400, "request.field_invalid", `systemPrompt must be at most ${MAX_SYSTEM_PROMPT_CHARS} characters`);
2879
396
  return null;
2880
397
  }
2881
398
  if (body.appendSystemPrompt !== undefined) {
2882
399
  if (typeof body.appendSystemPrompt !== "string" || body.appendSystemPrompt.length === 0) {
2883
- sendJson(res, 400, { error: "appendSystemPrompt must be a non-empty string" });
400
+ sendError(res, 400, "request.field_invalid", "appendSystemPrompt must be a non-empty string");
2884
401
  return null;
2885
402
  }
2886
403
  if (body.appendSystemPrompt.length > MAX_SYSTEM_PROMPT_CHARS) {
2887
- sendJson(res, 400, { error: `appendSystemPrompt must be at most ${MAX_SYSTEM_PROMPT_CHARS} characters` });
404
+ sendError(res, 400, "request.field_invalid", `appendSystemPrompt must be at most ${MAX_SYSTEM_PROMPT_CHARS} characters`);
2888
405
  return null;
2889
406
  }
2890
407
  }
2891
408
  if (body.cwd !== undefined && !isValidCwd(body.cwd)) {
2892
- sendJson(res, 400, { error: "cwd must be a non-empty absolute host path" });
409
+ sendError(res, 400, "request.field_invalid", "cwd must be a non-empty absolute host path");
2893
410
  return null;
2894
411
  }
2895
412
  const addDirsRaw = body.additionalDirectories;
2896
413
  if (addDirsRaw !== undefined) {
2897
414
  if (!Array.isArray(addDirsRaw)) {
2898
- sendJson(res, 400, { error: "additionalDirectories must be an array of absolute host paths" });
415
+ sendError(res, 400, "request.field_invalid", "additionalDirectories must be an array of absolute host paths");
2899
416
  return null;
2900
417
  }
2901
418
  if (addDirsRaw.length > MAX_ADDITIONAL_DIRS) {
2902
- sendJson(res, 400, { error: `additionalDirectories must have at most ${MAX_ADDITIONAL_DIRS} entries` });
419
+ sendError(res, 400, "request.field_invalid", `additionalDirectories must have at most ${MAX_ADDITIONAL_DIRS} entries`);
2903
420
  return null;
2904
421
  }
2905
422
  const bad = addDirsRaw.find((d) => !isValidCwd(d));
2906
423
  if (bad !== undefined) {
2907
- sendJson(res, 400, { error: "each additionalDirectories entry must be a non-empty absolute host path (no '..' segments)" });
424
+ sendError(res, 400, "request.field_invalid", "each additionalDirectories entry must be a non-empty absolute host path (no '..' segments)");
2908
425
  return null;
2909
426
  }
2910
427
  }
2911
428
  const settingsRaw = body.settings;
2912
429
  if (settingsRaw !== undefined && settingsRaw !== null) {
2913
430
  if (typeof settingsRaw !== "object" || Array.isArray(settingsRaw)) {
2914
- sendJson(res, 400, { error: "settings must be an object" });
431
+ sendError(res, 400, "request.field_invalid", "settings must be an object");
2915
432
  return null;
2916
433
  }
2917
434
  const st = settingsRaw;
2918
435
  if (typeof st.outputStyle === "string" && st.outputStyle.length > MAX_SETTINGS_OUTPUT_STYLE_CHARS) {
2919
- sendJson(res, 400, { error: `settings.outputStyle must be at most ${MAX_SETTINGS_OUTPUT_STYLE_CHARS} characters` });
436
+ sendError(res, 400, "request.field_invalid", `settings.outputStyle must be at most ${MAX_SETTINGS_OUTPUT_STYLE_CHARS} characters`);
2920
437
  return null;
2921
438
  }
2922
439
  if (typeof st.outputStyle === "string" &&
2923
440
  st.outputStyle.length > 0 &&
2924
441
  typeof body.appendSystemPrompt === "string" &&
2925
442
  body.appendSystemPrompt.length + 2 + st.outputStyle.length > MAX_SYSTEM_PROMPT_CHARS) {
2926
- sendJson(res, 400, { error: `appendSystemPrompt + settings.outputStyle fold into one system-prompt block — combined they must be at most ${MAX_SYSTEM_PROMPT_CHARS} characters (including the 2-char joiner)` });
443
+ sendError(res, 400, "request.field_conflict", `appendSystemPrompt + settings.outputStyle fold into one system-prompt block — combined they must be at most ${MAX_SYSTEM_PROMPT_CHARS} characters (including the 2-char joiner)`);
2927
444
  return null;
2928
445
  }
2929
446
  const perms = st.permissions;
@@ -2931,7 +448,7 @@ export function createHttpServer(rawDeps) {
2931
448
  for (const k of ["allow", "deny", "ask"]) {
2932
449
  const arr = perms[k];
2933
450
  if (Array.isArray(arr) && arr.length > MAX_SETTINGS_PERMISSION_RULES) {
2934
- sendJson(res, 400, { error: `settings.permissions.${k} must have at most ${MAX_SETTINGS_PERMISSION_RULES} entries` });
451
+ sendError(res, 400, "request.field_invalid", `settings.permissions.${k} must have at most ${MAX_SETTINGS_PERMISSION_RULES} entries`);
2935
452
  return null;
2936
453
  }
2937
454
  }
@@ -2939,47 +456,47 @@ export function createHttpServer(rawDeps) {
2939
456
  const envRaw = st.env;
2940
457
  if (envRaw !== undefined && envRaw !== null) {
2941
458
  if (typeof envRaw !== "object" || Array.isArray(envRaw)) {
2942
- sendJson(res, 400, { error: "settings.env must be an object of string→string" });
459
+ sendError(res, 400, "request.field_invalid", "settings.env must be an object of string→string");
2943
460
  return null;
2944
461
  }
2945
462
  const entries = Object.entries(envRaw);
2946
463
  if (entries.length > MAX_SETTINGS_ENV_VARS) {
2947
- sendJson(res, 400, { error: `settings.env must have at most ${MAX_SETTINGS_ENV_VARS} variables` });
464
+ sendError(res, 400, "request.field_invalid", `settings.env must have at most ${MAX_SETTINGS_ENV_VARS} variables`);
2948
465
  return null;
2949
466
  }
2950
467
  for (const [k, val] of entries) {
2951
468
  if (k.length > MAX_SETTINGS_ENV_KEY_CHARS || (typeof val === "string" && val.length > MAX_SETTINGS_ENV_VALUE_CHARS)) {
2952
- sendJson(res, 400, { error: `settings.env keys must be ≤${MAX_SETTINGS_ENV_KEY_CHARS} chars and values ≤${MAX_SETTINGS_ENV_VALUE_CHARS} chars` });
469
+ sendError(res, 400, "request.field_invalid", `settings.env keys must be ≤${MAX_SETTINGS_ENV_KEY_CHARS} chars and values ≤${MAX_SETTINGS_ENV_VALUE_CHARS} chars`);
2953
470
  return null;
2954
471
  }
2955
472
  }
2956
473
  }
2957
474
  if (st.ultracode !== undefined && typeof st.ultracode !== "boolean") {
2958
- sendJson(res, 400, { error: "settings.ultracode must be a boolean" });
475
+ sendError(res, 400, "request.field_invalid", "settings.ultracode must be a boolean");
2959
476
  return null;
2960
477
  }
2961
478
  if (st.hooks !== undefined && st.hooks !== null) {
2962
479
  const hooksParsed = parseHooksConfig(st.hooks);
2963
480
  if (!hooksParsed.config) {
2964
- sendJson(res, 400, { error: hooksParsed.error ?? "settings.hooks: invalid shape" });
481
+ sendError(res, 400, "request.field_invalid", hooksParsed.error ?? "settings.hooks: invalid shape");
2965
482
  return null;
2966
483
  }
2967
484
  }
2968
485
  }
2969
486
  const skillsErr = validateUserSkills(body.skills);
2970
487
  if (skillsErr) {
2971
- sendJson(res, 400, { error: skillsErr });
488
+ sendError(res, 400, "request.field_invalid", skillsErr);
2972
489
  return null;
2973
490
  }
2974
491
  if (Array.isArray(body.images)) {
2975
492
  if (body.images.length > MAX_IMAGES_PER_REQUEST) {
2976
- sendJson(res, 413, { error: `at most ${MAX_IMAGES_PER_REQUEST} images per request` });
493
+ sendError(res, 413, "request.payload_too_large", `at most ${MAX_IMAGES_PER_REQUEST} images per request`);
2977
494
  return null;
2978
495
  }
2979
496
  for (const img of body.images) {
2980
497
  const data = img.data;
2981
498
  if (typeof data === "string" && Buffer.byteLength(data, "utf8") > MAX_IMAGE_BASE64_BYTES) {
2982
- sendJson(res, 413, { error: `each inline image must be at most ${MAX_IMAGE_BASE64_BYTES} base64 bytes` });
499
+ sendError(res, 413, "request.payload_too_large", `each inline image must be at most ${MAX_IMAGE_BASE64_BYTES} base64 bytes`);
2983
500
  return null;
2984
501
  }
2985
502
  }
@@ -2987,135 +504,135 @@ export function createHttpServer(rawDeps) {
2987
504
  if (body.clientContext !== undefined) {
2988
505
  const cc = body.clientContext;
2989
506
  if (typeof cc !== "object" || cc === null || Array.isArray(cc)) {
2990
- sendJson(res, 400, { error: "clientContext must be an object" });
507
+ sendError(res, 400, "request.field_invalid", "clientContext must be an object");
2991
508
  return null;
2992
509
  }
2993
510
  if (cc.timeZone !== undefined && (typeof cc.timeZone !== "string" || cc.timeZone.length > 64)) {
2994
- sendJson(res, 400, { error: "clientContext.timeZone must be a string of at most 64 characters" });
511
+ sendError(res, 400, "request.field_invalid", "clientContext.timeZone must be a string of at most 64 characters");
2995
512
  return null;
2996
513
  }
2997
514
  if (cc.userEmail !== undefined && (typeof cc.userEmail !== "string" || cc.userEmail.length > 256)) {
2998
- sendJson(res, 400, { error: "clientContext.userEmail must be a string of at most 256 characters" });
515
+ sendError(res, 400, "request.field_invalid", "clientContext.userEmail must be a string of at most 256 characters");
2999
516
  return null;
3000
517
  }
3001
518
  }
3002
519
  if (body.sandboxImageProfile !== undefined && (typeof body.sandboxImageProfile !== "string" || body.sandboxImageProfile.length === 0 || body.sandboxImageProfile.length > 128)) {
3003
- sendJson(res, 400, { error: "sandboxImageProfile must be a non-empty string of at most 128 characters" });
520
+ sendError(res, 400, "request.field_invalid", "sandboxImageProfile must be a non-empty string of at most 128 characters");
3004
521
  return null;
3005
522
  }
3006
523
  if (body.capabilitiesNeeded !== undefined && (!Array.isArray(body.capabilitiesNeeded) || body.capabilitiesNeeded.some((c) => typeof c !== "string"))) {
3007
- sendJson(res, 400, { error: "capabilitiesNeeded must be an array of strings" });
524
+ sendError(res, 400, "request.field_invalid", "capabilitiesNeeded must be an array of strings");
3008
525
  return null;
3009
526
  }
3010
527
  if (Array.isArray(body.capabilitiesNeeded) && body.capabilitiesNeeded.length > 0 && (typeof body.sandboxImageProfile !== "string" || body.sandboxImageProfile.length === 0)) {
3011
- sendJson(res, 400, { error: "capabilitiesNeeded requires sandboxImageProfile (it constrains the selected image)" });
528
+ sendError(res, 400, "request.field_conflict", "capabilitiesNeeded requires sandboxImageProfile (it constrains the selected image)");
3012
529
  return null;
3013
530
  }
3014
531
  if (typeof body.sandboxImageProfile === "string" && body.sandboxImageProfile.length > 0 && (body.cascade === true || body.verify === true)) {
3015
- sendJson(res, 400, { error: "sandboxImageProfile is not supported together with cascade/verify in v1 (the per-task image would not bind to the cascade rungs / verifier sub-run)" });
532
+ sendError(res, 400, "request.field_conflict", "sandboxImageProfile is not supported together with cascade/verify in v1 (the per-task image would not bind to the cascade rungs / verifier sub-run)");
3016
533
  return null;
3017
534
  }
3018
535
  if (body.outputSchema !== undefined) {
3019
536
  if (typeof body.outputSchema !== "object" || body.outputSchema === null || Array.isArray(body.outputSchema)) {
3020
- sendJson(res, 400, { error: "outputSchema must be a JSON Schema object" });
537
+ sendError(res, 400, "request.field_invalid", "outputSchema must be a JSON Schema object");
3021
538
  return null;
3022
539
  }
3023
540
  if (JSON.stringify(body.outputSchema).length > MAX_OUTPUT_SCHEMA_CHARS) {
3024
- sendJson(res, 400, { error: `outputSchema must serialize to at most ${MAX_OUTPUT_SCHEMA_CHARS} characters` });
541
+ sendError(res, 400, "request.field_invalid", `outputSchema must serialize to at most ${MAX_OUTPUT_SCHEMA_CHARS} characters`);
3025
542
  return null;
3026
543
  }
3027
544
  }
3028
545
  if (body.reasoningEffort !== undefined && !isThinkingLevel(body.reasoningEffort)) {
3029
- sendJson(res, 400, { error: "reasoningEffort must be one of: off, minimal, low, medium, high, xhigh, max" });
546
+ sendError(res, 400, "request.field_invalid", "reasoningEffort must be one of: off, minimal, low, medium, high, xhigh, max");
3030
547
  return null;
3031
548
  }
3032
549
  if (body.resumeAt !== undefined) {
3033
550
  if (typeof body.resumeAt !== "string" || body.resumeAt.length === 0 || body.resumeAt.length > 64) {
3034
- sendJson(res, 400, { error: "resumeAt must be a non-empty message eventId string (≤64 chars)" });
551
+ sendError(res, 400, "request.field_invalid", "resumeAt must be a non-empty message eventId string (≤64 chars)");
3035
552
  return null;
3036
553
  }
3037
554
  if (body.verify === true || body.cascade === true) {
3038
- sendJson(res, 400, { error: "resumeAt is not supported with verify or cascade (those start a fresh/replaced session)" });
555
+ sendError(res, 400, "request.field_conflict", "resumeAt is not supported with verify or cascade (those start a fresh/replaced session)");
3039
556
  return null;
3040
557
  }
3041
558
  }
3042
559
  if (body.resumeAtMode !== undefined) {
3043
560
  if (body.resumeAtMode !== "at" && body.resumeAtMode !== "before") {
3044
- sendJson(res, 400, { error: 'resumeAtMode must be "at" or "before"' });
561
+ sendError(res, 400, "request.field_invalid", 'resumeAtMode must be "at" or "before"');
3045
562
  return null;
3046
563
  }
3047
564
  if (body.resumeAt === undefined) {
3048
- sendJson(res, 400, { error: "resumeAtMode requires resumeAt (it qualifies the rewind target)" });
565
+ sendError(res, 400, "request.field_conflict", "resumeAtMode requires resumeAt (it qualifies the rewind target)");
3049
566
  return null;
3050
567
  }
3051
568
  }
3052
569
  if (body.suggestNextPrompts !== undefined && typeof body.suggestNextPrompts !== "boolean") {
3053
570
  const s = body.suggestNextPrompts;
3054
571
  if (typeof s !== "object" || s === null || Array.isArray(s)) {
3055
- sendJson(res, 400, { error: "suggestNextPrompts must be a boolean or an object { count?, role? }" });
572
+ sendError(res, 400, "request.field_invalid", "suggestNextPrompts must be a boolean or an object { count?, role? }");
3056
573
  return null;
3057
574
  }
3058
575
  const o = s;
3059
576
  if (o.count !== undefined && (typeof o.count !== "number" || !Number.isFinite(o.count) || o.count < 1)) {
3060
- sendJson(res, 400, { error: "suggestNextPrompts.count must be a positive number" });
577
+ sendError(res, 400, "request.field_invalid", "suggestNextPrompts.count must be a positive number");
3061
578
  return null;
3062
579
  }
3063
580
  if (o.role !== undefined && typeof o.role !== "string") {
3064
- sendJson(res, 400, { error: "suggestNextPrompts.role must be a string (a model role name)" });
581
+ sendError(res, 400, "request.field_invalid", "suggestNextPrompts.role must be a string (a model role name)");
3065
582
  return null;
3066
583
  }
3067
584
  }
3068
585
  if (body.rewindFiles !== undefined && typeof body.rewindFiles !== "boolean") {
3069
- sendJson(res, 400, { error: "rewindFiles must be a boolean" });
586
+ sendError(res, 400, "request.field_invalid", "rewindFiles must be a boolean");
3070
587
  return null;
3071
588
  }
3072
589
  if (body.memoryWrite !== undefined && typeof body.memoryWrite !== "boolean") {
3073
- sendJson(res, 400, { error: "memoryWrite must be a boolean (false = pause memory writes for this run)" });
590
+ sendError(res, 400, "request.field_invalid", "memoryWrite must be a boolean (false = pause memory writes for this run)");
3074
591
  return null;
3075
592
  }
3076
593
  if (body.requireExistingSession !== undefined && typeof body.requireExistingSession !== "boolean") {
3077
- sendJson(res, 400, { error: "requireExistingSession must be a boolean" });
594
+ sendError(res, 400, "request.field_invalid", "requireExistingSession must be a boolean");
3078
595
  return null;
3079
596
  }
3080
597
  if (body.enableFork !== undefined && typeof body.enableFork !== "boolean") {
3081
- sendJson(res, 400, { error: "enableFork must be a boolean" });
598
+ sendError(res, 400, "request.field_invalid", "enableFork must be a boolean");
3082
599
  return null;
3083
600
  }
3084
601
  if (body.resilience !== undefined) {
3085
602
  const r = body.resilience;
3086
603
  if (typeof r !== "object" || r === null || Array.isArray(r)) {
3087
- sendJson(res, 400, { error: "resilience must be an object { allowDegrade?, allowFailover?, bypassBreaker? } (booleans)" });
604
+ sendError(res, 400, "request.field_invalid", "resilience must be an object { allowDegrade?, allowFailover?, bypassBreaker? } (booleans)");
3088
605
  return null;
3089
606
  }
3090
607
  for (const k of ["allowDegrade", "allowFailover", "bypassBreaker"]) {
3091
608
  const v = r[k];
3092
609
  if (v !== undefined && typeof v !== "boolean") {
3093
- sendJson(res, 400, { error: `resilience.${k} must be a boolean` });
610
+ sendError(res, 400, "request.field_invalid", `resilience.${k} must be a boolean`);
3094
611
  return null;
3095
612
  }
3096
613
  }
3097
614
  }
3098
615
  if (body.finalVerification !== undefined && typeof body.finalVerification !== "boolean") {
3099
- sendJson(res, 400, { error: "finalVerification must be a boolean" });
616
+ sendError(res, 400, "request.field_invalid", "finalVerification must be a boolean");
3100
617
  return null;
3101
618
  }
3102
619
  if (body.limits !== undefined) {
3103
620
  const l = body.limits;
3104
621
  if (typeof l !== "object" || l === null || Array.isArray(l)) {
3105
- sendJson(res, 400, { error: "limits must be an object { timeoutSec?, maxOutputTokens?, maxTurns? } (positive integers)" });
622
+ sendError(res, 400, "request.field_invalid", "limits must be an object { timeoutSec?, maxOutputTokens?, maxTurns? } (positive integers)");
3106
623
  return null;
3107
624
  }
3108
625
  for (const k of ["timeoutSec", "maxOutputTokens", "maxTurns"]) {
3109
626
  const v = l[k];
3110
627
  if (v !== undefined && (typeof v !== "number" || !Number.isInteger(v) || v < 1)) {
3111
- sendJson(res, 400, { error: `limits.${k} must be a positive integer` });
628
+ sendError(res, 400, "request.field_invalid", `limits.${k} must be a positive integer`);
3112
629
  return null;
3113
630
  }
3114
631
  }
3115
632
  for (const k of ["deadlineNudge", "callCapByDeadline", "gracefulFinalize"]) {
3116
633
  const v = body.limits[k];
3117
634
  if (v !== undefined && v !== false) {
3118
- sendJson(res, 400, { error: `limits.${k} accepts only literal false (it is an opt-out; omit to keep the default-on behavior)` });
635
+ sendError(res, 400, "request.field_invalid", `limits.${k} accepts only literal false (it is an opt-out; omit to keep the default-on behavior)`);
3119
636
  return null;
3120
637
  }
3121
638
  }
@@ -3123,88 +640,82 @@ export function createHttpServer(rawDeps) {
3123
640
  if (body.agents !== undefined) {
3124
641
  const agentsErr = validateTaskAgents(body.agents);
3125
642
  if (agentsErr) {
3126
- sendJson(res, 400, { error: agentsErr });
643
+ sendError(res, 400, "request.field_invalid", agentsErr);
3127
644
  return null;
3128
645
  }
3129
646
  }
3130
647
  if (body.interactiveTools !== undefined && typeof body.interactiveTools !== "boolean") {
3131
- sendJson(res, 400, { error: "interactiveTools must be a boolean" });
648
+ sendError(res, 400, "request.field_invalid", "interactiveTools must be a boolean");
3132
649
  return null;
3133
650
  }
3134
651
  if (body.retainBackgroundProcesses !== undefined && typeof body.retainBackgroundProcesses !== "boolean") {
3135
- sendJson(res, 400, { error: "retainBackgroundProcesses must be a boolean" });
652
+ sendError(res, 400, "request.field_invalid", "retainBackgroundProcesses must be a boolean");
3136
653
  return null;
3137
654
  }
3138
655
  for (const key of ["excludeTools", "deferTools"]) {
3139
656
  const v = body[key];
3140
657
  if (v !== undefined && (!Array.isArray(v) || v.some((n) => typeof n !== "string" || n.length === 0))) {
3141
- sendJson(res, 400, { error: `${key} must be an array of non-empty tool-name strings` });
658
+ sendError(res, 400, "request.field_invalid", `${key} must be an array of non-empty tool-name strings`);
3142
659
  return null;
3143
660
  }
3144
661
  }
3145
662
  {
3146
663
  const v = body.promptProfile;
3147
664
  if (v !== undefined && v !== "simple" && v !== "classic") {
3148
- sendJson(res, 400, { error: 'promptProfile must be "simple" or "classic"' });
665
+ sendError(res, 400, "request.field_invalid", 'promptProfile must be "simple" or "classic"');
3149
666
  return null;
3150
667
  }
3151
668
  }
3152
669
  if (body.model !== undefined) {
3153
670
  if (typeof body.model !== "string" || body.model.length === 0) {
3154
- sendJson(res, 400, { error: "model must be a non-empty string (a configured catalog name, tier word, or model id)" });
671
+ sendError(res, 400, "request.field_invalid", "model must be a non-empty string (a configured catalog name, tier word, or model id)");
3155
672
  return null;
3156
673
  }
3157
674
  const bare = deps.config.models ?? {};
3158
675
  const catalog = expandTiers(bare, deps.config.tiers ?? {}) ?? bare;
3159
676
  if (matchCatalogModel(body.model, catalog) === undefined) {
3160
- sendJson(res, 400, {
3161
- error: `unknown model "${body.model.slice(0, 120)}" — not in the configured catalog (name, tier word, or id)`,
3162
- available: Object.keys(catalog).filter((n) => n !== "default"),
3163
- });
677
+ sendError(res, 400, "request.unknown_reference", `unknown model "${body.model.slice(0, 120)}" — not in the configured catalog (name, tier word, or id)`, { available: Object.keys(catalog).filter((n) => n !== "default") });
3164
678
  return null;
3165
679
  }
3166
680
  }
3167
681
  if (body.compactionModel !== undefined) {
3168
682
  if (typeof body.compactionModel !== "string" || body.compactionModel.length === 0) {
3169
- sendJson(res, 400, { error: "compactionModel must be a non-empty string (a configured catalog name, tier word, or model id)" });
683
+ sendError(res, 400, "request.field_invalid", "compactionModel must be a non-empty string (a configured catalog name, tier word, or model id)");
3170
684
  return null;
3171
685
  }
3172
686
  const bare = deps.config.models ?? {};
3173
687
  const catalog = expandTiers(bare, deps.config.tiers ?? {}) ?? bare;
3174
688
  if (matchCatalogModel(body.compactionModel, catalog) === undefined) {
3175
- sendJson(res, 400, {
3176
- error: `unknown compactionModel "${body.compactionModel.slice(0, 120)}" — not in the configured catalog (name, tier word, or id)`,
3177
- available: Object.keys(catalog).filter((n) => n !== "default"),
3178
- });
689
+ sendError(res, 400, "request.unknown_reference", `unknown compactionModel "${body.compactionModel.slice(0, 120)}" — not in the configured catalog (name, tier word, or id)`, { available: Object.keys(catalog).filter((n) => n !== "default") });
3179
690
  return null;
3180
691
  }
3181
692
  }
3182
693
  if (body.attachments !== undefined) {
3183
694
  const a = body.attachments;
3184
695
  if (typeof a !== "object" || a === null || Array.isArray(a)) {
3185
- sendJson(res, 400, { error: "attachments must be an object { todoReminder?, todoReminderMode?, changedFiles?, planModeReminder?, budgetUsd?, backgroundTasks?, toolsDelta?, agentListing?, skillsListing?, mcpInstructions? }" });
696
+ sendError(res, 400, "request.field_invalid", "attachments must be an object { todoReminder?, todoReminderMode?, changedFiles?, planModeReminder?, budgetUsd?, backgroundTasks?, toolsDelta?, agentListing?, skillsListing?, mcpInstructions? }");
3186
697
  return null;
3187
698
  }
3188
699
  const rec = a;
3189
700
  for (const k of ["todoReminder", "planModeReminder", "budgetUsd", "backgroundTasks", "toolsDelta", "agentListing", "skillsListing", "mcpInstructions"]) {
3190
701
  if (rec[k] !== undefined && typeof rec[k] !== "boolean") {
3191
- sendJson(res, 400, { error: `attachments.${k} must be a boolean` });
702
+ sendError(res, 400, "request.field_invalid", `attachments.${k} must be a boolean`);
3192
703
  return null;
3193
704
  }
3194
705
  }
3195
706
  if (rec.todoReminderMode !== undefined && rec.todoReminderMode !== "baseline" && rec.todoReminderMode !== "off") {
3196
- sendJson(res, 400, { error: 'attachments.todoReminderMode must be "baseline" | "off"' });
707
+ sendError(res, 400, "request.field_invalid", 'attachments.todoReminderMode must be "baseline" | "off"');
3197
708
  return null;
3198
709
  }
3199
710
  const cf = rec.changedFiles;
3200
711
  if (cf !== undefined && typeof cf !== "boolean") {
3201
712
  if (typeof cf !== "object" || cf === null || Array.isArray(cf)) {
3202
- sendJson(res, 400, { error: "attachments.changedFiles must be a boolean or { maxFiles?: number }" });
713
+ sendError(res, 400, "request.field_invalid", "attachments.changedFiles must be a boolean or { maxFiles?: number }");
3203
714
  return null;
3204
715
  }
3205
716
  const mf = cf.maxFiles;
3206
717
  if (mf !== undefined && (typeof mf !== "number" || !Number.isFinite(mf) || mf < 1)) {
3207
- sendJson(res, 400, { error: "attachments.changedFiles.maxFiles must be a number ≥ 1" });
718
+ sendError(res, 400, "request.field_invalid", "attachments.changedFiles.maxFiles must be a number ≥ 1");
3208
719
  return null;
3209
720
  }
3210
721
  }
@@ -3217,15 +728,15 @@ export function createHttpServer(rawDeps) {
3217
728
  : undefined;
3218
729
  const cascade = body.cascade === true;
3219
730
  if (cascade && verify) {
3220
- sendJson(res, 400, { error: "verify and cascade are mutually exclusive (verify = adversarial gate; cascade = cheap→strong ladder)" });
731
+ sendError(res, 400, "request.field_conflict", "verify and cascade are mutually exclusive (verify = adversarial gate; cascade = cheap→strong ladder)");
3221
732
  return null;
3222
733
  }
3223
734
  if (body.suggestNextPrompts && (verify || cascade)) {
3224
- sendJson(res, 400, { error: "suggestNextPrompts is not supported with verify or cascade (those return a result, not a streamed run)" });
735
+ sendError(res, 400, "request.field_conflict", "suggestNextPrompts is not supported with verify or cascade (those return a result, not a streamed run)");
3225
736
  return null;
3226
737
  }
3227
738
  if (cascade && deps.config.cascadeLadder.length === 0) {
3228
- sendJson(res, 400, { error: "cascade requested but no ladder configured — set MODEL_CASCADE_LADDER (catalog model names, cheapest→strongest)" });
739
+ sendError(res, 400, "request.precondition_unmet", "cascade requested but no ladder configured — set MODEL_CASCADE_LADDER (catalog model names, cheapest→strongest)");
3229
740
  return null;
3230
741
  }
3231
742
  try {
@@ -3234,7 +745,7 @@ export function createHttpServer(rawDeps) {
3234
745
  }
3235
746
  catch (err) {
3236
747
  if (err instanceof HttpError) {
3237
- sendJson(res, err.status, { error: err.message, ...(err.code ? { code: err.code, errorCode: err.code } : {}), ...(err.extra ?? {}) });
748
+ sendError(res, err.status, httpErrorCode(err.status, err.code), err.message, { ...(err.code ? { code: err.code } : {}), ...(err.extra ?? {}) });
3238
749
  return null;
3239
750
  }
3240
751
  throw err;
@@ -3261,7 +772,7 @@ export function createHttpServer(rawDeps) {
3261
772
  }
3262
773
  if (!token) {
3263
774
  deps.logger?.info?.("approval_decide_no_pending", { sessionId });
3264
- return { status: 404, body: { error: "approval not found" } };
775
+ return { status: 404, body: { error: "approval not found", errorCode: "not_found.approval" } };
3265
776
  }
3266
777
  const cpPromise = cs.get(token);
3267
778
  const ctxPromise = cs.getCtx(sessionId);
@@ -3271,10 +782,10 @@ export function createHttpServer(rawDeps) {
3271
782
  }
3272
783
  catch (e) {
3273
784
  void ctxPromise.catch(() => { });
3274
- return { status: 409, body: { error: e instanceof Error ? e.message : String(e) } };
785
+ return { status: 409, body: { error: e instanceof Error ? e.message : String(e), errorCode: "conflict.checkpoint_unreadable" } };
3275
786
  }
3276
787
  if (!cp)
3277
- return { status: 404, body: { error: "checkpoint not found" } };
788
+ return { status: 404, body: { error: "checkpoint not found", errorCode: "not_found.checkpoint" } };
3278
789
  if (req !== undefined && deps.backgroundAgentStore !== undefined && deps.parkedReviveTool !== undefined) {
3279
790
  const parked = await decideParkedAgent({
3280
791
  agentStore: deps.backgroundAgentStore,
@@ -3301,10 +812,10 @@ export function createHttpServer(rawDeps) {
3301
812
  ctx = await ctxPromise;
3302
813
  }
3303
814
  catch (e) {
3304
- return { status: 409, body: { error: `resume context load failed: ${e instanceof Error ? e.message : String(e)}` } };
815
+ return { status: 409, body: { error: `resume context load failed: ${e instanceof Error ? e.message : String(e)}`, errorCode: "conflict.resume_context_unavailable" } };
3305
816
  }
3306
817
  if (!ctx)
3307
- return { status: 409, body: { error: "resume context missing — cannot rebuild task config" } };
818
+ return { status: 409, body: { error: "resume context missing — cannot rebuild task config", errorCode: "conflict.resume_context_unavailable" } };
3308
819
  const auth = { sessionId: cp.sessionId, principal: cp.scope === "_" ? undefined : cp.scope, memoryScope: ctx.memoryScope };
3309
820
  let spec;
3310
821
  try {
@@ -3333,14 +844,14 @@ export function createHttpServer(rawDeps) {
3333
844
  if (decision === "approve" && pendingToolCanon && !availableTools.has(pendingToolCanon)) {
3334
845
  return {
3335
846
  status: 422,
3336
- body: { error: `pending action no longer satisfiable: tool "${pendingToolCanon}" is not in the current task config (scenario changed since suspend)` },
847
+ body: { error: `pending action no longer satisfiable: tool "${pendingToolCanon}" is not in the current task config (scenario changed since suspend)`, errorCode: "conflict.pending_action_unsatisfiable" },
3337
848
  };
3338
849
  }
3339
850
  if (decision === "approve" && pendingTool === "AskUserQuestion" && !answer) {
3340
- return { status: 400, body: { error: "pending action is AskUserQuestion — approve must carry body.answer ({ answers: [{ header, selected: string[], note? }] })" } };
851
+ return { status: 400, body: { error: "pending action is AskUserQuestion — approve must carry body.answer ({ answers: [{ header, selected: string[], note? }] })", errorCode: "request.field_conflict" } };
3341
852
  }
3342
853
  if (answer && (decision !== "approve" || pendingTool !== "AskUserQuestion")) {
3343
- return { status: 400, body: { error: `body.answer is only valid when approving a pending AskUserQuestion (decision: "${decision}", pending: "${pendingTool ?? "unknown"}")` } };
854
+ return { status: 400, body: { error: `body.answer is only valid when approving a pending AskUserQuestion (decision: "${decision}", pending: "${pendingTool ?? "unknown"}")`, errorCode: "request.field_conflict" } };
3344
855
  }
3345
856
  const { objective: resumeObjective, sessionId: _sessionId, ...taskConfig } = spec;
3346
857
  if (answer)
@@ -3370,7 +881,7 @@ export function createHttpServer(rawDeps) {
3370
881
  const adm = await deps.fleetLease.admit(principal);
3371
882
  if (!adm.ok) {
3372
883
  deps.metrics?.inc("fleet_lease_rejected_total");
3373
- return { status: 429, body: { error: "budget exhausted (quota lease)", retryAfterSec: adm.retryAfterSec } };
884
+ return { status: 429, body: { error: "budget exhausted (quota lease)", errorCode: "quota_exhausted", retryAfterSec: adm.retryAfterSec } };
3374
885
  }
3375
886
  }
3376
887
  const taskId = await deps.runStore?.getActiveTaskId(sessionId);
@@ -3409,7 +920,7 @@ export function createHttpServer(rawDeps) {
3409
920
  if (taskId && deps.runStore) {
3410
921
  claimedRow = await deps.runStore.markResuming(taskId);
3411
922
  if (!claimedRow)
3412
- return { status: 409, body: { error: "run is not in a resumable (suspended) state (already resumed, decided, or expired)" } };
923
+ return { status: 409, body: { error: "run is not in a resumable (suspended) state (already resumed, decided, or expired)", errorCode: "conflict.not_resumable" } };
3413
924
  fleetPub?.onStart();
3414
925
  const rs = deps.runStore;
3415
926
  const owner = principal ?? null;
@@ -3589,7 +1100,7 @@ export function createHttpServer(rawDeps) {
3589
1100
  const st = ev;
3590
1101
  deps.metrics?.inc("brain_retry_total", { phase: String(st.phase) });
3591
1102
  await flush();
3592
- await append("brain_status", brainStatusEventData(st));
1103
+ await append("status", brainStatusEventData(st));
3593
1104
  break;
3594
1105
  }
3595
1106
  case "context_usage":
@@ -3760,23 +1271,23 @@ export function createHttpServer(rawDeps) {
3760
1271
  const cs = deps.checkpointStore;
3761
1272
  const token = await cs.findPendingTokenBySession(sessionId);
3762
1273
  if (!token)
3763
- return { status: 404, body: { error: "no resumable suspension for this task (already running, resumed, or expired)" } };
1274
+ return { status: 404, body: { error: "no resumable suspension for this task (already running, resumed, or expired)", errorCode: "not_found.suspension" } };
3764
1275
  let cp;
3765
1276
  try {
3766
1277
  cp = await cs.get(token);
3767
1278
  }
3768
1279
  catch (e) {
3769
- return { status: 409, body: { error: e instanceof Error ? e.message : String(e) } };
1280
+ return { status: 409, body: { error: e instanceof Error ? e.message : String(e), errorCode: "conflict.checkpoint_unreadable" } };
3770
1281
  }
3771
1282
  if (!cp)
3772
- return { status: 404, body: { error: "checkpoint not found" } };
1283
+ return { status: 404, body: { error: "checkpoint not found", errorCode: "not_found.checkpoint" } };
3773
1284
  const gateKind = cp.gate?.kind;
3774
1285
  if (gateKind !== "resource_limit") {
3775
1286
  return { status: 409, body: { error: `task is suspended on a '${gateKind ?? "unknown"}' gate, not a resumable resource/preempt suspension — an approval gate must be decided via POST /v1/approvals/:id/decide`, errorCode: "gate_not_resumable" } };
3776
1287
  }
3777
1288
  const ctx = await cs.getCtx(sessionId).catch(() => null);
3778
1289
  if (!ctx)
3779
- return { status: 409, body: { error: "resume context missing — cannot rebuild task config" } };
1290
+ return { status: 409, body: { error: "resume context missing — cannot rebuild task config", errorCode: "conflict.resume_context_unavailable" } };
3780
1291
  const auth = { sessionId: cp.sessionId, principal: cp.scope === "_" ? undefined : cp.scope, memoryScope: ctx.memoryScope };
3781
1292
  const spec = await deps.resolveSpec({ ...ctx.body, resumeAt: undefined }, req, auth);
3782
1293
  const { objective: resumeObjective, sessionId: _sessionId, ...taskConfig } = spec;
@@ -3792,12 +1303,12 @@ export function createHttpServer(rawDeps) {
3792
1303
  const cs = deps.checkpointStore;
3793
1304
  const token = await cs.findPendingTokenBySession(sessionId);
3794
1305
  if (!token)
3795
- return { status: 404, body: { error: "no parked checkpoint for this session (nothing to wake)" } };
1306
+ return { status: 404, body: { error: "no parked checkpoint for this session (nothing to wake)", errorCode: "not_found.parked_checkpoint" } };
3796
1307
  const operator = explicitOperatorOk(caller, deps.config.operatorPrincipals);
3797
1308
  if (!operator) {
3798
1309
  const scope = await cs.peekPendingScope(sessionId).catch(() => undefined);
3799
1310
  if (scope !== null && scope !== undefined && scope !== "_" && scope !== caller) {
3800
- return { status: 404, body: { error: "no parked checkpoint for this session (nothing to wake)" } };
1311
+ return { status: 404, body: { error: "no parked checkpoint for this session (nothing to wake)", errorCode: "not_found.parked_checkpoint" } };
3801
1312
  }
3802
1313
  }
3803
1314
  let cp;
@@ -3805,12 +1316,12 @@ export function createHttpServer(rawDeps) {
3805
1316
  cp = await cs.get(token);
3806
1317
  }
3807
1318
  catch (e) {
3808
- return { status: 409, body: { error: e instanceof Error ? e.message : String(e) } };
1319
+ return { status: 409, body: { error: e instanceof Error ? e.message : String(e), errorCode: "conflict.checkpoint_unreadable" } };
3809
1320
  }
3810
1321
  if (!cp)
3811
- return { status: 404, body: { error: "checkpoint not found" } };
1322
+ return { status: 404, body: { error: "checkpoint not found", errorCode: "not_found.checkpoint" } };
3812
1323
  if (!operator && cp.scope !== "_" && cp.scope !== caller) {
3813
- return { status: 404, body: { error: "no parked checkpoint for this session (nothing to wake)" } };
1324
+ return { status: 404, body: { error: "no parked checkpoint for this session (nothing to wake)", errorCode: "not_found.parked_checkpoint" } };
3814
1325
  }
3815
1326
  const gateKind = cp.gate?.kind;
3816
1327
  if (gateKind !== "task_done") {
@@ -3830,7 +1341,7 @@ export function createHttpServer(rawDeps) {
3830
1341
  }
3831
1342
  const ctx = await cs.getCtx(sessionId).catch(() => null);
3832
1343
  if (!ctx)
3833
- return { status: 409, body: { error: "resume context missing — cannot rebuild task config" } };
1344
+ return { status: 409, body: { error: "resume context missing — cannot rebuild task config", errorCode: "conflict.resume_context_unavailable" } };
3834
1345
  const auth = { sessionId: cp.sessionId, principal: cp.scope === "_" ? undefined : cp.scope, memoryScope: ctx.memoryScope };
3835
1346
  const spec = await deps.resolveSpec({ ...ctx.body, resumeAt: undefined }, req, auth);
3836
1347
  const { objective: resumeObjective, sessionId: _sessionId, ...taskConfig } = spec;
@@ -3849,16 +1360,16 @@ export function createHttpServer(rawDeps) {
3849
1360
  const cs = deps.checkpointStore;
3850
1361
  const token = await cs.findPendingTokenBySession(sessionId);
3851
1362
  if (!token)
3852
- return { status: 404, body: { error: "no pending plan_review for this session (already decided or expired)" } };
1363
+ return { status: 404, body: { error: "no pending plan_review for this session (already decided or expired)", errorCode: "not_found.plan_review" } };
3853
1364
  let cp;
3854
1365
  try {
3855
1366
  cp = await cs.get(token);
3856
1367
  }
3857
1368
  catch (e) {
3858
- return { status: 409, body: { error: e instanceof Error ? e.message : String(e) } };
1369
+ return { status: 409, body: { error: e instanceof Error ? e.message : String(e), errorCode: "conflict.checkpoint_unreadable" } };
3859
1370
  }
3860
1371
  if (!cp)
3861
- return { status: 404, body: { error: "checkpoint not found" } };
1372
+ return { status: 404, body: { error: "checkpoint not found", errorCode: "not_found.checkpoint" } };
3862
1373
  const gateKind = cp.gate?.kind;
3863
1374
  if (gateKind !== "plan_review") {
3864
1375
  return {
@@ -3871,7 +1382,7 @@ export function createHttpServer(rawDeps) {
3871
1382
  }
3872
1383
  const ctx = await cs.getCtx(sessionId).catch(() => null);
3873
1384
  if (!ctx)
3874
- return { status: 409, body: { error: "resume context missing — cannot rebuild task config" } };
1385
+ return { status: 409, body: { error: "resume context missing — cannot rebuild task config", errorCode: "conflict.resume_context_unavailable" } };
3875
1386
  const auth = { sessionId: cp.sessionId, principal: cp.scope === "_" ? undefined : cp.scope, memoryScope: ctx.memoryScope };
3876
1387
  const spec = await deps.resolveSpec({ ...ctx.body, resumeAt: undefined }, req, auth);
3877
1388
  const { objective: resumeObjective, sessionId: _sessionId, ...taskConfig } = spec;
@@ -3923,12 +1434,7 @@ export function createHttpServer(rawDeps) {
3923
1434
  return false;
3924
1435
  deps.metrics?.inc("cost_quota_rejected_total");
3925
1436
  res.setHeader("retry-after", String(d.retryAfterSec));
3926
- sendJson(res, 429, {
3927
- error: "cost quota exceeded",
3928
- usedMicroUsd: d.usedMicroUsd,
3929
- limitMicroUsd: d.limitMicroUsd,
3930
- retryAfterSec: d.retryAfterSec,
3931
- });
1437
+ sendError(res, 429, "limit.cost_quota_exceeded", "cost quota exceeded", { usedMicroUsd: d.usedMicroUsd, limitMicroUsd: d.limitMicroUsd, retryAfterSec: d.retryAfterSec });
3932
1438
  return true;
3933
1439
  }
3934
1440
  async function leaseDenied(req, res) {
@@ -3954,7 +1460,7 @@ export function createHttpServer(rawDeps) {
3954
1460
  return false;
3955
1461
  deps.metrics?.inc("rate_limited_total");
3956
1462
  res.setHeader("retry-after", String(d.retryAfterSec));
3957
- sendJson(res, 429, { error: "rate limit exceeded", retryAfterSec: d.retryAfterSec });
1463
+ sendError(res, 429, "limit.rate_exceeded", "rate limit exceeded", { retryAfterSec: d.retryAfterSec });
3958
1464
  return true;
3959
1465
  }
3960
1466
  function safeDecode(seg) {
@@ -3985,11 +1491,11 @@ export function createHttpServer(rawDeps) {
3985
1491
  function runOwnerOk(req, res, owner) {
3986
1492
  const principal = gatedPrincipal(req, deps.config);
3987
1493
  if (deps.config.requirePrincipal && !principal) {
3988
- sendJson(res, 401, { error: `missing principal header '${deps.config.principalHeader}'` });
1494
+ sendError(res, 401, "auth.principal_required", `missing principal header '${deps.config.principalHeader}'`);
3989
1495
  return false;
3990
1496
  }
3991
1497
  if (owner !== null && principal !== owner) {
3992
- sendJson(res, 404, { error: "run not found" });
1498
+ sendError(res, 404, "not_found.run", "run not found");
3993
1499
  return false;
3994
1500
  }
3995
1501
  return true;
@@ -4010,7 +1516,7 @@ export function createHttpServer(rawDeps) {
4010
1516
  return true;
4011
1517
  if (explicitOperatorOk(gatedPrincipal(req, deps.config), deps.config.operatorPrincipals))
4012
1518
  return true;
4013
- sendJson(res, 404, { error: notFoundError });
1519
+ sendError(res, 404, "not_found.run", notFoundError);
4014
1520
  return false;
4015
1521
  }
4016
1522
  async function readJson(req) {
@@ -4077,183 +1583,6 @@ export function createHttpServer(rawDeps) {
4077
1583
  }
4078
1584
  return Object.assign(server, { denyExpiredApprovals });
4079
1585
  }
4080
- function encodeCursor(c) {
4081
- return Buffer.from(JSON.stringify(c)).toString("base64url");
4082
- }
4083
- function decodeCursor(s) {
4084
- if (!s)
4085
- return undefined;
4086
- try {
4087
- const o = JSON.parse(Buffer.from(s, "base64url").toString());
4088
- return o.createdAt && o.taskId && !Number.isNaN(Date.parse(o.createdAt)) ? { createdAt: o.createdAt, taskId: o.taskId } : undefined;
4089
- }
4090
- catch {
4091
- return undefined;
4092
- }
4093
- }
4094
- async function handleTaskList(res, runStore, query, forceOwner) {
4095
- const status = query.get("status") ?? undefined;
4096
- const jobId = query.get("jobId") ?? undefined;
4097
- const source = query.get("source") ?? undefined;
4098
- const owner = forceOwner != null ? forceOwner : (query.get("owner") ?? undefined);
4099
- const limit = Math.min(100, Math.max(1, Number(query.get("limit") ?? 50) || 50));
4100
- const cursor = decodeCursor(query.get("cursor"));
4101
- const rows = await runStore.listRuns({ ...(status ? { status } : {}), ...(jobId ? { jobId } : {}), ...(source ? { source } : {}), ...(owner ? { owner } : {}), ...(cursor ? { cursor } : {}), limit: limit + 1 });
4102
- const hasMore = rows.length > limit;
4103
- const page = hasMore ? rows.slice(0, limit) : rows;
4104
- const tasks = page.map((r) => runSummary(r));
4105
- const last = page[page.length - 1];
4106
- const nextCursor = hasMore && last ? encodeCursor({ createdAt: last.createdAt, taskId: last.taskId }) : undefined;
4107
- sendJson(res, 200, { tasks, ...(nextCursor ? { nextCursor } : {}) });
4108
- }
4109
- async function handleTaskArtifacts(res, runStore, taskId) {
4110
- const run = await runStore.getRun(taskId);
4111
- if (!run) {
4112
- sendJson(res, 404, { error: "task not found" });
4113
- return;
4114
- }
4115
- const events = await runStore.getEvents(taskId, 0);
4116
- sendJson(res, 200, { artifacts: projectArtifacts(events, { taskId, jobId: run.jobId }) });
4117
- }
4118
- async function handleTaskTurns(res, runStore, taskId, query) {
4119
- const run = await runStore.getRun(taskId);
4120
- if (!run) {
4121
- sendJson(res, 404, { error: "task not found" });
4122
- return;
4123
- }
4124
- const limit = Math.min(100, Math.max(1, Number(query.get("limit") ?? 20) || 20));
4125
- const cursorSeq = Number(query.get("cursor") ?? 0) || 0;
4126
- const [events, retainedFrom] = await Promise.all([runStore.getEvents(taskId, 0), runStore.retainedFrom(taskId)]);
4127
- const all = projectEvents(events);
4128
- const eligible = cursorSeq > 0 ? all.filter((t) => t.seq < cursorSeq) : all;
4129
- const turns = eligible.slice(Math.max(0, eligible.length - limit));
4130
- const oldest = turns[0];
4131
- const nextCursor = eligible.length > limit && oldest ? String(oldest.seq) : undefined;
4132
- sendJson(res, 200, { turns, ...(nextCursor ? { nextCursor } : {}), retainedFrom });
4133
- }
4134
- async function streamTaskTrace(req, res, runStore, taskId, staleMs) {
4135
- const run0 = await runStore.getRun(taskId);
4136
- if (!run0) {
4137
- sendJson(res, 404, { error: "task not found" });
4138
- return;
4139
- }
4140
- const lastId = Number(req.headers["last-event-id"] ?? new URL(req.url ?? "", "http://x").searchParams.get("from") ?? 0);
4141
- let from = Number.isFinite(lastId) ? lastId : 0;
4142
- if (from > 0) {
4143
- const retainedFrom = await runStore.retainedFrom(taskId);
4144
- if (retainedFrom > from + 1) {
4145
- sendJson(res, 416, { error: "resume point evicted past retention", retainedFrom });
4146
- return;
4147
- }
4148
- }
4149
- sseHeaders(res);
4150
- res.write(`event: meta\ndata: ${JSON.stringify({ version: 1, mode: "delta", resumeFrom: from })}\n\n`);
4151
- let closed = false;
4152
- req.on("close", () => {
4153
- closed = true;
4154
- });
4155
- const start = Date.now();
4156
- const MAX_MS = 15 * 60 * 1000;
4157
- let lastBeat = Date.now();
4158
- const write = (events) => {
4159
- for (const ev of events) {
4160
- const out = mapTraceEvent(ev.type, ev.seq, (ev.data ?? {}));
4161
- if (out)
4162
- res.write(`id: ${ev.seq}\nevent: ${out.event}\ndata: ${JSON.stringify(out.data)}\n\n`);
4163
- from = ev.seq;
4164
- }
4165
- };
4166
- while (!closed) {
4167
- const [events, run] = await Promise.all([runStore.getEvents(taskId, from), runStore.getRun(taskId)]);
4168
- write(events);
4169
- if (!run)
4170
- break;
4171
- if (run.status !== "running") {
4172
- write(await runStore.getEvents(taskId, from));
4173
- break;
4174
- }
4175
- const stale = Date.now() - new Date(run.updatedAt).getTime() > staleMs;
4176
- if (stale && events.length === 0) {
4177
- res.write(`event: error\ndata: ${JSON.stringify({ code: "WORKER_DOWN", message: "run stalled (instance lost?)" })}\n\n`);
4178
- break;
4179
- }
4180
- if (Date.now() - start > MAX_MS) {
4181
- res.write(`event: error\ndata: ${JSON.stringify({ code: "STREAM_MAX_DURATION", message: "trace stream reached its 15-minute cap — reconnect with Last-Event-ID to continue (the run is still active)" })}\n\n`);
4182
- break;
4183
- }
4184
- if (events.length === 0) {
4185
- if (Date.now() - lastBeat > 15_000) {
4186
- res.write(`event: heartbeat\ndata: {}\n\n`);
4187
- lastBeat = Date.now();
4188
- }
4189
- await sleep(250);
4190
- }
4191
- }
4192
- res.end();
4193
- }
4194
- const APPROVALS_STREAM_POLL_MS = 3000;
4195
- export async function streamApprovals(req, res, cs, scope, pollMs = APPROVALS_STREAM_POLL_MS) {
4196
- const keyOf = (p) => JSON.stringify([p.sessionId, p.toolCallId ?? null]);
4197
- sseHeaders(res);
4198
- res.write(`event: meta\ndata: ${JSON.stringify({ type: "meta", version: 1, mode: "approvals-delta", pollMs: APPROVALS_STREAM_POLL_MS })}\n\n`);
4199
- let closed = false;
4200
- req.on("close", () => { closed = true; });
4201
- const start = Date.now();
4202
- const MAX_MS = 15 * 60 * 1000;
4203
- let lastBeat = Date.now();
4204
- let prev = new Map();
4205
- let first = true;
4206
- while (!closed) {
4207
- let pending;
4208
- try {
4209
- pending = await cs.listPending(scope);
4210
- }
4211
- catch {
4212
- if (!res.writableEnded)
4213
- res.write(`event: heartbeat\ndata: ${JSON.stringify({ type: "heartbeat" })}\n\n`);
4214
- await sleep(pollMs);
4215
- continue;
4216
- }
4217
- const cur = new Map(pending.map((p) => [keyOf(p), p]));
4218
- if (first) {
4219
- for (const p of pending)
4220
- res.write(`event: pending\ndata: ${JSON.stringify({ type: "pending", ...p })}\n\n`);
4221
- res.write(`event: synced\ndata: ${JSON.stringify({ type: "synced", count: pending.length })}\n\n`);
4222
- first = false;
4223
- }
4224
- else {
4225
- for (const [k, p] of cur)
4226
- if (!prev.has(k))
4227
- res.write(`event: pending\ndata: ${JSON.stringify({ type: "pending", ...p })}\n\n`);
4228
- for (const [k, p] of prev)
4229
- if (!cur.has(k))
4230
- res.write(`event: resolved\ndata: ${JSON.stringify({ type: "resolved", sessionId: p.sessionId, toolCallId: p.toolCallId })}\n\n`);
4231
- }
4232
- prev = cur;
4233
- if (Date.now() - start > MAX_MS) {
4234
- res.write(`event: error\ndata: ${JSON.stringify({ type: "error", code: "STREAM_MAX_DURATION", message: "approvals stream reached its 15-minute cap — reconnect to continue" })}\n\n`);
4235
- break;
4236
- }
4237
- if (Date.now() - lastBeat > 15_000) {
4238
- res.write(`event: heartbeat\ndata: ${JSON.stringify({ type: "heartbeat" })}\n\n`);
4239
- lastBeat = Date.now();
4240
- }
4241
- await sleep(pollMs);
4242
- }
4243
- res.end();
4244
- }
4245
- async function streamRunEvents(req, res, runStore, taskId, staleMs) {
4246
- await streamSseLog(req, res, {
4247
- statusOf: async (id) => {
4248
- const run = await runStore.getRun(id);
4249
- return run ? { status: run.status, updatedAt: run.updatedAt } : undefined;
4250
- },
4251
- getEvents: (id, after) => runStore.getEvents(id, after),
4252
- retainedFrom: (id) => runStore.retainedFrom(id),
4253
- formatEvent: (ev) => ({ id: ev.seq, event: ev.type, data: { type: ev.type, ...(ev.data ?? {}) } }),
4254
- staleFrame: () => ({ event: "failed", data: { type: "failed", errorMessage: "run stalled (instance lost?)" } }),
4255
- }, taskId, staleMs);
4256
- }
4257
1586
  function isBillableSubmitPath(url) {
4258
1587
  return (url === "/v1/side-query" ||
4259
1588
  url === "/v1/tasks" ||
@@ -4265,260 +1594,6 @@ function isBillableSubmitPath(url) {
4265
1594
  RUN_SUBAGENT_RESUME_RE.test(url) ||
4266
1595
  url.startsWith("/v1/approvals"));
4267
1596
  }
4268
- function summarizeWorkflowDetail(run) {
4269
- const isDone = (s) => s === "completed" || s === "failed";
4270
- const durationOf = (a) => a.startedAt === undefined || a.endedAt === undefined ? undefined : a.endedAt - a.startedAt;
4271
- const runPhases = run.phases ?? [];
4272
- const runAgents = run.agents ?? [];
4273
- const runGroups = run.groups ?? [];
4274
- const phases = runPhases.map((p) => {
4275
- const inPhase = runAgents.filter((a) => a.phase === p.title);
4276
- return {
4277
- title: redactSecrets(p.title),
4278
- status: p.status,
4279
- startedAt: p.startedAt,
4280
- endedAt: p.endedAt,
4281
- durationMs: durationOf(p),
4282
- done: inPhase.filter((a) => isDone(a.status)).length,
4283
- total: inPhase.length,
4284
- };
4285
- });
4286
- const phaseTitleSet = new Set(runPhases.map((p) => p.title));
4287
- const unphased = runAgents.filter((a) => a.phase === undefined || a.phase === "" || !phaseTitleSet.has(a.phase));
4288
- const unphasedView = unphased.length > 0
4289
- ? { done: unphased.filter((a) => isDone(a.status)).length, total: unphased.length }
4290
- : undefined;
4291
- const agents = runAgents.map((a) => ({
4292
- label: redactSecrets(a.label),
4293
- status: a.status,
4294
- displayStatus: deriveAgentDisplayStatus(a, run.status),
4295
- taskStatus: a.taskStatus,
4296
- callKey: a.callKey,
4297
- groupId: a.groupId,
4298
- phase: a.phase !== undefined ? redactSecrets(a.phase) : undefined,
4299
- model: a.model,
4300
- tokens: a.stats?.tokens,
4301
- turns: a.stats?.turns,
4302
- toolCalls: a.toolCalls,
4303
- activity: a.activity,
4304
- lastActivityAt: a.activity && a.activity.length > 0 ? a.activity[a.activity.length - 1].at : undefined,
4305
- durationMs: durationOf(a),
4306
- queuedAt: a.queuedAt,
4307
- startedAt: a.startedAt,
4308
- endedAt: a.endedAt,
4309
- replayed: a.replayed,
4310
- prompt: a.prompt,
4311
- output: a.output,
4312
- }));
4313
- const byId = new Map();
4314
- for (const g of runGroups) {
4315
- byId.set(g.groupId, {
4316
- groupId: g.groupId,
4317
- parentGroupId: g.parentGroupId,
4318
- status: g.status,
4319
- startedAt: g.startedAt,
4320
- endedAt: g.endedAt,
4321
- durationMs: durationOf(g),
4322
- agentCallKeys: runAgents.filter((a) => a.groupId === g.groupId).map((a) => a.callKey),
4323
- children: [],
4324
- });
4325
- }
4326
- const inCycle = (start) => {
4327
- const seen = new Set();
4328
- let cur = start;
4329
- while (cur && cur.parentGroupId !== undefined) {
4330
- if (seen.has(cur.groupId))
4331
- return true;
4332
- seen.add(cur.groupId);
4333
- cur = byId.get(cur.parentGroupId);
4334
- }
4335
- return false;
4336
- };
4337
- const roots = [];
4338
- for (const node of byId.values()) {
4339
- const parent = node.parentGroupId !== undefined ? byId.get(node.parentGroupId) : undefined;
4340
- if (parent && parent !== node && !inCycle(node))
4341
- parent.children.push(node);
4342
- else
4343
- roots.push(node);
4344
- }
4345
- return {
4346
- id: run.id,
4347
- scope: run.scope,
4348
- status: run.status,
4349
- ...(run.name !== undefined ? { name: redactSecrets(run.name) } : {}),
4350
- ...(run.description !== undefined ? { description: redactSecrets(run.description) } : {}),
4351
- stats: run.stats,
4352
- ...(run.agentFailures !== undefined ? { agentFailures: run.agentFailures } : {}),
4353
- startedAt: run.startedAt,
4354
- endedAt: run.endedAt,
4355
- createdAt: run.createdAt,
4356
- durationMs: durationOf(run),
4357
- rev: run.rev,
4358
- error: run.error,
4359
- ...(run.result !== undefined ? { result: redactSecrets(run.result) } : {}),
4360
- phases,
4361
- ...(unphasedView ? { unphased: unphasedView } : {}),
4362
- agents,
4363
- groups: roots,
4364
- };
4365
- }
4366
- async function streamWorkflowRun(req, res, runId, scope) {
4367
- sseHeaders(res);
4368
- res.write(`event: meta\ndata: ${JSON.stringify({ version: 1, runId })}\n\n`);
4369
- let closed = false;
4370
- const it = subscribeWorkflow(runId, scope)[Symbol.asyncIterator]();
4371
- const hb = setInterval(() => {
4372
- if (!res.writableEnded)
4373
- res.write(`event: heartbeat\ndata: {}\n\n`);
4374
- }, 15_000);
4375
- if (typeof hb.unref === "function")
4376
- hb.unref();
4377
- const shutdown = () => {
4378
- if (closed)
4379
- return;
4380
- closed = true;
4381
- clearInterval(hb);
4382
- void it.return?.(undefined);
4383
- if (!res.writableEnded)
4384
- res.end();
4385
- };
4386
- req.on("close", shutdown);
4387
- res.on("close", shutdown);
4388
- try {
4389
- for (let next = await it.next(); !next.done; next = await it.next()) {
4390
- if (closed || res.writableEnded)
4391
- break;
4392
- const ev = next.value;
4393
- res.write(`event: ${ev.type ?? "event"}\ndata: ${JSON.stringify(ev)}\n\n`);
4394
- }
4395
- }
4396
- catch {
4397
- if (!closed && !res.writableEnded)
4398
- res.write(`event: error\ndata: ${JSON.stringify({ message: "workflow stream error" })}\n\n`);
4399
- }
4400
- finally {
4401
- clearInterval(hb);
4402
- void it.return?.(undefined);
4403
- }
4404
- if (!res.writableEnded)
4405
- res.end();
4406
- }
4407
- function streamFleet(req, res, bus, callerScope, callerSession = null, completionInbox) {
4408
- return new Promise((resolve) => {
4409
- sseHeaders(res);
4410
- res.write(`event: meta\ndata: ${JSON.stringify({ version: 1, scoped: callerScope !== null, sessionScoped: callerSession !== null, bgNotifyFailClosed: true })}\n\n`);
4411
- const seenTasks = new Set();
4412
- const seenWf = new Set();
4413
- const sessionOk = (rowSession) => callerSession === null || rowSession === undefined || rowSession === callerSession;
4414
- const visT = (r) => (callerScope === null || r.scope === callerScope) && sessionOk(r.sessionId);
4415
- const visW = (r) => (callerScope === null || r.scope === callerScope) && sessionOk(r.sessionId);
4416
- const stripT = (r) => {
4417
- const { scope: _s, sessionId: _sid, ...rest } = r;
4418
- if (callerSession !== null && r.sessionId === undefined) {
4419
- const { description: _d, agentType: _at, agentName: _an, currentAction: _ca, ...keep } = rest;
4420
- return { ...keep, name: "(unattributed)" };
4421
- }
4422
- return rest;
4423
- };
4424
- const stripW = (r) => {
4425
- const { scope: _s, sessionId: _sid, ...rest } = r;
4426
- if (callerSession !== null && r.sessionId === undefined) {
4427
- const { description: _d, ...keep } = rest;
4428
- return { ...keep, name: "(unattributed)" };
4429
- }
4430
- return rest;
4431
- };
4432
- const send = (event, data) => { if (!res.writableEnded)
4433
- res.write(`event: ${event}\ndata: ${JSON.stringify(data)}\n\n`); };
4434
- const snap = bus.snapshot();
4435
- const tasks = snap.tasks.filter(visT);
4436
- const workflows = snap.workflows.filter(visW);
4437
- tasks.forEach((r) => seenTasks.add(r.id));
4438
- workflows.forEach((r) => seenWf.add(r.id));
4439
- send("snapshot", { tasks: tasks.map(stripT), workflows: workflows.map(stripW), ts: snap.ts });
4440
- const unsubscribe = bus.subscribe((frame) => {
4441
- switch (frame.type) {
4442
- case "task":
4443
- if (visT(frame.row)) {
4444
- seenTasks.add(frame.row.id);
4445
- send("task", { row: stripT(frame.row), ts: frame.ts });
4446
- }
4447
- break;
4448
- case "workflow":
4449
- if (visW(frame.row)) {
4450
- seenWf.add(frame.row.id);
4451
- send("workflow", { row: stripW(frame.row), ts: frame.ts });
4452
- }
4453
- break;
4454
- case "task_remove":
4455
- if (seenTasks.delete(frame.id))
4456
- send("task_remove", { id: frame.id, ts: frame.ts });
4457
- break;
4458
- case "workflow_remove":
4459
- if (seenWf.delete(frame.id))
4460
- send("workflow_remove", { id: frame.id, ts: frame.ts });
4461
- break;
4462
- case "hook_notice": {
4463
- const hn = frame.notice;
4464
- const hScopeOk = callerScope === null || hn.ownerScope === callerScope;
4465
- const hSessionOk = callerSession === null || hn.ownerSessionId === callerSession;
4466
- if (hScopeOk && hSessionOk) {
4467
- const { ownerScope: _hs, ownerSessionId: _hsid, ...wire } = hn;
4468
- send("hook_notice", { type: "hook_notice", ...wire, ts: frame.ts });
4469
- }
4470
- break;
4471
- }
4472
- case "bg_notification": {
4473
- const n = frame.notification;
4474
- const scopeOk = callerScope === null || n.ownerScope === callerScope;
4475
- const sessionInjectOk = callerSession === null || n.ownerSessionId === callerSession;
4476
- if (scopeOk && sessionInjectOk) {
4477
- const { ownerScope: _os, ownerSessionId: _osid, ...wire } = n;
4478
- send("bg_notification", { type: "bg_notification", ...wire, ts: frame.ts });
4479
- if (completionInbox && n.ownerSessionId && callerSession !== null && callerSession === n.ownerSessionId && !res.writableEnded) {
4480
- const ownerSid = n.ownerSessionId;
4481
- const family = taskNotificationFoldKey({ task_id: n.taskId, status: n.status, ...(n.seq !== undefined ? { seq: n.seq } : {}) });
4482
- const familyUuid = n.sessionId !== undefined && n.sessionId !== n.taskId
4483
- ? taskNotificationFoldKey({ task_id: n.sessionId, status: n.status, ...(n.seq !== undefined ? { seq: n.seq } : {}) })
4484
- : undefined;
4485
- const ackIfAlive = () => {
4486
- if (res.writableEnded || res.socket?.destroyed)
4487
- return;
4488
- void completionInbox.markTerminalServed(ownerSid, family).catch(() => undefined);
4489
- if (familyUuid !== undefined)
4490
- void completionInbox.markTerminalServed(ownerSid, familyUuid).catch(() => undefined);
4491
- };
4492
- res.write(`: ack
4493
-
4494
- `, ackIfAlive);
4495
- }
4496
- }
4497
- break;
4498
- }
4499
- default:
4500
- break;
4501
- }
4502
- });
4503
- const hb = setInterval(() => { if (!res.writableEnded)
4504
- res.write(`event: heartbeat\ndata: {}\n\n`); }, 15_000);
4505
- if (typeof hb.unref === "function")
4506
- hb.unref();
4507
- let done = false;
4508
- const shutdown = () => {
4509
- if (done)
4510
- return;
4511
- done = true;
4512
- clearInterval(hb);
4513
- unsubscribe();
4514
- if (!res.writableEnded)
4515
- res.end();
4516
- resolve();
4517
- };
4518
- req.on("close", shutdown);
4519
- res.on("close", shutdown);
4520
- });
4521
- }
4522
1597
  function routeLabel(_method, url) {
4523
1598
  if (RUN_STEER_RE.test(url))
4524
1599
  return "/v1/runs/:id/steer";