@sema-agent/server 7.115.0 → 7.116.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/MIGRATION.md +167 -4
- package/USAGE.md +6 -3
- package/dist/approval-ask-audit-store.d.ts +25 -6
- package/dist/approval-ask-audit-store.js +6 -1
- package/dist/approval-card.d.ts +8 -4
- package/dist/approval-card.js +2 -0
- package/dist/approval-crash-convergence.d.ts +60 -0
- package/dist/approval-crash-convergence.js +75 -0
- package/dist/approval-deny-reasons.d.ts +13 -0
- package/dist/approval-deny-reasons.js +2 -0
- package/dist/approval.js +2 -1
- package/dist/boot/coordinators.js +0 -12
- package/dist/boot/execution-env.js +2 -1
- package/dist/boot/resolve-spec.js +1 -0
- package/dist/boot/stage-07-capability-layer.js +21 -15
- package/dist/boot/stores.js +4 -4
- package/dist/capabilities/hands-lane.d.ts +0 -23
- package/dist/capabilities/hands-lane.js +3 -2
- package/dist/capabilities/send-user-file-tool.d.ts +10 -22
- package/dist/capabilities/send-user-file-tool.js +1 -8
- package/dist/capability-facts.d.ts +296 -0
- package/dist/capability-facts.js +354 -0
- package/dist/config-center/apply-effective.js +1 -1
- package/dist/config.d.ts +16 -13
- package/dist/config.js +12 -12
- package/dist/http/admission.d.ts +6 -0
- package/dist/http/admission.js +22 -5
- package/dist/http/principal-gate.d.ts +3 -0
- package/dist/http/principal-gate.js +3 -0
- package/dist/http/resume-legs.js +10 -1
- package/dist/http/retired-skill-manifest.d.ts +23 -0
- package/dist/http/retired-skill-manifest.js +22 -0
- package/dist/http/route-table.d.ts +3 -2
- package/dist/http/route-table.js +5 -5
- package/dist/http/routes/a2a-serve.js +14 -21
- package/dist/http/routes/admin-config-refresh.js +7 -2
- package/dist/http/routes/admin-drain.js +7 -2
- package/dist/http/routes/admin-memory-optout.js +20 -6
- package/dist/http/routes/adoption.js +19 -5
- package/dist/http/routes/agents-roster.js +4 -2
- package/dist/http/routes/approvals-assistant.js +41 -18
- package/dist/http/routes/background.d.ts +0 -2
- package/dist/http/routes/background.js +4 -5
- package/dist/http/routes/capabilities.d.ts +2 -1
- package/dist/http/routes/capabilities.js +62 -65
- package/dist/http/routes/config-catalog.d.ts +4 -5
- package/dist/http/routes/config-catalog.js +16 -2
- package/dist/http/routes/devices.js +17 -11
- package/dist/http/routes/diagnostics.js +6 -1
- package/dist/http/routes/fleet.js +4 -2
- package/dist/http/routes/images.js +30 -3
- package/dist/http/routes/leader.js +7 -1
- package/dist/http/routes/mcp-reconnect.js +17 -10
- package/dist/http/routes/memory-bundle.js +20 -6
- package/dist/http/routes/memory-compliance.js +19 -5
- package/dist/http/routes/memory-consolidation.d.ts +9 -12
- package/dist/http/routes/memory-consolidation.js +17 -3
- package/dist/http/routes/memory-origin.js +19 -5
- package/dist/http/routes/notify-wake.js +5 -3
- package/dist/http/routes/observability.js +19 -5
- package/dist/http/routes/retention-ops.js +6 -1
- package/dist/http/routes/rules.js +49 -21
- package/dist/http/routes/run-memory-capture.js +5 -2
- package/dist/http/routes/runs.js +90 -60
- package/dist/http/routes/session-sync.js +64 -53
- package/dist/http/routes/sessions-list.js +8 -5
- package/dist/http/routes/sessions.js +39 -24
- package/dist/http/routes/shared-memory.js +5 -3
- package/dist/http/routes/tasks.js +2 -1
- package/dist/http/routes/trace-usage.js +18 -11
- package/dist/http/routes/workflows.js +36 -13
- package/dist/http/server.js +4 -1
- package/dist/http/wire-types.d.ts +6 -0
- package/dist/namespaced-rule-name.d.ts +54 -0
- package/dist/namespaced-rule-name.js +33 -0
- package/dist/observability/corrupt-read-seat.d.ts +4 -3
- package/dist/observability/corrupt-read-seat.js +4 -3
- package/dist/observability/fail-open.d.ts +5 -1
- package/dist/observability/fail-open.js +5 -1
- package/dist/observability/permission-rule-events.d.ts +3 -0
- package/dist/observability/permission-rule-events.js +4 -0
- package/dist/plugins/file-run-store.d.ts +7 -1
- package/dist/plugins/file-run-store.js +20 -7
- package/dist/plugins/memory-run-store.js +2 -1
- package/dist/plugins/one-time-migrations.d.ts +4 -2
- package/dist/plugins/one-time-migrations.js +26 -3
- package/dist/plugins/pg-session-storage.js +1 -1
- package/dist/plugins/run-store-sql.js +4 -4
- package/dist/plugins/session-policy-store-sql.js +4 -2
- package/dist/plugins/store-backend.d.ts +6 -1
- package/dist/plugins/store-backend.js +2 -1
- package/dist/plugins/store-contracts.d.ts +27 -8
- package/dist/plugins/store-contracts.js +12 -3
- package/dist/plugins/tidb-session-store.js +1 -1
- package/dist/request-key-closure.js +1 -1
- package/dist/run-local.d.ts +37 -3
- package/dist/run-local.js +48 -5
- package/dist/runs.d.ts +0 -58
- package/dist/runs.js +15 -20
- package/dist/session-policy-wire.d.ts +40 -60
- package/dist/session-policy-wire.js +60 -32
- package/dist/spec-fields.d.ts +32 -0
- package/dist/spec-fields.js +31 -16
- package/dist/standing-answer-grant.d.ts +8 -0
- package/dist/standing-answer-grant.js +5 -0
- package/dist/task-settings.d.ts +13 -3
- package/dist/tool-approval.d.ts +16 -0
- package/dist/tool-approval.js +29 -7
- package/dist/trace/core-keyset-guard.d.ts +4 -4
- package/dist/trace/self-minted-wire.d.ts +42 -0
- package/dist/trace/self-minted-wire.js +13 -0
- package/dist/user-input-ledger.d.ts +8 -5
- package/package.json +4 -2
- package/scripts/drain-check-local.mjs +115 -25
- package/scripts/drain-check.mysql.sql +67 -0
- package/scripts/drain-check.pg.sql +71 -0
package/dist/http/routes/runs.js
CHANGED
|
@@ -2,7 +2,7 @@ import { persistedPlaneOf } from "../../terminal.js";
|
|
|
2
2
|
import { memoRedactedRead, memoRedactTerminalResult, redactErrorMessage, redactHead, redactLedgerEventForRead, redactTerminalResult } from "../../observability/run-terminal.js";
|
|
3
3
|
import { z } from "zod";
|
|
4
4
|
import { uuidv7, mintCheckpointToken, CheckpointError, validatePendingSteer, TASK_OUTPUT_REPLAY_KINDS } from "@sema-agent/core";
|
|
5
|
-
import { isTerminalRunStatus, isParkedRunStatus, runProvenElsewhere } from "../../plugins/store-contracts.js";
|
|
5
|
+
import { RUN_HOLDER_DEATH_CODES, isTerminalRunStatus, isParkedRunStatus, runProvenElsewhere } from "../../plugins/store-contracts.js";
|
|
6
6
|
import { LIVE_RUN_GATE_KINDS, TERMINAL_PARK_GATE_KINDS } from "../../plugins/checkpoint-store-sql.js";
|
|
7
7
|
import { redactSecrets } from "../../trace/redact.js";
|
|
8
8
|
import { HttpError, verifiedPrincipal, isUuidV7, encodeCheckpointScope } from "../../security.js";
|
|
@@ -24,6 +24,7 @@ import { mintCancelledResult } from "../../run-cancel-context.js";
|
|
|
24
24
|
import { buildCrossSliceUsage, readResourceWindow } from "../../resource-window.js";
|
|
25
25
|
import { headerStr, gatedPrincipal, explicitOperatorOk, sendPrincipalRefusal } from "../principal-gate.js";
|
|
26
26
|
import { ActorAssertionWire } from "@sema-agent/settings-schema";
|
|
27
|
+
import * as facts from "../../capability-facts.js";
|
|
27
28
|
import { assertNever } from "../route-ctx.js";
|
|
28
29
|
import { mintAndDriveDurableRun } from "../durable-run-leg.js";
|
|
29
30
|
import { RUN_NOT_FOUND_MESSAGE, runNotFoundMessage } from "../route-ctx.js";
|
|
@@ -167,10 +168,12 @@ export async function handleRuns(req, res, match, ctx) {
|
|
|
167
168
|
const { prepareSpec } = ctx.legs;
|
|
168
169
|
const source = ctx.req.source;
|
|
169
170
|
if (match.decl.id === "runs-submit") {
|
|
170
|
-
|
|
171
|
+
const runs = facts.durableRuns(deps);
|
|
172
|
+
if (!runs.available) {
|
|
171
173
|
sendError(res, 501, "capability.run_store_required", "async runs require a durable run store (DB_BACKEND=mysql|pg|local)");
|
|
172
174
|
return;
|
|
173
175
|
}
|
|
176
|
+
const runStore = runs.runStore;
|
|
174
177
|
const rawIdem = headerStr(req.headers["idempotency-key"]);
|
|
175
178
|
const idemKey = rawIdem ? scopedIdempotencyKey(rawIdem, source, gatedPrincipal(req, deps.config)) : undefined;
|
|
176
179
|
const cached = idemKey ? idemCache.peek(idemKey) : undefined;
|
|
@@ -191,16 +194,18 @@ export async function handleRuns(req, res, match, ctx) {
|
|
|
191
194
|
return;
|
|
192
195
|
}
|
|
193
196
|
if (match.decl.id === "run-detail" || match.decl.id === "run-events") {
|
|
194
|
-
|
|
197
|
+
const runs = facts.durableRuns(deps);
|
|
198
|
+
if (!runs.available) {
|
|
195
199
|
sendError(res, 501, "capability.run_store_required", "async runs require a durable run store (DB_BACKEND=mysql|pg|local)");
|
|
196
200
|
return;
|
|
197
201
|
}
|
|
202
|
+
const runStore = runs.runStore;
|
|
198
203
|
const taskId = match.params.taskId;
|
|
199
204
|
if (deps.config.requirePrincipal && !gatedPrincipal(req, deps.config)) {
|
|
200
205
|
sendPrincipalRefusal(res, req, deps.config);
|
|
201
206
|
return;
|
|
202
207
|
}
|
|
203
|
-
const run = await
|
|
208
|
+
const run = await runStore.getRun(taskId);
|
|
204
209
|
if (!run) {
|
|
205
210
|
sendError(res, 404, "not_found.run", runNotFoundMessage(taskId));
|
|
206
211
|
return;
|
|
@@ -210,7 +215,7 @@ export async function handleRuns(req, res, match, ctx) {
|
|
|
210
215
|
if (!runSessionAcceptOk(req, res, run, match.decl.id === "run-events" ? "run.events" : "run.poll"))
|
|
211
216
|
return;
|
|
212
217
|
if (match.decl.id === "run-events") {
|
|
213
|
-
await streamRunEvents(req, res, deps,
|
|
218
|
+
await streamRunEvents(req, res, deps, runStore, taskId, deps.config.runStaleSec * 1000);
|
|
214
219
|
}
|
|
215
220
|
else {
|
|
216
221
|
const stale = run.status === "running" && Date.now() - new Date(run.updatedAt).getTime() > deps.config.runStaleSec * 1000;
|
|
@@ -221,8 +226,8 @@ export async function handleRuns(req, res, match, ctx) {
|
|
|
221
226
|
let events;
|
|
222
227
|
let eventsUnavailable = false;
|
|
223
228
|
if (needCost || needSuggestions || resourceWindow !== undefined) {
|
|
224
|
-
if (
|
|
225
|
-
events = await
|
|
229
|
+
if (runStore.getEvents) {
|
|
230
|
+
events = await runStore.getEvents(taskId, 0).catch(() => {
|
|
226
231
|
eventsUnavailable = true;
|
|
227
232
|
return [];
|
|
228
233
|
});
|
|
@@ -262,7 +267,7 @@ export async function handleRuns(req, res, match, ctx) {
|
|
|
262
267
|
result: isTerminalRunStatus(run.status) ? memoRedactTerminalResult("run.result", run.result ?? undefined) : redactTerminalResult(run.result ?? undefined),
|
|
263
268
|
supervisorCost,
|
|
264
269
|
suggestions,
|
|
265
|
-
errorCode: stale ?
|
|
270
|
+
errorCode: stale ? RUN_HOLDER_DEATH_CODES.stalled : (persistedPlaneOf(run.result).errorCode ?? run.errorCode ?? undefined),
|
|
266
271
|
error: stale ? "run stalled (instance lost?)" : isTerminalRunStatus(run.status) ? memoRedactedRead("run.error", run.error ?? undefined, redactErrorMessage) : redactErrorMessage(run.error ?? undefined),
|
|
267
272
|
jobId: run.jobId ?? undefined,
|
|
268
273
|
source: run.source ?? undefined,
|
|
@@ -273,16 +278,18 @@ export async function handleRuns(req, res, match, ctx) {
|
|
|
273
278
|
return;
|
|
274
279
|
}
|
|
275
280
|
if (match.decl.id === "run-cancel") {
|
|
276
|
-
|
|
281
|
+
const runs = facts.durableRuns(deps);
|
|
282
|
+
if (!runs.available) {
|
|
277
283
|
sendError(res, 501, "capability.run_store_required", "async runs require a durable run store (DB_BACKEND=mysql|pg|local)");
|
|
278
284
|
return;
|
|
279
285
|
}
|
|
286
|
+
const runStore = runs.runStore;
|
|
280
287
|
if (deps.config.requirePrincipal && !gatedPrincipal(req, deps.config)) {
|
|
281
288
|
sendPrincipalRefusal(res, req, deps.config);
|
|
282
289
|
return;
|
|
283
290
|
}
|
|
284
291
|
const taskId = match.params.taskId;
|
|
285
|
-
const run = await
|
|
292
|
+
const run = await runStore.getRun(taskId);
|
|
286
293
|
if (!run) {
|
|
287
294
|
sendError(res, 404, "not_found.run", runNotFoundMessage(taskId));
|
|
288
295
|
return;
|
|
@@ -291,7 +298,7 @@ export async function handleRuns(req, res, match, ctx) {
|
|
|
291
298
|
return;
|
|
292
299
|
if (!runSessionAcceptOk(req, res, run, "run.cancel"))
|
|
293
300
|
return;
|
|
294
|
-
const rs =
|
|
301
|
+
const rs = runStore;
|
|
295
302
|
const cancelSuspended = async () => {
|
|
296
303
|
const cs = deps.checkpointStore;
|
|
297
304
|
let note = "cancelled while suspended (pending approval settled)";
|
|
@@ -363,16 +370,16 @@ export async function handleRuns(req, res, match, ctx) {
|
|
|
363
370
|
sendJson(res, 202, { taskId, status: "failed", errorCode: "cancelled", cancelRequested: false, note });
|
|
364
371
|
};
|
|
365
372
|
if (run.status === "running") {
|
|
366
|
-
let flagged = await
|
|
373
|
+
let flagged = await runStore.requestCancel(taskId, run.owner);
|
|
367
374
|
const CANCEL_RECLASSIFY_ATTEMPTS = 3;
|
|
368
375
|
for (let attempt = 0; !flagged; attempt++) {
|
|
369
|
-
const now = await
|
|
376
|
+
const now = await runStore.getRun(taskId);
|
|
370
377
|
if (now !== undefined && isParkedRunStatus(now.status)) {
|
|
371
378
|
await cancelSuspended();
|
|
372
379
|
return;
|
|
373
380
|
}
|
|
374
381
|
if (now?.status === "running" && attempt < CANCEL_RECLASSIFY_ATTEMPTS) {
|
|
375
|
-
flagged = await
|
|
382
|
+
flagged = await runStore.requestCancel(taskId, run.owner);
|
|
376
383
|
continue;
|
|
377
384
|
}
|
|
378
385
|
if (now?.status === "running") {
|
|
@@ -396,10 +403,12 @@ export async function handleRuns(req, res, match, ctx) {
|
|
|
396
403
|
if (match.decl.id === "run-steer") {
|
|
397
404
|
if (rateLimited(req, res) || quotaExceeded(req, res) || (await leaseDenied(req, res)))
|
|
398
405
|
return;
|
|
399
|
-
|
|
406
|
+
const runs = facts.durableRuns(deps);
|
|
407
|
+
if (!runs.available) {
|
|
400
408
|
sendError(res, 501, "capability.run_store_required", "async runs require a durable run store (DB_BACKEND=mysql|pg|local)");
|
|
401
409
|
return;
|
|
402
410
|
}
|
|
411
|
+
const runStore = runs.runStore;
|
|
403
412
|
const principal = gatedPrincipal(req, deps.config);
|
|
404
413
|
if (deps.config.requirePrincipal && principal === undefined) {
|
|
405
414
|
sendPrincipalRefusal(res, req, deps.config);
|
|
@@ -437,8 +446,14 @@ export async function handleRuns(req, res, match, ctx) {
|
|
|
437
446
|
const steerInputId = steerScoped ? `idem-${createHash("sha256").update(steerScoped).digest("hex").slice(0, 32)}` : undefined;
|
|
438
447
|
const messageId = steerInputId ?? uuidv7();
|
|
439
448
|
const trusted = explicitOperatorOk(principal, deps.config.operatorPrincipals);
|
|
449
|
+
const steerInput = () => ({
|
|
450
|
+
text,
|
|
451
|
+
trusted,
|
|
452
|
+
...(steerInputId !== undefined ? { inputId: steerInputId } : {}),
|
|
453
|
+
...(priority !== undefined ? { priority } : {}),
|
|
454
|
+
});
|
|
440
455
|
try {
|
|
441
|
-
validatePendingSteer(
|
|
456
|
+
validatePendingSteer(steerInput());
|
|
442
457
|
}
|
|
443
458
|
catch (e) {
|
|
444
459
|
if (e instanceof CheckpointError && e.code === "steering.invalid_content") {
|
|
@@ -447,7 +462,7 @@ export async function handleRuns(req, res, match, ctx) {
|
|
|
447
462
|
}
|
|
448
463
|
throw e;
|
|
449
464
|
}
|
|
450
|
-
const run = await
|
|
465
|
+
const run = await runStore.getRun(taskId);
|
|
451
466
|
if (!run) {
|
|
452
467
|
sendError(res, 404, "not_found.run", runNotFoundMessage(taskId));
|
|
453
468
|
return;
|
|
@@ -458,11 +473,11 @@ export async function handleRuns(req, res, match, ctx) {
|
|
|
458
473
|
return;
|
|
459
474
|
let parkQueueFull;
|
|
460
475
|
let parkDuplicateKey;
|
|
461
|
-
const steerInput = () => ({ text, trusted, ...(steerInputId !== undefined ? { inputId: steerInputId } : {}) });
|
|
462
476
|
const tryPark = async () => {
|
|
463
|
-
const
|
|
464
|
-
if (!
|
|
477
|
+
const park = facts.checkpointPark(deps);
|
|
478
|
+
if (!park.available)
|
|
465
479
|
return "no-store";
|
|
480
|
+
const cs = park.checkpointStore;
|
|
466
481
|
const parkBinding = { gateKinds: LIVE_RUN_GATE_KINDS };
|
|
467
482
|
const scope = await cs.peekPendingScope(run.sessionId, parkBinding);
|
|
468
483
|
if (scope === null || scope === undefined)
|
|
@@ -495,7 +510,7 @@ export async function handleRuns(req, res, match, ctx) {
|
|
|
495
510
|
if (acceptance === "refused")
|
|
496
511
|
return;
|
|
497
512
|
try {
|
|
498
|
-
await relayAcceptedInput(deps, acceptance, () => live.steer(text, { trusted, inputId: messageId }));
|
|
513
|
+
await relayAcceptedInput(deps, acceptance, () => live.steer(text, { trusted, inputId: messageId, ...(priority !== undefined ? { priority } : {}) }));
|
|
499
514
|
sendJson(res, 200, { taskId, status: "running", delivery: "applied", messageId, ...(priority ? { priority } : {}) });
|
|
500
515
|
return;
|
|
501
516
|
}
|
|
@@ -516,7 +531,7 @@ export async function handleRuns(req, res, match, ctx) {
|
|
|
516
531
|
}
|
|
517
532
|
if (code !== "steering.not_running")
|
|
518
533
|
throw e;
|
|
519
|
-
const freshBeforePark = await
|
|
534
|
+
const freshBeforePark = await runStore.getRun(taskId).catch(() => undefined);
|
|
520
535
|
const raced = await tryPark();
|
|
521
536
|
if (raced === "parked") {
|
|
522
537
|
sendParked(freshBeforePark?.status ?? run.status);
|
|
@@ -535,11 +550,11 @@ export async function handleRuns(req, res, match, ctx) {
|
|
|
535
550
|
}
|
|
536
551
|
}
|
|
537
552
|
if (isParkedRunStatus(run.status)) {
|
|
538
|
-
|
|
539
|
-
if (outcome === "no-store") {
|
|
553
|
+
if (!facts.checkpointPark(deps).available) {
|
|
540
554
|
sendError(res, 501, "capability.checkpoint_store_required", "steering a suspended run requires the checkpoint store");
|
|
541
555
|
return;
|
|
542
556
|
}
|
|
557
|
+
const outcome = await tryPark();
|
|
543
558
|
if (outcome === "parked") {
|
|
544
559
|
sendParked(run.status);
|
|
545
560
|
return;
|
|
@@ -566,7 +581,7 @@ export async function handleRuns(req, res, match, ctx) {
|
|
|
566
581
|
return;
|
|
567
582
|
}
|
|
568
583
|
if (deps.checkpointStore && deps.sessionStorage?.getLeafId) {
|
|
569
|
-
const activeTaskId = await
|
|
584
|
+
const activeTaskId = await runStore.getActiveTaskId?.(run.sessionId).catch(() => undefined);
|
|
570
585
|
if (activeTaskId !== undefined && activeTaskId !== null && activeTaskId !== taskId) {
|
|
571
586
|
sendNotRunning(`run is ${run.status} and its session has an ACTIVE run (${activeTaskId}) — steer that run instead`);
|
|
572
587
|
return;
|
|
@@ -688,10 +703,12 @@ export async function handleRuns(req, res, match, ctx) {
|
|
|
688
703
|
if (match.decl.id === "run-interrupt") {
|
|
689
704
|
if (rateLimited(req, res) || quotaExceeded(req, res) || (await leaseDenied(req, res)))
|
|
690
705
|
return;
|
|
691
|
-
|
|
706
|
+
const runs = facts.durableRuns(deps);
|
|
707
|
+
if (!runs.available) {
|
|
692
708
|
sendError(res, 501, "capability.run_store_required", "async runs require a durable run store (DB_BACKEND=mysql|pg|local)");
|
|
693
709
|
return;
|
|
694
710
|
}
|
|
711
|
+
const runStore = runs.runStore;
|
|
695
712
|
const principal = gatedPrincipal(req, deps.config);
|
|
696
713
|
if (deps.config.requirePrincipal && principal === undefined) {
|
|
697
714
|
sendPrincipalRefusal(res, req, deps.config);
|
|
@@ -746,7 +763,7 @@ export async function handleRuns(req, res, match, ctx) {
|
|
|
746
763
|
throw e;
|
|
747
764
|
}
|
|
748
765
|
}
|
|
749
|
-
const run = await
|
|
766
|
+
const run = await runStore.getRun(taskId);
|
|
750
767
|
if (!run) {
|
|
751
768
|
sendError(res, 404, "not_found.run", runNotFoundMessage(taskId));
|
|
752
769
|
return;
|
|
@@ -790,7 +807,7 @@ export async function handleRuns(req, res, match, ctx) {
|
|
|
790
807
|
catch (e) {
|
|
791
808
|
if (e.code !== "steering.not_running")
|
|
792
809
|
throw e;
|
|
793
|
-
const freshRun = await
|
|
810
|
+
const freshRun = await runStore.getRun(taskId).catch(() => undefined);
|
|
794
811
|
sendNoLiveTurn(freshRun?.status ?? run.status, true);
|
|
795
812
|
return;
|
|
796
813
|
}
|
|
@@ -824,7 +841,7 @@ export async function handleRuns(req, res, match, ctx) {
|
|
|
824
841
|
}
|
|
825
842
|
if (code !== "steering.not_running")
|
|
826
843
|
throw e;
|
|
827
|
-
const freshRun = await
|
|
844
|
+
const freshRun = await runStore.getRun(taskId).catch(() => undefined);
|
|
828
845
|
sendNoLiveTurn(freshRun?.status ?? run.status, true);
|
|
829
846
|
return;
|
|
830
847
|
}
|
|
@@ -834,17 +851,19 @@ export async function handleRuns(req, res, match, ctx) {
|
|
|
834
851
|
if (match.decl.id === "run-compact") {
|
|
835
852
|
if (rateLimited(req, res) || quotaExceeded(req, res) || (await leaseDenied(req, res)))
|
|
836
853
|
return;
|
|
837
|
-
|
|
854
|
+
const runs = facts.durableRuns(deps);
|
|
855
|
+
if (!runs.available) {
|
|
838
856
|
sendError(res, 501, "capability.run_store_required", "async runs require a durable run store (DB_BACKEND=mysql|pg|local)");
|
|
839
857
|
return;
|
|
840
858
|
}
|
|
859
|
+
const runStore = runs.runStore;
|
|
841
860
|
const principal = gatedPrincipal(req, deps.config);
|
|
842
861
|
if (deps.config.requirePrincipal && principal === undefined) {
|
|
843
862
|
sendPrincipalRefusal(res, req, deps.config);
|
|
844
863
|
return;
|
|
845
864
|
}
|
|
846
865
|
const taskId = match.params.taskId;
|
|
847
|
-
const run = await
|
|
866
|
+
const run = await runStore.getRun(taskId);
|
|
848
867
|
if (!run) {
|
|
849
868
|
sendError(res, 404, "not_found.run", RUN_NOT_FOUND_MESSAGE);
|
|
850
869
|
return;
|
|
@@ -898,10 +917,12 @@ export async function handleRuns(req, res, match, ctx) {
|
|
|
898
917
|
if (match.decl.id === "run-detach") {
|
|
899
918
|
if (rateLimited(req, res))
|
|
900
919
|
return;
|
|
901
|
-
|
|
920
|
+
const runs = facts.durableRuns(deps);
|
|
921
|
+
if (!runs.available) {
|
|
902
922
|
sendError(res, 501, "capability.run_store_required", "async runs require a durable run store (DB_BACKEND=mysql|pg|local)");
|
|
903
923
|
return;
|
|
904
924
|
}
|
|
925
|
+
const runStore = runs.runStore;
|
|
905
926
|
const principal = gatedPrincipal(req, deps.config);
|
|
906
927
|
if (deps.config.requirePrincipal && principal === undefined) {
|
|
907
928
|
sendPrincipalRefusal(res, req, deps.config);
|
|
@@ -924,7 +945,7 @@ export async function handleRuns(req, res, match, ctx) {
|
|
|
924
945
|
sendError(res, 400, "request.body_shape", "body must be { toolCallId: string (non-empty, ≤256 chars) }");
|
|
925
946
|
return;
|
|
926
947
|
}
|
|
927
|
-
const run = await
|
|
948
|
+
const run = await runStore.getRun(taskId);
|
|
928
949
|
if (!run) {
|
|
929
950
|
sendError(res, 404, "not_found.run", RUN_NOT_FOUND_MESSAGE);
|
|
930
951
|
return;
|
|
@@ -949,10 +970,13 @@ export async function handleRunVerbs(req, res, match, ctx) {
|
|
|
949
970
|
const { deps } = ctx;
|
|
950
971
|
const { readJson, rateLimited, quotaExceeded, ownerQuotaDenied, leaseDenied, ownerLeaseDenied, safeDecode, runOwnerOk, runSessionAcceptOk } = ctx.helpers;
|
|
951
972
|
if (match.decl.id === "run-subagent-output") {
|
|
952
|
-
|
|
973
|
+
const outputFact = facts.subagentOutput(deps);
|
|
974
|
+
if (!outputFact.available) {
|
|
953
975
|
sendError(res, 501, "capability.run_store_required", "subagent output reads require the run store");
|
|
954
976
|
return;
|
|
955
977
|
}
|
|
978
|
+
const runStore = outputFact.runStore;
|
|
979
|
+
const subagentTaskOutput = outputFact.output;
|
|
956
980
|
const runId = safeDecode(match.params.runId);
|
|
957
981
|
const target = safeDecode(match.params.sub);
|
|
958
982
|
if (runId === null || target === null) {
|
|
@@ -963,7 +987,7 @@ export async function handleRunVerbs(req, res, match, ctx) {
|
|
|
963
987
|
if (!gated)
|
|
964
988
|
return;
|
|
965
989
|
const { run } = gated;
|
|
966
|
-
const out = await
|
|
990
|
+
const out = await subagentTaskOutput(target, { owner: runId, scope: run.owner ?? "default", ...(run.sessionId ? { sessionId: run.sessionId } : {}) });
|
|
967
991
|
const details = out.details;
|
|
968
992
|
if (details?.error === "not_found" || details?.type !== "background_agent") {
|
|
969
993
|
if (!sendHandleMiss(res, run, deps.instanceId))
|
|
@@ -974,10 +998,13 @@ export async function handleRunVerbs(req, res, match, ctx) {
|
|
|
974
998
|
return;
|
|
975
999
|
}
|
|
976
1000
|
if (match.decl.id === "run-subagent-stream") {
|
|
977
|
-
|
|
1001
|
+
const outputFact = facts.subagentOutput(deps);
|
|
1002
|
+
if (!outputFact.available) {
|
|
978
1003
|
sendError(res, 501, "capability.run_store_required", "subagent streams require the run store");
|
|
979
1004
|
return;
|
|
980
1005
|
}
|
|
1006
|
+
const runStore = outputFact.runStore;
|
|
1007
|
+
const subagentTaskOutput = outputFact.output;
|
|
981
1008
|
const runId = safeDecode(match.params.runId);
|
|
982
1009
|
const target = safeDecode(match.params.sub);
|
|
983
1010
|
if (runId === null || target === null) {
|
|
@@ -989,7 +1016,7 @@ export async function handleRunVerbs(req, res, match, ctx) {
|
|
|
989
1016
|
return;
|
|
990
1017
|
const { run } = gatedTail;
|
|
991
1018
|
const it = defaultSubagentTailBus.subscribe(target);
|
|
992
|
-
const probe = await
|
|
1019
|
+
const probe = await subagentTaskOutput(target, { owner: runId, scope: run.owner ?? "default", ...(run.sessionId ? { sessionId: run.sessionId } : {}) });
|
|
993
1020
|
const probeDetails = probe.details;
|
|
994
1021
|
if (probeDetails?.error === "not_found" || probeDetails?.type !== "background_agent") {
|
|
995
1022
|
void it.return?.();
|
|
@@ -1037,10 +1064,12 @@ export async function handleRunVerbs(req, res, match, ctx) {
|
|
|
1037
1064
|
}
|
|
1038
1065
|
if (match.decl.id === "run-task-output" || match.decl.id === "run-task-stop") {
|
|
1039
1066
|
const stopVerb = match.decl.id === "run-task-stop";
|
|
1040
|
-
|
|
1067
|
+
const handlesFact = facts.taskHandles(deps);
|
|
1068
|
+
if (!handlesFact.available) {
|
|
1041
1069
|
sendError(res, 501, "capability.run_store_required", "task-handle verbs require the run store and the task registry");
|
|
1042
1070
|
return;
|
|
1043
1071
|
}
|
|
1072
|
+
const runStore = handlesFact.runStore;
|
|
1044
1073
|
if (stopVerb && rateLimited(req, res))
|
|
1045
1074
|
return;
|
|
1046
1075
|
const runId = safeDecode(match.params.runId);
|
|
@@ -1060,8 +1089,8 @@ export async function handleRunVerbs(req, res, match, ctx) {
|
|
|
1060
1089
|
return;
|
|
1061
1090
|
}
|
|
1062
1091
|
const out = stopVerb
|
|
1063
|
-
? await
|
|
1064
|
-
: await
|
|
1092
|
+
? await handlesFact.stop(target, access)
|
|
1093
|
+
: await handlesFact.output(target, access);
|
|
1065
1094
|
const details = out.details;
|
|
1066
1095
|
if (details?.error === "not_found") {
|
|
1067
1096
|
if (!sendHandleMiss(res, run, deps.instanceId))
|
|
@@ -1108,10 +1137,12 @@ export async function handleRunVerbs(req, res, match, ctx) {
|
|
|
1108
1137
|
const verb = match.decl.id === "run-subagent-resume" ? "resume" : "steer";
|
|
1109
1138
|
if (rateLimited(req, res) || quotaExceeded(req, res) || (await leaseDenied(req, res)))
|
|
1110
1139
|
return;
|
|
1111
|
-
|
|
1140
|
+
const steerFact = facts.subagentSteer(deps);
|
|
1141
|
+
if (!steerFact.available) {
|
|
1112
1142
|
sendError(res, 501, "capability.run_store_required", "async runs require a durable run store (DB_BACKEND=mysql|pg|local)");
|
|
1113
1143
|
return;
|
|
1114
1144
|
}
|
|
1145
|
+
const runStore = steerFact.runStore;
|
|
1115
1146
|
const principal = gatedPrincipal(req, deps.config);
|
|
1116
1147
|
if (deps.config.requirePrincipal && principal === undefined) {
|
|
1117
1148
|
sendPrincipalRefusal(res, req, deps.config);
|
|
@@ -1144,7 +1175,7 @@ export async function handleRunVerbs(req, res, match, ctx) {
|
|
|
1144
1175
|
return;
|
|
1145
1176
|
}
|
|
1146
1177
|
const trusted = explicitOperatorOk(principal, deps.config.operatorPrincipals);
|
|
1147
|
-
const run = await
|
|
1178
|
+
const run = await runStore.getRun(runId);
|
|
1148
1179
|
if (!run) {
|
|
1149
1180
|
sendError(res, 404, "not_found.run", RUN_NOT_FOUND_MESSAGE);
|
|
1150
1181
|
return;
|
|
@@ -1155,7 +1186,7 @@ export async function handleRunVerbs(req, res, match, ctx) {
|
|
|
1155
1186
|
return;
|
|
1156
1187
|
const send409 = (errorCode, error) => sendError(res, 409, errorCode, error);
|
|
1157
1188
|
const redacted = redactSteerIn(body.content, target);
|
|
1158
|
-
const resolution =
|
|
1189
|
+
const resolution = steerFact.registry.resolve(runId, target);
|
|
1159
1190
|
if (resolution && resolution.count > 1) {
|
|
1160
1191
|
send409("steering.ambiguous_target", `${resolution.count} sub-agents match '${target}' in this run — address by parentToolCallId`);
|
|
1161
1192
|
return;
|
|
@@ -1215,7 +1246,8 @@ export async function handleRunVerbs(req, res, match, ctx) {
|
|
|
1215
1246
|
if (match.decl.id === "elicitation-respond") {
|
|
1216
1247
|
if (rateLimited(req, res))
|
|
1217
1248
|
return;
|
|
1218
|
-
|
|
1249
|
+
const elicitFact = facts.elicitation(deps);
|
|
1250
|
+
if (!elicitFact.available) {
|
|
1219
1251
|
sendError(res, 501, "feature.elicitation_disabled", "inbound MCP elicitation is not enabled on this worker (MCP_ELICITATION_ENABLED)");
|
|
1220
1252
|
return;
|
|
1221
1253
|
}
|
|
@@ -1233,14 +1265,15 @@ export async function handleRunVerbs(req, res, match, ctx) {
|
|
|
1233
1265
|
sendError(res, 400, "request.invalid_json", "invalid JSON body");
|
|
1234
1266
|
return;
|
|
1235
1267
|
}
|
|
1236
|
-
const { status, body: respBody } =
|
|
1268
|
+
const { status, body: respBody } = elicitFact.coordinator.respond(id, principal, body);
|
|
1237
1269
|
sendJson(res, status, respBody);
|
|
1238
1270
|
return;
|
|
1239
1271
|
}
|
|
1240
1272
|
if (match.decl.id === "question-respond") {
|
|
1241
1273
|
if (rateLimited(req, res))
|
|
1242
1274
|
return;
|
|
1243
|
-
|
|
1275
|
+
const questionFact = facts.askUserQuestion(deps);
|
|
1276
|
+
if (!questionFact.available) {
|
|
1244
1277
|
sendError(res, 501, "feature.ask_question_disabled", "AskUserQuestion live HITL is not enabled on this worker (ASK_QUESTION_ENABLED)");
|
|
1245
1278
|
return;
|
|
1246
1279
|
}
|
|
@@ -1258,14 +1291,15 @@ export async function handleRunVerbs(req, res, match, ctx) {
|
|
|
1258
1291
|
sendError(res, 400, "request.invalid_json", "invalid JSON body");
|
|
1259
1292
|
return;
|
|
1260
1293
|
}
|
|
1261
|
-
const { status, body: respBody } =
|
|
1294
|
+
const { status, body: respBody } = questionFact.coordinator.respond(id, principal, body);
|
|
1262
1295
|
sendJson(res, status, respBody);
|
|
1263
1296
|
return;
|
|
1264
1297
|
}
|
|
1265
1298
|
if (match.decl.id === "tool-approval-respond") {
|
|
1266
1299
|
if (rateLimited(req, res))
|
|
1267
1300
|
return;
|
|
1268
|
-
|
|
1301
|
+
const approvalFact = facts.toolApproval(deps);
|
|
1302
|
+
if (!approvalFact.available) {
|
|
1269
1303
|
sendError(res, 501, "feature.tool_approval_disabled", "live tool-approval HITL is not enabled on this worker (TOOL_APPROVAL_ENABLED)");
|
|
1270
1304
|
return;
|
|
1271
1305
|
}
|
|
@@ -1283,7 +1317,7 @@ export async function handleRunVerbs(req, res, match, ctx) {
|
|
|
1283
1317
|
sendError(res, 400, "request.invalid_json", "invalid JSON body");
|
|
1284
1318
|
return;
|
|
1285
1319
|
}
|
|
1286
|
-
const { status, body: respBody } = await
|
|
1320
|
+
const { status, body: respBody } = await approvalFact.coordinator.respond(id, principal, body, req);
|
|
1287
1321
|
const retryAfterSec = respBody.retryAfterSec;
|
|
1288
1322
|
if (typeof retryAfterSec === "number" && Number.isFinite(retryAfterSec) && retryAfterSec >= 0) {
|
|
1289
1323
|
res.setHeader("retry-after", String(Math.ceil(retryAfterSec)));
|
|
@@ -1294,17 +1328,12 @@ export async function handleRunVerbs(req, res, match, ctx) {
|
|
|
1294
1328
|
if (match.decl.id === "task-ask-decision") {
|
|
1295
1329
|
if (rateLimited(req, res))
|
|
1296
1330
|
return;
|
|
1297
|
-
const
|
|
1298
|
-
|
|
1299
|
-
streamApprovalEnabled: deps.config.streamApproval?.enabled === true,
|
|
1300
|
-
backend: deps.backend,
|
|
1301
|
-
parkFacility: deps.checkpointStore !== undefined,
|
|
1302
|
-
});
|
|
1303
|
-
const askStore = gate.active ? gate.askStore : undefined;
|
|
1304
|
-
if (!askStore) {
|
|
1331
|
+
const streamFact = facts.streamApproval(deps);
|
|
1332
|
+
if (!streamFact.available) {
|
|
1305
1333
|
sendError(res, 501, "feature.approval_ask_disabled", "the durable stream-approval ask ledger is not enabled on this worker (needs STREAM_APPROVAL_ENABLED plus a DURABLE store backend — the local backend keeps asks in memory and would lose accepted decisions on restart)");
|
|
1306
1334
|
return;
|
|
1307
1335
|
}
|
|
1336
|
+
const askStore = streamFact.askStore;
|
|
1308
1337
|
const principal = gatedPrincipal(req, deps.config);
|
|
1309
1338
|
if (deps.config.requirePrincipal && principal === undefined) {
|
|
1310
1339
|
sendPrincipalRefusal(res, req, deps.config);
|
|
@@ -1434,7 +1463,7 @@ export async function handleRunVerbs(req, res, match, ctx) {
|
|
|
1434
1463
|
return;
|
|
1435
1464
|
case "DENIED":
|
|
1436
1465
|
case "VOID":
|
|
1437
|
-
sendError(res, 410, "gone.ask", "this ask reached a terminal state without a human decision (routing failure or batch abort) and can no longer be decided", {
|
|
1466
|
+
sendError(res, 410, "gone.ask", "this ask reached a terminal state without a human decision (deniedReason says why: a routing failure, a cancellation or batch abort, or the process that held it ended) and can no longer be decided", {
|
|
1438
1467
|
state: row.state,
|
|
1439
1468
|
...(row.deniedReason !== null ? { deniedReason: row.deniedReason } : {}),
|
|
1440
1469
|
});
|
|
@@ -1566,11 +1595,12 @@ export async function createDurableRun(req, res, ctx, prepared, idemKey) {
|
|
|
1566
1595
|
const { idemCache, inflightRuns, preemptableRuns, steerableRuns } = ctx.registry;
|
|
1567
1596
|
const { leaseDenied, usageWindowDenied } = ctx.helpers;
|
|
1568
1597
|
const source = ctx.req.source;
|
|
1569
|
-
|
|
1598
|
+
const runs = facts.durableRuns(deps);
|
|
1599
|
+
if (!runs.available) {
|
|
1570
1600
|
sendError(res, 501, "capability.run_store_required", "async runs require a durable run store (DB_BACKEND=mysql|pg|local)");
|
|
1571
1601
|
return null;
|
|
1572
1602
|
}
|
|
1573
|
-
const runStore =
|
|
1603
|
+
const runStore = runs.runStore;
|
|
1574
1604
|
const rawClientTaskId = prepared.body.taskId;
|
|
1575
1605
|
if (rawClientTaskId !== undefined && (typeof rawClientTaskId !== "string" || !isUuidV7(rawClientTaskId))) {
|
|
1576
1606
|
sendError(res, 400, "request.id_invalid", "body.taskId must be a uuidv7 string (caller-minted idempotency key)");
|