@sema-agent/server 7.62.0 → 7.64.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/MIGRATION.md +8 -0
- package/README.md +2 -0
- package/README.zh-CN.md +2 -0
- package/USAGE.md +48 -15
- package/deploy/sema-up/smoke.sh +2 -2
- package/dist/approval-ask-audit-store.d.ts +149 -3
- package/dist/approval-ask-audit-store.js +11 -4
- package/dist/bench/s1/arms.d.ts +2 -2
- package/dist/bench/s1/arms.js +20 -15
- package/dist/bench/s1/live-deps.js +3 -1
- package/dist/bench/s1/run-firm.js +2 -2
- package/dist/boot/runner-deps.d.ts +2 -2
- package/dist/boot/runner-deps.js +1 -0
- package/dist/config-catalog.js +2 -0
- package/dist/config-types.d.ts +27 -2
- package/dist/config.js +83 -2
- package/dist/degenerate-instrument.d.ts +0 -33
- package/dist/degenerate-instrument.js +2 -1
- package/dist/fleet/fleet-bus.d.ts +9 -4
- package/dist/fleet/fleet-bus.js +2 -1
- package/dist/host-decision.d.ts +39 -0
- package/dist/host-decision.js +6 -0
- package/dist/http/principal-gate.d.ts +9 -0
- package/dist/http/principal-gate.js +12 -0
- package/dist/http/route-ctx.d.ts +8 -5
- package/dist/http/routes/a2a-serve.js +4 -2
- package/dist/http/routes/approvals-assistant.d.ts +0 -13
- package/dist/http/routes/approvals-assistant.js +2 -1
- package/dist/http/routes/capabilities.js +2 -0
- package/dist/http/routes/diagnostics.js +7 -4
- package/dist/http/routes/runs.js +3 -2
- package/dist/http/routes/tasks.js +33 -29
- package/dist/http/routes/workflows.d.ts +0 -11
- package/dist/http/routes/workflows.js +18 -14
- package/dist/http/server.d.ts +7 -0
- package/dist/http/server.js +50 -46
- package/dist/leader/fanout.d.ts +6 -23
- package/dist/leader/fanout.js +8 -19
- package/dist/leader/leader.d.ts +13 -10
- package/dist/leader/leader.js +30 -22
- package/dist/leader/repair-wire.d.ts +4 -1
- package/dist/leader/repair-wire.js +7 -5
- package/dist/leader/wire.js +5 -4
- package/dist/main.js +7 -3
- package/dist/observability/fail-open.d.ts +8 -0
- package/dist/observability/fail-open.js +8 -0
- package/dist/observability/metrics.js +1 -1
- package/dist/observability/run-terminal-log.d.ts +25 -7
- package/dist/observability/run-terminal-log.js +15 -4
- package/dist/observability/tool-trace.d.ts +13 -12
- package/dist/observability/tool-trace.js +8 -13
- package/dist/parked-decide.d.ts +8 -4
- package/dist/parked-decide.js +1 -1
- package/dist/plugins/file-run-store.js +12 -21
- package/dist/plugins/memory-run-store.js +3 -2
- package/dist/plugins/run-store-sql.js +3 -2
- package/dist/run-local.js +6 -4
- package/dist/runs.d.ts +4 -19
- package/dist/runs.js +49 -47
- package/dist/terminal.d.ts +100 -0
- package/dist/terminal.js +55 -0
- package/dist/trace/core-keyset-guard.d.ts +20 -4
- package/dist/trace/ledger-sink.d.ts +2 -1
- package/dist/trace/ledger-sink.js +14 -7
- package/dist/trace/project.d.ts +9 -12
- package/dist/trace/project.js +54 -9
- package/dist/write-protection.d.ts +80 -0
- package/dist/write-protection.js +30 -0
- package/package.json +4 -4
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { redactErrorMessage, redactHead, redactTerminalResult } from "../../observability/run-terminal-log.js";
|
|
2
|
-
import { runWithVerification, runCascade, uuidv7 } from "@sema-agent/core";
|
|
3
|
-
import { graftParkToolCallId, markChildrenStoppedByUserOnAbort, resumeAtHttpStatus,
|
|
2
|
+
import { runWithVerification, runCascade, uuidv7, terminalProjection } from "@sema-agent/core";
|
|
3
|
+
import { graftParkToolCallId, markChildrenStoppedByUserOnAbort, resumeAtHttpStatus, HEARTBEAT_MS } from "../../runs.js";
|
|
4
|
+
import { stripResumeToken } from "../../terminal.js";
|
|
4
5
|
import { withPrincipal } from "../../observability/principal-context.js";
|
|
5
6
|
import { fleetRunPublisher, fleetRunLabels, fleetRunResiduals, isFleetAgentTerminalNotification } from "../../fleet/fleet-bus.js";
|
|
6
7
|
import { defaultSubagentTailBus, projectTailFrame } from "../../fleet/subagent-tail-bus.js";
|
|
@@ -50,11 +51,12 @@ async function handleTasksBody(req, res, url, ctx, miss) {
|
|
|
50
51
|
const reqState = ctx.req;
|
|
51
52
|
const source = ctx.req.source;
|
|
52
53
|
const noteRunOutcome = (r) => {
|
|
53
|
-
if (!r?.
|
|
54
|
+
if (!r?.terminal)
|
|
54
55
|
return;
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
56
|
+
const plane = terminalProjection(r.terminal);
|
|
57
|
+
reqState.runStatus = plane.status;
|
|
58
|
+
if (plane.errorCode)
|
|
59
|
+
reqState.runErrorCode = plane.errorCode;
|
|
58
60
|
};
|
|
59
61
|
if (req.method === "POST" && (url === "/v1/tasks" || url === "/v1/tasks/stream")) {
|
|
60
62
|
const rawIdem = headerStr(req.headers["idempotency-key"]);
|
|
@@ -193,10 +195,8 @@ async function handleTasksBody(req, res, url, ctx, miss) {
|
|
|
193
195
|
return {
|
|
194
196
|
taskId: durableTaskId ?? "",
|
|
195
197
|
sessionId: prepared.spec.sessionId ?? "",
|
|
196
|
-
|
|
198
|
+
terminal: { kind: "failed", code: "cancelled", message: errorMessage },
|
|
197
199
|
result: "",
|
|
198
|
-
errorCode: "cancelled",
|
|
199
|
-
errorMessage,
|
|
200
200
|
stats: { turns: 0, tokens: 0 },
|
|
201
201
|
...(ctx ? { cancelContext: ctx } : {}),
|
|
202
202
|
};
|
|
@@ -403,16 +403,16 @@ async function handleTasksBody(req, res, url, ctx, miss) {
|
|
|
403
403
|
if (ev.type === "message_committed" && ev.role === "user")
|
|
404
404
|
userMsgEntryId = ev.entryId;
|
|
405
405
|
if (ev.type === "done") {
|
|
406
|
-
finalResult = redactTerminalResult(
|
|
406
|
+
finalResult = redactTerminalResult(stripResumeToken(await graftParkToolCallId(ev.result, deps.checkpointStore)));
|
|
407
407
|
if (durableTaskId && finalResult)
|
|
408
408
|
finalResult = { ...finalResult, taskId: durableTaskId };
|
|
409
|
-
if (durableTaskId && cancelledViaVerb.has(durableTaskId) && finalResult?.
|
|
410
|
-
finalResult = { ...finalResult,
|
|
409
|
+
if (durableTaskId && cancelledViaVerb.has(durableTaskId) && finalResult?.terminal.kind === "failed" && !finalResult.terminal.code) {
|
|
410
|
+
finalResult = { ...finalResult, terminal: { ...finalResult.terminal, code: "cancelled" } };
|
|
411
411
|
}
|
|
412
412
|
finalizeTaskResult(ev.result, principal, prepared.spec.objective, prepared.spec.sessionId);
|
|
413
413
|
noteRunOutcome(finalResult);
|
|
414
414
|
putRewindAnchor();
|
|
415
|
-
settleFleet(finalResult
|
|
415
|
+
settleFleet(finalResult ? terminalProjection(finalResult.terminal).status : "completed", fleetRunResiduals(finalResult));
|
|
416
416
|
if (ledgerSink && finalResult) {
|
|
417
417
|
if (usageKey && deps.modelUsage && deps.runStore && durableTaskId) {
|
|
418
418
|
const rs2 = deps.runStore;
|
|
@@ -622,12 +622,13 @@ async function handleTasksBody(req, res, url, ctx, miss) {
|
|
|
622
622
|
}, withApproval)
|
|
623
623
|
: withApproval());
|
|
624
624
|
if (durableTaskId && deps.runStore) {
|
|
625
|
-
|
|
625
|
+
const finalPlane = finalResult ? terminalProjection(finalResult.terminal) : undefined;
|
|
626
|
+
if (finalPlane?.status === "suspended" && deps.checkpointStore)
|
|
626
627
|
await deps.runStore.setSuspended(durableTaskId);
|
|
627
|
-
else if (
|
|
628
|
+
else if (finalPlane?.status === "needs_review" && deps.checkpointStore)
|
|
628
629
|
await deps.runStore.setNeedsReview(durableTaskId);
|
|
629
|
-
else if (finalResult)
|
|
630
|
-
await deps.runStore.setTerminal(durableTaskId,
|
|
630
|
+
else if (finalResult && finalPlane)
|
|
631
|
+
await deps.runStore.setTerminal(durableTaskId, finalPlane.status, finalResult, finalPlane.errorMessage ?? null);
|
|
631
632
|
else {
|
|
632
633
|
putRewindAnchor();
|
|
633
634
|
const pending = await deps.checkpointStore?.findPendingTokenBySession(prepared.spec.sessionId);
|
|
@@ -688,8 +689,9 @@ async function handleTasksBody(req, res, url, ctx, miss) {
|
|
|
688
689
|
ledgerTerminal = true;
|
|
689
690
|
await ledgerSink.onDone(finalResult).catch(() => undefined);
|
|
690
691
|
}
|
|
691
|
-
|
|
692
|
-
|
|
692
|
+
const retryPlane = terminalProjection(finalResult.terminal);
|
|
693
|
+
await deps.runStore.setTerminal(durableTaskId, retryPlane.status, finalResult, retryPlane.errorMessage ?? null);
|
|
694
|
+
settleFleet(retryPlane.status, fleetRunResiduals(finalResult));
|
|
693
695
|
}
|
|
694
696
|
else if (ac.signal.aborted) {
|
|
695
697
|
const err = "stream aborted before completion (run cancelled)";
|
|
@@ -721,7 +723,7 @@ async function handleTasksBody(req, res, url, ctx, miss) {
|
|
|
721
723
|
};
|
|
722
724
|
counters.uncountedBillableInflight++;
|
|
723
725
|
try {
|
|
724
|
-
resp = await idemCache.run(idemKey, runStreamSubmitLeg, (r) => r.body?.
|
|
726
|
+
resp = await idemCache.run(idemKey, runStreamSubmitLeg, (r) => r.body?.terminal.kind === "completed");
|
|
725
727
|
}
|
|
726
728
|
catch (e) {
|
|
727
729
|
deps.logger?.error("stream_error", { closed, err: redactHead(e instanceof Error ? e.message : String(e), 2000) });
|
|
@@ -805,33 +807,35 @@ async function handleTasksBody(req, res, url, ctx, miss) {
|
|
|
805
807
|
cancelledViaVerb.delete(durableTaskId);
|
|
806
808
|
}
|
|
807
809
|
}
|
|
808
|
-
if (cancelVerbLabel && result.
|
|
809
|
-
result = { ...result,
|
|
810
|
+
if (cancelVerbLabel && result.terminal.kind === "failed" && !result.terminal.code) {
|
|
811
|
+
result = { ...result, terminal: { ...result.terminal, code: "cancelled" } };
|
|
810
812
|
}
|
|
811
813
|
const resumeAtStatus = resumeAtHttpStatus(result);
|
|
812
814
|
if (resumeAtStatus) {
|
|
815
|
+
const resumeAtPlane = terminalProjection(result.terminal);
|
|
813
816
|
if (durableTaskId && deps.runStore)
|
|
814
|
-
await deps.runStore.setTerminal(durableTaskId, "failed",
|
|
815
|
-
const retryAfterSec =
|
|
817
|
+
await deps.runStore.setTerminal(durableTaskId, "failed", stripResumeToken(result), resumeAtPlane.errorMessage ?? null);
|
|
818
|
+
const retryAfterSec = resumeAtPlane.errorCode === "memory.admission_required" && result.retryAfterMs !== undefined
|
|
816
819
|
? Math.max(1, Math.ceil(result.retryAfterMs / 1000))
|
|
817
820
|
: undefined;
|
|
818
|
-
return { status: resumeAtStatus, body: { errorCode:
|
|
821
|
+
return { status: resumeAtStatus, body: { errorCode: resumeAtPlane.errorCode, error: redactErrorMessage(resumeAtPlane.errorMessage) ?? "resume-at failed", ...(retryAfterSec !== undefined ? { retryAfterSec } : {}) } };
|
|
819
822
|
}
|
|
820
|
-
|
|
823
|
+
const syncPlane = terminalProjection(result.terminal);
|
|
824
|
+
if (syncPlane.status === "suspended" && deps.checkpointStore && deps.runStore && prepared.spec.sessionId) {
|
|
821
825
|
if (durableTaskId)
|
|
822
826
|
await deps.runStore.setSuspended(durableTaskId);
|
|
823
827
|
return { status: 200, body: withDiscoveryNotice({ taskId: durableTaskId, sessionId: prepared.spec.sessionId, status: "suspended" }) };
|
|
824
828
|
}
|
|
825
|
-
if (
|
|
829
|
+
if (syncPlane.status === "needs_review" && deps.checkpointStore && deps.runStore && prepared.spec.sessionId) {
|
|
826
830
|
if (durableTaskId)
|
|
827
831
|
await deps.runStore.setNeedsReview(durableTaskId);
|
|
828
832
|
return { status: 200, body: withDiscoveryNotice({ taskId: durableTaskId, sessionId: prepared.spec.sessionId, status: "needs_review" }) };
|
|
829
833
|
}
|
|
830
834
|
if (durableTaskId && deps.runStore)
|
|
831
|
-
await deps.runStore.setTerminal(durableTaskId,
|
|
835
|
+
await deps.runStore.setTerminal(durableTaskId, syncPlane.status, stripResumeToken(result), syncPlane.errorMessage ?? null);
|
|
832
836
|
finalizeTaskResult(result, principal, prepared.spec.objective, prepared.spec.sessionId);
|
|
833
837
|
noteRunOutcome(result);
|
|
834
|
-
return { status: 200, body: withDiscoveryNotice(redactTerminalResult(
|
|
838
|
+
return { status: 200, body: withDiscoveryNotice(redactTerminalResult(stripResumeToken(result))) };
|
|
835
839
|
};
|
|
836
840
|
const resp = await idemCache.run(idemKey, runSyncSubmitLeg, (r) => r.status < 400);
|
|
837
841
|
sendSubmitResult(res, resp);
|
|
@@ -1,14 +1,3 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* [ref] A9 域模块 —— S8 自编排 workflow 面([ref]/98)。
|
|
3
|
-
*
|
|
4
|
-
* 路由:`GET /v1/workflows`(列表)· `GET /v1/workflows/:id[/journal]`(详情)· `GET /v1/workflows/:id/stream`
|
|
5
|
-
* (在进程内 live SSE)· `POST /v1/workflows/:id/agents/:label/steer`(SVC-5:给 workflow 里某个仍在跑的
|
|
6
|
-
* agent 注入引导)。
|
|
7
|
-
*
|
|
8
|
-
* ⚠️ 只读三路跑在全局 service-credential 门**之前**(与 trace/fleet 同列),因此块内那道按 scope 的 per-tenant
|
|
9
|
-
* 门就是这些读的唯一租户边界;steer 那路在门之后。两段在 `handle()` 里原本相隔很远(中间隔着 tasks/runs 两域),
|
|
10
|
-
* 拆分保持了各自的相对位置 —— 本文件里的先后只是同域归并,不代表路由顺序变化。本域零可变状态。
|
|
11
|
-
*/
|
|
12
1
|
import type { IncomingMessage, ServerResponse } from "node:http";
|
|
13
2
|
import type { RouteCtx } from "../route-ctx.js";
|
|
14
3
|
export declare function handleWorkflows(req: IncomingMessage, res: ServerResponse, url: string, ctx: RouteCtx): Promise<boolean>;
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { persistedPlaneOf } from "../../terminal.js";
|
|
1
2
|
import { z } from "zod";
|
|
2
3
|
import { getWorkflowRun, subscribeWorkflow, deriveAgentDisplayStatus, callKeyOrdinal } from "@sema-agent/core";
|
|
3
4
|
import { redactSteerIn, STEER_IN_MAX_CHARS, STEER_IN_MAX_REQUEST_CHARS } from "../../orchestration/workflow-agent-steer.js";
|
|
@@ -85,20 +86,23 @@ async function handleWorkflowsReadBody(req, res, url, ctx, miss) {
|
|
|
85
86
|
journalProjectionMemo.set(memoKey, out);
|
|
86
87
|
return out;
|
|
87
88
|
};
|
|
88
|
-
const projectResultUncached = (callKey, r) =>
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
? {
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
89
|
+
const projectResultUncached = (callKey, r) => {
|
|
90
|
+
const plane = persistedPlaneOf(r);
|
|
91
|
+
return {
|
|
92
|
+
callKey,
|
|
93
|
+
ordinal: callKeyOrdinal(callKey),
|
|
94
|
+
status: plane.status,
|
|
95
|
+
...(plane.errorMessage ? { error: firstLine(plane.errorMessage) } : {}),
|
|
96
|
+
...(r.result
|
|
97
|
+
? {
|
|
98
|
+
result: redactHead(r.result, MAX_RESULT_CHARS),
|
|
99
|
+
...(r.result.length > MAX_RESULT_CHARS ? { resultTruncated: true, resultChars: r.result.length } : {}),
|
|
100
|
+
}
|
|
101
|
+
: {}),
|
|
102
|
+
...(r.stats ? { tokens: r.stats.tokens, turns: r.stats.turns } : {}),
|
|
103
|
+
...(r.haltedOnUserRejection === true ? { haltedOnUserRejection: true } : {}),
|
|
104
|
+
};
|
|
105
|
+
};
|
|
102
106
|
const js = deps.workflowJournalStore;
|
|
103
107
|
let projected;
|
|
104
108
|
if (js.loadPage) {
|
package/dist/http/server.d.ts
CHANGED
|
@@ -660,6 +660,13 @@ export interface ServiceDeploymentDeps {
|
|
|
660
660
|
* 的 `sql` 位(窄投影)。getter 而不是快照:池重连后驱动会重新初始化并刷新事实,快照会说谎。
|
|
661
661
|
* 缺席 / 返回 null = 本部署没有 SQL 后端(env-only worker、local 文件后端)—— 诚实缺席,不铸空壳。 */
|
|
662
662
|
sqlEngineFacts?: () => import("../plugins/sql-driver.js").SqlEngineFacts | null;
|
|
663
|
+
/** S-138:**写保护名表**的姿态(`write-protection.ts` 的 boot 期单一推导点 —— 与递给每只 Runner 的
|
|
664
|
+
* `writeProtectedPaths` 座同一份 config 决定)。两个读面共用:`GET /v1/diagnostics/wiring` 的
|
|
665
|
+
* `writeProtection` 段(operator-only,**逐行**表 + 来源 + 被丢的缺省行名)与 `GET /v1/capabilities`
|
|
666
|
+
* 的 `writeProtection` 位(窄投影 `{armed, rows, replaced}`,**不含行内容**)。
|
|
667
|
+
* boot 期算一次的**存活对象**(纯 config 派生,运行期不变;core 座没有热改腿)。
|
|
668
|
+
* 缺席 = 这个进程不是由 composition root 装起来的(测试夹具形)⇒ 诊断面报 null、能力位报 null。 */
|
|
669
|
+
writeProtection?: import("../write-protection.js").WriteProtectionPosture;
|
|
663
670
|
/** ① core ruling — SPLIT: `capabilities.workflows` = the ENGINE-CAN axis, boot-computed from core's own
|
|
664
671
|
* `workflowsCapability(deps)` (hardened script runner ∧ governance), NOT the `Boolean(workflowRunStore)` store
|
|
665
672
|
* proxy. Orthogonal to `workflowsList` (the durable-list axis = `workflowRunStore`). Falls back to the store
|
package/dist/http/server.js
CHANGED
|
@@ -2,7 +2,7 @@ import http from "node:http";
|
|
|
2
2
|
import { redactErrorMessage, redactHead } from "../observability/run-terminal-log.js";
|
|
3
3
|
import { once } from "node:events";
|
|
4
4
|
import { createHash } from "node:crypto";
|
|
5
|
-
import { uuidv7, isThinkingLevel, expandTiers, resumeWithVerification, CheckpointError, defaultTaskRegistry, validatePendingSteer, subscribeWorkflow, GLOBAL_USAGE_KEY, usageRetryAfterMs, RUN_WORKFLOW_TOOL_NAME } from "@sema-agent/core";
|
|
5
|
+
import { uuidv7, isThinkingLevel, expandTiers, resumeWithVerification, CheckpointError, defaultTaskRegistry, validatePendingSteer, subscribeWorkflow, GLOBAL_USAGE_KEY, usageRetryAfterMs, RUN_WORKFLOW_TOOL_NAME, terminalProjection, PAUSE_REGISTRY } from "@sema-agent/core";
|
|
6
6
|
import { decideParkedAgent, findParkedAgentForCheckpoint } from "../parked-decide.js";
|
|
7
7
|
import { isModelAllowlisted, matchCatalogModel } from "../model-select.js";
|
|
8
8
|
import {} from "../config-center/facade.js";
|
|
@@ -15,7 +15,9 @@ import { centerEntitlementSourceWired } from "../runtime-caps-resolver.js";
|
|
|
15
15
|
import { parseHooksConfig } from "../hooks/hook-runner.js";
|
|
16
16
|
import { normalizeApproachNotice, unknownTaskAgentModelRef, validateTaskAgents } from "../spec-fields.js";
|
|
17
17
|
import { isValidCwd, MAX_ADDITIONAL_DIRS } from "../task-cwd.js";
|
|
18
|
-
import { runInBackground, evictIfConflict,
|
|
18
|
+
import { runInBackground, evictIfConflict, parkToolCallId, TurnAnchorCapture, HEARTBEAT_MS, markChildrenStoppedByUserOnAbort } from "../runs.js";
|
|
19
|
+
import { failureCodeOf, nestedPauseOf, stripResumeToken } from "../terminal.js";
|
|
20
|
+
import { personDecision } from "../host-decision.js";
|
|
19
21
|
import { readTurnActivityMs, recordTurnActivity } from "../turn-activity.js";
|
|
20
22
|
import { recordResumeResourceWindow } from "../resource-window.js";
|
|
21
23
|
import { publicStoreProbeError } from "../store-live-probe.js";
|
|
@@ -122,17 +124,14 @@ function mentionableNames(config, catalog) {
|
|
|
122
124
|
return Object.keys(catalog).filter((n) => n !== "default" && isModelAllowlisted(n, catalog, allowlist));
|
|
123
125
|
}
|
|
124
126
|
function isCheckpointReopenedFailure(r) {
|
|
125
|
-
|
|
127
|
+
const code = failureCodeOf(r.terminal);
|
|
128
|
+
return code === "resume.env_failed" || code === "resume.tool_unavailable";
|
|
126
129
|
}
|
|
127
130
|
function nestedParkStatusOf(r) {
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
return "suspended";
|
|
133
|
-
default:
|
|
134
|
-
return undefined;
|
|
135
|
-
}
|
|
131
|
+
const paused = nestedPauseOf(r.terminal);
|
|
132
|
+
if (paused === undefined)
|
|
133
|
+
return undefined;
|
|
134
|
+
return PAUSE_REGISTRY[paused.gate.kind].taskStatus;
|
|
136
135
|
}
|
|
137
136
|
const ROUTE_DOMAINS = [
|
|
138
137
|
handleTraceUsage,
|
|
@@ -1014,7 +1013,7 @@ export function createHttpServer(rawDeps) {
|
|
|
1014
1013
|
}
|
|
1015
1014
|
}
|
|
1016
1015
|
const resumeFleetScope = (req, auth) => req ? (gatedPrincipal(req, deps.config) ?? "default") : (auth.principal ?? "default");
|
|
1017
|
-
async function resumeCheckpoint(sessionId, decision, reason,
|
|
1016
|
+
async function resumeCheckpoint(sessionId, decision, reason, hostDecision, req, answer, binding, onResumeCommitted, acceptEarly, decider) {
|
|
1018
1017
|
const cs = deps.checkpointStore;
|
|
1019
1018
|
const token = (await cs.findPendingTokenBySession(sessionId, undefined, { gateKinds: APPROVAL_GATE_KINDS })) ?? (await cs.findPendingTokenBySession(sessionId));
|
|
1020
1019
|
if (binding?.checkpointToken && binding.checkpointToken !== token) {
|
|
@@ -1162,7 +1161,7 @@ export function createHttpServer(rawDeps) {
|
|
|
1162
1161
|
if (out.body?.errorCode === "cancelled")
|
|
1163
1162
|
return;
|
|
1164
1163
|
try {
|
|
1165
|
-
lateSettleSink.recordLateSettle({ ...lateSettleAnchor, outcome: decision === "approve" ? "allowed" : "denied",
|
|
1164
|
+
lateSettleSink.recordLateSettle({ ...lateSettleAnchor, outcome: decision === "approve" ? "allowed" : "denied", decidedBy: hostDecision.decidedBy, ...(hostDecision.approver !== undefined ? { approver: hostDecision.approver } : {}), settledAtMs: Date.now() });
|
|
1166
1165
|
}
|
|
1167
1166
|
catch (e) {
|
|
1168
1167
|
recordFailOpen("server.approval-ask-audit.append-failed", `kind=late_settle err=${e instanceof Error ? e.message : String(e)}`);
|
|
@@ -1188,7 +1187,7 @@ export function createHttpServer(rawDeps) {
|
|
|
1188
1187
|
...(deps.logger ? { warn: (event, fields) => deps.logger?.warn?.(event, fields) } : {}),
|
|
1189
1188
|
}, {
|
|
1190
1189
|
token, scope: cp.scope, sessionId: cp.sessionId, pendingAction: cp.pendingAction,
|
|
1191
|
-
decision,
|
|
1190
|
+
decision, hostDecision, ...(reason !== undefined ? { reason } : {}),
|
|
1192
1191
|
...(binding !== undefined ? { binding: { ...(binding.boundCallId !== undefined ? { boundCallId: binding.boundCallId } : {}), ...(binding.boundInputHash !== undefined ? { boundInputHash: binding.boundInputHash } : {}), ...(binding.updatedInput !== undefined ? { updatedInput: binding.updatedInput } : {}) } } : {}),
|
|
1193
1192
|
...(boundGrant ? { grantRemember: (rootSessionId) => boundGrant(rootSessionId) } : {}),
|
|
1194
1193
|
...(answer !== undefined ? { answer } : {}),
|
|
@@ -1257,7 +1256,7 @@ export function createHttpServer(rawDeps) {
|
|
|
1257
1256
|
...(decision === "approve" && binding?.updatedInput !== undefined ? { updatedInput: binding.updatedInput } : {}),
|
|
1258
1257
|
...(reason ? { reason } : {}),
|
|
1259
1258
|
...(answer !== undefined ? { answer } : {}),
|
|
1260
|
-
|
|
1259
|
+
hostDecision,
|
|
1261
1260
|
};
|
|
1262
1261
|
const verifyRounds = verifyRoundsFromBody(ctx.body);
|
|
1263
1262
|
const driven = await driveResumeIntoRunLog({ runner: runnerFor(spec), token, sessionId: cp.sessionId, principal: auth.principal, fleetScope: resumeFleetScope(req, auth), taskConfig, resumeObjective, outcome, verifyRounds, ...(boundGrant ? { onResumeCommitted: boundGrant } : {}), ...(acceptEarly === true ? { acceptEarly: true } : {}), ...(req !== undefined ? { callerInitiated: true } : {}), ...(ownerLeaseAdmittedHere ? { ownerLeaseAdmitted: true } : {}) });
|
|
@@ -1477,9 +1476,10 @@ export function createHttpServer(rawDeps) {
|
|
|
1477
1476
|
};
|
|
1478
1477
|
resumeLegLive = true;
|
|
1479
1478
|
const vr = await resumeWithVerification(legRunner, token, outcome, { ...resumeTaskConfig, signal: cancelCtrl.signal, preemptSignal: preemptCtrl.signal }, resumeObjective, verifyRounds, verifyInternals);
|
|
1480
|
-
let safe =
|
|
1479
|
+
let safe = stripResumeToken(vr);
|
|
1480
|
+
const vrNested = nestedPauseOf(vr.terminal);
|
|
1481
1481
|
const reParkStatus = nestedParkStatusOf(vr);
|
|
1482
|
-
const reSuspended =
|
|
1482
|
+
const reSuspended = vrNested !== undefined;
|
|
1483
1483
|
const vrReopened = isCheckpointReopenedFailure(vr) && (await confirmCheckpointReopened());
|
|
1484
1484
|
if (vrReopened)
|
|
1485
1485
|
reopenConfirmedPark = true;
|
|
@@ -1490,8 +1490,8 @@ export function createHttpServer(rawDeps) {
|
|
|
1490
1490
|
await appendPromptManifest(ap, deps.promptManifests, taskId);
|
|
1491
1491
|
if (reSuspended || vrReopened) {
|
|
1492
1492
|
await appendModelUsageDelta(ap, deps.modelUsage, taskId);
|
|
1493
|
-
const vrParkCallId =
|
|
1494
|
-
await ap(
|
|
1493
|
+
const vrParkCallId = vrNested !== undefined ? await parkToolCallId({ terminal: vrNested }, deps.checkpointStore) : undefined;
|
|
1494
|
+
await ap(reParkStatus ?? "suspended", { gate: vrNested?.gate ?? null, ...(vrParkCallId !== undefined ? { toolCallId: vrParkCallId } : {}), ...(vrReopened ? { reopened: failureCodeOf(vr.terminal) ?? null } : {}) });
|
|
1495
1495
|
}
|
|
1496
1496
|
else {
|
|
1497
1497
|
safe = await attachModelUsage(safe, { append: ap, getEvents: (id, a) => rs2.getEvents(id, a), modelUsage: deps.modelUsage, taskId });
|
|
@@ -1503,8 +1503,8 @@ export function createHttpServer(rawDeps) {
|
|
|
1503
1503
|
throw e;
|
|
1504
1504
|
}
|
|
1505
1505
|
}
|
|
1506
|
-
if (
|
|
1507
|
-
return { ...safe,
|
|
1506
|
+
if (vrNested !== undefined)
|
|
1507
|
+
return { ...safe, terminal: vrNested };
|
|
1508
1508
|
if (vr.verification?.verdict)
|
|
1509
1509
|
deps.metrics?.inc("verifications_total", { verdict: vr.verification.verdict });
|
|
1510
1510
|
return safe;
|
|
@@ -1699,30 +1699,31 @@ export function createHttpServer(rawDeps) {
|
|
|
1699
1699
|
}
|
|
1700
1700
|
await flush();
|
|
1701
1701
|
const r = await stream.result();
|
|
1702
|
-
|
|
1702
|
+
const rPark = r.terminal.kind === "paused" ? terminalProjection(r.terminal).status : undefined;
|
|
1703
|
+
if (rPark === "suspended") {
|
|
1703
1704
|
const parkCallId = await parkToolCallId(r, deps.checkpointStore);
|
|
1704
|
-
await append("suspended", { gate: r.
|
|
1705
|
+
await append("suspended", { gate: r.terminal.kind === "paused" ? r.terminal.gate : null, ...(parkCallId !== undefined ? { toolCallId: parkCallId } : {}) });
|
|
1705
1706
|
await appendModelUsageDelta(append, deps.modelUsage, taskId);
|
|
1706
1707
|
return r;
|
|
1707
1708
|
}
|
|
1708
|
-
if (
|
|
1709
|
-
await append("needs_review", { gate: r.
|
|
1709
|
+
if (rPark === "needs_review") {
|
|
1710
|
+
await append("needs_review", { gate: r.terminal.kind === "paused" ? r.terminal.gate : null });
|
|
1710
1711
|
await appendModelUsageDelta(append, deps.modelUsage, taskId);
|
|
1711
1712
|
return r;
|
|
1712
1713
|
}
|
|
1713
1714
|
if (isCheckpointReopenedFailure(r) && (await confirmCheckpointReopened())) {
|
|
1714
1715
|
reopenConfirmedPark = true;
|
|
1715
1716
|
try {
|
|
1716
|
-
await append("suspended", { gate: null, reopened: r.
|
|
1717
|
+
await append("suspended", { gate: null, reopened: failureCodeOf(r.terminal) ?? null });
|
|
1717
1718
|
await appendModelUsageDelta(append, deps.modelUsage, taskId);
|
|
1718
1719
|
}
|
|
1719
1720
|
catch {
|
|
1720
1721
|
}
|
|
1721
1722
|
return r;
|
|
1722
1723
|
}
|
|
1723
|
-
const safe = await attachModelUsage(
|
|
1724
|
+
const safe = await attachModelUsage(stripResumeToken(r), { append, getEvents: (id, a) => rs.getEvents(id, a), modelUsage: deps.modelUsage, taskId });
|
|
1724
1725
|
await append("done", { result: safe });
|
|
1725
|
-
if (taskConfig.suggestNextPrompts && safe.
|
|
1726
|
+
if (taskConfig.suggestNextPrompts && safe.terminal.kind === "completed") {
|
|
1726
1727
|
const suggestions = await stream.suggestions().catch(() => []);
|
|
1727
1728
|
if (suggestions.length)
|
|
1728
1729
|
await append("suggestions", { suggestions: suggestions.map((s) => redactSecrets(s)) });
|
|
@@ -1772,8 +1773,8 @@ export function createHttpServer(rawDeps) {
|
|
|
1772
1773
|
}
|
|
1773
1774
|
}
|
|
1774
1775
|
if (claimedRow && taskId && deps.runStore) {
|
|
1775
|
-
const c = { taskId, sessionId,
|
|
1776
|
-
await deps.runStore.setTerminal(taskId, "failed", c,
|
|
1776
|
+
const c = { taskId, sessionId, terminal: { kind: "failed", code: "cancelled", message: e.message }, stats: { turns: 0, tokens: 0 } };
|
|
1777
|
+
await deps.runStore.setTerminal(taskId, "failed", c, e.message).catch(() => undefined);
|
|
1777
1778
|
}
|
|
1778
1779
|
settleFleet("failed");
|
|
1779
1780
|
return { status: 200, body: { taskId, sessionId, status: "failed", errorCode: "cancelled" } };
|
|
@@ -1793,8 +1794,8 @@ export function createHttpServer(rawDeps) {
|
|
|
1793
1794
|
(e.code === "checkpoint.unsupported_version" && checkpointRowRedeemableElsewhere(e.detail?.reason));
|
|
1794
1795
|
if (claimedRow && taskId && deps.runStore) {
|
|
1795
1796
|
if (retriable && freshRunClaimed) {
|
|
1796
|
-
const c = { taskId, sessionId,
|
|
1797
|
-
await deps.runStore.setTerminal(taskId, "failed", c,
|
|
1797
|
+
const c = { taskId, sessionId, terminal: { kind: "failed", code: e.code, message: e.message }, stats: { turns: 0, tokens: 0 } };
|
|
1798
|
+
await deps.runStore.setTerminal(taskId, "failed", c, e.message).catch((err) => {
|
|
1798
1799
|
deps.logger?.warn?.("wake_synthetic_run_release_failed", { taskId, sessionId, errorCode: e.code, error: err instanceof Error ? err.message : String(err), note: "the synthetic run row for this refused fresh-run resume could not be settled — its task_active claim is still held and the next attempt on this session will 409 conflict.session_active_run until reapStale collects it (runStaleSec)" });
|
|
1799
1800
|
});
|
|
1800
1801
|
settleFleet("failed");
|
|
@@ -1827,8 +1828,8 @@ export function createHttpServer(rawDeps) {
|
|
|
1827
1828
|
}
|
|
1828
1829
|
if (cancelCtrl.signal.aborted) {
|
|
1829
1830
|
if (taskId && deps.runStore) {
|
|
1830
|
-
const c = { taskId, sessionId,
|
|
1831
|
-
await deps.runStore.setTerminal(taskId, "failed", c,
|
|
1831
|
+
const c = { taskId, sessionId, terminal: { kind: "failed", code: "cancelled", message: "cancelled by user" }, stats: { turns: 0, tokens: 0 } };
|
|
1832
|
+
await deps.runStore.setTerminal(taskId, "failed", c, "cancelled by user").catch(() => undefined);
|
|
1832
1833
|
}
|
|
1833
1834
|
settleFleet("failed");
|
|
1834
1835
|
return { status: 200, body: { taskId, sessionId, status: "failed", errorCode: "cancelled" } };
|
|
@@ -1859,22 +1860,23 @@ export function createHttpServer(rawDeps) {
|
|
|
1859
1860
|
}
|
|
1860
1861
|
evictIfConflict(deps.runner, sessionId, result);
|
|
1861
1862
|
const checkpointReopened = reopenConfirmedPark;
|
|
1863
|
+
const resumePlane = terminalProjection(result.terminal);
|
|
1862
1864
|
if (taskId && deps.runStore) {
|
|
1863
|
-
if (
|
|
1865
|
+
if (resumePlane.status === "suspended" || checkpointReopened)
|
|
1864
1866
|
await deps.runStore.setSuspended(taskId);
|
|
1865
|
-
else if (
|
|
1867
|
+
else if (resumePlane.status === "needs_review")
|
|
1866
1868
|
await deps.runStore.setNeedsReview(taskId);
|
|
1867
1869
|
else
|
|
1868
|
-
await deps.runStore.setTerminal(taskId,
|
|
1870
|
+
await deps.runStore.setTerminal(taskId, resumePlane.status, stripResumeToken(result), resumePlane.errorMessage ?? null);
|
|
1869
1871
|
}
|
|
1870
|
-
settleFleet(checkpointReopened ? "suspended" :
|
|
1872
|
+
settleFleet(checkpointReopened ? "suspended" : resumePlane.status, fleetRunResiduals(result));
|
|
1871
1873
|
return {
|
|
1872
1874
|
status: 200,
|
|
1873
1875
|
body: {
|
|
1874
|
-
taskId, sessionId, status:
|
|
1876
|
+
taskId, sessionId, status: resumePlane.status,
|
|
1875
1877
|
bindingEnforced: true,
|
|
1876
|
-
...(
|
|
1877
|
-
...(
|
|
1878
|
+
...(resumePlane.errorCode ? { errorCode: resumePlane.errorCode } : {}),
|
|
1879
|
+
...(resumePlane.errorMessage ? { errorMessage: redactErrorMessage(resumePlane.errorMessage) ?? "" } : {}),
|
|
1878
1880
|
...(checkpointReopened ? { retriable: true } : {}),
|
|
1879
1881
|
},
|
|
1880
1882
|
};
|
|
@@ -2091,12 +2093,13 @@ export function createHttpServer(rawDeps) {
|
|
|
2091
2093
|
return driveResumeIntoRunLog({ runner: runnerFor(spec), token, sessionId: cp.sessionId, principal: auth.principal, fleetScope: resumeFleetScope(req, auth), taskConfig, resumeObjective, outcome, verifyRounds, ...(acceptEarly === true ? { acceptEarly: true } : {}), ...(req !== undefined ? { callerInitiated: true } : {}) });
|
|
2092
2094
|
}
|
|
2093
2095
|
function recordTaskResult(result) {
|
|
2094
|
-
|
|
2096
|
+
const plane = terminalProjection(result.terminal);
|
|
2097
|
+
deps.metrics?.inc("tasks_total", { status: plane.status });
|
|
2095
2098
|
if (result.stats.tokens)
|
|
2096
2099
|
deps.metrics?.inc("task_tokens_total", {}, result.stats.tokens);
|
|
2097
2100
|
if (result.stats.cacheHitRate !== undefined)
|
|
2098
2101
|
deps.metrics?.observe("task_cache_hit_rate", result.stats.cacheHitRate);
|
|
2099
|
-
const code =
|
|
2102
|
+
const code = plane.errorCode;
|
|
2100
2103
|
if (code && code.startsWith("limits.")) {
|
|
2101
2104
|
deps.metrics?.inc("budget_exceeded_total", { code });
|
|
2102
2105
|
}
|
|
@@ -2110,7 +2113,7 @@ export function createHttpServer(rawDeps) {
|
|
|
2110
2113
|
}
|
|
2111
2114
|
function finalizeTaskResult(result, principal, objective, sessionId) {
|
|
2112
2115
|
recordTaskResult(result);
|
|
2113
|
-
deps.planCacheProbe?.record(principal, objective, result.
|
|
2116
|
+
deps.planCacheProbe?.record(principal, objective, result.terminal.kind === "completed");
|
|
2114
2117
|
evictIfConflict(deps.runner, sessionId, result);
|
|
2115
2118
|
}
|
|
2116
2119
|
function quotaExceeded(req, res) {
|
|
@@ -2347,7 +2350,7 @@ export function createHttpServer(rawDeps) {
|
|
|
2347
2350
|
parkedSkipped += 1;
|
|
2348
2351
|
continue;
|
|
2349
2352
|
}
|
|
2350
|
-
await resumeCheckpoint(sessionId, "deny", "approval SLA expired — auto-denied", "
|
|
2353
|
+
await resumeCheckpoint(sessionId, "deny", "approval SLA expired — auto-denied", { decidedBy: "sla_timeout" }).then((r) => { if (r.status >= 400)
|
|
2351
2354
|
deps.logger?.warn?.("deny_sweep_resume_blocked", { sessionId, status: r.status, errorCode: r.body?.errorCode }); }, (e) => deps.logger?.warn?.("deny_sweep_resume_failed", { sessionId, err: redactHead(e instanceof Error ? e.message : String(e), 2000) }));
|
|
2352
2355
|
}
|
|
2353
2356
|
if (parkedSkipped > 0)
|
|
@@ -2420,7 +2423,7 @@ export function createHttpServer(rawDeps) {
|
|
|
2420
2423
|
...(req.gateBoundInputHash !== null ? { boundInputHash: req.gateBoundInputHash } : {}),
|
|
2421
2424
|
...(req.updatedInput !== undefined ? { updatedInput: req.updatedInput } : {}),
|
|
2422
2425
|
};
|
|
2423
|
-
const out = await resumeCheckpoint(cp.sessionId, req.decision, req.note,
|
|
2426
|
+
const out = await resumeCheckpoint(cp.sessionId, req.decision, req.note, personDecision(req.principal), req.httpReq, undefined, binding, undefined, true, { principal: req.principal, explicitOperator: false });
|
|
2424
2427
|
return { status: out.status, body: out.body };
|
|
2425
2428
|
}
|
|
2426
2429
|
const isRunLiveOnThisReplica = (taskId) => inflightRuns.has(taskId) || steerableRuns.has(taskId);
|
|
@@ -2430,6 +2433,7 @@ const WORKER_SWAP_REDEEMABLE = {
|
|
|
2430
2433
|
version_newer: true,
|
|
2431
2434
|
env_factory_missing: true,
|
|
2432
2435
|
governed_unwired: true,
|
|
2436
|
+
origin_missing: true,
|
|
2433
2437
|
real_approval_damaged: false,
|
|
2434
2438
|
real_approval_forged: false,
|
|
2435
2439
|
read_face_damaged: false,
|
package/dist/leader/fanout.d.ts
CHANGED
|
@@ -27,7 +27,7 @@
|
|
|
27
27
|
* MVP scope: the COMPOSITION mechanics, unit-tested with a mock runner. Real E2B workers + branch/baseSha from
|
|
28
28
|
* a worker's actual git state are wired at M4/M5 (joint dogfood).
|
|
29
29
|
*/
|
|
30
|
-
import type
|
|
30
|
+
import { type TaskSpec, type TaskResult, type TaskStream, type RepairTerminal, type CheckpointGate } from "@sema-agent/core";
|
|
31
31
|
/** What the Coordinator gets back per worker (council#6 — merge(b) routes on this). */
|
|
32
32
|
export interface WorkerReport {
|
|
33
33
|
workerId: string;
|
|
@@ -48,10 +48,11 @@ export interface WorkerReport {
|
|
|
48
48
|
/** [ref] Slice 6: the terminal checkpoint gate when a worker is still SUSPENDED (the leader auto-resumed
|
|
49
49
|
* it up to its bound but it didn't fully complete — e.g. the overall deadline hit or the slice cap). Present
|
|
50
50
|
* ⇒ resumable: the operator/leader can escalate (more budget) or accept the belt-diff partial progress. */
|
|
51
|
-
checkpointGate?:
|
|
52
|
-
/** [ref] Slice 6: the worker's terminal
|
|
53
|
-
* exhausted resume, `limits.max_turns_exceeded` — core 5.8.0 码名轴) — surfaced for /v1/leader observability.
|
|
54
|
-
|
|
51
|
+
checkpointGate?: CheckpointGate;
|
|
52
|
+
/** [ref] Slice 6: the worker's terminal failure code (e.g. `limits.max_cost_exceeded` on an
|
|
53
|
+
* exhausted resume, `limits.max_turns_exceeded` — core 5.8.0 码名轴) — surfaced for /v1/leader observability.
|
|
54
|
+
* S-136:源从 `TaskResult.errorCode` 换成因由 `failed` 臂的 `code`(本报告面的键名与形不变)。 */
|
|
55
|
+
errorCode?: string;
|
|
55
56
|
/** LEADER-REPAIRLOOP-INTEGRATION §10.6 (THE push chokepoint): the `runRepairLoop` terminal a single-agent
|
|
56
57
|
* worker resolved to, when the repair loop ran on this worker (LEADER_REPAIR_LOOP on). It MUST ride on the
|
|
57
58
|
* report because a SAFE-tier `candidate_only` maps to core verdict "PASS" → this report's `status` collapses
|
|
@@ -105,24 +106,6 @@ export interface FanOutResult {
|
|
|
105
106
|
/** True if the overall deadline fired and cancelAll ran. */
|
|
106
107
|
cancelled: boolean;
|
|
107
108
|
}
|
|
108
|
-
/**
|
|
109
|
-
* 「这条 worker 的 TaskResult 停在门上(durable park)了吗」——`TaskStatus` 的穷举判据。
|
|
110
|
-
*
|
|
111
|
-
* 🔴 park 是**两个词**(扫描P2,status/kind 同名词混淆病族):`suspended`(工具审批 / human /
|
|
112
|
-
* resource_limit 门)与 `needs_review`(`plan_review` 与 dry-run 拦截门)。真源 = core
|
|
113
|
-
* `agents/suspend-guard.js` 的 `isDurablePause`(逐字 `status === "suspended" || status === "needs_review"`),
|
|
114
|
-
* `mapNestedSuspend` / `verify.js` 的 `unverifiedReason` 两处都按这张表分词。
|
|
115
|
-
*
|
|
116
|
-
* 只认 `suspended` 的后果不是「少一个标签」:一条 plan_review 停下的 worker 会被报成 `failed` ⇒
|
|
117
|
-
* fanOut 的 C4 抢占({@link FanOutOptions.cancelOnSuspend})不触发(其余 worker 继续烧钱)、leader 的
|
|
118
|
-
* replan「suspended 波」看不见它、`suspendedSessions` 的豁免名单漏掉它 ⇒ 它那张还能被决议的 checkpoint
|
|
119
|
-
* 与暂停中的环境会被当失败清掉。带验证器的那条腿早已改用「还带着 checkpointToken 就是 park」的判据
|
|
120
|
-
* (leader.ts round-3 review BLOCKER),这里是它当时漏掉的另外半场。
|
|
121
|
-
*
|
|
122
|
-
* `never` 收敛 = core `TaskStatus` 增删成员的**编译期**执行点(同 `store-contracts.ts` 两条判据的姿势);
|
|
123
|
-
* 运行时未知词保守判**非 park**(与既有 `else → failed` 同行为,不静默扩大 park 面)。
|
|
124
|
-
*/
|
|
125
|
-
export declare function isDurablePauseStatus(s: TaskResult["status"]): boolean;
|
|
126
109
|
/**
|
|
127
110
|
* Fan out `subtasks` in parallel. Never throws — every worker resolves to a WorkerReport (failures captured).
|
|
128
111
|
* The caller (Coordinator) merges only `status==="completed"` reports.
|
package/dist/leader/fanout.js
CHANGED
|
@@ -1,4 +1,6 @@
|
|
|
1
|
+
import { terminalProjection } from "@sema-agent/core";
|
|
1
2
|
import { redactErrorMessage, redactWorkerReportError } from "../observability/run-terminal-log.js";
|
|
3
|
+
import { pausedCauseOf } from "../terminal.js";
|
|
2
4
|
class Semaphore {
|
|
3
5
|
max;
|
|
4
6
|
active = 0;
|
|
@@ -19,24 +21,10 @@ class Semaphore {
|
|
|
19
21
|
}
|
|
20
22
|
}
|
|
21
23
|
}
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
return true;
|
|
27
|
-
case "completed":
|
|
28
|
-
case "failed":
|
|
29
|
-
case "blocked":
|
|
30
|
-
return false;
|
|
31
|
-
default: {
|
|
32
|
-
const unhandled = s;
|
|
33
|
-
void unhandled;
|
|
34
|
-
return false;
|
|
35
|
-
}
|
|
36
|
-
}
|
|
37
|
-
}
|
|
38
|
-
function mapStatus(s) {
|
|
39
|
-
return s === "completed" ? "completed" : isDurablePauseStatus(s) ? "suspended" : "failed";
|
|
24
|
+
function mapStatus(terminal) {
|
|
25
|
+
if (terminal.kind === "completed")
|
|
26
|
+
return "completed";
|
|
27
|
+
return pausedCauseOf(terminal) !== undefined ? "suspended" : "failed";
|
|
40
28
|
}
|
|
41
29
|
export async function fanOut(subtasks, opts) {
|
|
42
30
|
const sharedAC = new AbortController();
|
|
@@ -80,7 +68,8 @@ export async function fanOut(subtasks, opts) {
|
|
|
80
68
|
let report;
|
|
81
69
|
try {
|
|
82
70
|
const result = await stream.result();
|
|
83
|
-
|
|
71
|
+
const plane = terminalProjection(result.terminal);
|
|
72
|
+
report = { ...base, status: mapStatus(result.terminal), stats: result.stats, ...(plane.errorMessage ? { error: redactErrorMessage(plane.errorMessage) ?? "" } : {}) };
|
|
84
73
|
}
|
|
85
74
|
catch (e) {
|
|
86
75
|
report = { ...base, status: cancelled ? "timeout" : "failed", error: redactErrorMessage(e instanceof Error ? e.message : String(e)) ?? "" };
|
package/dist/leader/leader.d.ts
CHANGED
|
@@ -15,18 +15,21 @@
|
|
|
15
15
|
import { type RunnerLike, type WorkerReport } from "./fanout.js";
|
|
16
16
|
import { type ExecStreamLike } from "./diffout.js";
|
|
17
17
|
import { type MergeDeps, type MergeResult } from "./merge.js";
|
|
18
|
-
import { type TaskSpec, type VerifyConfig, type CheckpointToken, type ToolSpec, type RepairTerminal, type OracleTier, type RepairResult } from "@sema-agent/core";
|
|
18
|
+
import { type TaskSpec, type VerifyConfig, type CheckpointToken, type TerminalCause, type ToolSpec, type RepairTerminal, type OracleTier, type RepairResult } from "@sema-agent/core";
|
|
19
19
|
/** [ref] Slice 6: drive a worker's resource-suspend auto-resume to a terminal (or bound) state. Exported
|
|
20
|
-
* for unit tests. Loops `resume(token, slice)` WHILE the result
|
|
21
|
-
*
|
|
22
|
-
* (
|
|
23
|
-
*
|
|
24
|
-
*
|
|
20
|
+
* for unit tests. Loops `resume(token, slice)` WHILE the result's terminal cause is a park on a
|
|
21
|
+
* `resource_limit` gate. Exits on: a terminal cause that is not a park, a park on any OTHER gate
|
|
22
|
+
* (a human approval / a review — never auto-resumed), an exhausted budget (core fails the resume fast, so
|
|
23
|
+
* the cause is `failed` with no nested park), `signal.aborted`, or `maxSlices` (a belt; core's ledger
|
|
24
|
+
* fail-fast + per-task suspend-loop cap are the primary bounds). Returns the final result.
|
|
25
|
+
*
|
|
26
|
+
* 🔴 S-136(core 7.6.0 D-8):判据从「两个平面键合读」(`checkpointToken` 在场 ∧ `checkpointGate.kind`)
|
|
27
|
+
* 换成**一条因由**({@link pausedCauseOf}) —— 这条循环从一开始就刻意**不按 status 判**(顶注原话:
|
|
28
|
+
* 「the verify wrapper may surface a suspend as failed-with-token」),因为嵌套边界会把 park 报成 failed;
|
|
29
|
+
* 现在那件事是形状:两条路径同一个读法,而 token 与 gate 在同一个 `PausedCause` 上,不可能只有一半。
|
|
30
|
+
*/
|
|
25
31
|
export declare function autoResumeOnResource<T extends {
|
|
26
|
-
|
|
27
|
-
checkpointGate?: {
|
|
28
|
-
kind: string;
|
|
29
|
-
};
|
|
32
|
+
terminal: TerminalCause;
|
|
30
33
|
}>(initial: T, signal: {
|
|
31
34
|
readonly aborted: boolean;
|
|
32
35
|
}, resume: (token: CheckpointToken, slice: number) => Promise<T>, maxSlices?: number): Promise<T>;
|