@sema-agent/server 7.148.0 → 7.149.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/MIGRATION.md +429 -0
- package/README.md +16 -1
- package/README.zh-CN.md +4 -0
- package/USAGE.md +89 -55
- package/dist/adoption/wire.d.ts +3 -3
- package/dist/approval-card.d.ts +32 -32
- package/dist/approval-leg-origin.d.ts +33 -0
- package/dist/approval-leg-origin.js +4 -0
- package/dist/auth-bridge.d.ts +9 -1
- package/dist/auth-bridge.js +4 -4
- package/dist/auth-keys.d.ts +10 -1
- package/dist/auth-keys.js +8 -0
- package/dist/boot/execution-env.js +10 -11
- package/dist/boot/execution-lane.d.ts +6 -4
- package/dist/boot/execution-lane.js +3 -1
- package/dist/boot/org-delivery.d.ts +3 -2
- package/dist/boot/resolve-spec.js +6 -4
- package/dist/boot/runner-deps.js +3 -1
- package/dist/boot/stage-01-config.d.ts +1 -0
- package/dist/boot/stage-01-config.js +4 -3
- package/dist/boot/stage-02-web-admin-pull.d.ts +1 -0
- package/dist/boot/stage-02-web-admin-pull.js +10 -2
- package/dist/boot/stage-03-store-backend.d.ts +1 -0
- package/dist/boot/stage-03-store-backend.js +2 -2
- package/dist/boot/stage-04-stores.d.ts +1 -0
- package/dist/boot/stage-05-execution-env.d.ts +1 -0
- package/dist/boot/stage-06-runners.d.ts +2 -1
- package/dist/boot/stage-07-capability-layer.d.ts +2 -1
- package/dist/boot/stage-08-reapers.d.ts +2 -1
- package/dist/boot/stage-09-leader.d.ts +2 -1
- package/dist/boot/stage-10-http-server.d.ts +2 -1
- package/dist/boot/stores.d.ts +3 -0
- package/dist/boot/stores.js +22 -19
- package/dist/capabilities/tool-defer.d.ts +1 -1
- package/dist/capability-facts.d.ts +5 -4
- package/dist/capability-facts.js +3 -14
- package/dist/card-settle-stored-scope.d.ts +56 -0
- package/dist/card-settle-stored-scope.js +27 -0
- package/dist/card-void.d.ts +7 -12
- package/dist/card-void.js +2 -7
- package/dist/config-catalog.d.ts +12 -0
- package/dist/config-catalog.js +36 -32
- package/dist/config-invariants.d.ts +5 -4
- package/dist/config-invariants.js +4 -4
- package/dist/config-types.d.ts +37 -5
- package/dist/config.d.ts +6 -5
- package/dist/config.js +35 -24
- package/dist/context-window-notice.d.ts +31 -0
- package/dist/context-window-notice.js +28 -0
- package/dist/deployment-shape.d.ts +20 -11
- package/dist/deployment-shape.js +1 -0
- package/dist/env-name-allowlist-knobs.d.ts +6 -5
- package/dist/env-name-allowlist-knobs.js +5 -0
- package/dist/execution-lane-caps.d.ts +20 -5
- package/dist/execution-lane-caps.js +18 -4
- package/dist/hooks/command-spawn.d.ts +7 -2
- package/dist/hooks/command-spawn.js +13 -6
- package/dist/hooks/hook-runner.d.ts +5 -0
- package/dist/hooks/hook-runner.js +11 -7
- package/dist/http/admission.js +3 -3
- package/dist/http/approval-sweeps.js +3 -2
- package/dist/http/claim-heartbeat.d.ts +3 -3
- package/dist/http/claim-heartbeat.js +3 -3
- package/dist/http/claimed-run.d.ts +16 -9
- package/dist/http/claimed-run.js +8 -0
- package/dist/http/dispatch.js +5 -3
- package/dist/http/durable-run-leg.d.ts +8 -1
- package/dist/http/durable-run-leg.js +30 -11
- package/dist/http/execution-context.d.ts +115 -14
- package/dist/http/execution-context.js +130 -40
- package/dist/http/owner-access.d.ts +3 -2
- package/dist/http/owner-access.js +2 -3
- package/dist/http/principal-gate.d.ts +18 -11
- package/dist/http/principal-gate.js +3 -1
- package/dist/http/resume-ctx-read.d.ts +14 -3
- package/dist/http/resume-ctx-read.js +8 -2
- package/dist/http/resume-legs.d.ts +35 -3
- package/dist/http/resume-legs.js +244 -92
- package/dist/http/route-ctx.d.ts +6 -0
- package/dist/http/routes/a2a-serve.js +2 -0
- package/dist/http/routes/approvals-assistant.js +3 -2
- package/dist/http/routes/capabilities.js +1 -1
- package/dist/http/routes/diagnostics.d.ts +3 -1
- package/dist/http/routes/diagnostics.js +30 -5
- package/dist/http/routes/principal-inbox.d.ts +3 -1
- package/dist/http/routes/principal-inbox.js +3 -2
- package/dist/http/routes/runs.js +10 -7
- package/dist/http/routes/session-sync.d.ts +24 -5
- package/dist/http/routes/session-sync.js +46 -34
- package/dist/http/routes/sessions-list.d.ts +1 -1
- package/dist/http/routes/tasks.js +127 -93
- package/dist/http/server.d.ts +4 -5
- package/dist/http/server.js +30 -5
- package/dist/http/submit-ctx.d.ts +1 -0
- package/dist/http/submit-ctx.js +2 -3
- package/dist/index.d.ts +3 -1
- package/dist/index.js +1 -0
- package/dist/leg-facts.d.ts +7 -2
- package/dist/leg-facts.js +1 -0
- package/dist/local-store-declaration.d.ts +54 -0
- package/dist/local-store-declaration.js +59 -0
- package/dist/mcp-assertion-header.d.ts +1 -1
- package/dist/mcp-assertion-header.js +1 -1
- package/dist/memory-project-layer.js +1 -1
- package/dist/memory-scope.d.ts +4 -4
- package/dist/memory-scope.js +4 -4
- package/dist/memory-sync-client.js +2 -2
- package/dist/model-context-window.d.ts +50 -11
- package/dist/model-context-window.js +20 -1
- package/dist/model-entry-refusal.d.ts +5 -4
- package/dist/model-entry-refusal.js +17 -3
- package/dist/model-reach-scope.d.ts +2 -1
- package/dist/observability/fail-open.d.ts +14 -2
- package/dist/observability/fail-open.js +14 -2
- package/dist/observability/metrics.js +1 -1
- package/dist/observability/run-terminal.js +1 -0
- package/dist/observability/secret-env-scrub.d.ts +1 -1
- package/dist/observability/secret-env-scrub.js +1 -1
- package/dist/org-delivery-gate.d.ts +31 -6
- package/dist/org-delivery-gate.js +7 -4
- package/dist/owner-faces.d.ts +3 -2
- package/dist/park-row-commit.d.ts +67 -0
- package/dist/park-row-commit.js +76 -0
- package/dist/parked-decide.d.ts +9 -7
- package/dist/parked-decide.js +25 -6
- package/dist/peer-directory.d.ts +1 -1
- package/dist/plugins/adoption-log-sql.d.ts +3 -3
- package/dist/plugins/adoption-log-sql.js +19 -19
- package/dist/plugins/approval-ask-store-sql.d.ts +1 -1
- package/dist/plugins/approval-ask-store-sql.js +3 -5
- package/dist/plugins/approval-exemption-skill-cutoff.d.ts +64 -0
- package/dist/plugins/approval-exemption-skill-cutoff.js +122 -0
- package/dist/plugins/approval-exemption-skill-m46.d.ts +0 -6
- package/dist/plugins/approval-exemption-skill-m46.js +6 -10
- package/dist/plugins/approval-exemption-store.d.ts +26 -1
- package/dist/plugins/approval-exemption-store.js +28 -19
- package/dist/plugins/ensure-schema-lock-policy.d.ts +36 -3
- package/dist/plugins/ensure-schema-lock-policy.js +75 -9
- package/dist/plugins/file-run-store.d.ts +22 -9
- package/dist/plugins/file-run-store.js +50 -22
- package/dist/plugins/local-checkpoint-store.d.ts +4 -0
- package/dist/plugins/local-checkpoint-store.js +2 -1
- package/dist/plugins/local-data-root-admission.d.ts +52 -0
- package/dist/plugins/local-data-root-admission.js +100 -0
- package/dist/plugins/local-engine-high-water.d.ts +28 -7
- package/dist/plugins/local-engine-high-water.js +27 -22
- package/dist/plugins/local-session-store.d.ts +134 -49
- package/dist/plugins/local-session-store.js +149 -108
- package/dist/plugins/memory-run-store.d.ts +14 -9
- package/dist/plugins/memory-run-store.js +47 -21
- package/dist/plugins/one-time-migration-marker.d.ts +19 -3
- package/dist/plugins/one-time-migration-marker.js +7 -1
- package/dist/plugins/one-time-migrations.d.ts +46 -20
- package/dist/plugins/one-time-migrations.js +102 -79
- package/dist/plugins/org-delivery-quota-store.d.ts +34 -11
- package/dist/plugins/org-delivery-quota-store.js +29 -14
- package/dist/plugins/permission-rule-store-sql.d.ts +1 -1
- package/dist/plugins/permission-rule-store-sql.js +5 -9
- package/dist/plugins/pg-pool.d.ts +1 -1
- package/dist/plugins/pg-pool.js +10 -12
- package/dist/plugins/remote-env-device.d.ts +3 -2
- package/dist/plugins/remote-env-e2b.d.ts +1 -7
- package/dist/plugins/remote-env-e2b.js +10 -7
- package/dist/plugins/remote-env-host.d.ts +50 -9
- package/dist/plugins/remote-env-host.js +77 -20
- package/dist/plugins/remote-env-k8s.js +4 -2
- package/dist/plugins/remote-env-local-docker.d.ts +6 -3
- package/dist/plugins/remote-env-local-docker.js +8 -23
- package/dist/plugins/run-store-sql.d.ts +26 -7
- package/dist/plugins/run-store-sql.js +44 -16
- package/dist/plugins/scenario-prefix-switch-m42.d.ts +3 -1
- package/dist/plugins/scenario-prefix-switch-m42.js +4 -3
- package/dist/plugins/schema-consistency.d.ts +89 -5
- package/dist/plugins/schema-consistency.js +176 -7
- package/dist/plugins/schema-drift-sentence.d.ts +6 -5
- package/dist/plugins/schema-drift-sentence.js +13 -5
- package/dist/plugins/session-placement.d.ts +5 -4
- package/dist/plugins/shell-sentinel-owner-m48.d.ts +22 -0
- package/dist/plugins/shell-sentinel-owner-m48.js +6 -0
- package/dist/plugins/stop-then-start-sentence.d.ts +16 -0
- package/dist/plugins/stop-then-start-sentence.js +7 -0
- package/dist/plugins/store-backend.d.ts +22 -8
- package/dist/plugins/store-backend.js +58 -24
- package/dist/plugins/store-contracts.d.ts +33 -3
- package/dist/plugins/store-contracts.js +16 -0
- package/dist/plugins/tidb-pool.d.ts +11 -6
- package/dist/plugins/tidb-pool.js +18 -12
- package/dist/plugins/workspace-custody.d.ts +44 -0
- package/dist/plugins/workspace-custody.js +47 -0
- package/dist/project-identity.d.ts +8 -7
- package/dist/project-identity.js +40 -17
- package/dist/project-layer-key.d.ts +13 -2
- package/dist/project-layer-key.js +9 -1
- package/dist/read-deny-floor-notice.d.ts +36 -0
- package/dist/read-deny-floor-notice.js +40 -0
- package/dist/{plain-string-record.d.ts → request-declaration-disposition.d.ts} +9 -23
- package/dist/request-declaration-disposition.js +39 -0
- package/dist/run-cancel-card.d.ts +28 -0
- package/dist/run-cancel-card.js +19 -0
- package/dist/run-cancel-context.d.ts +10 -0
- package/dist/run-cancel-context.js +1 -0
- package/dist/run-local.js +28 -13
- package/dist/runs.d.ts +31 -7
- package/dist/runs.js +34 -16
- package/dist/security.d.ts +2 -2
- package/dist/server-secret-env.d.ts +34 -20
- package/dist/server-secret-env.js +61 -18
- package/dist/session-agent.d.ts +3 -2
- package/dist/session-children-retire.d.ts +6 -3
- package/dist/session-children-retire.js +2 -5
- package/dist/session-workspace.d.ts +3 -3
- package/dist/store-admission.d.ts +33 -0
- package/dist/store-admission.js +49 -0
- package/dist/store-posture.d.ts +5 -0
- package/dist/store-posture.js +18 -1
- package/dist/stores-sql/checkpoint-store-sql.d.ts +1 -1
- package/dist/stores-sql/checkpoint-store-sql.js +3 -5
- package/dist/stores-sql/file-history-store-sql.d.ts +1 -1
- package/dist/stores-sql/file-history-store-sql.js +3 -5
- package/dist/stores-sql/tool-result-store-sql.d.ts +1 -1
- package/dist/stores-sql/tool-result-store-sql.js +4 -6
- package/dist/task-a2a.d.ts +3 -8
- package/dist/task-a2a.js +25 -27
- package/dist/task-cwd.d.ts +23 -7
- package/dist/task-cwd.js +3 -0
- package/dist/task-mcp.d.ts +3 -3
- package/dist/task-mcp.js +1 -1
- package/dist/task-settings.js +1 -0
- package/dist/terminal.d.ts +2 -0
- package/dist/terminal.js +21 -6
- package/dist/tool-approval.d.ts +44 -36
- package/dist/tool-approval.js +66 -47
- package/dist/trace/engine-notice-wire.d.ts +24 -21
- package/dist/trace/engine-notice-wire.js +7 -3
- package/dist/trace/ledger-sink.d.ts +3 -1
- package/dist/trace/project.d.ts +4 -2
- package/dist/trace/project.js +8 -7
- package/dist/trace/redact.d.ts +12 -0
- package/dist/trace/redact.js +5 -1
- package/dist/trace/self-minted-wire.d.ts +131 -52
- package/dist/trace/self-minted-wire.js +71 -39
- package/dist/web-admin-client/apply-effective.js +22 -9
- package/docs/error-text-lexicon.json +317 -0
- package/package.json +3 -1
- package/scripts/embed-minimal.mjs +183 -0
- package/wire-changes/server@7.149.0.json +372 -0
- package/dist/plain-string-record.js +0 -70
package/dist/http/resume-legs.js
CHANGED
|
@@ -10,10 +10,11 @@ import { HttpError, checkpointPartsOf, decodeCheckpointScope, scopeCellOf, CHECK
|
|
|
10
10
|
import { keyAdmits } from "../owner-faces.js";
|
|
11
11
|
import { personOf } from "../owner-brands.js";
|
|
12
12
|
import { recoveredSubject } from "../owner-faces.js";
|
|
13
|
-
import { beatRunHeartbeat, graftParkToolCallId, parkToolCallId, TurnAnchorCapture, HEARTBEAT_MS, markChildrenStoppedByUserOnAbort, voidHeldParkCard } from "../runs.js";
|
|
13
|
+
import { beatRunHeartbeat, graftParkToolCallId, heldParkOf, parkToolCallId, TurnAnchorCapture, HEARTBEAT_MS, markChildrenStoppedByUserOnAbort, voidHeldParkCard } from "../runs.js";
|
|
14
|
+
import { commitParkRow } from "../park-row-commit.js";
|
|
14
15
|
import { atHostBoundary, failureCodeOf, nestedPauseOf, sealResultForExit, outwardPlane, outwardThrownMessage } from "../terminal.js";
|
|
15
16
|
import { voidVerifierPauseCard } from "../verifier-pause-card.js";
|
|
16
|
-
import { buildCancelledLedgerRow, CARD_SETTLED_BY_CANCEL_MESSAGE, createRunAbort, mintCancelledResult, withCardSettledByCancelMessage } from "../run-cancel-context.js";
|
|
17
|
+
import { buildCancelledLedgerRow, CANCELLED_CODE, CARD_SETTLED_BY_CANCEL_MESSAGE, createRunAbort, mintCancelledResult, withCardSettledByCancelMessage } from "../run-cancel-context.js";
|
|
17
18
|
import { isTerminalRunStatus, runProvenElsewhere } from "../plugins/store-contracts.js";
|
|
18
19
|
import { readTurnActivityMs, recordTurnActivity } from "../turn-activity.js";
|
|
19
20
|
import { recordResumeResourceWindow } from "../resource-window.js";
|
|
@@ -27,15 +28,19 @@ import { buildCheckpointAnchorHash } from "../approval-ask-audit-store.js";
|
|
|
27
28
|
import { APPROVAL_ENDING_CAUSES } from "../approval-deny-reasons.js";
|
|
28
29
|
import { serviceVersion } from "../service-version.js";
|
|
29
30
|
import { TERMINAL_PARK_GATE_KINDS, UnreadableCheckpointRowError } from "../plugins/checkpoint-store-face.js";
|
|
31
|
+
import { cancelSettlesCard } from "../run-cancel-card.js";
|
|
32
|
+
import { CARD_EXPIRE_OUTCOME, expireCardUnderStoredScope } from "../card-settle-stored-scope.js";
|
|
30
33
|
import { withSubject } from "../observability/principal-context.js";
|
|
31
34
|
import { withModelReachScope } from "../model-reach-scope.js";
|
|
35
|
+
import { approvalLegOriginOf } from "../approval-leg-origin.js";
|
|
32
36
|
import { legFactsOf } from "../leg-facts.js";
|
|
37
|
+
import { compositionLegRoot } from "../task-cwd.js";
|
|
33
38
|
import { createInstructionSelectionLeg } from "../instruction-selection.js";
|
|
34
39
|
import { modelTextEventData, appendParkRow, turnEndEventData, contextUsageEventData, toolStartEventData, toolEndEventData, taskProgressEventData, taskStartedEventData, taskNotificationEventData, compactedEventData, diagnosticsEventData, brainStatusEventData, steeringInjectedEventData, compactionOutcomeEventData, compactionStartedEventData, workspaceChangedEventData, wiringManifestEventData, toolRosterDeltaEventData, toolDisclosureEventData, humanInputEventData, appendModelUsageDelta, attachModelUsage, createPromptManifestSeat } from "../trace/project.js";
|
|
35
40
|
import { redactSecrets } from "../trace/redact.js";
|
|
36
41
|
import { buildRunObjectivePreview } from "../user-authored-text.js";
|
|
37
42
|
import { createForwardAppendFailedNote, createLedgerSink, createSerialLedgerAppend } from "../trace/ledger-sink.js";
|
|
38
|
-
import { registerEngineNoticeLeg } from "../trace/engine-notice-wire.js";
|
|
43
|
+
import { defaultEngineNoticeRouter, registerEngineNoticeLeg } from "../trace/engine-notice-wire.js";
|
|
39
44
|
import { blockText, reasoningDeltaBytes, reasoningSegmentEventId } from "../trace/model-segment.js";
|
|
40
45
|
import { errorText } from "../observability/err-text.js";
|
|
41
46
|
import { recordFailOpen } from "../observability/fail-open.js";
|
|
@@ -52,13 +57,14 @@ import { createImagesLocal } from "./routes/images.js";
|
|
|
52
57
|
import { createCapabilitiesLocal } from "./routes/capabilities.js";
|
|
53
58
|
import { gatedSubject, explicitOperatorOk, runIdentityOf } from "./principal-gate.js";
|
|
54
59
|
import { buildActiveRunConflict, resumeEntryForGate } from "./active-run-conflict.js";
|
|
55
|
-
import {
|
|
60
|
+
import { refusalAfterLostClaim } from "./claim-heartbeat.js";
|
|
61
|
+
import { CONTINUATION_CLAIM_FACE, ClaimFaultAnswer, commitExecutionContext, faultedAnswerOf, isRootSessionGoneError, resolveBeforeClaim, rootSessionAbsentAnswer, stampSpecRoot, withdrawClaim } from "./execution-context.js";
|
|
56
62
|
import { timeClaim } from "./claim-latency.js";
|
|
57
63
|
import { commitSessionInheritEnv } from "./session-inherit-env.js";
|
|
58
64
|
import { ASK_USER_QUESTION_TOOL_NAME, isApprovalGateKind, isEngineQuestionRow } from "../approval-content-kind.js";
|
|
59
65
|
import { deploymentListAsks } from "../approval.js";
|
|
60
66
|
import { isParkLedgerEventType } from "../trace/ledger-events.js";
|
|
61
|
-
import { readResumeCtx, resumeCtxCorruptAnswer, resumeCtxUnreadableAnswer, resumeCtxUnsealableAnswer,
|
|
67
|
+
import { readResumeCtx, resumeCtxCorruptAnswer, resumeCtxUnreadableAnswer, resumeCtxUnsealableAnswer, resumeStoreFailureOf } from "./resume-ctx-read.js";
|
|
62
68
|
import { objectiveWithheldIn } from "./submit-ctx.js";
|
|
63
69
|
import { buildCheckpointVersionKeys, resumeAssemblyRefusalAnswer, resumeRefusalWithCard, sealCheckpointCtx, storedBodyReplayAnswer, storedBodyReplayLogFields, withPermissionModeRewritten } from "./checkpoint-ctx.js";
|
|
64
70
|
import { refusalBody, resumeRefusalFacts } from "./refusal-body.js";
|
|
@@ -204,6 +210,9 @@ function decisionRefusalCauseOf(e, gate) {
|
|
|
204
210
|
function notResumableAnswer() {
|
|
205
211
|
return { status: 409, body: refusalBody({ errorCode: "conflict.not_resumable", message: "run is not in a resumable (suspended) state (already resumed, decided, or expired)" }) };
|
|
206
212
|
}
|
|
213
|
+
function resumeClaimFaultAnswerOf(e) {
|
|
214
|
+
return faultedAnswerOf(e, CONTINUATION_CLAIM_FACE.faulted);
|
|
215
|
+
}
|
|
207
216
|
function approvalStaleAnswer(facts) {
|
|
208
217
|
return {
|
|
209
218
|
status: 409,
|
|
@@ -217,13 +226,41 @@ function approvalStaleAnswer(facts) {
|
|
|
217
226
|
}),
|
|
218
227
|
};
|
|
219
228
|
}
|
|
220
|
-
function rowRefusalAnswer(verdict, diagnostic, ctx) {
|
|
229
|
+
function rowRefusalAnswer(verdict, diagnostic, ctx, wayOut) {
|
|
221
230
|
const refusal = new CheckpointError(verdict.code, diagnostic === undefined ? verdict.sentence : `${verdict.sentence} Diagnostic: ${diagnostic}`, verdict.reason === undefined ? undefined : { reason: verdict.reason });
|
|
222
|
-
return checkpointRefusalAnswer(refusal, { gate: "policy_ask", retriable: resumeRefusalDisposition(refusal) === "retriable", versionKeys: refusalVersionKeys(refusal, ctx) });
|
|
231
|
+
return checkpointRefusalAnswer(refusal, { gate: "policy_ask", retriable: resumeRefusalDisposition(refusal) === "retriable", versionKeys: refusalVersionKeys(refusal, ctx), ...(wayOut !== undefined ? { wayOut } : {}) });
|
|
232
|
+
}
|
|
233
|
+
export async function rowWayOutOf(seat, kind) {
|
|
234
|
+
switch (kind) {
|
|
235
|
+
case "lacking":
|
|
236
|
+
return "newer_worker";
|
|
237
|
+
case "damaged":
|
|
238
|
+
try {
|
|
239
|
+
return (await cancelSettlesCard(seat.cs, seat.runs, seat.token)) ? "cancel_run" : undefined;
|
|
240
|
+
}
|
|
241
|
+
catch (e) {
|
|
242
|
+
seat.logger?.warn?.("row_way_out_probe_failed", { note: "the store could not answer whether cancel would settle this card — no wayOut offered", err: errorText(e) });
|
|
243
|
+
return undefined;
|
|
244
|
+
}
|
|
245
|
+
default: {
|
|
246
|
+
const unhandled = kind;
|
|
247
|
+
throw new Error(`row verdict ${String(unhandled)} has no way out`);
|
|
248
|
+
}
|
|
249
|
+
}
|
|
250
|
+
}
|
|
251
|
+
async function loadedRowWayOut(cp, seat) {
|
|
252
|
+
const verdict = judgeCheckpointRow(cp);
|
|
253
|
+
return verdict.kind === "ok" ? undefined : rowWayOutOf(seat, verdict.kind);
|
|
254
|
+
}
|
|
255
|
+
async function loadedRowRefusal(cp, seat, writer) {
|
|
256
|
+
const verdict = judgeCheckpointRow(cp);
|
|
257
|
+
if (verdict.kind === "ok")
|
|
258
|
+
return undefined;
|
|
259
|
+
return rowRefusalAnswer(verdict, undefined, verdict.code === "checkpoint.unsupported_version" ? await writer() : {}, await rowWayOutOf(seat, verdict.kind));
|
|
223
260
|
}
|
|
224
|
-
export function rowReadRefusalAnswer(e) {
|
|
261
|
+
export async function rowReadRefusalAnswer(e, seat) {
|
|
225
262
|
if (e instanceof UnreadableCheckpointRowError && e.rowVerdict.kind !== "ok")
|
|
226
|
-
return rowRefusalAnswer(e.rowVerdict, errorText(e), {});
|
|
263
|
+
return rowRefusalAnswer(e.rowVerdict, errorText(e), {}, seat === undefined ? undefined : await rowWayOutOf(seat, e.rowVerdict.kind));
|
|
227
264
|
return { status: 409, body: refusalBody({ errorCode: "conflict.checkpoint_unreadable", message: errorText(e) }) };
|
|
228
265
|
}
|
|
229
266
|
function refusalVersionKeys(e, ctx) {
|
|
@@ -252,6 +289,7 @@ export function checkpointRefusalAnswer(e, at) {
|
|
|
252
289
|
: {}),
|
|
253
290
|
...at.versionKeys,
|
|
254
291
|
...(e.code === "resume.constraint_rejected" ? acceptableFormFacts(detail) : {}),
|
|
292
|
+
...(at.wayOut !== undefined ? { wayOut: at.wayOut } : {}),
|
|
255
293
|
},
|
|
256
294
|
}),
|
|
257
295
|
}, at.retriable);
|
|
@@ -335,7 +373,7 @@ export function createResumeLegs(ctx) {
|
|
|
335
373
|
return { status: 404, body: refusalBody({ errorCode: "not_found.approval", message: "approval not found" }) };
|
|
336
374
|
}
|
|
337
375
|
const cpPromise = cs.get(token);
|
|
338
|
-
const runSessionPromise = cs.hostSessionOfCard(token).then((id) => (typeof id === "string" ? { kind: "owned", id } : { kind: "unowned" }), (e) => ({ kind: "failed", failure:
|
|
376
|
+
const runSessionPromise = cs.hostSessionOfCard(token).then((id) => (typeof id === "string" ? { kind: "owned", id } : { kind: "unowned" }), (e) => ({ kind: "failed", failure: resumeStoreFailureOf(e) }));
|
|
339
377
|
const runCtxOf = async () => {
|
|
340
378
|
const owner = await runSessionPromise;
|
|
341
379
|
if (owner.kind === "unowned")
|
|
@@ -352,7 +390,6 @@ export function createResumeLegs(ctx) {
|
|
|
352
390
|
const ctxPromise = runCtxOf();
|
|
353
391
|
void ctxPromise.catch(() => { });
|
|
354
392
|
const noApproval = { status: 404, body: refusalBody({ errorCode: "not_found.approval", message: "approval not found" }) };
|
|
355
|
-
const rowOwnRefusal = async (verdict) => rowRefusalAnswer(verdict, undefined, verdict.code === "checkpoint.unsupported_version" ? await ctxPromise.then((r) => (r !== undefined && "ctx" in r ? (r.ctx ?? {}) : {}), () => ({})) : {});
|
|
356
393
|
let cp;
|
|
357
394
|
try {
|
|
358
395
|
cp = await cpPromise;
|
|
@@ -363,7 +400,7 @@ export function createResumeLegs(ctx) {
|
|
|
363
400
|
deps.logger?.warn?.("approval_decide_unreadable_row_withheld", { sessionId, note: "row unreadable and the caller's ownership of THAT token is unproven — folded into the uniform 404" });
|
|
364
401
|
return noApproval;
|
|
365
402
|
}
|
|
366
|
-
return rowReadRefusalAnswer(e);
|
|
403
|
+
return rowReadRefusalAnswer(e, { cs, runs: deps.runStore, token, logger: deps.logger });
|
|
367
404
|
}
|
|
368
405
|
if (!cp) {
|
|
369
406
|
if (!(await mayDiscloseRowError(cs, token, decider)))
|
|
@@ -374,9 +411,9 @@ export function createResumeLegs(ctx) {
|
|
|
374
411
|
deps.logger?.warn?.("approval_decide_row_scope_mismatch", { sessionId, note: "loaded pending row is foreign to the verified caller — TOCTOU fold" });
|
|
375
412
|
return { status: 404, body: refusalBody({ errorCode: "not_found.approval", message: "approval not found" }) };
|
|
376
413
|
}
|
|
377
|
-
const
|
|
378
|
-
if (
|
|
379
|
-
return
|
|
414
|
+
const rowOwn = await loadedRowRefusal(cp, { cs, runs: deps.runStore, token, logger: deps.logger }, () => ctxPromise.then((r) => (r !== undefined && "ctx" in r ? (r.ctx ?? {}) : {}), () => ({})));
|
|
415
|
+
if (rowOwn !== undefined)
|
|
416
|
+
return rowOwn;
|
|
380
417
|
const decideGateKind = cp.gate?.kind;
|
|
381
418
|
if (!isApprovalGateKind(decideGateKind)) {
|
|
382
419
|
if (typeof decideGateKind !== "string" || decideGateKind === "") {
|
|
@@ -491,12 +528,14 @@ export function createResumeLegs(ctx) {
|
|
|
491
528
|
...(deps.parkedReviveInheritedGate ? { rebuildInheritedGate: deps.parkedReviveInheritedGate } : {}),
|
|
492
529
|
...(deps.toolApproval ? { approverSeatFor: windowZeroSeatOf(deps.toolApproval) } : {}),
|
|
493
530
|
execution: {
|
|
494
|
-
resolve: async (root) => {
|
|
531
|
+
resolve: async (root, face) => {
|
|
495
532
|
let ctx;
|
|
496
533
|
try {
|
|
497
|
-
ctx = await
|
|
534
|
+
ctx = await resolveBeforeClaim(deps, { kind: "continuation", root, principal: checkpointPartsOf(cp.scope)?.person }, face);
|
|
498
535
|
}
|
|
499
536
|
catch (e) {
|
|
537
|
+
if (e instanceof ClaimFaultAnswer)
|
|
538
|
+
return { ok: false, gone: false, answer: e.answer };
|
|
500
539
|
if (isRootSessionGoneError(e))
|
|
501
540
|
return { ok: false, gone: true };
|
|
502
541
|
if (e instanceof HttpError)
|
|
@@ -548,29 +587,52 @@ export function createResumeLegs(ctx) {
|
|
|
548
587
|
if (replay !== undefined)
|
|
549
588
|
return replay;
|
|
550
589
|
let recordFailure;
|
|
590
|
+
const answerOfRecordFailure = async (failed) => {
|
|
591
|
+
if (failed.kind === "faulted")
|
|
592
|
+
return faultedAnswerOf(failed.err, CONTINUATION_CLAIM_FACE.faulted);
|
|
593
|
+
try {
|
|
594
|
+
return await refusalAfterLostClaim(deps, runStore, plan.hostSessionId, failed.deliveredTaskId);
|
|
595
|
+
}
|
|
596
|
+
catch (e) {
|
|
597
|
+
return faultedAnswerOf(e, CONTINUATION_CLAIM_FACE.faulted);
|
|
598
|
+
}
|
|
599
|
+
};
|
|
551
600
|
const delivery = {
|
|
552
601
|
carried: [{ runId: plan.runId, token, outcome: plan.outcome }],
|
|
553
602
|
commitBeforeLaunch: async (deliveredTaskId) => {
|
|
603
|
+
let recorded;
|
|
554
604
|
try {
|
|
555
|
-
|
|
556
|
-
if (!recorded)
|
|
557
|
-
throw new Error(DELIVERY_NOT_RECORDED);
|
|
605
|
+
recorded = await decisions.record({ token, runId: plan.runId, hostSessionId: plan.hostSessionId, outcome: plan.outcome, fingerprint, deliveredTaskId, acceptedAtMs: Date.now() });
|
|
558
606
|
}
|
|
559
607
|
catch (err) {
|
|
560
|
-
recordFailure = { err };
|
|
608
|
+
recordFailure = { kind: "faulted", err };
|
|
561
609
|
throw err;
|
|
562
610
|
}
|
|
611
|
+
if (!recorded) {
|
|
612
|
+
recordFailure = { kind: "not_recorded", deliveredTaskId };
|
|
613
|
+
throw new Error(DELIVERY_NOT_RECORDED);
|
|
614
|
+
}
|
|
563
615
|
},
|
|
564
616
|
};
|
|
565
617
|
const hostToken = await cs
|
|
566
618
|
.findPendingTokenBySession(plan.hostSessionId, undefined, { gateKinds: TERMINAL_PARK_GATE_KINDS, pick: "oldest" })
|
|
567
619
|
.catch(() => undefined);
|
|
568
620
|
const admittedPayer = decodeCheckpointScope(cp.scope);
|
|
569
|
-
|
|
570
|
-
|
|
571
|
-
|
|
572
|
-
|
|
573
|
-
|
|
621
|
+
let out;
|
|
622
|
+
try {
|
|
623
|
+
out = hostToken
|
|
624
|
+
? await resumeWake(plan.hostSessionId, plan.message, decider?.subject, req, acceptEarly, delivery, admittedPayer, restatement)
|
|
625
|
+
: await startFreshHostTurn(plan.hostSessionId, plan.message.text, admittedPayer, req, delivery, plan.runId, restatement);
|
|
626
|
+
}
|
|
627
|
+
catch (err) {
|
|
628
|
+
const failed = recordFailure;
|
|
629
|
+
if (failed === undefined)
|
|
630
|
+
throw err;
|
|
631
|
+
return await answerOfRecordFailure(failed);
|
|
632
|
+
}
|
|
633
|
+
const failed = recordFailure;
|
|
634
|
+
if (failed !== undefined)
|
|
635
|
+
return await answerOfRecordFailure(failed);
|
|
574
636
|
if (acceptedTaskIdOf(out) === undefined)
|
|
575
637
|
return (await liveReceiptOf(await decisions.get(token))) ?? out;
|
|
576
638
|
return out;
|
|
@@ -623,7 +685,7 @@ export function createResumeLegs(ctx) {
|
|
|
623
685
|
const ONE_SHOT_SUBMIT_STRIP = Object.fromEntries(ONE_SHOT_SUBMIT_KEYS.map((k) => [k, undefined]));
|
|
624
686
|
const resolveResumeSpec = async (ctx, req, auth, lease) => {
|
|
625
687
|
const engineNotices = [];
|
|
626
|
-
const execution = await
|
|
688
|
+
const execution = await resolveBeforeClaim(deps, { kind: "continuation", root: auth.sessionId, principal: auth.subject === undefined ? undefined : personOf(recoveredSubject(auth.subject)) }, CONTINUATION_CLAIM_FACE);
|
|
627
689
|
const resolved = await deps.resolveSpec({ ...ctx.body, ...ONE_SHOT_SUBMIT_STRIP }, req, auth, {
|
|
628
690
|
leg: "resume",
|
|
629
691
|
onNotice: (n) => engineNotices.push(n),
|
|
@@ -633,7 +695,7 @@ export function createResumeLegs(ctx) {
|
|
|
633
695
|
});
|
|
634
696
|
const legFacts = legFactsOf(resolved);
|
|
635
697
|
const runner = runnerFor(resolved);
|
|
636
|
-
const spec = stampSpecRoot(resolved, execution, ctx.body);
|
|
698
|
+
const spec = stampSpecRoot(resolved, execution, { use: "session", requireExistingSession: ctx.body.requireExistingSession });
|
|
637
699
|
return { spec, runner, legFacts, engineNotices, execution };
|
|
638
700
|
};
|
|
639
701
|
const refuseStoredBodyReplay = (sessionId, refusal) => {
|
|
@@ -651,6 +713,8 @@ export function createResumeLegs(ctx) {
|
|
|
651
713
|
}
|
|
652
714
|
catch (e) {
|
|
653
715
|
lease.release();
|
|
716
|
+
if (e instanceof ClaimFaultAnswer)
|
|
717
|
+
return { ok: false, answer: e.answer };
|
|
654
718
|
if (e instanceof HttpError) {
|
|
655
719
|
return { ok: false, answer: resumeRefusalWithCard(resumeAssemblyRefusalAnswer(e.status, httpErrorRefusal(e)), true) };
|
|
656
720
|
}
|
|
@@ -684,6 +748,14 @@ export function createResumeLegs(ctx) {
|
|
|
684
748
|
const payer = legSubject === undefined ? undefined : personOf(legSubject);
|
|
685
749
|
const heldObjective = objectiveWithheldIn(resumeCtx.ctx) ? undefined : resumeObjective;
|
|
686
750
|
const reparkStatus = outcome.gate === "plan_review" ? "needs_review" : "suspended";
|
|
751
|
+
const commitResumePark = (runStore, runId, word, parkResult, cardToken) => {
|
|
752
|
+
if (heldLeg === undefined)
|
|
753
|
+
throw new Error(`resume leg: run ${runId} parked without the holding leg its claim handed back — the park cannot be committed`);
|
|
754
|
+
return commitParkRow({
|
|
755
|
+
runStore, taskId: runId, word, result: parkResult, card: { token: cardToken, sessionId }, cards: deps.checkpointStore, leg: "resume",
|
|
756
|
+
legId: heldLeg, staleWindowMs: deps.config?.runStaleSec === undefined ? undefined : deps.config.runStaleSec * 1000,
|
|
757
|
+
});
|
|
758
|
+
};
|
|
687
759
|
const resumeCtxNotPersisted = (why) => {
|
|
688
760
|
deps.metrics?.inc("resume_ctx_not_persisted_total");
|
|
689
761
|
deps.logger?.warn?.("resume_ctx_not_persisted", { sessionId: resumeCtx.sessionId, gate: outcome.gate, error: why, note: "this continuation runs with the body it was given; the durable rebuild input still holds the PREVIOUS body, so the NEXT continuation of this session rebuilds from that one — its permission mode is never wider than this leg's, but if this leg restated its MCP / A2A peer tables (mcpServers / a2aPeers on the resume request), a later continuation that does not restate them runs with the previous tables again" });
|
|
@@ -720,16 +792,27 @@ export function createResumeLegs(ctx) {
|
|
|
720
792
|
const heldDecisions = await heldWorkflowParkDecisions(deps, sessionId, args.workflowDecision?.carried);
|
|
721
793
|
let taskId = await deps.runStore?.getActiveTaskId(sessionId);
|
|
722
794
|
let freshRunClaimed = false;
|
|
795
|
+
let heldLeg;
|
|
796
|
+
let legClaim;
|
|
797
|
+
let engineHandedOff = false;
|
|
798
|
+
let restored;
|
|
799
|
+
const cancelledInClaimWindow = () => restored?.kind === "ended" && restored.row?.errorCode === CANCELLED_CODE;
|
|
723
800
|
if (args.mintFreshRun && taskId && deps.runStore) {
|
|
724
801
|
return { status: 409, body: await buildActiveRunConflict({ runStore: deps.runStore, checkpointStore: deps.checkpointStore, governance: deps.config, turnActivity: readTurnActivityMs }, sessionId, taskId) };
|
|
725
802
|
}
|
|
726
803
|
if (!taskId && args.mintFreshRun && deps.runStore) {
|
|
727
804
|
const freshId = uuidv7();
|
|
728
|
-
|
|
729
|
-
|
|
730
|
-
|
|
731
|
-
|
|
732
|
-
|
|
805
|
+
let created;
|
|
806
|
+
try {
|
|
807
|
+
created = await timeClaim(deps.metrics, "run_claim", deps.runStore.createRun(freshId, sessionId, subject ?? null, deps.instanceId ?? "default", {
|
|
808
|
+
source: args.mintFreshRun.source,
|
|
809
|
+
identity: args.mintFreshRun.identity,
|
|
810
|
+
objectivePreview: heldObjective === undefined ? null : buildRunObjectivePreview(heldObjective),
|
|
811
|
+
}));
|
|
812
|
+
}
|
|
813
|
+
catch (e) {
|
|
814
|
+
return resumeClaimFaultAnswerOf(e);
|
|
815
|
+
}
|
|
733
816
|
if (created.existing) {
|
|
734
817
|
throw new Error(`wake fresh run: createRun observed an existing row for the freshly minted taskId ${freshId} (no idempotency key was sent) — refusing to treat it as a busy session`);
|
|
735
818
|
}
|
|
@@ -738,11 +821,20 @@ export function createResumeLegs(ctx) {
|
|
|
738
821
|
if (!created.ok) {
|
|
739
822
|
return { status: 409, body: await buildActiveRunConflict({ runStore: deps.runStore, checkpointStore: deps.checkpointStore, governance: deps.config, turnActivity: readTurnActivityMs }, sessionId, created.activeTaskId) };
|
|
740
823
|
}
|
|
741
|
-
|
|
824
|
+
legClaim = { kind: "run", runStore: deps.runStore, taskId: freshId, legId: created.legId };
|
|
825
|
+
let placementReject;
|
|
826
|
+
try {
|
|
827
|
+
const refusal = await commitExecutionContext(deps, legClaim, execution);
|
|
828
|
+
placementReject = refusal === null ? null : CONTINUATION_CLAIM_FACE.refused(refusal);
|
|
829
|
+
}
|
|
830
|
+
catch (e) {
|
|
831
|
+
placementReject = resumeClaimFaultAnswerOf(e);
|
|
832
|
+
}
|
|
742
833
|
if (placementReject)
|
|
743
834
|
return placementReject;
|
|
744
835
|
taskId = freshId;
|
|
745
836
|
freshRunClaimed = true;
|
|
837
|
+
heldLeg = created.legId;
|
|
746
838
|
}
|
|
747
839
|
const fleetPub = taskId && deps.fleetBus
|
|
748
840
|
? fleetRunPublisher(deps.fleetBus, { runId: taskId, scope: fleetScope, rootTaskId: taskId, sessionId, ...fleetRunLabels(heldObjective) })
|
|
@@ -751,12 +843,12 @@ export function createResumeLegs(ctx) {
|
|
|
751
843
|
if (deps.toolApproval && taskId) {
|
|
752
844
|
const toolApproval = deps.toolApproval;
|
|
753
845
|
resumeTaskConfig.onAsk = approvalLeg.unattended
|
|
754
|
-
? toolApproval.boundUnattendedAsk({ owner: subject ?? null, taskId, sessionId,
|
|
846
|
+
? toolApproval.boundUnattendedAsk({ owner: subject ?? null, taskId, sessionId, ...approvalLegOriginOf(execution) }, approvalLeg.unattended)
|
|
755
847
|
: toolApproval.boundAsk({
|
|
756
848
|
owner: subject ?? null,
|
|
757
849
|
taskId,
|
|
758
850
|
sessionId,
|
|
759
|
-
|
|
851
|
+
...approvalLegOriginOf(execution),
|
|
760
852
|
...(approvalLeg.active ? { legKey, ...(legDeadlineMonotonic !== undefined ? { legDeadlineMonotonic } : {}) } : {}),
|
|
761
853
|
});
|
|
762
854
|
}
|
|
@@ -791,12 +883,9 @@ export function createResumeLegs(ctx) {
|
|
|
791
883
|
if (!cs2)
|
|
792
884
|
return false;
|
|
793
885
|
try {
|
|
794
|
-
|
|
795
|
-
if (live?.status !== "pending")
|
|
796
|
-
return false;
|
|
797
|
-
if (await cs2.expire(token, live.scope))
|
|
886
|
+
if ((await expireCardUnderStoredScope(cs2, token)) === CARD_EXPIRE_OUTCOME.expired)
|
|
798
887
|
return true;
|
|
799
|
-
deps.logger?.
|
|
888
|
+
deps.logger?.info?.("resume_cancel_checkpoint_not_settled", { taskId, sessionId });
|
|
800
889
|
return false;
|
|
801
890
|
}
|
|
802
891
|
catch (cancelSettleErr) {
|
|
@@ -826,19 +915,45 @@ export function createResumeLegs(ctx) {
|
|
|
826
915
|
let cancelLedgerAppend;
|
|
827
916
|
const preemptCtrl = new AbortController();
|
|
828
917
|
if (taskId && deps.runStore) {
|
|
829
|
-
|
|
918
|
+
if (freshRunClaimed)
|
|
919
|
+
claimedRow = true;
|
|
920
|
+
else {
|
|
921
|
+
try {
|
|
922
|
+
heldLeg = await deps.runStore.markResuming(taskId, deps.instanceId ?? "default");
|
|
923
|
+
}
|
|
924
|
+
catch (e) {
|
|
925
|
+
return resumeClaimFaultAnswerOf(e);
|
|
926
|
+
}
|
|
927
|
+
claimedRow = heldLeg !== undefined;
|
|
928
|
+
}
|
|
830
929
|
if (!claimedRow)
|
|
831
930
|
return notResumableAnswer();
|
|
931
|
+
inflightRuns.set(taskId, cancelCtrl);
|
|
832
932
|
if (!freshRunClaimed) {
|
|
833
|
-
const
|
|
834
|
-
|
|
835
|
-
|
|
933
|
+
const rs0 = deps.runStore;
|
|
934
|
+
const claimedTask = taskId;
|
|
935
|
+
const restoreClaim = async () => {
|
|
936
|
+
restored = await commitResumePark(rs0, claimedTask, reparkStatus, null, token);
|
|
937
|
+
};
|
|
938
|
+
legClaim = { kind: "resumed", runStore: rs0, taskId: claimedTask, restore: restoreClaim };
|
|
939
|
+
let placementReject;
|
|
940
|
+
try {
|
|
941
|
+
const refusal = await commitExecutionContext(deps, legClaim, execution);
|
|
942
|
+
placementReject = refusal === null ? null : CONTINUATION_CLAIM_FACE.refused(refusal);
|
|
943
|
+
}
|
|
944
|
+
catch (e) {
|
|
945
|
+
placementReject = resumeClaimFaultAnswerOf(e);
|
|
946
|
+
}
|
|
947
|
+
if (placementReject) {
|
|
948
|
+
if (inflightRuns.get(claimedTask) === cancelCtrl)
|
|
949
|
+
inflightRuns.delete(claimedTask);
|
|
950
|
+
return cancelledInClaimWindow() ? { status: 200, body: { taskId: claimedTask, sessionId, status: "failed", errorCode: "cancelled" } } : placementReject;
|
|
951
|
+
}
|
|
836
952
|
}
|
|
837
953
|
markChildrenStoppedByUserOnAbort(cancelCtrl.signal, taskId, subject);
|
|
838
954
|
fleetPub?.onStart();
|
|
839
955
|
const rs = deps.runStore;
|
|
840
956
|
const owner = subject ?? null;
|
|
841
|
-
inflightRuns.set(taskId, cancelCtrl);
|
|
842
957
|
preemptableRuns.set(taskId, preemptCtrl);
|
|
843
958
|
resumeHeartbeat = setInterval(() => {
|
|
844
959
|
beatRunHeartbeat(rs, taskId, owner, () => cancelCtrl.abort("stall_verdict"), () => undefined);
|
|
@@ -867,6 +982,10 @@ export function createResumeLegs(ctx) {
|
|
|
867
982
|
if (args.onResumeCommitted)
|
|
868
983
|
await args.onResumeCommitted();
|
|
869
984
|
let unregisterEngineNotice;
|
|
985
|
+
const handToEngine = (start) => {
|
|
986
|
+
engineHandedOff = true;
|
|
987
|
+
return start();
|
|
988
|
+
};
|
|
870
989
|
try {
|
|
871
990
|
const instructionLeg = createInstructionSelectionLeg();
|
|
872
991
|
const legWiringSections = () => ({ instructions: instructionLeg.segment(), skills: args.legFacts?.skills });
|
|
@@ -885,6 +1004,21 @@ export function createResumeLegs(ctx) {
|
|
|
885
1004
|
return write;
|
|
886
1005
|
}, { route: "resume-open", connection: "durable-append", log: (m, x) => deps.logger?.info?.(m, x) });
|
|
887
1006
|
}
|
|
1007
|
+
const openEngineNoticePort = async (append) => {
|
|
1008
|
+
unregisterEngineNotice = registerEngineNoticeLeg({
|
|
1009
|
+
sessionId,
|
|
1010
|
+
...(taskId !== undefined ? { taskId } : {}),
|
|
1011
|
+
durable: (row) => append("engine_notice", row).then(() => true, (err) => {
|
|
1012
|
+
recordFailOpen("server.engine-notice.durable-append-failed", `leg=resume code=${row.code} task=${taskId ?? ""}`);
|
|
1013
|
+
deps.logger?.warn?.("engine_notice_append_failed", { taskId, noticeCode: row.code, sessionId: row.sessionId, err: err instanceof Error ? err.message : String(err) });
|
|
1014
|
+
return false;
|
|
1015
|
+
}),
|
|
1016
|
+
});
|
|
1017
|
+
for (const notice of args.engineNotices ?? [])
|
|
1018
|
+
await defaultEngineNoticeRouter.routeConfirmed(notice);
|
|
1019
|
+
if (deps.userInputLedger && taskId)
|
|
1020
|
+
await deps.userInputLedger.announceUndrained({ sessionId, taskId });
|
|
1021
|
+
};
|
|
888
1022
|
if (composition !== undefined) {
|
|
889
1023
|
const rsV = deps.runStore;
|
|
890
1024
|
const vSeqBase = rsV && taskId ? await rsV.maxSeq(taskId) : 0;
|
|
@@ -897,9 +1031,11 @@ export function createResumeLegs(ctx) {
|
|
|
897
1031
|
});
|
|
898
1032
|
const vAppend = (type, data) => verifySink.append(type, data);
|
|
899
1033
|
cancelLedgerAppend = vAppend;
|
|
1034
|
+
await openEngineNoticePort(vAppend);
|
|
900
1035
|
const verifyTailContentMode = resumeTaskConfig.forwardSubagentEvents === true ? "on" : "progress_only";
|
|
901
1036
|
const verifyInternals = {
|
|
902
1037
|
...launchInternals,
|
|
1038
|
+
...compositionLegRoot(args.legFacts?.registeredCwd),
|
|
903
1039
|
...(taskId ? { onActivity: () => recordTurnActivity(taskId) } : {}),
|
|
904
1040
|
onForwardEvent: (e) => {
|
|
905
1041
|
fleetPub?.onForwardEvent(e);
|
|
@@ -949,10 +1085,10 @@ export function createResumeLegs(ctx) {
|
|
|
949
1085
|
let vrEngine;
|
|
950
1086
|
switch (composition.entry) {
|
|
951
1087
|
case "verify":
|
|
952
|
-
vrEngine = await resumeWithVerification(legRunner, token, outcome, legSpec, resumeObjective, composition.rounds, verifyInternals);
|
|
1088
|
+
vrEngine = await handToEngine(() => resumeWithVerification(legRunner, token, outcome, legSpec, resumeObjective, composition.rounds, verifyInternals));
|
|
953
1089
|
break;
|
|
954
1090
|
case "cascade":
|
|
955
|
-
vrEngine = await resumeCascade(legRunner, token, outcome, stampSpecRoot({ ...legSpec, ...composition.objectiveInputs, objective: resumeObjective }, execution,
|
|
1091
|
+
vrEngine = await handToEngine(() => resumeCascade(legRunner, token, outcome, stampSpecRoot({ ...legSpec, ...composition.objectiveInputs, objective: resumeObjective }, execution, { use: "ladder_host" }), composition.config, composition.parkedLeg, verifyInternals));
|
|
956
1092
|
break;
|
|
957
1093
|
default: {
|
|
958
1094
|
const unhandled = composition;
|
|
@@ -989,8 +1125,9 @@ export function createResumeLegs(ctx) {
|
|
|
989
1125
|
}
|
|
990
1126
|
}
|
|
991
1127
|
catch (e) {
|
|
992
|
-
if (!vrReopened)
|
|
1128
|
+
if (!reSuspended && !vrReopened)
|
|
993
1129
|
throw e;
|
|
1130
|
+
recordFailOpen("server.runs.terminal-ledger-write-failed", `arm=resume-park-projection ${errorText(e)}`);
|
|
994
1131
|
}
|
|
995
1132
|
}
|
|
996
1133
|
if (reSuspended)
|
|
@@ -1009,17 +1146,7 @@ export function createResumeLegs(ctx) {
|
|
|
1009
1146
|
const serialAppend = createSerialLedgerAppend((s, type, data) => rs.appendEvent(taskId, s, type, data), startSeq);
|
|
1010
1147
|
const append = (type, data) => (rs && taskId ? serialAppend(type, data) : Promise.resolve());
|
|
1011
1148
|
cancelLedgerAppend = append;
|
|
1012
|
-
|
|
1013
|
-
sessionId,
|
|
1014
|
-
...(args.engineNotices !== undefined && args.engineNotices.length > 0 ? { pending: [...args.engineNotices] } : {}),
|
|
1015
|
-
durable: (row) => append("engine_notice", row).then(() => true, (err) => {
|
|
1016
|
-
recordFailOpen("server.engine-notice.durable-append-failed", `leg=resume code=${row.code} task=${taskId ?? ""}`);
|
|
1017
|
-
deps.logger?.warn?.("engine_notice_append_failed", { taskId, noticeCode: row.code, sessionId: row.sessionId, err: err instanceof Error ? err.message : String(err) });
|
|
1018
|
-
return false;
|
|
1019
|
-
}),
|
|
1020
|
-
});
|
|
1021
|
-
if (deps.userInputLedger && taskId)
|
|
1022
|
-
await deps.userInputLedger.announceUndrained({ sessionId, taskId });
|
|
1149
|
+
await openEngineNoticePort(append);
|
|
1023
1150
|
const anchorOwner = subject ?? null;
|
|
1024
1151
|
const headOf = headReadOf(deps.sessionStorage);
|
|
1025
1152
|
const captureTurnAnchor = deps.resumeAnchorStore && headOf
|
|
@@ -1047,7 +1174,7 @@ export function createResumeLegs(ctx) {
|
|
|
1047
1174
|
resumeLegLive = true;
|
|
1048
1175
|
const tailContentMode = resumeTaskConfig.forwardSubagentEvents === true ? "on" : "progress_only";
|
|
1049
1176
|
const resumeWorkspace = createSessionWorkspaceRecorder(sessionWorkspaceStoreOf(deps.sessionStorage), args.legFacts?.sessionWitness);
|
|
1050
|
-
const stream = await legRunner.resumeStream(token, outcome, { ...resumeTaskConfig, signal: cancelCtrl.signal, preemptSignal: preemptCtrl.signal }, {
|
|
1177
|
+
const stream = await handToEngine(() => legRunner.resumeStream(token, outcome, { ...resumeTaskConfig, signal: cancelCtrl.signal, preemptSignal: preemptCtrl.signal }, {
|
|
1051
1178
|
...launchInternals,
|
|
1052
1179
|
onForwardEvent: (e) => {
|
|
1053
1180
|
fleetPub?.onForwardEvent(e);
|
|
@@ -1102,7 +1229,7 @@ export function createResumeLegs(ctx) {
|
|
|
1102
1229
|
},
|
|
1103
1230
|
}
|
|
1104
1231
|
: {}),
|
|
1105
|
-
});
|
|
1232
|
+
}));
|
|
1106
1233
|
await commitResumeCtx();
|
|
1107
1234
|
resumeStreamRef = stream;
|
|
1108
1235
|
if (taskId)
|
|
@@ -1260,11 +1387,7 @@ export function createResumeLegs(ctx) {
|
|
|
1260
1387
|
const settledLeg = await reopenedAfterLeg(r, causeAtDone !== undefined);
|
|
1261
1388
|
if (settledLeg.reopened) {
|
|
1262
1389
|
reopenConfirmedPark = true;
|
|
1263
|
-
|
|
1264
|
-
await appendParkRow(append, { modelUsage: deps.modelUsage, taskId }, reparkStatus, r, failureCodeOf(r.terminal) ?? null);
|
|
1265
|
-
}
|
|
1266
|
-
catch {
|
|
1267
|
-
}
|
|
1390
|
+
await appendParkRow(append, { modelUsage: deps.modelUsage, taskId }, reparkStatus, r, failureCodeOf(r.terminal) ?? null);
|
|
1268
1391
|
return r;
|
|
1269
1392
|
}
|
|
1270
1393
|
const safe = await attachModelUsage(sealResultForExit(settledLeg.leg), { append, getEvents: (id, a) => rs.getEvents(id, a), modelUsage: deps.modelUsage, taskId });
|
|
@@ -1283,7 +1406,7 @@ export function createResumeLegs(ctx) {
|
|
|
1283
1406
|
owner: subject ?? null,
|
|
1284
1407
|
abortSignal: cancelCtrl.signal,
|
|
1285
1408
|
sessionId,
|
|
1286
|
-
|
|
1409
|
+
...approvalLegOriginOf(execution),
|
|
1287
1410
|
family: "approval_request",
|
|
1288
1411
|
transport: hitlTransport,
|
|
1289
1412
|
legKey,
|
|
@@ -1315,6 +1438,7 @@ export function createResumeLegs(ctx) {
|
|
|
1315
1438
|
return { status: 200, body: { taskId, sessionId, status: "failed", errorCode: "cancelled" } };
|
|
1316
1439
|
}
|
|
1317
1440
|
const retriable = disposition === "retriable";
|
|
1441
|
+
let reparkEnded = false;
|
|
1318
1442
|
if (claimedRow && taskId && deps.runStore) {
|
|
1319
1443
|
if (retriable && freshRunClaimed) {
|
|
1320
1444
|
const c = { taskId, sessionId, terminal: { kind: "failed", code: e.code, message: e.message }, stats: { turns: 0, tokens: 0, usageMissing: true } };
|
|
@@ -1324,15 +1448,17 @@ export function createResumeLegs(ctx) {
|
|
|
1324
1448
|
settleFleet("failed");
|
|
1325
1449
|
}
|
|
1326
1450
|
else if (retriable) {
|
|
1327
|
-
|
|
1328
|
-
|
|
1451
|
+
const parked = await commitResumePark(deps.runStore, taskId, reparkStatus, null, token);
|
|
1452
|
+
reparkEnded = parked.kind === "ended";
|
|
1453
|
+
settleFleet(reparkEnded ? "failed" : reparkStatus);
|
|
1329
1454
|
}
|
|
1330
1455
|
else {
|
|
1331
1456
|
await deps.runStore.setTerminal(taskId, "failed", null, e.message).catch(() => undefined);
|
|
1332
1457
|
settleFleet("failed");
|
|
1333
1458
|
}
|
|
1334
1459
|
}
|
|
1335
|
-
|
|
1460
|
+
const answerRetriable = retriable && !reparkEnded;
|
|
1461
|
+
return checkpointRefusalAnswer(e, { gate: outcome.gate, retriable: answerRetriable, ...(taskId !== undefined ? { taskId } : {}), versionKeys: refusalVersionKeys(e, resumeCtx.ctx), ...(answerRetriable && args.rowWayOut !== undefined ? { wayOut: args.rowWayOut } : {}) });
|
|
1336
1462
|
}
|
|
1337
1463
|
const thrownCause = cancelCtrl.cause();
|
|
1338
1464
|
if (thrownCause !== undefined) {
|
|
@@ -1345,12 +1471,28 @@ export function createResumeLegs(ctx) {
|
|
|
1345
1471
|
settleFleet("failed");
|
|
1346
1472
|
return { status: 200, body: { taskId, sessionId, status: "failed", errorCode: "cancelled" } };
|
|
1347
1473
|
}
|
|
1348
|
-
if (
|
|
1349
|
-
|
|
1350
|
-
await deps
|
|
1474
|
+
if (!engineHandedOff && legClaim !== undefined) {
|
|
1475
|
+
const claim = legClaim;
|
|
1476
|
+
await withdrawClaim(deps, claim, `the resume could not be handed to the engine: ${outwardThrownMessage(e, errorText(e))}`, "faulted");
|
|
1477
|
+
const parkedAgain = claim.kind === "resumed" && restored?.kind !== "ended";
|
|
1478
|
+
settleFleet(parkedAgain ? reparkStatus : "failed");
|
|
1479
|
+
if (cancelledInClaimWindow())
|
|
1480
|
+
return { status: 200, body: { taskId, sessionId, status: "failed", errorCode: "cancelled" } };
|
|
1481
|
+
return resumeClaimFaultAnswerOf(e);
|
|
1482
|
+
}
|
|
1483
|
+
const heldParkWord = heldPark === undefined ? undefined : heldParkOf(heldPark);
|
|
1484
|
+
if (!(claimedRow && taskId && deps.runStore && heldPark !== undefined && heldParkWord !== undefined)) {
|
|
1485
|
+
if (claimedRow && taskId && deps.runStore) {
|
|
1486
|
+
await voidHeldParkCard(heldPark, sessionId, deps.checkpointStore, "resume");
|
|
1487
|
+
await deps.runStore.setTerminal(taskId, "failed", null, outwardThrownMessage(e, errorText(e))).catch(() => undefined);
|
|
1488
|
+
}
|
|
1489
|
+
settleFleet("failed");
|
|
1490
|
+
throw e;
|
|
1351
1491
|
}
|
|
1352
|
-
|
|
1353
|
-
|
|
1492
|
+
recordFailOpen("server.runs.terminal-ledger-write-failed", `arm=resume-park-held ${errorText(e)}`);
|
|
1493
|
+
if (cancelLedgerAppend !== undefined)
|
|
1494
|
+
await appendParkRow(cancelLedgerAppend, { modelUsage: deps.modelUsage, taskId }, heldParkWord.word, await graftParkToolCallId(heldPark, deps.checkpointStore));
|
|
1495
|
+
result = heldPark;
|
|
1354
1496
|
}
|
|
1355
1497
|
finally {
|
|
1356
1498
|
unregisterEngineNotice?.();
|
|
@@ -1375,25 +1517,32 @@ export function createResumeLegs(ctx) {
|
|
|
1375
1517
|
const checkpointReopened = reopenConfirmedPark;
|
|
1376
1518
|
const sealed = sealResultForExit(result);
|
|
1377
1519
|
const resumePlane = outwardPlane(sealed.terminal);
|
|
1520
|
+
let parkEnded;
|
|
1378
1521
|
if (taskId && deps.runStore) {
|
|
1379
|
-
|
|
1380
|
-
|
|
1381
|
-
|
|
1382
|
-
|
|
1383
|
-
|
|
1384
|
-
await deps.runStore
|
|
1522
|
+
const park = checkpointReopened && reparkStatus === "needs_review" ? { word: "needs_review", result: null }
|
|
1523
|
+
: resumePlane.status === "suspended" || checkpointReopened ? { word: "suspended", result: sealed.terminal.kind === "paused" ? sealed : null }
|
|
1524
|
+
: resumePlane.status === "needs_review" ? { word: "needs_review", result: sealed }
|
|
1525
|
+
: undefined;
|
|
1526
|
+
if (park !== undefined) {
|
|
1527
|
+
const parked = await commitResumePark(deps.runStore, taskId, park.word, park.result, heldPark?.terminal.kind === "paused" ? heldPark.terminal.token : token);
|
|
1528
|
+
if (parked.kind === "ended")
|
|
1529
|
+
parkEnded = { row: parked.row };
|
|
1530
|
+
}
|
|
1385
1531
|
else
|
|
1386
1532
|
await deps.runStore.setTerminal(taskId, resumePlane.status, sealed, resumePlane.errorMessage ?? null);
|
|
1387
1533
|
}
|
|
1388
|
-
settleFleet(checkpointReopened ? reparkStatus : resumePlane.status, fleetRunResiduals(sealed, sealed ? failureCodeOf(sealed.terminal) : undefined));
|
|
1534
|
+
settleFleet(parkEnded !== undefined ? "failed" : checkpointReopened ? reparkStatus : resumePlane.status, fleetRunResiduals(sealed, sealed ? failureCodeOf(sealed.terminal) : undefined));
|
|
1535
|
+
const answered = parkEnded !== undefined
|
|
1536
|
+
? { status: parkEnded.row?.status ?? "failed", errorCode: parkEnded.row?.errorCode ?? undefined, errorMessage: parkEnded.row?.error ?? undefined }
|
|
1537
|
+
: { status: resumePlane.status, errorCode: resumePlane.errorCode, errorMessage: resumePlane.errorMessage };
|
|
1389
1538
|
return {
|
|
1390
1539
|
status: 200,
|
|
1391
1540
|
body: {
|
|
1392
|
-
taskId, sessionId, status:
|
|
1541
|
+
taskId, sessionId, status: answered.status,
|
|
1393
1542
|
bindingEnforced: true,
|
|
1394
|
-
...(
|
|
1395
|
-
...(
|
|
1396
|
-
...(checkpointReopened ? { retriable: true } : {}),
|
|
1543
|
+
...(answered.errorCode ? { errorCode: answered.errorCode } : {}),
|
|
1544
|
+
...(answered.errorMessage ? { errorMessage: redactErrorMessage(answered.errorMessage) ?? "" } : {}),
|
|
1545
|
+
...(checkpointReopened && parkEnded === undefined ? { retriable: true } : {}),
|
|
1397
1546
|
...buildSettlementEvidence({ at: "engine_terminal", commitment: args.commitment }),
|
|
1398
1547
|
},
|
|
1399
1548
|
};
|
|
@@ -1446,7 +1595,7 @@ export function createResumeLegs(ctx) {
|
|
|
1446
1595
|
deps.logger?.warn?.("assistant_resume_unreadable_row_withheld", { sessionId, note: "row unreadable and the caller's ownership of THAT token is unproven — folded into the uniform 404" });
|
|
1447
1596
|
return noSuspension;
|
|
1448
1597
|
}
|
|
1449
|
-
return rowReadRefusalAnswer(e);
|
|
1598
|
+
return rowReadRefusalAnswer(e, { cs, runs: deps.runStore, token, logger: deps.logger });
|
|
1450
1599
|
}
|
|
1451
1600
|
if (!cp) {
|
|
1452
1601
|
if (!(await mayDiscloseRowError(cs, token, decider)))
|
|
@@ -1482,7 +1631,8 @@ export function createResumeLegs(ctx) {
|
|
|
1482
1631
|
if (composition?.entry === "cascade_unlinked")
|
|
1483
1632
|
return compositionUnlinkedAnswer();
|
|
1484
1633
|
const selectedGate = durableRowIdentity("decision-receipt", cp.checkpointId, token);
|
|
1485
|
-
|
|
1634
|
+
const rowWayOut = await loadedRowWayOut(cp, { cs, runs: deps.runStore, token, logger: deps.logger });
|
|
1635
|
+
return await driveResumeIntoRunLog({ peerLease: resolved.peerLease, execution: resolved.execution, runner: resolved.runner, legFacts: resolved.legFacts, token, sessionId, subject: auth.subject, fleetScope: resumeFleetScope(req, auth), taskConfig, resumeObjective, outcome: { gate: "resource_limit", decision: "continue" }, commitment: { kind: "gate_resolved", ...selectedGate }, composition, resumeCtx: { sessionId, ctx }, ...(rowWayOut !== undefined ? { rowWayOut } : {}), ...(engineNotices.length > 0 ? { engineNotices } : {}), ...(acceptEarly === true ? { acceptEarly: true } : {}), ...(req !== undefined ? { callerInitiated: true } : {}) });
|
|
1486
1636
|
}
|
|
1487
1637
|
finally {
|
|
1488
1638
|
resolved.peerLease.release();
|
|
@@ -1532,6 +1682,7 @@ export function createResumeLegs(ctx) {
|
|
|
1532
1682
|
source: "workflow_parked_decide",
|
|
1533
1683
|
identity: runIdentityOf(req, deps.config, subject ?? null),
|
|
1534
1684
|
workflowDecision,
|
|
1685
|
+
face: CONTINUATION_CLAIM_FACE,
|
|
1535
1686
|
});
|
|
1536
1687
|
if (out.existing) {
|
|
1537
1688
|
throw new Error(`decide host turn: createRun observed an existing row ${out.existing.taskId} for a freshly minted taskId (no idempotency key was sent)`);
|
|
@@ -1567,7 +1718,7 @@ export function createResumeLegs(ctx) {
|
|
|
1567
1718
|
deps.logger?.warn?.("wake_unreadable_row_withheld", { sessionId, note: "row unreadable and the caller's ownership of THAT token is unproven — folded into the uniform 404" });
|
|
1568
1719
|
return { status: 404, body: refusalBody({ errorCode: "not_found.parked_checkpoint", message: "no parked checkpoint for this session (nothing to wake)" }) };
|
|
1569
1720
|
}
|
|
1570
|
-
return rowReadRefusalAnswer(e);
|
|
1721
|
+
return rowReadRefusalAnswer(e, undefined);
|
|
1571
1722
|
}
|
|
1572
1723
|
if (!cp) {
|
|
1573
1724
|
if (!(await mayDiscloseRowError(cs, token, who))) {
|
|
@@ -1672,7 +1823,7 @@ export function createResumeLegs(ctx) {
|
|
|
1672
1823
|
deps.logger?.warn?.("plan_review_unreadable_row_withheld", { sessionId, note: "row unreadable and the caller's ownership of THAT token is unproven — folded into the uniform 404" });
|
|
1673
1824
|
return refuse(noPending);
|
|
1674
1825
|
}
|
|
1675
|
-
return refuse(rowReadRefusalAnswer(e));
|
|
1826
|
+
return refuse(await rowReadRefusalAnswer(e, { cs, runs: deps.runStore, token, logger: deps.logger }));
|
|
1676
1827
|
}
|
|
1677
1828
|
if (!cp) {
|
|
1678
1829
|
log404("checkpoint-missing");
|
|
@@ -1744,7 +1895,8 @@ export function createResumeLegs(ctx) {
|
|
|
1744
1895
|
hostDecision,
|
|
1745
1896
|
};
|
|
1746
1897
|
const selectedGate = durableRowIdentity("decision-receipt", cp.checkpointId, token);
|
|
1747
|
-
const
|
|
1898
|
+
const rowWayOut = await loadedRowWayOut(cp, { cs: deps.checkpointStore, runs: deps.runStore, token, logger: deps.logger });
|
|
1899
|
+
const out = await driveResumeIntoRunLog({ peerLease: resolved.peerLease, execution: resolved.execution, runner: resolved.runner, legFacts: resolved.legFacts, token, sessionId, subject: auth.subject, fleetScope: resumeFleetScope(req, auth), taskConfig, resumeObjective, outcome, commitment: { kind: "gate_resolved", ...selectedGate }, composition, resumeCtx: { sessionId, ctx: rebuildCtx }, ...(rowWayOut !== undefined ? { rowWayOut } : {}), ...(engineNotices.length > 0 ? { engineNotices } : {}), ...(acceptEarly === true ? { acceptEarly: true } : {}), ...(req !== undefined ? { callerInitiated: true } : {}) });
|
|
1748
1900
|
return out;
|
|
1749
1901
|
}
|
|
1750
1902
|
finally {
|