intentdna 1.9.0 → 1.9.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +2 -2
- package/.claude-plugin/plugin.json +1 -1
- package/README.md +227 -87
- package/dist/cli/commands/run-lifecycle.d.ts +61 -11
- package/dist/cli/commands/run-lifecycle.js +184 -50
- package/dist/cli/commands/run-observer.d.ts +15 -0
- package/dist/cli/commands/run-observer.js +210 -0
- package/dist/cli/commands/run.d.ts +16 -20
- package/dist/cli/commands/run.js +461 -659
- package/dist/cli/commands/sync.js +16 -12
- package/dist/cli/index.js +14 -5
- package/dist/compiler/compile.d.ts +1 -0
- package/dist/compiler/compile.js +1 -1
- package/dist/compiler/controller.d.ts +16 -0
- package/dist/compiler/controller.js +693 -0
- package/dist/compiler/input-resolver.d.ts +2 -0
- package/dist/compiler/input-resolver.js +22 -6
- package/dist/hooks/cli.d.ts +45 -0
- package/dist/hooks/cli.js +263 -31
- package/dist/hooks/host-event.d.ts +57 -0
- package/dist/hooks/host-event.js +187 -0
- package/dist/hooks/index.d.ts +1 -0
- package/dist/hooks/index.js +1 -0
- package/dist/hooks/protocol.d.ts +7 -0
- package/dist/hooks/schema.d.ts +1 -0
- package/dist/hooks/schema.js +89 -1
- package/dist/hooks/state.d.ts +23 -1
- package/dist/hooks/state.js +9 -0
- package/dist/index.d.ts +1 -0
- package/dist/index.js +1 -0
- package/dist/mcp/index.js +2 -0
- package/dist/mcp/tools-run.d.ts +4 -0
- package/dist/mcp/tools-run.js +419 -0
- package/dist/mcp/tools-state.d.ts +1 -1
- package/dist/mcp/tools-state.js +13 -241
- package/dist/runtime/artifact-store.d.ts +146 -0
- package/dist/runtime/artifact-store.js +1436 -0
- package/dist/runtime/canonical-attempt-outcome.d.ts +12 -0
- package/dist/runtime/canonical-attempt-outcome.js +195 -0
- package/dist/runtime/canonical-json.d.ts +10 -0
- package/dist/runtime/canonical-json.js +105 -0
- package/dist/runtime/canonical-run-application.d.ts +93 -0
- package/dist/runtime/canonical-run-application.js +229 -0
- package/dist/runtime/canonical-run-service.d.ts +454 -0
- package/dist/runtime/canonical-run-service.js +4760 -0
- package/dist/runtime/canonical-runtime-composition.d.ts +22 -0
- package/dist/runtime/canonical-runtime-composition.js +34 -0
- package/dist/runtime/canonical-runtime-projection.d.ts +3 -0
- package/dist/runtime/canonical-runtime-projection.js +4 -0
- package/dist/runtime/canonical-target-compiler.d.ts +46 -0
- package/dist/runtime/canonical-target-compiler.js +514 -0
- package/dist/runtime/claude-agent-identity.d.ts +2 -0
- package/dist/runtime/claude-agent-identity.js +21 -0
- package/dist/runtime/executable-run-plan.d.ts +328 -0
- package/dist/runtime/executable-run-plan.js +1485 -0
- package/dist/runtime/execution-authority.d.ts +83 -0
- package/dist/runtime/execution-authority.js +98 -0
- package/dist/runtime/handoff-resolver.js +4 -1
- package/dist/runtime/harness-pull-adapter.d.ts +146 -0
- package/dist/runtime/harness-pull-adapter.js +315 -0
- package/dist/runtime/index.d.ts +39 -14
- package/dist/runtime/index.js +29 -7
- package/dist/runtime/local-execution-authority.d.ts +194 -0
- package/dist/runtime/local-execution-authority.js +2897 -0
- package/dist/runtime/local-execution-reconciliation.d.ts +20 -0
- package/dist/runtime/local-execution-reconciliation.js +107 -0
- package/dist/runtime/local-execution-supervisor-script.d.ts +7 -0
- package/dist/runtime/local-execution-supervisor-script.js +793 -0
- package/dist/runtime/local-provider-sandbox.d.ts +42 -0
- package/dist/runtime/local-provider-sandbox.js +154 -0
- package/dist/runtime/local-verifier-execution.d.ts +24 -0
- package/dist/runtime/local-verifier-execution.js +290 -0
- package/dist/runtime/local-windows-execution-supervisor-script.d.ts +6 -0
- package/dist/runtime/local-windows-execution-supervisor-script.js +788 -0
- package/dist/runtime/plan-store.d.ts +30 -0
- package/dist/runtime/plan-store.js +231 -0
- package/dist/runtime/process-tree.d.ts +9 -0
- package/dist/runtime/process-tree.js +97 -24
- package/dist/runtime/providers/codex.js +2 -2
- package/dist/runtime/push-driver.d.ts +184 -0
- package/dist/runtime/push-driver.js +1002 -0
- package/dist/runtime/result-store.d.ts +34 -3
- package/dist/runtime/result-store.js +1073 -2
- package/dist/runtime/run-binding.d.ts +68 -0
- package/dist/runtime/run-binding.js +278 -0
- package/dist/runtime/run-contracts.d.ts +575 -0
- package/dist/runtime/run-contracts.js +58 -7
- package/dist/runtime/run-controller.d.ts +1 -6
- package/dist/runtime/run-controller.js +0 -41
- package/dist/runtime/run-store.d.ts +85 -7
- package/dist/runtime/run-store.js +2528 -186
- package/dist/runtime/skill-adapter.d.ts +18 -7
- package/dist/runtime/skill-adapter.js +100 -742
- package/dist/runtime/structured-output-validator.d.ts +44 -0
- package/dist/runtime/structured-output-validator.js +171 -0
- package/dist/runtime/verifier-command-binding.d.ts +22 -0
- package/dist/runtime/verifier-command-binding.js +162 -0
- package/dist/runtime/verifier.d.ts +46 -6
- package/dist/runtime/verifier.js +355 -53
- package/dist/runtime/windows-job-keeper.d.ts +47 -0
- package/dist/runtime/windows-job-keeper.js +231 -0
- package/dist/runtime/worker-executor.d.ts +17 -1
- package/dist/runtime/worker-executor.js +26 -5
- package/dist/runtime/workflow-plan-adapter.d.ts +19 -0
- package/dist/runtime/workflow-plan-adapter.js +502 -6
- package/dist/runtime/workflow-runtime-manifest.d.ts +1 -0
- package/dist/runtime/workflow-runtime-manifest.js +5 -0
- package/dist/runtime/workspace-isolation.d.ts +29 -2
- package/dist/runtime/workspace-isolation.js +488 -11
- package/dist/runtime/workspace-observation.d.ts +44 -0
- package/dist/runtime/workspace-observation.js +216 -0
- package/dist/schema/controller-registry.d.ts +0 -7
- package/dist/schema/controller-registry.js +0 -8
- package/dist/schema/types.d.ts +84 -10
- package/dist/schema/types.js +2 -0
- package/dist/schema/validate.js +151 -1
- package/dist/schema/validators/controllers.d.ts +1 -1
- package/dist/schema/validators/controllers.js +126 -41
- package/dist/schema/workflow-authoring-contract.js +2 -0
- package/dist/schema/yaml-parser.js +1 -1
- package/dist/templates/flutter-rewrite-new.dna.yaml +1 -0
- package/dist/templates/flutter-rewrite.dna.yaml +87 -60
- package/dist/templates/safe-refactoring.dna.yaml +1 -0
- package/native/windows-job-keeper/README.md +80 -0
- package/native/windows-job-keeper/bin/aarch64/intentdna-windows-job-keeper.exe +0 -0
- package/native/windows-job-keeper/bin/x86_64/intentdna-windows-job-keeper.exe +0 -0
- package/native/windows-job-keeper/keeper.c +979 -0
- package/native/windows-job-keeper/manifest.json +33 -0
- package/package.json +6 -2
- package/scripts/build-windows-job-keeper.mjs +172 -0
- package/scripts/verify-windows-job-keeper.mjs +184 -0
|
@@ -0,0 +1,4760 @@
|
|
|
1
|
+
import { createHash, randomUUID } from "node:crypto";
|
|
2
|
+
import { resolve } from "node:path";
|
|
3
|
+
import { canonicalizeJson } from "./canonical-json.js";
|
|
4
|
+
import { CANONICAL_RESULT_SCHEMA_VERSION, CANONICAL_VERIFIER_EXECUTION_RECEIPT_SCHEMA_VERSION, CANONICAL_WORKSPACE_OBSERVATION_SCHEMA_VERSION, } from "./run-contracts.js";
|
|
5
|
+
import { canonicalAttemptOutcomeStatus, validateCanonicalAttemptOutcome, } from "./canonical-attempt-outcome.js";
|
|
6
|
+
import { parseStoredRunBinding } from "./run-binding.js";
|
|
7
|
+
import { RunStoreError, } from "./run-store.js";
|
|
8
|
+
import { ResultStoreError, } from "./result-store.js";
|
|
9
|
+
import { ArtifactMaterializer, } from "./artifact-store.js";
|
|
10
|
+
import { parseEvaluatorResult } from "./evaluator.js";
|
|
11
|
+
import { assertVerifierCommandExecutionResult, execVerifierCommand, MAX_VERIFIER_DIAGNOSTIC_BYTES, MAX_VERIFIER_EVIDENCE_BYTES, verifierCommandExecutionPassed, } from "./verifier.js";
|
|
12
|
+
import { workerNonlaunchAttestationMatches, workerStopReceiptMatchesExecution, workerStopReceiptMatchesStop, workerAlreadyStoppedEvidenceMatchesStop, } from "./execution-authority.js";
|
|
13
|
+
import { observeWorkspace, workspaceObservationPolicy, WorkspaceObservationError, } from "./workspace-observation.js";
|
|
14
|
+
import { DEFAULT_STRUCTURED_OUTPUT_SCHEMA_REGISTRY, validatePinnedStructuredOutput, } from "./structured-output-validator.js";
|
|
15
|
+
import { bindVerifierCommand, TRUSTED_FULL_VERIFIER_COMMAND_SCHEMA_VERSION, } from "./verifier-command-binding.js";
|
|
16
|
+
import { canonicalWorkspaceDirectory, WorkspaceIsolationError, } from "./workspace-isolation.js";
|
|
17
|
+
export class CanonicalRunServiceError extends Error {
|
|
18
|
+
code;
|
|
19
|
+
constructor(code, message, options) {
|
|
20
|
+
super(message, options);
|
|
21
|
+
this.name = "CanonicalRunServiceError";
|
|
22
|
+
this.code = code;
|
|
23
|
+
}
|
|
24
|
+
}
|
|
25
|
+
export function canonicalVerifierEnvironment(supplied, platform = process.platform, hostEnvironment = process.env) {
|
|
26
|
+
const base = platform === "win32"
|
|
27
|
+
? {
|
|
28
|
+
PATH: hostEnvironment.PATH ?? "",
|
|
29
|
+
PATHEXT: hostEnvironment.PATHEXT ?? ".COM;.EXE;.BAT;.CMD",
|
|
30
|
+
SystemRoot: hostEnvironment.SystemRoot ?? hostEnvironment.SYSTEMROOT ?? "C:\\Windows",
|
|
31
|
+
ComSpec: hostEnvironment.ComSpec ?? hostEnvironment.COMSPEC ?? "C:\\Windows\\System32\\cmd.exe",
|
|
32
|
+
}
|
|
33
|
+
: { PATH: "/opt/homebrew/bin:/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin" };
|
|
34
|
+
return { ...base, ...supplied };
|
|
35
|
+
}
|
|
36
|
+
function isExecutionDeadlineTimeout(execution) {
|
|
37
|
+
return execution.outcome.kind === "worker_process"
|
|
38
|
+
&& execution.outcome.status === "failed"
|
|
39
|
+
&& execution.outcome.reason_code === "timeout"
|
|
40
|
+
&& execution.outcome.process.kind === "timeout";
|
|
41
|
+
}
|
|
42
|
+
const TERMINAL_ACTIVATION_STATES = new Set(["succeeded", "failed", "skipped", "cancelled"]);
|
|
43
|
+
const ATTEMPT_NODE_KINDS = new Set(["worker", "system", "interaction"]);
|
|
44
|
+
const DEFAULT_LEASE_DURATION_MS = 30_000;
|
|
45
|
+
const DEFAULT_EXECUTION_TIMEOUT_MS = 30 * 60_000;
|
|
46
|
+
function digest(value) {
|
|
47
|
+
return `sha256:${createHash("sha256")
|
|
48
|
+
.update(canonicalizeJson(value), "utf8")
|
|
49
|
+
.digest("hex")}`;
|
|
50
|
+
}
|
|
51
|
+
function textDigest(value) {
|
|
52
|
+
return `sha256:${createHash("sha256").update(value, "utf8").digest("hex")}`;
|
|
53
|
+
}
|
|
54
|
+
const SHA256_DIGEST_PATTERN = /^sha256:[a-f0-9]{64}$/u;
|
|
55
|
+
function snapshotVerifierAuthorityDescriptor(value) {
|
|
56
|
+
let snapshot;
|
|
57
|
+
try {
|
|
58
|
+
snapshot = JSON.parse(canonicalizeJson(value));
|
|
59
|
+
}
|
|
60
|
+
catch (error) {
|
|
61
|
+
throw new CanonicalRunServiceError("invalid_request", "verifier execution authority descriptor must be canonical JSON data", { cause: error });
|
|
62
|
+
}
|
|
63
|
+
if (typeof snapshot !== "object" || snapshot === null || Array.isArray(snapshot)) {
|
|
64
|
+
throw new CanonicalRunServiceError("invalid_request", "verifier execution authority descriptor must be an object");
|
|
65
|
+
}
|
|
66
|
+
const descriptor = snapshot;
|
|
67
|
+
const expectedKeys = [
|
|
68
|
+
"schema_version",
|
|
69
|
+
"authority_instance_id",
|
|
70
|
+
"adapter_kind",
|
|
71
|
+
"adapter_version",
|
|
72
|
+
"configuration_digest",
|
|
73
|
+
"process_tree_containment",
|
|
74
|
+
"network_policies",
|
|
75
|
+
].sort();
|
|
76
|
+
const actualKeys = Object.keys(descriptor).sort();
|
|
77
|
+
const fieldsMatch = actualKeys.length === expectedKeys.length
|
|
78
|
+
&& actualKeys.every((key, index) => key === expectedKeys[index]);
|
|
79
|
+
const policies = descriptor.network_policies;
|
|
80
|
+
if (!fieldsMatch
|
|
81
|
+
|| descriptor.schema_version !== "intentdna.verifier_execution_authority.v1"
|
|
82
|
+
|| typeof descriptor.authority_instance_id !== "string"
|
|
83
|
+
|| descriptor.authority_instance_id.trim().length === 0
|
|
84
|
+
|| typeof descriptor.adapter_kind !== "string"
|
|
85
|
+
|| descriptor.adapter_kind.trim().length === 0
|
|
86
|
+
|| typeof descriptor.adapter_version !== "string"
|
|
87
|
+
|| descriptor.adapter_version.trim().length === 0
|
|
88
|
+
|| typeof descriptor.configuration_digest !== "string"
|
|
89
|
+
|| !SHA256_DIGEST_PATTERN.test(descriptor.configuration_digest)
|
|
90
|
+
|| typeof descriptor.process_tree_containment !== "boolean"
|
|
91
|
+
|| !Array.isArray(policies)
|
|
92
|
+
|| policies.length === 0
|
|
93
|
+
|| policies.some((policy) => policy !== "allow" && policy !== "deny_all")
|
|
94
|
+
|| new Set(policies).size !== policies.length) {
|
|
95
|
+
throw new CanonicalRunServiceError("invalid_request", "verifier execution authority descriptor is invalid");
|
|
96
|
+
}
|
|
97
|
+
Object.freeze(policies);
|
|
98
|
+
return Object.freeze(snapshot);
|
|
99
|
+
}
|
|
100
|
+
async function observeAttemptWorkspace(workspaceRoot, workspaceId) {
|
|
101
|
+
try {
|
|
102
|
+
const observation = await observeWorkspace({ workspace_root: workspaceRoot });
|
|
103
|
+
return {
|
|
104
|
+
schema_version: CANONICAL_WORKSPACE_OBSERVATION_SCHEMA_VERSION,
|
|
105
|
+
workspace_id: workspaceId,
|
|
106
|
+
scope: {
|
|
107
|
+
kind: "attempt_workspace",
|
|
108
|
+
relative_root: ".",
|
|
109
|
+
excluded_relative_paths: observation.excluded_relative_paths,
|
|
110
|
+
},
|
|
111
|
+
observed_digest: observation.observed_digest,
|
|
112
|
+
entry_count: observation.entry_count,
|
|
113
|
+
byte_count: observation.byte_count,
|
|
114
|
+
git: observation.git,
|
|
115
|
+
};
|
|
116
|
+
}
|
|
117
|
+
catch (error) {
|
|
118
|
+
if (!(error instanceof WorkspaceObservationError))
|
|
119
|
+
throw error;
|
|
120
|
+
throw new CanonicalRunServiceError(error.code, error.message, { cause: error });
|
|
121
|
+
}
|
|
122
|
+
}
|
|
123
|
+
function addMilliseconds(iso, milliseconds) {
|
|
124
|
+
return new Date(Date.parse(iso) + milliseconds).toISOString();
|
|
125
|
+
}
|
|
126
|
+
function defaultIdentity(kind, seed) {
|
|
127
|
+
const hex = createHash("sha256").update(`${kind}\0${seed}`, "utf8").digest("hex");
|
|
128
|
+
return `${kind}:${hex}`;
|
|
129
|
+
}
|
|
130
|
+
function publicAuditEvent(event) {
|
|
131
|
+
if (event.type !== "attempt_heartbeat")
|
|
132
|
+
return event;
|
|
133
|
+
const key = `claim:${event.attempt_id ?? "unknown"}`
|
|
134
|
+
+ `:epoch:${event.payload.claim_epoch}:heartbeat:${event.payload.heartbeat_at}`;
|
|
135
|
+
return {
|
|
136
|
+
...event,
|
|
137
|
+
event_id: defaultIdentity("event", key),
|
|
138
|
+
idempotency_key: key,
|
|
139
|
+
};
|
|
140
|
+
}
|
|
141
|
+
function requireNonEmpty(value, label) {
|
|
142
|
+
if (!value.trim()) {
|
|
143
|
+
throw new CanonicalRunServiceError("invalid_request", `${label} must not be empty`);
|
|
144
|
+
}
|
|
145
|
+
}
|
|
146
|
+
function resultIdFor(identities, attemptId) {
|
|
147
|
+
return identities.create("result", attemptId);
|
|
148
|
+
}
|
|
149
|
+
function runtimeTemplateValue(value) {
|
|
150
|
+
return typeof value === "string"
|
|
151
|
+
? value
|
|
152
|
+
: canonicalizeJson(value);
|
|
153
|
+
}
|
|
154
|
+
function renderPacketPrompt(template, values) {
|
|
155
|
+
const malformed = template.replace(/\{\{[^{}]*\}\}/gu, "");
|
|
156
|
+
if (malformed.includes("{{")) {
|
|
157
|
+
throw new CanonicalRunServiceError("invalid_request", "packet prompt contains a malformed runtime placeholder");
|
|
158
|
+
}
|
|
159
|
+
return template.replace(/\{\{([^{}]+)\}\}|(^|[^\\])\$ARGUMENTS/gu, (_match, name, prefix) => {
|
|
160
|
+
if (name !== undefined) {
|
|
161
|
+
const value = values.get(name);
|
|
162
|
+
if (value === undefined) {
|
|
163
|
+
throw new CanonicalRunServiceError("invalid_request", `packet prompt contains unresolved runtime placeholder {{${name}}}`);
|
|
164
|
+
}
|
|
165
|
+
return value;
|
|
166
|
+
}
|
|
167
|
+
const value = values.get("ARGUMENTS");
|
|
168
|
+
if (value === undefined) {
|
|
169
|
+
throw new CanonicalRunServiceError("invalid_request", "packet prompt contains unresolved runtime placeholder $ARGUMENTS");
|
|
170
|
+
}
|
|
171
|
+
return `${prefix ?? ""}${value}`;
|
|
172
|
+
});
|
|
173
|
+
}
|
|
174
|
+
function resolveVerifierCommand(command, bindings, runtimeInputs, inputDefinitions) {
|
|
175
|
+
let context = "unquoted";
|
|
176
|
+
let backtickReturn = "unquoted";
|
|
177
|
+
let resolved = "";
|
|
178
|
+
const environment = new Map();
|
|
179
|
+
const environmentNames = new Map();
|
|
180
|
+
const renderPlaceholder = (name) => {
|
|
181
|
+
const handoff = /^handoff\.(.+)\.(materialized_path|commit_oid|artifact_id)$/u.exec(name);
|
|
182
|
+
if (handoff === null) {
|
|
183
|
+
if (name.startsWith("handoff.")) {
|
|
184
|
+
throw new CanonicalRunServiceError("verification_failed", `verifier command contains unsupported handoff placeholder {{${name}}}`);
|
|
185
|
+
}
|
|
186
|
+
return `{{${name}}}`;
|
|
187
|
+
}
|
|
188
|
+
if (context === "single" || context === "backtick") {
|
|
189
|
+
throw new CanonicalRunServiceError("verification_failed", `verifier handoff placeholder {{${name}}} is not allowed in ${context} quotes`);
|
|
190
|
+
}
|
|
191
|
+
const binding = bindings.find((candidate) => candidate.binding_id === handoff[1]);
|
|
192
|
+
if (binding?.mode !== "reference") {
|
|
193
|
+
throw new CanonicalRunServiceError("verification_failed", `verifier handoff ${handoff[1]} has no immutable reference`);
|
|
194
|
+
}
|
|
195
|
+
const field = handoff[2];
|
|
196
|
+
if ((field === "materialized_path" && binding.reference.kind === "artifact")
|
|
197
|
+
|| (field === "commit_oid" && binding.reference.kind !== "artifact")) {
|
|
198
|
+
throw new CanonicalRunServiceError("verification_failed", `verifier handoff ${handoff[1]} does not provide ${field}`);
|
|
199
|
+
}
|
|
200
|
+
const value = field === "artifact_id"
|
|
201
|
+
? binding.reference.artifact_id
|
|
202
|
+
: binding.reference.value;
|
|
203
|
+
if (value === null) {
|
|
204
|
+
throw new CanonicalRunServiceError("verification_failed", `verifier handoff ${handoff[1]} has no immutable artifact identity`);
|
|
205
|
+
}
|
|
206
|
+
const environmentKey = `${binding.binding_id}:${field}`;
|
|
207
|
+
let environmentName = environmentNames.get(environmentKey);
|
|
208
|
+
if (environmentName === undefined) {
|
|
209
|
+
environmentName = `INTENTDNA_HANDOFF_${environmentNames.size + 1}`;
|
|
210
|
+
environmentNames.set(environmentKey, environmentName);
|
|
211
|
+
environment.set(environmentName, value);
|
|
212
|
+
}
|
|
213
|
+
return context === "double"
|
|
214
|
+
? `\${${environmentName}}`
|
|
215
|
+
: `"\${${environmentName}}"`;
|
|
216
|
+
};
|
|
217
|
+
for (let index = 0; index < command.length;) {
|
|
218
|
+
if (command.startsWith("{{", index)) {
|
|
219
|
+
const end = command.indexOf("}}", index + 2);
|
|
220
|
+
if (end < 0) {
|
|
221
|
+
throw new CanonicalRunServiceError("verification_failed", "verifier command contains a malformed runtime placeholder");
|
|
222
|
+
}
|
|
223
|
+
resolved += renderPlaceholder(command.slice(index + 2, end));
|
|
224
|
+
index = end + 2;
|
|
225
|
+
continue;
|
|
226
|
+
}
|
|
227
|
+
if (command.startsWith("$ARGUMENTS", index)) {
|
|
228
|
+
resolved += "$ARGUMENTS";
|
|
229
|
+
index += "$ARGUMENTS".length;
|
|
230
|
+
continue;
|
|
231
|
+
}
|
|
232
|
+
const character = command[index];
|
|
233
|
+
resolved += character;
|
|
234
|
+
if (character === "\\" && context !== "single" && index + 1 < command.length) {
|
|
235
|
+
resolved += command[index + 1];
|
|
236
|
+
index += 2;
|
|
237
|
+
continue;
|
|
238
|
+
}
|
|
239
|
+
if (context === "unquoted") {
|
|
240
|
+
if (character === "'")
|
|
241
|
+
context = "single";
|
|
242
|
+
else if (character === "\"")
|
|
243
|
+
context = "double";
|
|
244
|
+
else if (character === "`") {
|
|
245
|
+
backtickReturn = "unquoted";
|
|
246
|
+
context = "backtick";
|
|
247
|
+
}
|
|
248
|
+
}
|
|
249
|
+
else if (context === "single") {
|
|
250
|
+
if (character === "'")
|
|
251
|
+
context = "unquoted";
|
|
252
|
+
}
|
|
253
|
+
else if (context === "double") {
|
|
254
|
+
if (character === "\"")
|
|
255
|
+
context = "unquoted";
|
|
256
|
+
else if (character === "`") {
|
|
257
|
+
backtickReturn = "double";
|
|
258
|
+
context = "backtick";
|
|
259
|
+
}
|
|
260
|
+
}
|
|
261
|
+
else if (character === "`") {
|
|
262
|
+
context = backtickReturn;
|
|
263
|
+
}
|
|
264
|
+
index += 1;
|
|
265
|
+
}
|
|
266
|
+
const runtimeValues = Object.fromEntries(Object.entries(runtimeInputs).map(([name, value]) => [name, runtimeTemplateValue(value)]));
|
|
267
|
+
const trimmed = command.trim();
|
|
268
|
+
const namedWholeCommand = /^\{\{([A-Za-z_][A-Za-z0-9_]*)\}\}$/u.exec(trimmed)?.[1];
|
|
269
|
+
const wholeCommandInput = namedWholeCommand ?? (trimmed === "$ARGUMENTS" ? "ARGUMENTS" : null);
|
|
270
|
+
const wholeCommandDefinition = wholeCommandInput === null
|
|
271
|
+
? undefined
|
|
272
|
+
: inputDefinitions.find((definition) => definition.name === wholeCommandInput);
|
|
273
|
+
const trustedWholeCommand = wholeCommandDefinition?.type === "trusted_verifier_command"
|
|
274
|
+
&& typeof wholeCommandDefinition.default_value === "string"
|
|
275
|
+
&& runtimeInputs[wholeCommandDefinition.name] === wholeCommandDefinition.default_value;
|
|
276
|
+
const bound = bindVerifierCommand({
|
|
277
|
+
command_template: resolved,
|
|
278
|
+
runtime_values: runtimeValues,
|
|
279
|
+
full_command_trust: trustedWholeCommand
|
|
280
|
+
? {
|
|
281
|
+
schema_version: TRUSTED_FULL_VERIFIER_COMMAND_SCHEMA_VERSION,
|
|
282
|
+
source: "compiled_plan",
|
|
283
|
+
}
|
|
284
|
+
: undefined,
|
|
285
|
+
});
|
|
286
|
+
if (!bound.bound) {
|
|
287
|
+
throw new CanonicalRunServiceError("verification_failed", bound.message);
|
|
288
|
+
}
|
|
289
|
+
if (/\{\{[^{}]+\}\}/u.test(bound.command)) {
|
|
290
|
+
throw new CanonicalRunServiceError("verification_failed", "verifier command contains an unresolved runtime placeholder");
|
|
291
|
+
}
|
|
292
|
+
return {
|
|
293
|
+
command: bound.command,
|
|
294
|
+
environment: { ...Object.fromEntries(environment), ...bound.env },
|
|
295
|
+
};
|
|
296
|
+
}
|
|
297
|
+
function nodeById(plan, nodeId) {
|
|
298
|
+
const node = plan.nodes.find((candidate) => candidate.node_id === nodeId);
|
|
299
|
+
if (!node) {
|
|
300
|
+
throw new CanonicalRunServiceError("invalid_request", `plan node ${nodeId} was not found`);
|
|
301
|
+
}
|
|
302
|
+
return node;
|
|
303
|
+
}
|
|
304
|
+
class ArtifactOutputCapture {
|
|
305
|
+
store;
|
|
306
|
+
constructor(store) {
|
|
307
|
+
this.store = store;
|
|
308
|
+
}
|
|
309
|
+
async capture(request) {
|
|
310
|
+
const captured = [];
|
|
311
|
+
const captures = [];
|
|
312
|
+
for (const output of request.execution.outputs) {
|
|
313
|
+
if (output.kind === "text") {
|
|
314
|
+
captured.push({ name: output.name, kind: "text", text: output.text });
|
|
315
|
+
continue;
|
|
316
|
+
}
|
|
317
|
+
if (output.kind === "structured") {
|
|
318
|
+
canonicalizeJson(output.value);
|
|
319
|
+
captured.push({
|
|
320
|
+
name: output.name,
|
|
321
|
+
kind: "structured",
|
|
322
|
+
value: output.value,
|
|
323
|
+
schema_ref: output.schema_ref,
|
|
324
|
+
});
|
|
325
|
+
continue;
|
|
326
|
+
}
|
|
327
|
+
const response = await this.store.capture({
|
|
328
|
+
kind: output.artifact_kind,
|
|
329
|
+
workspace_root: request.execution.workspace.root,
|
|
330
|
+
source_path: output.source_path,
|
|
331
|
+
run_id: request.run_id,
|
|
332
|
+
activation_id: request.activation_id,
|
|
333
|
+
attempt_id: request.attempt_id,
|
|
334
|
+
output_name: output.name,
|
|
335
|
+
source_workspace_id: request.execution.workspace.workspace_id,
|
|
336
|
+
});
|
|
337
|
+
captures.push(response.capture);
|
|
338
|
+
captured.push({ name: output.name, kind: "artifact", capture: response.capture });
|
|
339
|
+
}
|
|
340
|
+
return {
|
|
341
|
+
outputs: captured,
|
|
342
|
+
captures,
|
|
343
|
+
revalidate: async () => {
|
|
344
|
+
for (const output of request.execution.outputs) {
|
|
345
|
+
if (output.kind !== "reference")
|
|
346
|
+
continue;
|
|
347
|
+
const original = captures.find((capture) => capture.output_name === output.name);
|
|
348
|
+
const response = await this.store.capture({
|
|
349
|
+
kind: output.artifact_kind,
|
|
350
|
+
workspace_root: request.execution.workspace.root,
|
|
351
|
+
source_path: output.source_path,
|
|
352
|
+
run_id: request.run_id,
|
|
353
|
+
activation_id: request.activation_id,
|
|
354
|
+
attempt_id: request.attempt_id,
|
|
355
|
+
output_name: output.name,
|
|
356
|
+
source_workspace_id: request.execution.workspace.workspace_id,
|
|
357
|
+
});
|
|
358
|
+
if (!original || response.capture.artifact_id !== original.artifact_id) {
|
|
359
|
+
throw new CanonicalRunServiceError("source_drift", `output ${output.name} changed after verification`);
|
|
360
|
+
}
|
|
361
|
+
}
|
|
362
|
+
},
|
|
363
|
+
};
|
|
364
|
+
}
|
|
365
|
+
}
|
|
366
|
+
export class LocalVerifierExecutionPort {
|
|
367
|
+
clock;
|
|
368
|
+
authority_descriptor;
|
|
369
|
+
constructor(clock, authorityInstanceId = "local-verifier-execution") {
|
|
370
|
+
this.clock = clock;
|
|
371
|
+
this.authority_descriptor = {
|
|
372
|
+
schema_version: "intentdna.verifier_execution_authority.v1",
|
|
373
|
+
authority_instance_id: authorityInstanceId,
|
|
374
|
+
adapter_kind: "local_process",
|
|
375
|
+
adapter_version: "v1",
|
|
376
|
+
configuration_digest: digest({ shell: "bash", arguments: ["--noprofile", "--norc", "-c"], network: "allow" }),
|
|
377
|
+
process_tree_containment: false,
|
|
378
|
+
network_policies: ["allow"],
|
|
379
|
+
};
|
|
380
|
+
}
|
|
381
|
+
async execute(request) {
|
|
382
|
+
if (request.network_policy !== "allow") {
|
|
383
|
+
throw new CanonicalRunServiceError("unsupported", "local verifier execution cannot enforce deny_all network policy");
|
|
384
|
+
}
|
|
385
|
+
const execution = await execVerifierCommand(request.cwd, request.resolved_command, request.timeout_ms, request.environment, () => this.clock.now());
|
|
386
|
+
if (!execution.processTreeDrained) {
|
|
387
|
+
throw new CanonicalRunServiceError("verification_failed", "local verifier process tree could not be drained");
|
|
388
|
+
}
|
|
389
|
+
return {
|
|
390
|
+
execution,
|
|
391
|
+
authority: {
|
|
392
|
+
authority_instance_id: this.authority_descriptor.authority_instance_id,
|
|
393
|
+
request_digest: request.request_digest,
|
|
394
|
+
authority_descriptor_digest: digest(this.authority_descriptor),
|
|
395
|
+
launch_descriptor_digest: request.request_digest,
|
|
396
|
+
process_tree_drained: true,
|
|
397
|
+
network_enforced: false,
|
|
398
|
+
enforcement_kind: "not_required",
|
|
399
|
+
enforcement_digest: null,
|
|
400
|
+
enforcement_descriptor: null,
|
|
401
|
+
},
|
|
402
|
+
};
|
|
403
|
+
}
|
|
404
|
+
}
|
|
405
|
+
class ContractVerifier {
|
|
406
|
+
artifactStore;
|
|
407
|
+
serviceInstanceId;
|
|
408
|
+
clock;
|
|
409
|
+
structuredOutputSchemaRegistry;
|
|
410
|
+
executionPort;
|
|
411
|
+
authorityDescriptor;
|
|
412
|
+
authorityDescriptorDigest;
|
|
413
|
+
constructor(artifactStore, serviceInstanceId, clock, structuredOutputSchemaRegistry, executionPort, authorityDescriptor, authorityDescriptorDigest) {
|
|
414
|
+
this.artifactStore = artifactStore;
|
|
415
|
+
this.serviceInstanceId = serviceInstanceId;
|
|
416
|
+
this.clock = clock;
|
|
417
|
+
this.structuredOutputSchemaRegistry = structuredOutputSchemaRegistry;
|
|
418
|
+
this.executionPort = executionPort;
|
|
419
|
+
this.authorityDescriptor = authorityDescriptor;
|
|
420
|
+
this.authorityDescriptorDigest = authorityDescriptorDigest;
|
|
421
|
+
}
|
|
422
|
+
async verify(request) {
|
|
423
|
+
const node = nodeById(request.plan.plan, request.activation.step_id);
|
|
424
|
+
if (node.kind !== "worker")
|
|
425
|
+
return [];
|
|
426
|
+
const byName = new Map(request.captured_outputs.map((output) => [output.name, output]));
|
|
427
|
+
const contractResults = node.packet_template.output_contract.map((contract) => ({
|
|
428
|
+
verifier_id: `output_contract:${contract.name}`,
|
|
429
|
+
passed: !contract.required || byName.has(contract.name),
|
|
430
|
+
reason_code: !contract.required || byName.has(contract.name)
|
|
431
|
+
? "output_contract_satisfied"
|
|
432
|
+
: "required_output_missing",
|
|
433
|
+
}));
|
|
434
|
+
const gitCommitContracts = node.packet_template.output_contract.filter((contract) => contract.artifact_kind === "git_commit");
|
|
435
|
+
if (gitCommitContracts.length > 0) {
|
|
436
|
+
const presentOutputs = gitCommitContracts
|
|
437
|
+
.map((contract) => byName.get(contract.name))
|
|
438
|
+
.filter((output) => (output?.kind === "artifact" && output.capture.artifact_kind === "git_commit"));
|
|
439
|
+
const observation = presentOutputs.length === 0
|
|
440
|
+
? null
|
|
441
|
+
: await observeAttemptWorkspace(request.execution.workspace.root, request.execution.workspace.workspace_id);
|
|
442
|
+
for (const contract of gitCommitContracts) {
|
|
443
|
+
const verifierId = `output_contract:${contract.name}:git_commit_provenance`;
|
|
444
|
+
const output = byName.get(contract.name);
|
|
445
|
+
if (output?.kind !== "artifact" || output.capture.artifact_kind !== "git_commit") {
|
|
446
|
+
contractResults.push({
|
|
447
|
+
verifier_id: verifierId,
|
|
448
|
+
passed: !contract.required && output === undefined,
|
|
449
|
+
reason_code: !contract.required && output === undefined
|
|
450
|
+
? "optional_git_commit_absent"
|
|
451
|
+
: "git_commit_provenance_invalid",
|
|
452
|
+
});
|
|
453
|
+
continue;
|
|
454
|
+
}
|
|
455
|
+
if (observation === null) {
|
|
456
|
+
throw new CanonicalRunServiceError("verification_failed", `git_commit output ${output.name} has no stopped workspace observation`);
|
|
457
|
+
}
|
|
458
|
+
let passed = false;
|
|
459
|
+
if (observation.git !== null
|
|
460
|
+
&& request.workspace_allocation.observed_head !== null
|
|
461
|
+
&& observation.git.head === output.capture.source_path) {
|
|
462
|
+
try {
|
|
463
|
+
await this.artifactStore.verifyGitCommitProducedByAttempt({
|
|
464
|
+
artifact_id: output.capture.artifact_id,
|
|
465
|
+
artifact_kind: "git_commit",
|
|
466
|
+
}, request.execution.workspace.root, request.workspace_allocation.observed_head, request.workspace_allocation.observed_branch_ref);
|
|
467
|
+
passed = true;
|
|
468
|
+
}
|
|
469
|
+
catch {
|
|
470
|
+
passed = false;
|
|
471
|
+
}
|
|
472
|
+
}
|
|
473
|
+
const evidence = {
|
|
474
|
+
kind: "verifier_evidence",
|
|
475
|
+
verifier_id: verifierId,
|
|
476
|
+
evidence_digest: digest({
|
|
477
|
+
artifact_capture: output.capture,
|
|
478
|
+
workspace_observation: observation,
|
|
479
|
+
}),
|
|
480
|
+
artifact_capture: output.capture,
|
|
481
|
+
workspace_observation: observation,
|
|
482
|
+
execution_receipt: null,
|
|
483
|
+
};
|
|
484
|
+
contractResults.push({
|
|
485
|
+
verifier_id: verifierId,
|
|
486
|
+
passed,
|
|
487
|
+
reason_code: passed
|
|
488
|
+
? "git_commit_produced_by_attempt"
|
|
489
|
+
: "git_commit_provenance_invalid",
|
|
490
|
+
evidence_refs: [evidence],
|
|
491
|
+
});
|
|
492
|
+
}
|
|
493
|
+
}
|
|
494
|
+
const planResults = [];
|
|
495
|
+
const workspaceVerifierIds = new Set();
|
|
496
|
+
const commandExecutions = new Map();
|
|
497
|
+
for (const verifier of request.plan.plan.verifier_plan.filter((item) => item.subject_node_id === node.node_id)) {
|
|
498
|
+
if (verifier.kind === "structured_output") {
|
|
499
|
+
const output = byName.get(verifier.output_name);
|
|
500
|
+
const validation = output?.kind === "structured"
|
|
501
|
+
? validatePinnedStructuredOutput({
|
|
502
|
+
plan: request.plan.plan,
|
|
503
|
+
node_id: verifier.node_id,
|
|
504
|
+
output_name: verifier.output_name,
|
|
505
|
+
schema_ref: output.schema_ref,
|
|
506
|
+
value: output.value,
|
|
507
|
+
}, this.structuredOutputSchemaRegistry)
|
|
508
|
+
: null;
|
|
509
|
+
const passed = output?.kind === "structured"
|
|
510
|
+
&& output.schema_ref === verifier.schema_ref
|
|
511
|
+
&& validation?.valid === true;
|
|
512
|
+
planResults.push({
|
|
513
|
+
verifier_id: verifier.verifier_id,
|
|
514
|
+
passed,
|
|
515
|
+
reason_code: passed
|
|
516
|
+
? "structured_output_present"
|
|
517
|
+
: "structured_output_invalid",
|
|
518
|
+
});
|
|
519
|
+
continue;
|
|
520
|
+
}
|
|
521
|
+
if (verifier.kind === "artifact_exists"
|
|
522
|
+
|| verifier.kind === "artifact_non_empty"
|
|
523
|
+
|| verifier.kind === "artifact_contains"
|
|
524
|
+
|| verifier.kind === "evaluator_result") {
|
|
525
|
+
const path = verifier.relative_path_template;
|
|
526
|
+
const output = request.captured_outputs.find((candidate) => (candidate.kind === "artifact" && candidate.capture.source_path === path));
|
|
527
|
+
let passed = output?.kind === "artifact";
|
|
528
|
+
if (passed && output?.kind === "artifact" && verifier.kind !== "artifact_exists") {
|
|
529
|
+
const object = await this.artifactStore.require({
|
|
530
|
+
artifact_id: output.capture.artifact_id,
|
|
531
|
+
artifact_kind: output.capture.artifact_kind,
|
|
532
|
+
});
|
|
533
|
+
passed = object.size_bytes > 0;
|
|
534
|
+
if (passed && verifier.kind === "artifact_contains") {
|
|
535
|
+
if (output.capture.artifact_kind !== "file") {
|
|
536
|
+
passed = false;
|
|
537
|
+
}
|
|
538
|
+
else {
|
|
539
|
+
try {
|
|
540
|
+
passed = new RegExp(verifier.pattern, "u").test((await this.artifactStore.readFileBytes(output.capture.artifact_id)).toString("utf8"));
|
|
541
|
+
}
|
|
542
|
+
catch {
|
|
543
|
+
passed = false;
|
|
544
|
+
}
|
|
545
|
+
}
|
|
546
|
+
}
|
|
547
|
+
if (passed && verifier.kind === "evaluator_result") {
|
|
548
|
+
if (output.capture.artifact_kind !== "file") {
|
|
549
|
+
passed = false;
|
|
550
|
+
}
|
|
551
|
+
else {
|
|
552
|
+
try {
|
|
553
|
+
passed = parseEvaluatorResult((await this.artifactStore.readFileBytes(output.capture.artifact_id)).toString("utf8"), { require_score: verifier.require_score }).pass;
|
|
554
|
+
}
|
|
555
|
+
catch {
|
|
556
|
+
passed = false;
|
|
557
|
+
}
|
|
558
|
+
}
|
|
559
|
+
}
|
|
560
|
+
}
|
|
561
|
+
planResults.push({
|
|
562
|
+
verifier_id: verifier.verifier_id,
|
|
563
|
+
passed,
|
|
564
|
+
reason_code: passed ? "immutable_artifact_verified" : "immutable_artifact_verification_failed",
|
|
565
|
+
});
|
|
566
|
+
continue;
|
|
567
|
+
}
|
|
568
|
+
if (verifier.kind === "command" || verifier.kind === "checkpoint") {
|
|
569
|
+
if (verifier.observation_policy === "current_provenance") {
|
|
570
|
+
throw new CanonicalRunServiceError("unsupported", `verifier ${verifier.verifier_id} requires current provenance without typed immutable observations`);
|
|
571
|
+
}
|
|
572
|
+
const command = verifier.command;
|
|
573
|
+
if (command === null) {
|
|
574
|
+
planResults.push({
|
|
575
|
+
verifier_id: verifier.verifier_id,
|
|
576
|
+
passed: false,
|
|
577
|
+
reason_code: "checkpoint_command_required",
|
|
578
|
+
});
|
|
579
|
+
continue;
|
|
580
|
+
}
|
|
581
|
+
const relativeWorkingDirectory = verifier.kind === "command"
|
|
582
|
+
? verifier.relative_working_directory
|
|
583
|
+
: ".";
|
|
584
|
+
let workingDirectory;
|
|
585
|
+
try {
|
|
586
|
+
workingDirectory = await canonicalWorkspaceDirectory(request.execution.workspace.root, relativeWorkingDirectory, `verifier ${verifier.verifier_id} working_directory`);
|
|
587
|
+
}
|
|
588
|
+
catch (error) {
|
|
589
|
+
if (!(error instanceof WorkspaceIsolationError))
|
|
590
|
+
throw error;
|
|
591
|
+
throw new CanonicalRunServiceError("verification_failed", error.message, { cause: error });
|
|
592
|
+
}
|
|
593
|
+
const resolvedCommand = resolveVerifierCommand(command, request.handoff_bindings, request.run.run_binding.binding.validated_inputs, request.plan.plan.input_contract.inputs);
|
|
594
|
+
const environment = canonicalVerifierEnvironment(resolvedCommand.environment);
|
|
595
|
+
const authorityDescriptorDigest = this.authorityDescriptorDigest;
|
|
596
|
+
const cwdIdentity = {
|
|
597
|
+
workspace_id: request.execution.workspace.workspace_id,
|
|
598
|
+
workspace_allocation_digest: digest(request.workspace_allocation),
|
|
599
|
+
relative_working_directory: relativeWorkingDirectory,
|
|
600
|
+
absolute_path_digest: textDigest(workingDirectory),
|
|
601
|
+
};
|
|
602
|
+
const environmentBindingDigest = digest(Object.fromEntries(Object.entries(environment)
|
|
603
|
+
.sort(([left], [right]) => left.localeCompare(right))
|
|
604
|
+
.map(([name, value]) => [name, textDigest(value)])));
|
|
605
|
+
const requestDescriptor = {
|
|
606
|
+
schema_version: "intentdna.verifier_execution_request.v1",
|
|
607
|
+
plan_id: request.plan.plan_id,
|
|
608
|
+
verifier_id: verifier.verifier_id,
|
|
609
|
+
verifier_definition_digest: digest(verifier),
|
|
610
|
+
raw_command_digest: textDigest(resolvedCommand.command),
|
|
611
|
+
cwd: cwdIdentity,
|
|
612
|
+
supplied_environment: {
|
|
613
|
+
binding_digest: environmentBindingDigest,
|
|
614
|
+
variable_names: Object.keys(environment).sort(),
|
|
615
|
+
inherit_parent: false,
|
|
616
|
+
startup_files: false,
|
|
617
|
+
},
|
|
618
|
+
shell: { executable: "bash", arguments: ["--noprofile", "--norc", "-c"] },
|
|
619
|
+
timeout_ms: 30_000,
|
|
620
|
+
requested_at: request.execution.completed_at,
|
|
621
|
+
network_mode: request.plan.plan.workspace_policy.allow_network ? "allow" : "deny_all",
|
|
622
|
+
expected_authority_descriptor_digest: authorityDescriptorDigest,
|
|
623
|
+
};
|
|
624
|
+
const executionRequest = {
|
|
625
|
+
request_digest: digest(requestDescriptor),
|
|
626
|
+
descriptor: requestDescriptor,
|
|
627
|
+
plan_id: requestDescriptor.plan_id,
|
|
628
|
+
verifier_id: requestDescriptor.verifier_id,
|
|
629
|
+
verifier_definition_digest: requestDescriptor.verifier_definition_digest,
|
|
630
|
+
resolved_command_digest: requestDescriptor.raw_command_digest,
|
|
631
|
+
resolved_command: resolvedCommand.command,
|
|
632
|
+
cwd: workingDirectory,
|
|
633
|
+
cwd_identity: cwdIdentity,
|
|
634
|
+
environment,
|
|
635
|
+
environment_binding_digest: environmentBindingDigest,
|
|
636
|
+
timeout_ms: requestDescriptor.timeout_ms,
|
|
637
|
+
requested_at: requestDescriptor.requested_at,
|
|
638
|
+
network_policy: requestDescriptor.network_mode,
|
|
639
|
+
inherit_parent_environment: false,
|
|
640
|
+
load_startup_files: false,
|
|
641
|
+
};
|
|
642
|
+
const executed = await this.executionPort.execute(executionRequest);
|
|
643
|
+
try {
|
|
644
|
+
assertVerifierCommandExecutionResult(executed.execution);
|
|
645
|
+
}
|
|
646
|
+
catch (error) {
|
|
647
|
+
throw new CanonicalRunServiceError("verification_failed", "verifier execution returned an invalid process outcome", { cause: error });
|
|
648
|
+
}
|
|
649
|
+
if (executed.execution.processTreeDrained !== true) {
|
|
650
|
+
throw new CanonicalRunServiceError("verification_failed", "verifier execution did not prove a drained process tree");
|
|
651
|
+
}
|
|
652
|
+
this.assertExecutionAuthority(executionRequest, executed.authority);
|
|
653
|
+
const authoritativePassed = verifierCommandExecutionPassed(executed.execution);
|
|
654
|
+
workspaceVerifierIds.add(verifier.verifier_id);
|
|
655
|
+
commandExecutions.set(verifier.verifier_id, {
|
|
656
|
+
verifier,
|
|
657
|
+
resolved: resolvedCommand,
|
|
658
|
+
execution: executed.execution,
|
|
659
|
+
relative_working_directory: relativeWorkingDirectory,
|
|
660
|
+
request: executionRequest,
|
|
661
|
+
authority: executed.authority,
|
|
662
|
+
});
|
|
663
|
+
planResults.push({
|
|
664
|
+
verifier_id: verifier.verifier_id,
|
|
665
|
+
passed: authoritativePassed,
|
|
666
|
+
reason_code: authoritativePassed
|
|
667
|
+
? "verifier_command_passed"
|
|
668
|
+
: executed.execution.outcome.kind === "timed_out"
|
|
669
|
+
? "verifier_command_timeout"
|
|
670
|
+
: executed.execution.outcome.kind === "launch_error"
|
|
671
|
+
? "verifier_command_launch_error"
|
|
672
|
+
: "verifier_command_failed",
|
|
673
|
+
});
|
|
674
|
+
continue;
|
|
675
|
+
}
|
|
676
|
+
planResults.push({
|
|
677
|
+
verifier_id: verifier.verifier_id,
|
|
678
|
+
passed: false,
|
|
679
|
+
reason_code: "verifier_kind_requires_injected_runtime",
|
|
680
|
+
});
|
|
681
|
+
}
|
|
682
|
+
if (workspaceVerifierIds.size > 0) {
|
|
683
|
+
const observation = await observeAttemptWorkspace(request.execution.workspace.root, request.execution.workspace.workspace_id);
|
|
684
|
+
for (let index = 0; index < planResults.length; index += 1) {
|
|
685
|
+
const result = planResults[index];
|
|
686
|
+
if (!workspaceVerifierIds.has(result.verifier_id))
|
|
687
|
+
continue;
|
|
688
|
+
const commandExecution = commandExecutions.get(result.verifier_id);
|
|
689
|
+
if (commandExecution === undefined) {
|
|
690
|
+
throw new CanonicalRunServiceError("verification_failed", `verifier ${result.verifier_id} has no historical execution record`);
|
|
691
|
+
}
|
|
692
|
+
const receipt = {
|
|
693
|
+
schema_version: CANONICAL_VERIFIER_EXECUTION_RECEIPT_SCHEMA_VERSION,
|
|
694
|
+
evidence_mode: "historical_execution",
|
|
695
|
+
plan_id: request.plan.plan_id,
|
|
696
|
+
verifier_id: result.verifier_id,
|
|
697
|
+
verifier_definition_digest: digest(commandExecution.verifier),
|
|
698
|
+
resolved_command_digest: textDigest(commandExecution.resolved.command),
|
|
699
|
+
environment_binding_digest: commandExecution.request.environment_binding_digest,
|
|
700
|
+
cwd_identity: {
|
|
701
|
+
workspace_id: request.execution.workspace.workspace_id,
|
|
702
|
+
workspace_allocation_digest: digest(request.workspace_allocation),
|
|
703
|
+
relative_working_directory: commandExecution.relative_working_directory,
|
|
704
|
+
absolute_path_digest: commandExecution.request.cwd_identity.absolute_path_digest,
|
|
705
|
+
},
|
|
706
|
+
started_at: commandExecution.execution.startedAt,
|
|
707
|
+
requested_at: commandExecution.request.requested_at,
|
|
708
|
+
completed_at: commandExecution.execution.completedAt,
|
|
709
|
+
process_outcome: {
|
|
710
|
+
kind: commandExecution.execution.outcome.kind,
|
|
711
|
+
exit_code: commandExecution.execution.outcome.exitCode,
|
|
712
|
+
signal: commandExecution.execution.outcome.signal,
|
|
713
|
+
timed_out: commandExecution.execution.outcome.timedOut,
|
|
714
|
+
launch_error: commandExecution.execution.outcome.launchError === null
|
|
715
|
+
? null
|
|
716
|
+
: {
|
|
717
|
+
code: commandExecution.execution.outcome.launchError.code,
|
|
718
|
+
message_digest: commandExecution.execution.outcome.launchError.messageDigest,
|
|
719
|
+
},
|
|
720
|
+
},
|
|
721
|
+
stdout: {
|
|
722
|
+
digest: commandExecution.execution.stdoutDigest,
|
|
723
|
+
byte_length: commandExecution.execution.stdoutBytes,
|
|
724
|
+
truncated: commandExecution.execution.stdoutBytes > 0,
|
|
725
|
+
},
|
|
726
|
+
stderr: {
|
|
727
|
+
digest: commandExecution.execution.stderrDigest,
|
|
728
|
+
byte_length: commandExecution.execution.stderrBytes,
|
|
729
|
+
truncated: commandExecution.execution.stderrBytes > 0,
|
|
730
|
+
},
|
|
731
|
+
truncation_policy: {
|
|
732
|
+
kind: "digest_full_stream_diagnostics_only",
|
|
733
|
+
raw_output_retained_bytes: 0,
|
|
734
|
+
diagnostic_summary_max_bytes: MAX_VERIFIER_EVIDENCE_BYTES,
|
|
735
|
+
diagnostic_line_max_bytes: MAX_VERIFIER_DIAGNOSTIC_BYTES,
|
|
736
|
+
},
|
|
737
|
+
observation_policy_digest: digest(workspaceObservationPolicy()),
|
|
738
|
+
workspace_observation_digest: observation.observed_digest,
|
|
739
|
+
service_invocation: {
|
|
740
|
+
service_instance_id: this.serviceInstanceId,
|
|
741
|
+
invocation_id: `verifier-invocation:${randomUUID()}`,
|
|
742
|
+
},
|
|
743
|
+
request_identity: {
|
|
744
|
+
request_digest: commandExecution.request.request_digest,
|
|
745
|
+
command_digest: commandExecution.request.resolved_command_digest,
|
|
746
|
+
environment_digest: commandExecution.request.environment_binding_digest,
|
|
747
|
+
authority_descriptor_digest: commandExecution.authority.authority_descriptor_digest,
|
|
748
|
+
launch_descriptor_digest: commandExecution.authority.launch_descriptor_digest,
|
|
749
|
+
},
|
|
750
|
+
supplied_environment: {
|
|
751
|
+
inherit_parent: false,
|
|
752
|
+
startup_files: false,
|
|
753
|
+
variable_names: Object.keys(commandExecution.request.environment).sort(),
|
|
754
|
+
},
|
|
755
|
+
shell_policy: {
|
|
756
|
+
executable: "bash",
|
|
757
|
+
arguments: ["--noprofile", "--norc", "-c"],
|
|
758
|
+
},
|
|
759
|
+
network_policy: {
|
|
760
|
+
mode: commandExecution.request.network_policy,
|
|
761
|
+
enforced: commandExecution.authority.network_enforced,
|
|
762
|
+
enforcement_kind: commandExecution.authority.enforcement_kind,
|
|
763
|
+
enforcement_digest: commandExecution.authority.enforcement_digest,
|
|
764
|
+
enforcement_descriptor: commandExecution.authority.enforcement_descriptor,
|
|
765
|
+
},
|
|
766
|
+
execution_authority: {
|
|
767
|
+
authority_instance_id: commandExecution.authority.authority_instance_id,
|
|
768
|
+
authority_descriptor_digest: commandExecution.authority.authority_descriptor_digest,
|
|
769
|
+
adapter_kind: this.authorityDescriptor.adapter_kind,
|
|
770
|
+
adapter_version: this.authorityDescriptor.adapter_version,
|
|
771
|
+
configuration_digest: this.authorityDescriptor.configuration_digest,
|
|
772
|
+
process_tree_drained: commandExecution.authority.process_tree_drained,
|
|
773
|
+
},
|
|
774
|
+
};
|
|
775
|
+
const evidence = {
|
|
776
|
+
kind: "verifier_evidence",
|
|
777
|
+
verifier_id: result.verifier_id,
|
|
778
|
+
evidence_digest: digest({
|
|
779
|
+
execution_receipt: receipt,
|
|
780
|
+
workspace_observation: observation,
|
|
781
|
+
}),
|
|
782
|
+
artifact_capture: null,
|
|
783
|
+
workspace_observation: observation,
|
|
784
|
+
execution_receipt: receipt,
|
|
785
|
+
};
|
|
786
|
+
planResults[index] = { ...result, evidence_refs: [evidence] };
|
|
787
|
+
}
|
|
788
|
+
}
|
|
789
|
+
return [...contractResults, ...planResults];
|
|
790
|
+
}
|
|
791
|
+
assertExecutionAuthority(request, authority) {
|
|
792
|
+
if (!authority.authority_instance_id
|
|
793
|
+
|| authority.request_digest !== request.request_digest
|
|
794
|
+
|| authority.authority_descriptor_digest !== this.authorityDescriptorDigest
|
|
795
|
+
|| authority.authority_descriptor_digest !== request.descriptor.expected_authority_descriptor_digest
|
|
796
|
+
|| authority.launch_descriptor_digest !== request.request_digest
|
|
797
|
+
|| authority.process_tree_drained !== true
|
|
798
|
+
|| authority.authority_instance_id !== this.authorityDescriptor.authority_instance_id
|
|
799
|
+
|| (request.network_policy === "deny_all" && (authority.network_enforced !== true
|
|
800
|
+
|| authority.enforcement_kind !== "configured_adapter"
|
|
801
|
+
|| authority.enforcement_digest === null
|
|
802
|
+
|| authority.enforcement_descriptor === null
|
|
803
|
+
|| authority.enforcement_digest !== digest(authority.enforcement_descriptor)
|
|
804
|
+
|| authority.enforcement_descriptor.mode !== "deny_all"
|
|
805
|
+
|| authority.enforcement_descriptor.adapter_kind !== this.authorityDescriptor.adapter_kind
|
|
806
|
+
|| authority.enforcement_descriptor.adapter_version !== this.authorityDescriptor.adapter_version
|
|
807
|
+
|| authority.enforcement_descriptor.configuration_digest !== this.authorityDescriptor.configuration_digest
|
|
808
|
+
|| authority.enforcement_descriptor.authority_descriptor_digest !== authority.authority_descriptor_digest
|
|
809
|
+
|| authority.enforcement_descriptor.request_digest !== request.request_digest))
|
|
810
|
+
|| (request.network_policy === "allow" && (authority.network_enforced !== false
|
|
811
|
+
|| authority.enforcement_kind !== "not_required"
|
|
812
|
+
|| authority.enforcement_digest !== null
|
|
813
|
+
|| authority.enforcement_descriptor !== null))) {
|
|
814
|
+
throw new CanonicalRunServiceError("verification_failed", "verifier execution authority does not match its request");
|
|
815
|
+
}
|
|
816
|
+
}
|
|
817
|
+
}
|
|
818
|
+
export class CanonicalRunService {
|
|
819
|
+
planStore;
|
|
820
|
+
runStore;
|
|
821
|
+
resultStore;
|
|
822
|
+
artifactStore;
|
|
823
|
+
artifactMaterializer;
|
|
824
|
+
clock;
|
|
825
|
+
identities;
|
|
826
|
+
outputCapture;
|
|
827
|
+
verifier;
|
|
828
|
+
additionalVerifier;
|
|
829
|
+
verifierExecutionPort;
|
|
830
|
+
verifierAuthorityDescriptor;
|
|
831
|
+
verifierAuthorityDescriptorDigest;
|
|
832
|
+
workspaceAuthority;
|
|
833
|
+
executionAuthority;
|
|
834
|
+
handoffRootDirectory;
|
|
835
|
+
hooks;
|
|
836
|
+
leaseDurationMs;
|
|
837
|
+
defaultExecutionTimeoutMs;
|
|
838
|
+
serviceInstanceId;
|
|
839
|
+
structuredOutputSchemaRegistry;
|
|
840
|
+
constructor(options) {
|
|
841
|
+
this.planStore = options.plan_store;
|
|
842
|
+
this.runStore = options.run_store;
|
|
843
|
+
this.resultStore = options.result_store;
|
|
844
|
+
this.artifactStore = options.artifact_store;
|
|
845
|
+
this.artifactMaterializer = new ArtifactMaterializer(options.artifact_store);
|
|
846
|
+
this.clock = options.clock ?? { now: () => new Date() };
|
|
847
|
+
this.identities = options.identities ?? { create: defaultIdentity };
|
|
848
|
+
this.serviceInstanceId = options.service_instance_id ?? `service:${randomUUID()}`;
|
|
849
|
+
this.structuredOutputSchemaRegistry = options.structured_output_schema_registry
|
|
850
|
+
?? DEFAULT_STRUCTURED_OUTPUT_SCHEMA_REGISTRY;
|
|
851
|
+
this.outputCapture = options.output_capture
|
|
852
|
+
?? new ArtifactOutputCapture(options.artifact_store);
|
|
853
|
+
this.verifierExecutionPort = options.verifier_execution_port ?? new LocalVerifierExecutionPort(this.clock);
|
|
854
|
+
this.verifierAuthorityDescriptor = snapshotVerifierAuthorityDescriptor(this.verifierExecutionPort.authority_descriptor);
|
|
855
|
+
this.verifierAuthorityDescriptorDigest = digest(this.verifierAuthorityDescriptor);
|
|
856
|
+
this.verifier = new ContractVerifier(options.artifact_store, this.serviceInstanceId, this.clock, this.structuredOutputSchemaRegistry, this.verifierExecutionPort, this.verifierAuthorityDescriptor, this.verifierAuthorityDescriptorDigest);
|
|
857
|
+
this.additionalVerifier = options.verifier ?? null;
|
|
858
|
+
this.workspaceAuthority = options.workspace_authority ?? null;
|
|
859
|
+
this.executionAuthority = options.execution_authority ?? null;
|
|
860
|
+
this.handoffRootDirectory = resolve(options.handoff_root_directory ?? this.runStore.serviceOwnedRuntimeDirectory("handoffs"));
|
|
861
|
+
this.hooks = options.hooks ?? {};
|
|
862
|
+
this.leaseDurationMs = options.lease_duration_ms ?? DEFAULT_LEASE_DURATION_MS;
|
|
863
|
+
this.defaultExecutionTimeoutMs = options.default_execution_timeout_ms
|
|
864
|
+
?? DEFAULT_EXECUTION_TIMEOUT_MS;
|
|
865
|
+
if (!Number.isSafeInteger(this.leaseDurationMs) || this.leaseDurationMs <= 0) {
|
|
866
|
+
throw new CanonicalRunServiceError("invalid_request", "lease_duration_ms must be positive");
|
|
867
|
+
}
|
|
868
|
+
if (!Number.isSafeInteger(this.defaultExecutionTimeoutMs) || this.defaultExecutionTimeoutMs <= 0) {
|
|
869
|
+
throw new CanonicalRunServiceError("invalid_request", "default_execution_timeout_ms must be positive");
|
|
870
|
+
}
|
|
871
|
+
}
|
|
872
|
+
async start(request) {
|
|
873
|
+
this.assertVerifierExecutionSupported(request.plan_payload);
|
|
874
|
+
const storedPlan = await this.planStore.commitPlan(request.plan_payload);
|
|
875
|
+
const binding = parseStoredRunBinding(request.run_binding);
|
|
876
|
+
if (binding.binding.plan_id !== storedPlan.plan_id) {
|
|
877
|
+
throw new CanonicalRunServiceError("invalid_request", "run binding does not reference the committed plan");
|
|
878
|
+
}
|
|
879
|
+
const now = this.now();
|
|
880
|
+
const runId = request.run_id ?? this.identities.create("run", `${storedPlan.plan_id}\0${binding.binding_id}\0${now}\0${randomUUID()}`);
|
|
881
|
+
requireNonEmpty(runId, "run_id");
|
|
882
|
+
const incomingRouteTargets = new Set(storedPlan.plan.routes.map((route) => route.to_node_id));
|
|
883
|
+
const activations = storedPlan.plan.nodes
|
|
884
|
+
.filter((node) => ATTEMPT_NODE_KINDS.has(node.kind))
|
|
885
|
+
.map((node) => ({
|
|
886
|
+
run_id: runId,
|
|
887
|
+
step_id: node.node_id,
|
|
888
|
+
activation_id: this.activationId(runId, node.node_id),
|
|
889
|
+
workflow_round: 1,
|
|
890
|
+
state: incomingRouteTargets.has(node.node_id)
|
|
891
|
+
? "dormant"
|
|
892
|
+
: node.depends_on.length === 0 ? "ready" : "dependency_blocked",
|
|
893
|
+
attempt_count: 0,
|
|
894
|
+
current_attempt_id: null,
|
|
895
|
+
committed_result_id: null,
|
|
896
|
+
created_at: now,
|
|
897
|
+
updated_at: now,
|
|
898
|
+
}));
|
|
899
|
+
const run = {
|
|
900
|
+
run_id: runId,
|
|
901
|
+
target: storedPlan.plan.target,
|
|
902
|
+
plan_id: storedPlan.plan_id,
|
|
903
|
+
plan_digest: storedPlan.plan_digest,
|
|
904
|
+
run_binding: binding,
|
|
905
|
+
status: "active",
|
|
906
|
+
cancellation: null,
|
|
907
|
+
terminal: null,
|
|
908
|
+
metadata: request.metadata ?? {},
|
|
909
|
+
created_at: now,
|
|
910
|
+
updated_at: now,
|
|
911
|
+
};
|
|
912
|
+
const requireSameStartIdentity = (existing) => {
|
|
913
|
+
const sameIdentity = existing.run.plan_id === run.plan_id
|
|
914
|
+
&& existing.run.run_binding.binding_id === run.run_binding.binding_id
|
|
915
|
+
&& canonicalizeJson(existing.run.metadata) === canonicalizeJson(run.metadata)
|
|
916
|
+
&& existing.run.target.kind === run.target.kind
|
|
917
|
+
&& existing.run.target.key === run.target.key;
|
|
918
|
+
if (!sameIdentity) {
|
|
919
|
+
throw new CanonicalRunServiceError("submission_conflict", `run ${runId} already exists with different start identity`);
|
|
920
|
+
}
|
|
921
|
+
return existing;
|
|
922
|
+
};
|
|
923
|
+
if (request.run_id !== undefined) {
|
|
924
|
+
const existing = await this.runStore.loadCanonicalRun(runId);
|
|
925
|
+
if (existing) {
|
|
926
|
+
const drained = await this.drainReadySystemNodes(requireSameStartIdentity(existing), storedPlan);
|
|
927
|
+
return {
|
|
928
|
+
run_id: runId,
|
|
929
|
+
plan_id: storedPlan.plan_id,
|
|
930
|
+
view: this.toView(drained, storedPlan),
|
|
931
|
+
};
|
|
932
|
+
}
|
|
933
|
+
}
|
|
934
|
+
let snapshot;
|
|
935
|
+
try {
|
|
936
|
+
snapshot = await this.runStore.createCanonicalRun({ run, activations });
|
|
937
|
+
}
|
|
938
|
+
catch (error) {
|
|
939
|
+
if (request.run_id === undefined
|
|
940
|
+
|| !(error instanceof RunStoreError)
|
|
941
|
+
|| error.code !== "run_exists") {
|
|
942
|
+
throw error;
|
|
943
|
+
}
|
|
944
|
+
const existing = await this.runStore.loadCanonicalRun(runId);
|
|
945
|
+
if (!existing)
|
|
946
|
+
throw error;
|
|
947
|
+
snapshot = requireSameStartIdentity(existing);
|
|
948
|
+
}
|
|
949
|
+
snapshot = await this.reduceWithoutResult(snapshot, storedPlan);
|
|
950
|
+
snapshot = await this.drainReadySystemNodes(snapshot, storedPlan);
|
|
951
|
+
return {
|
|
952
|
+
run_id: runId,
|
|
953
|
+
plan_id: storedPlan.plan_id,
|
|
954
|
+
view: this.toView(snapshot, storedPlan),
|
|
955
|
+
};
|
|
956
|
+
}
|
|
957
|
+
async acquireAttempt(request) {
|
|
958
|
+
requireNonEmpty(request.driver_id, "driver_id");
|
|
959
|
+
for (;;) {
|
|
960
|
+
let snapshot = await this.requireSnapshot(request.run_id);
|
|
961
|
+
const plan = await this.requirePinnedPlan(snapshot);
|
|
962
|
+
this.assertVerifierExecutionSupported(plan.plan);
|
|
963
|
+
if (snapshot.events.some((event) => event.type === "attempt_stop_requested" && snapshot.attempts.find((attempt) => attempt.attempt_id === event.attempt_id)?.phase !== "terminal")) {
|
|
964
|
+
snapshot = await this.continueDurableStops(snapshot);
|
|
965
|
+
}
|
|
966
|
+
snapshot = await this.drainReadySystemNodes(snapshot, plan);
|
|
967
|
+
if (snapshot.run.terminal !== null) {
|
|
968
|
+
return { kind: "terminal", run: this.toView(snapshot, plan) };
|
|
969
|
+
}
|
|
970
|
+
if (snapshot.run.status === "cancelling") {
|
|
971
|
+
return {
|
|
972
|
+
kind: "wait",
|
|
973
|
+
reason_code: "run_cancelling",
|
|
974
|
+
run: this.toView(snapshot, plan),
|
|
975
|
+
};
|
|
976
|
+
}
|
|
977
|
+
snapshot = await this.reduceWithoutResult(snapshot, plan);
|
|
978
|
+
if (snapshot.run.terminal !== null) {
|
|
979
|
+
return { kind: "terminal", run: this.toView(snapshot, plan) };
|
|
980
|
+
}
|
|
981
|
+
const now = this.now();
|
|
982
|
+
const activeAttempts = snapshot.attempts.filter((attempt) => attempt.phase !== "terminal");
|
|
983
|
+
let recoveredExpiredAttempt = false;
|
|
984
|
+
for (const activeAttempt of activeAttempts) {
|
|
985
|
+
const claims = this.claimsFor(snapshot, activeAttempt.attempt_id);
|
|
986
|
+
const latest = claims.at(-1);
|
|
987
|
+
if (Date.parse(latest.execution_deadline_at) <= Date.parse(now)) {
|
|
988
|
+
await this.recoverExpiredAttempt(snapshot, plan, activeAttempt, now);
|
|
989
|
+
recoveredExpiredAttempt = true;
|
|
990
|
+
break;
|
|
991
|
+
}
|
|
992
|
+
}
|
|
993
|
+
if (recoveredExpiredAttempt)
|
|
994
|
+
continue;
|
|
995
|
+
const maxConcurrency = snapshot.run.run_binding.binding.max_concurrency ?? 1;
|
|
996
|
+
const activation = activeAttempts.length < maxConcurrency
|
|
997
|
+
&& (activeAttempts.length === 0 || plan.plan.required_capabilities.parallelism)
|
|
998
|
+
? snapshot.activations.find((item) => item.state === "ready")
|
|
999
|
+
: undefined;
|
|
1000
|
+
if (!activation && activeAttempts.length > 0) {
|
|
1001
|
+
let reclaimableAttempt;
|
|
1002
|
+
for (const attempt of activeAttempts) {
|
|
1003
|
+
const latest = this.claimsFor(snapshot, attempt.attempt_id).at(-1);
|
|
1004
|
+
if (Date.parse(latest.execution_deadline_at) <= Date.parse(now))
|
|
1005
|
+
continue;
|
|
1006
|
+
if (attempt.phase === "claimed"
|
|
1007
|
+
&& Date.parse(latest.driver_claim_lease.expires_at) <= Date.parse(now)) {
|
|
1008
|
+
reclaimableAttempt = attempt;
|
|
1009
|
+
break;
|
|
1010
|
+
}
|
|
1011
|
+
if (attempt.phase === "running"
|
|
1012
|
+
&& request.recover_running === true
|
|
1013
|
+
&& await this.runningAttemptIsRecoverable(snapshot, attempt, latest, now, request.driver_id)) {
|
|
1014
|
+
reclaimableAttempt = attempt;
|
|
1015
|
+
break;
|
|
1016
|
+
}
|
|
1017
|
+
}
|
|
1018
|
+
if (reclaimableAttempt) {
|
|
1019
|
+
const latest = this.claimsFor(snapshot, reclaimableAttempt.attempt_id).at(-1);
|
|
1020
|
+
const claim = this.createClaim(snapshot, plan, reclaimableAttempt, request.driver_id, now, latest.claim_epoch + 1);
|
|
1021
|
+
const event = this.event(snapshot, 0, {
|
|
1022
|
+
type: "attempt_claimed",
|
|
1023
|
+
step_id: reclaimableAttempt.step_id,
|
|
1024
|
+
activation_id: reclaimableAttempt.activation_id,
|
|
1025
|
+
attempt_id: reclaimableAttempt.attempt_id,
|
|
1026
|
+
result_id: null,
|
|
1027
|
+
key: `attempt:${reclaimableAttempt.attempt_id}:claim:${claim.claim_epoch}`,
|
|
1028
|
+
at: now,
|
|
1029
|
+
payload: {
|
|
1030
|
+
claim_epoch: claim.claim_epoch,
|
|
1031
|
+
owner_id: claim.owner_id,
|
|
1032
|
+
claim_expires_at: claim.driver_claim_lease.expires_at,
|
|
1033
|
+
execution_deadline_at: claim.execution_deadline_at,
|
|
1034
|
+
},
|
|
1035
|
+
});
|
|
1036
|
+
try {
|
|
1037
|
+
snapshot = (await this.runStore.updateCanonicalRun({
|
|
1038
|
+
run_id: snapshot.run.run_id,
|
|
1039
|
+
expected_record_version: snapshot.record_version,
|
|
1040
|
+
claims: [claim],
|
|
1041
|
+
events: [event],
|
|
1042
|
+
})).snapshot;
|
|
1043
|
+
return {
|
|
1044
|
+
kind: "attempt",
|
|
1045
|
+
lease: this.toLease(snapshot, reclaimableAttempt, claim),
|
|
1046
|
+
recovery: reclaimableAttempt.phase === "running"
|
|
1047
|
+
? this.attemptRecovery(snapshot, reclaimableAttempt)
|
|
1048
|
+
: null,
|
|
1049
|
+
};
|
|
1050
|
+
}
|
|
1051
|
+
catch (error) {
|
|
1052
|
+
if (this.isVersionConflict(error))
|
|
1053
|
+
continue;
|
|
1054
|
+
throw error;
|
|
1055
|
+
}
|
|
1056
|
+
}
|
|
1057
|
+
return {
|
|
1058
|
+
kind: "wait",
|
|
1059
|
+
reason_code: "attempt_active",
|
|
1060
|
+
run: this.toView(snapshot, plan),
|
|
1061
|
+
};
|
|
1062
|
+
}
|
|
1063
|
+
if (!activation) {
|
|
1064
|
+
return { kind: "wait", reason_code: "dependencies_pending", run: this.toView(snapshot, plan) };
|
|
1065
|
+
}
|
|
1066
|
+
const retrySegmentEvent = [...snapshot.events].reverse().find((event) => (event.type === "retry_segment_scheduled"
|
|
1067
|
+
&& event.payload.retry_from_activation_id === activation.activation_id));
|
|
1068
|
+
if (retrySegmentEvent
|
|
1069
|
+
&& Date.parse(now) < Date.parse(retrySegmentEvent.payload.ready_at)) {
|
|
1070
|
+
return { kind: "wait", reason_code: "retry_backoff", run: this.toView(snapshot, plan) };
|
|
1071
|
+
}
|
|
1072
|
+
const attemptNumber = activation.attempt_count + 1;
|
|
1073
|
+
if (attemptNumber > 1) {
|
|
1074
|
+
const priorAttempt = snapshot.attempts.find((candidate) => (candidate.activation_id === activation.activation_id
|
|
1075
|
+
&& candidate.attempt_number === attemptNumber - 1));
|
|
1076
|
+
const retryNode = nodeById(plan.plan, activation.step_id);
|
|
1077
|
+
if (priorAttempt?.completed_at && retryNode.kind === "worker") {
|
|
1078
|
+
const eligibleAt = Date.parse(priorAttempt.completed_at)
|
|
1079
|
+
+ this.retryDelay(retryNode, priorAttempt.attempt_number);
|
|
1080
|
+
if (Date.parse(now) < eligibleAt) {
|
|
1081
|
+
return {
|
|
1082
|
+
kind: "wait",
|
|
1083
|
+
reason_code: "retry_backoff",
|
|
1084
|
+
run: this.toView(snapshot, plan),
|
|
1085
|
+
};
|
|
1086
|
+
}
|
|
1087
|
+
}
|
|
1088
|
+
}
|
|
1089
|
+
const attemptId = this.identities.create("attempt", `${activation.activation_id}\0${attemptNumber}`);
|
|
1090
|
+
const attempt = {
|
|
1091
|
+
run_id: snapshot.run.run_id,
|
|
1092
|
+
step_id: activation.step_id,
|
|
1093
|
+
activation_id: activation.activation_id,
|
|
1094
|
+
attempt_id: attemptId,
|
|
1095
|
+
attempt_number: attemptNumber,
|
|
1096
|
+
phase: "claimed",
|
|
1097
|
+
executor: null,
|
|
1098
|
+
started_at: null,
|
|
1099
|
+
completed_at: null,
|
|
1100
|
+
terminal_outcome: null,
|
|
1101
|
+
};
|
|
1102
|
+
const claim = this.createClaim(snapshot, plan, attempt, request.driver_id, now, 1);
|
|
1103
|
+
const claimedActivation = {
|
|
1104
|
+
...activation,
|
|
1105
|
+
state: "claimed",
|
|
1106
|
+
attempt_count: attemptNumber,
|
|
1107
|
+
current_attempt_id: attemptId,
|
|
1108
|
+
updated_at: now,
|
|
1109
|
+
};
|
|
1110
|
+
const events = [
|
|
1111
|
+
this.activationEvent(snapshot, 0, activation, claimedActivation, "driver_claimed", now),
|
|
1112
|
+
this.event(snapshot, 1, {
|
|
1113
|
+
type: "attempt_claimed",
|
|
1114
|
+
step_id: attempt.step_id,
|
|
1115
|
+
activation_id: attempt.activation_id,
|
|
1116
|
+
attempt_id: attempt.attempt_id,
|
|
1117
|
+
result_id: null,
|
|
1118
|
+
key: `attempt:${attempt.attempt_id}:claim:1`,
|
|
1119
|
+
at: now,
|
|
1120
|
+
payload: {
|
|
1121
|
+
claim_epoch: 1,
|
|
1122
|
+
owner_id: request.driver_id,
|
|
1123
|
+
claim_expires_at: claim.driver_claim_lease.expires_at,
|
|
1124
|
+
execution_deadline_at: claim.execution_deadline_at,
|
|
1125
|
+
},
|
|
1126
|
+
}),
|
|
1127
|
+
];
|
|
1128
|
+
if (attemptNumber > 1) {
|
|
1129
|
+
const priorAttempt = snapshot.attempts.find((candidate) => (candidate.activation_id === activation.activation_id
|
|
1130
|
+
&& candidate.attempt_number === attemptNumber - 1));
|
|
1131
|
+
if (priorAttempt) {
|
|
1132
|
+
events.push(this.event(snapshot, events.length, {
|
|
1133
|
+
type: "retry_scheduled",
|
|
1134
|
+
step_id: attempt.step_id,
|
|
1135
|
+
activation_id: attempt.activation_id,
|
|
1136
|
+
attempt_id: priorAttempt.attempt_id,
|
|
1137
|
+
result_id: null,
|
|
1138
|
+
key: `attempt:${priorAttempt.attempt_id}:retry:${attempt.attempt_id}`,
|
|
1139
|
+
at: now,
|
|
1140
|
+
payload: {
|
|
1141
|
+
prior_attempt_id: priorAttempt.attempt_id,
|
|
1142
|
+
replacement_attempt_id: attempt.attempt_id,
|
|
1143
|
+
next_attempt_number: attemptNumber,
|
|
1144
|
+
delay_ms: nodeById(plan.plan, attempt.step_id).kind === "worker"
|
|
1145
|
+
? this.retryDelay(nodeById(plan.plan, attempt.step_id), priorAttempt.attempt_number)
|
|
1146
|
+
: 0,
|
|
1147
|
+
reason_code: "retry_claimed",
|
|
1148
|
+
},
|
|
1149
|
+
}));
|
|
1150
|
+
}
|
|
1151
|
+
}
|
|
1152
|
+
try {
|
|
1153
|
+
snapshot = (await this.runStore.updateCanonicalRun({
|
|
1154
|
+
run_id: snapshot.run.run_id,
|
|
1155
|
+
expected_record_version: snapshot.record_version,
|
|
1156
|
+
activations: [claimedActivation],
|
|
1157
|
+
attempts: [attempt],
|
|
1158
|
+
claims: [claim],
|
|
1159
|
+
events,
|
|
1160
|
+
})).snapshot;
|
|
1161
|
+
return {
|
|
1162
|
+
kind: "attempt",
|
|
1163
|
+
lease: this.toLease(snapshot, attempt, claim),
|
|
1164
|
+
recovery: null,
|
|
1165
|
+
};
|
|
1166
|
+
}
|
|
1167
|
+
catch (error) {
|
|
1168
|
+
if (this.isVersionConflict(error))
|
|
1169
|
+
continue;
|
|
1170
|
+
throw error;
|
|
1171
|
+
}
|
|
1172
|
+
}
|
|
1173
|
+
}
|
|
1174
|
+
async attachAttempt(request) {
|
|
1175
|
+
const located = await this.locateLease(request.lease_token, request.owner_id, false);
|
|
1176
|
+
this.assertRunAttachable(located.snapshot);
|
|
1177
|
+
const { activation, plan } = located;
|
|
1178
|
+
this.assertVerifierExecutionSupported(plan.plan);
|
|
1179
|
+
this.assertCapabilities(plan.plan.required_capabilities, request.capabilities);
|
|
1180
|
+
const executor = typeof request.worker_identity === "string"
|
|
1181
|
+
? { kind: "worker", worker_session_id: request.worker_identity }
|
|
1182
|
+
: request.worker_identity;
|
|
1183
|
+
const node = nodeById(plan.plan, activation.step_id);
|
|
1184
|
+
const expectedExecutorKind = node.kind === "worker"
|
|
1185
|
+
? "worker"
|
|
1186
|
+
: node.kind === "system" ? "system" : node.kind === "interaction" ? "human" : null;
|
|
1187
|
+
if (expectedExecutorKind === null || executor.kind !== expectedExecutorKind) {
|
|
1188
|
+
throw new CanonicalRunServiceError(executor.kind === "worker" ? "attempt_state_conflict" : "unsupported", executor.kind === "worker"
|
|
1189
|
+
? `${node.kind} node requires ${expectedExecutorKind ?? "no"} executor`
|
|
1190
|
+
: `external ${executor.kind} attach is unsupported in the canonical service`);
|
|
1191
|
+
}
|
|
1192
|
+
if (node.kind === "worker" && executor.kind === "worker") {
|
|
1193
|
+
return this.attachWorkerAttempt(request, executor);
|
|
1194
|
+
}
|
|
1195
|
+
const { snapshot, claim, attempt } = located;
|
|
1196
|
+
if (attempt.phase === "running") {
|
|
1197
|
+
if (attempt.executor === null || canonicalizeJson(attempt.executor) !== canonicalizeJson(executor)) {
|
|
1198
|
+
throw new CanonicalRunServiceError("attempt_state_conflict", "attempt is attached to a different executor");
|
|
1199
|
+
}
|
|
1200
|
+
return { lease: this.toLeaseView(snapshot, attempt, claim), attempt, activation, packet: null };
|
|
1201
|
+
}
|
|
1202
|
+
if (attempt.phase !== "claimed" || activation.state !== "claimed") {
|
|
1203
|
+
throw new CanonicalRunServiceError("attempt_state_conflict", `attempt ${attempt.attempt_id} is not claimable`);
|
|
1204
|
+
}
|
|
1205
|
+
const now = this.now();
|
|
1206
|
+
const runningAttempt = {
|
|
1207
|
+
...attempt,
|
|
1208
|
+
phase: "running",
|
|
1209
|
+
executor,
|
|
1210
|
+
started_at: now,
|
|
1211
|
+
};
|
|
1212
|
+
const runningActivation = {
|
|
1213
|
+
...activation,
|
|
1214
|
+
state: activation.state === "claimed" && nodeById(plan.plan, activation.step_id).kind === "interaction"
|
|
1215
|
+
? "waiting_for_human"
|
|
1216
|
+
: "running",
|
|
1217
|
+
updated_at: now,
|
|
1218
|
+
};
|
|
1219
|
+
const packet = null;
|
|
1220
|
+
const events = [
|
|
1221
|
+
this.activationEvent(snapshot, 0, activation, runningActivation, "attempt_attached", now),
|
|
1222
|
+
this.event(snapshot, 1, {
|
|
1223
|
+
type: "attempt_attached",
|
|
1224
|
+
step_id: attempt.step_id,
|
|
1225
|
+
activation_id: attempt.activation_id,
|
|
1226
|
+
attempt_id: attempt.attempt_id,
|
|
1227
|
+
result_id: null,
|
|
1228
|
+
key: `attempt:${attempt.attempt_id}:attached`,
|
|
1229
|
+
at: now,
|
|
1230
|
+
payload: { claim_epoch: claim.claim_epoch, executor },
|
|
1231
|
+
}),
|
|
1232
|
+
this.event(snapshot, 2, {
|
|
1233
|
+
type: "attempt_started",
|
|
1234
|
+
step_id: attempt.step_id,
|
|
1235
|
+
activation_id: attempt.activation_id,
|
|
1236
|
+
attempt_id: attempt.attempt_id,
|
|
1237
|
+
result_id: null,
|
|
1238
|
+
key: `attempt:${attempt.attempt_id}:started`,
|
|
1239
|
+
at: now,
|
|
1240
|
+
payload: { executor },
|
|
1241
|
+
}),
|
|
1242
|
+
];
|
|
1243
|
+
events.push(...this.handoffResolutionEvents(snapshot, runningAttempt, packet, events.length, now));
|
|
1244
|
+
if (node.kind === "interaction" && executor.kind === "human") {
|
|
1245
|
+
events.push(this.event(snapshot, events.length, {
|
|
1246
|
+
type: "interaction_requested",
|
|
1247
|
+
step_id: attempt.step_id,
|
|
1248
|
+
activation_id: attempt.activation_id,
|
|
1249
|
+
attempt_id: attempt.attempt_id,
|
|
1250
|
+
result_id: null,
|
|
1251
|
+
key: `interaction:${node.interaction_id}:requested:${attempt.attempt_id}`,
|
|
1252
|
+
at: now,
|
|
1253
|
+
payload: {
|
|
1254
|
+
interaction_id: node.interaction_id,
|
|
1255
|
+
expected_principal_ref: executor.principal_ref,
|
|
1256
|
+
},
|
|
1257
|
+
}));
|
|
1258
|
+
}
|
|
1259
|
+
try {
|
|
1260
|
+
const updated = (await this.runStore.updateCanonicalRun({
|
|
1261
|
+
run_id: snapshot.run.run_id,
|
|
1262
|
+
expected_record_version: snapshot.record_version,
|
|
1263
|
+
activations: [runningActivation],
|
|
1264
|
+
attempts: [runningAttempt],
|
|
1265
|
+
events,
|
|
1266
|
+
})).snapshot;
|
|
1267
|
+
return {
|
|
1268
|
+
lease: this.toLeaseView(updated, runningAttempt, claim),
|
|
1269
|
+
attempt: runningAttempt,
|
|
1270
|
+
activation: runningActivation,
|
|
1271
|
+
packet,
|
|
1272
|
+
};
|
|
1273
|
+
}
|
|
1274
|
+
catch (error) {
|
|
1275
|
+
if (this.isVersionConflict(error)) {
|
|
1276
|
+
const current = await this.locateLease(request.lease_token, request.owner_id, false);
|
|
1277
|
+
this.assertRunAttachable(current.snapshot);
|
|
1278
|
+
throw new CanonicalRunServiceError("stale_lease", "attempt changed while attaching", { cause: error });
|
|
1279
|
+
}
|
|
1280
|
+
throw error;
|
|
1281
|
+
}
|
|
1282
|
+
}
|
|
1283
|
+
async attachWorkerAttempt(request, executor) {
|
|
1284
|
+
if (this.workspaceAuthority === null) {
|
|
1285
|
+
throw new CanonicalRunServiceError("unsupported", "attempt workspace authority is unavailable");
|
|
1286
|
+
}
|
|
1287
|
+
for (;;) {
|
|
1288
|
+
const located = await this.locateLease(request.lease_token, request.owner_id, false);
|
|
1289
|
+
let { snapshot, claim, attempt, activation, plan } = located;
|
|
1290
|
+
this.assertRunAttachable(snapshot);
|
|
1291
|
+
const node = nodeById(plan.plan, activation.step_id);
|
|
1292
|
+
if (node.kind !== "worker")
|
|
1293
|
+
throw new CanonicalRunServiceError("attempt_state_conflict", "attempt is not a worker node");
|
|
1294
|
+
if (attempt.executor !== null && canonicalizeJson(attempt.executor) !== canonicalizeJson(executor)) {
|
|
1295
|
+
throw new CanonicalRunServiceError("attempt_state_conflict", "attempt is attached to a different executor");
|
|
1296
|
+
}
|
|
1297
|
+
if (attempt.phase === "running") {
|
|
1298
|
+
const allocation = this.requireCommittedWorkspace(snapshot, attempt, node);
|
|
1299
|
+
return this.withWorkspaceAuthority(allocation, "prepare_inputs", false, async () => ({
|
|
1300
|
+
lease: this.toLeaseView(snapshot, attempt, claim),
|
|
1301
|
+
attempt,
|
|
1302
|
+
activation,
|
|
1303
|
+
packet: await this.materializePacket(snapshot, plan, activation, attempt, executor, allocation),
|
|
1304
|
+
}));
|
|
1305
|
+
}
|
|
1306
|
+
if (attempt.phase !== "claimed" || activation.state !== "claimed") {
|
|
1307
|
+
throw new CanonicalRunServiceError("attempt_state_conflict", `attempt ${attempt.attempt_id} is not claimable`);
|
|
1308
|
+
}
|
|
1309
|
+
let workspacePlan = this.workspacePlan(snapshot, attempt.attempt_id);
|
|
1310
|
+
if (workspacePlan === null) {
|
|
1311
|
+
workspacePlan = await this.planWorkspace(plan, snapshot.run, activation, attempt);
|
|
1312
|
+
const now = this.now();
|
|
1313
|
+
const events = [
|
|
1314
|
+
this.event(snapshot, 0, {
|
|
1315
|
+
type: "workspace_planned", step_id: attempt.step_id, activation_id: attempt.activation_id,
|
|
1316
|
+
attempt_id: attempt.attempt_id, result_id: null, key: `attempt:${attempt.attempt_id}:workspace:planned`, at: now,
|
|
1317
|
+
payload: { workspace_plan_id: workspacePlan.workspace_plan_id, plan: workspacePlan },
|
|
1318
|
+
}),
|
|
1319
|
+
];
|
|
1320
|
+
try {
|
|
1321
|
+
snapshot = (await this.runStore.updateCanonicalRun({
|
|
1322
|
+
run_id: snapshot.run.run_id, expected_record_version: snapshot.record_version,
|
|
1323
|
+
events,
|
|
1324
|
+
})).snapshot;
|
|
1325
|
+
}
|
|
1326
|
+
catch (error) {
|
|
1327
|
+
if (this.isVersionConflict(error))
|
|
1328
|
+
continue;
|
|
1329
|
+
throw error;
|
|
1330
|
+
}
|
|
1331
|
+
}
|
|
1332
|
+
let allocation = this.workspaceAllocation(snapshot, attempt.attempt_id);
|
|
1333
|
+
if (allocation === null) {
|
|
1334
|
+
try {
|
|
1335
|
+
snapshot = await this.workspaceAuthority.ensureWithExclusiveAuthority(workspacePlan, async (ensured) => {
|
|
1336
|
+
this.validateWorkspaceAllocation(ensured, attempt, node);
|
|
1337
|
+
const now = this.now();
|
|
1338
|
+
return (await this.runStore.updateCanonicalRun({
|
|
1339
|
+
run_id: snapshot.run.run_id,
|
|
1340
|
+
expected_record_version: snapshot.record_version,
|
|
1341
|
+
events: [this.event(snapshot, 0, {
|
|
1342
|
+
type: "workspace_allocated", step_id: attempt.step_id, activation_id: attempt.activation_id,
|
|
1343
|
+
attempt_id: attempt.attempt_id, result_id: null, key: `attempt:${attempt.attempt_id}:workspace:allocated`, at: now,
|
|
1344
|
+
payload: { workspace_plan_id: workspacePlan.workspace_plan_id, allocation: ensured },
|
|
1345
|
+
})],
|
|
1346
|
+
})).snapshot;
|
|
1347
|
+
});
|
|
1348
|
+
allocation = this.requireCommittedWorkspace(snapshot, attempt, node);
|
|
1349
|
+
}
|
|
1350
|
+
catch (error) {
|
|
1351
|
+
if (this.isVersionConflict(error))
|
|
1352
|
+
continue;
|
|
1353
|
+
throw this.workspaceError(error, "attempt workspace ensure failed");
|
|
1354
|
+
}
|
|
1355
|
+
}
|
|
1356
|
+
try {
|
|
1357
|
+
return await this.withWorkspaceAuthority(allocation, "prepare_inputs", true, async () => {
|
|
1358
|
+
const now = this.now();
|
|
1359
|
+
const runningAttempt = { ...attempt, phase: "running", executor, started_at: now };
|
|
1360
|
+
const runningActivation = { ...activation, state: "running", updated_at: now };
|
|
1361
|
+
const packet = await this.materializePacket(snapshot, plan, runningActivation, runningAttempt, executor, allocation);
|
|
1362
|
+
const events = [
|
|
1363
|
+
this.activationEvent(snapshot, 0, activation, runningActivation, "attempt_attached", now),
|
|
1364
|
+
this.event(snapshot, 1, {
|
|
1365
|
+
type: "attempt_attached", step_id: attempt.step_id, activation_id: attempt.activation_id,
|
|
1366
|
+
attempt_id: attempt.attempt_id, result_id: null, key: `attempt:${attempt.attempt_id}:attached`, at: now,
|
|
1367
|
+
payload: { claim_epoch: claim.claim_epoch, executor },
|
|
1368
|
+
}),
|
|
1369
|
+
this.event(snapshot, 2, {
|
|
1370
|
+
type: "attempt_started", step_id: attempt.step_id, activation_id: attempt.activation_id,
|
|
1371
|
+
attempt_id: attempt.attempt_id, result_id: null, key: `attempt:${attempt.attempt_id}:started`, at: now,
|
|
1372
|
+
payload: { executor },
|
|
1373
|
+
}),
|
|
1374
|
+
...this.handoffResolutionEvents(snapshot, runningAttempt, packet, 3, now),
|
|
1375
|
+
];
|
|
1376
|
+
const updated = (await this.runStore.updateCanonicalRun({
|
|
1377
|
+
run_id: snapshot.run.run_id, expected_record_version: snapshot.record_version,
|
|
1378
|
+
activations: [runningActivation], attempts: [runningAttempt], events,
|
|
1379
|
+
})).snapshot;
|
|
1380
|
+
return { lease: this.toLeaseView(updated, runningAttempt, claim), attempt: runningAttempt, activation: runningActivation, packet };
|
|
1381
|
+
});
|
|
1382
|
+
}
|
|
1383
|
+
catch (error) {
|
|
1384
|
+
if (this.isVersionConflict(error))
|
|
1385
|
+
continue;
|
|
1386
|
+
throw error;
|
|
1387
|
+
}
|
|
1388
|
+
}
|
|
1389
|
+
}
|
|
1390
|
+
async heartbeat(request) {
|
|
1391
|
+
for (;;) {
|
|
1392
|
+
const { snapshot, claim, attempt } = await this.locateLease(request.lease_token, request.owner_id, false);
|
|
1393
|
+
if (this.stopIntentFor(snapshot, attempt.attempt_id) !== null) {
|
|
1394
|
+
throw new CanonicalRunServiceError("stale_lease", "attempt stop has been requested");
|
|
1395
|
+
}
|
|
1396
|
+
if (attempt.phase === "terminal") {
|
|
1397
|
+
throw new CanonicalRunServiceError("attempt_state_conflict", "terminal attempt cannot heartbeat");
|
|
1398
|
+
}
|
|
1399
|
+
const now = this.now();
|
|
1400
|
+
if (Date.parse(now) < Date.parse(claim.driver_claim_lease.heartbeat_at)) {
|
|
1401
|
+
throw new CanonicalRunServiceError("invalid_request", "heartbeat clock moved backward");
|
|
1402
|
+
}
|
|
1403
|
+
if (now === claim.driver_claim_lease.heartbeat_at) {
|
|
1404
|
+
return { ...this.toLeaseView(snapshot, attempt, claim), heartbeat_at: now };
|
|
1405
|
+
}
|
|
1406
|
+
const expiresAt = new Date(Math.min(Date.parse(claim.execution_deadline_at), Date.parse(now) + this.leaseDurationMs)).toISOString();
|
|
1407
|
+
const renewed = {
|
|
1408
|
+
...claim,
|
|
1409
|
+
driver_claim_lease: {
|
|
1410
|
+
...claim.driver_claim_lease,
|
|
1411
|
+
heartbeat_at: now,
|
|
1412
|
+
expires_at: expiresAt,
|
|
1413
|
+
},
|
|
1414
|
+
};
|
|
1415
|
+
const event = this.event(snapshot, 0, {
|
|
1416
|
+
type: "attempt_heartbeat",
|
|
1417
|
+
step_id: attempt.step_id,
|
|
1418
|
+
activation_id: attempt.activation_id,
|
|
1419
|
+
attempt_id: attempt.attempt_id,
|
|
1420
|
+
result_id: null,
|
|
1421
|
+
key: `claim:${attempt.attempt_id}:epoch:${claim.claim_epoch}:heartbeat:${now}`,
|
|
1422
|
+
at: now,
|
|
1423
|
+
payload: {
|
|
1424
|
+
claim_epoch: claim.claim_epoch,
|
|
1425
|
+
heartbeat_at: now,
|
|
1426
|
+
claim_expires_at: expiresAt,
|
|
1427
|
+
},
|
|
1428
|
+
});
|
|
1429
|
+
try {
|
|
1430
|
+
const updated = (await this.runStore.updateCanonicalRun({
|
|
1431
|
+
run_id: snapshot.run.run_id,
|
|
1432
|
+
expected_record_version: snapshot.record_version,
|
|
1433
|
+
claims: [renewed],
|
|
1434
|
+
events: [event],
|
|
1435
|
+
})).snapshot;
|
|
1436
|
+
return { ...this.toLeaseView(updated, attempt, renewed), heartbeat_at: now };
|
|
1437
|
+
}
|
|
1438
|
+
catch (error) {
|
|
1439
|
+
if (this.isVersionConflict(error))
|
|
1440
|
+
continue;
|
|
1441
|
+
throw error;
|
|
1442
|
+
}
|
|
1443
|
+
}
|
|
1444
|
+
}
|
|
1445
|
+
async submitAttempt(request) {
|
|
1446
|
+
requireNonEmpty(request.submission_key, "submission_key");
|
|
1447
|
+
if (request.execution.stdout_reference != null
|
|
1448
|
+
|| request.execution.stderr_reference != null) {
|
|
1449
|
+
throw new CanonicalRunServiceError("invalid_request", "caller cannot submit service-owned stdout or stderr evidence");
|
|
1450
|
+
}
|
|
1451
|
+
for (const evidence of request.evidence_refs ?? []) {
|
|
1452
|
+
if (["submission", "verifier_evidence", "artifact_capture", "worker_stopped"].includes(evidence.kind)) {
|
|
1453
|
+
throw new CanonicalRunServiceError("invalid_request", `caller cannot submit service-owned ${evidence.kind} evidence`);
|
|
1454
|
+
}
|
|
1455
|
+
}
|
|
1456
|
+
const submissionKeyDigest = digest({ submission_key: request.submission_key });
|
|
1457
|
+
const submissionDigest = digest({
|
|
1458
|
+
execution: request.execution,
|
|
1459
|
+
evidence_refs: request.evidence_refs ?? [],
|
|
1460
|
+
});
|
|
1461
|
+
const initial = await this.locateLease(request.lease_token, request.owner_id, true, true);
|
|
1462
|
+
this.assertVerifierExecutionSupported(initial.plan.plan);
|
|
1463
|
+
const resultId = resultIdFor(this.identities, initial.attempt.attempt_id);
|
|
1464
|
+
const replay = await this.findReplay(initial.snapshot, initial.plan, initial.attempt.attempt_id, resultId, submissionKeyDigest, submissionDigest);
|
|
1465
|
+
if (replay) {
|
|
1466
|
+
const drained = await this.drainReadySystemNodes(initial.snapshot, initial.plan);
|
|
1467
|
+
return { ...replay, run: this.toView(drained, initial.plan) };
|
|
1468
|
+
}
|
|
1469
|
+
if (initial.snapshot.run.terminal !== null || initial.snapshot.run.status === "cancelling") {
|
|
1470
|
+
throw new CanonicalRunServiceError("run_terminal", initial.snapshot.run.status === "cancelling" ? "run cancellation has been requested" : "run is terminal");
|
|
1471
|
+
}
|
|
1472
|
+
const orphan = await this.resultStore.loadPublished(resultId);
|
|
1473
|
+
if (orphan) {
|
|
1474
|
+
const submission = this.submissionEvidence(orphan.result);
|
|
1475
|
+
if (submission.submission_key_digest !== submissionKeyDigest
|
|
1476
|
+
|| submission.submission_digest !== submissionDigest) {
|
|
1477
|
+
throw new CanonicalRunServiceError("submission_conflict", "attempt has a different published submission");
|
|
1478
|
+
}
|
|
1479
|
+
const reconciled = await this.reconcilePublishedResult(initial.snapshot, initial.plan, orphan.result);
|
|
1480
|
+
const drained = await this.drainReadySystemNodes(reconciled, initial.plan);
|
|
1481
|
+
return {
|
|
1482
|
+
result: orphan.result,
|
|
1483
|
+
replayed: true,
|
|
1484
|
+
run: this.toView(drained, initial.plan),
|
|
1485
|
+
};
|
|
1486
|
+
}
|
|
1487
|
+
if (initial.attempt.phase !== "running" || initial.attempt.executor === null) {
|
|
1488
|
+
throw new CanonicalRunServiceError("attempt_state_conflict", "attempt must be attached before submit");
|
|
1489
|
+
}
|
|
1490
|
+
if (canonicalizeJson(initial.attempt.executor) !== canonicalizeJson(request.execution.executor)) {
|
|
1491
|
+
throw new CanonicalRunServiceError("stale_lease", "submission executor does not match attached executor");
|
|
1492
|
+
}
|
|
1493
|
+
if (request.execution.started_at !== initial.attempt.started_at) {
|
|
1494
|
+
throw new CanonicalRunServiceError("submission_conflict", "execution.started_at does not match the durable attempt");
|
|
1495
|
+
}
|
|
1496
|
+
const submittedAt = this.clock.now().getTime();
|
|
1497
|
+
const completedAt = Date.parse(request.execution.completed_at);
|
|
1498
|
+
if (!Number.isFinite(completedAt)
|
|
1499
|
+
|| completedAt < Date.parse(request.execution.started_at)
|
|
1500
|
+
|| completedAt > submittedAt) {
|
|
1501
|
+
throw new CanonicalRunServiceError("invalid_request", "execution completion time is invalid");
|
|
1502
|
+
}
|
|
1503
|
+
const executionDeadline = Date.parse(initial.claim.execution_deadline_at);
|
|
1504
|
+
if (submittedAt >= executionDeadline) {
|
|
1505
|
+
const node = nodeById(initial.plan.plan, initial.attempt.step_id);
|
|
1506
|
+
const finalizationDeadline = executionDeadline + (node.kind === "worker" ? node.attempt_policy.cancellation_grace_ms : 0);
|
|
1507
|
+
if (!isExecutionDeadlineTimeout(request.execution)
|
|
1508
|
+
|| submittedAt > finalizationDeadline
|
|
1509
|
+
|| completedAt > finalizationDeadline) {
|
|
1510
|
+
throw new CanonicalRunServiceError("stale_lease", "attempt execution deadline has elapsed");
|
|
1511
|
+
}
|
|
1512
|
+
}
|
|
1513
|
+
await this.validateCallerAttestationIndependence(initial.snapshot, initial.plan, initial.activation, request.evidence_refs ?? []);
|
|
1514
|
+
const allocation = this.requireCommittedWorkspace(initial.snapshot, initial.attempt, nodeById(initial.plan.plan, initial.activation.step_id));
|
|
1515
|
+
const authoritativeWorkspace = allocation.working_directory;
|
|
1516
|
+
if (resolve(request.execution.workspace.root) !== resolve(authoritativeWorkspace)
|
|
1517
|
+
|| request.execution.workspace.workspace_id !== initial.attempt.attempt_id) {
|
|
1518
|
+
throw new CanonicalRunServiceError("submission_conflict", "execution workspace does not match the attempt allocation");
|
|
1519
|
+
}
|
|
1520
|
+
const finalized = await this.withStoppedExecution(initial.snapshot, initial.claim, initial.attempt, request.execution, "finalize_result", null, async (executionAuthority) => this.withWorkspaceAuthority(allocation, "finalize_result", false, async (workspaceAuthority) => {
|
|
1521
|
+
validateCanonicalAttemptOutcome(request.execution.executor, request.execution.outcome);
|
|
1522
|
+
this.validateSubmittedOutputs(initial.snapshot, initial.plan, initial.activation, request.execution);
|
|
1523
|
+
const captureRequest = {
|
|
1524
|
+
run_id: initial.snapshot.run.run_id,
|
|
1525
|
+
activation_id: initial.activation.activation_id,
|
|
1526
|
+
attempt_id: initial.attempt.attempt_id,
|
|
1527
|
+
execution: request.execution,
|
|
1528
|
+
};
|
|
1529
|
+
const captured = await this.outputCapture.capture(captureRequest);
|
|
1530
|
+
await this.hooks.after_output_capture?.(captureRequest);
|
|
1531
|
+
const verifierBindings = await this.verifierHandoffBindings(initial.snapshot, initial.plan, initial.activation, initial.attempt, allocation);
|
|
1532
|
+
const verifyRequest = {
|
|
1533
|
+
plan: initial.plan,
|
|
1534
|
+
run: initial.snapshot.run,
|
|
1535
|
+
activation: initial.activation,
|
|
1536
|
+
attempt: initial.attempt,
|
|
1537
|
+
execution: request.execution,
|
|
1538
|
+
workspace_allocation: allocation,
|
|
1539
|
+
captured_outputs: captured.outputs,
|
|
1540
|
+
handoff_bindings: verifierBindings,
|
|
1541
|
+
};
|
|
1542
|
+
const mandatoryVerification = await this.verifier.verify(verifyRequest);
|
|
1543
|
+
const additionalVerification = await this.additionalVerifier?.verify(verifyRequest) ?? [];
|
|
1544
|
+
this.validateVerificationLanes(verifyRequest, mandatoryVerification, additionalVerification);
|
|
1545
|
+
const invalidGitCommit = mandatoryVerification.find((result) => (result.verifier_id.endsWith(":git_commit_provenance") && !result.passed));
|
|
1546
|
+
if (invalidGitCommit) {
|
|
1547
|
+
throw new CanonicalRunServiceError("verification_failed", `git_commit output failed attempt provenance verification: ${invalidGitCommit.verifier_id}`);
|
|
1548
|
+
}
|
|
1549
|
+
const verification = [...mandatoryVerification, ...additionalVerification];
|
|
1550
|
+
await this.validateCommandVerifierReceipts(verifyRequest, mandatoryVerification);
|
|
1551
|
+
await this.hooks.after_verification?.(verifyRequest);
|
|
1552
|
+
await captured.revalidate();
|
|
1553
|
+
this.validateDecisionOutputs(initial.plan.plan, initial.activation.step_id, captured.outputs);
|
|
1554
|
+
const verifierEvidence = verification.flatMap((item) => item.evidence_refs ?? []);
|
|
1555
|
+
if (verification.some((item) => ((item.evidence_refs ?? []).some((evidence) => evidence.verifier_id !== item.verifier_id)))) {
|
|
1556
|
+
throw new CanonicalRunServiceError("verification_failed", "verifier evidence identity does not match its parent result");
|
|
1557
|
+
}
|
|
1558
|
+
const verificationResults = [
|
|
1559
|
+
...mandatoryVerification.map((item) => ({
|
|
1560
|
+
verifier_id: item.verifier_id,
|
|
1561
|
+
authority: "service",
|
|
1562
|
+
passed: item.passed,
|
|
1563
|
+
reason_code: item.reason_code,
|
|
1564
|
+
evidence_refs: item.evidence_refs ?? [],
|
|
1565
|
+
domain_value: item.domain_value ?? null,
|
|
1566
|
+
})),
|
|
1567
|
+
...additionalVerification.map((item) => ({
|
|
1568
|
+
verifier_id: item.verifier_id,
|
|
1569
|
+
authority: "additional",
|
|
1570
|
+
passed: item.passed,
|
|
1571
|
+
reason_code: item.reason_code,
|
|
1572
|
+
evidence_refs: item.evidence_refs ?? [],
|
|
1573
|
+
domain_value: item.domain_value ?? null,
|
|
1574
|
+
})),
|
|
1575
|
+
];
|
|
1576
|
+
const now = this.now();
|
|
1577
|
+
const result = {
|
|
1578
|
+
schema_version: CANONICAL_RESULT_SCHEMA_VERSION,
|
|
1579
|
+
run_id: initial.snapshot.run.run_id,
|
|
1580
|
+
step_id: initial.attempt.step_id,
|
|
1581
|
+
activation_id: initial.attempt.activation_id,
|
|
1582
|
+
attempt_id: initial.attempt.attempt_id,
|
|
1583
|
+
result_id: resultId,
|
|
1584
|
+
executor: request.execution.executor,
|
|
1585
|
+
terminal_outcome: request.execution.outcome,
|
|
1586
|
+
outputs: captured.outputs,
|
|
1587
|
+
verification_results: verificationResults,
|
|
1588
|
+
started_at: request.execution.started_at,
|
|
1589
|
+
completed_at: request.execution.completed_at,
|
|
1590
|
+
committed_at: now,
|
|
1591
|
+
stdout_reference: request.execution.stdout_reference ?? null,
|
|
1592
|
+
stderr_reference: request.execution.stderr_reference ?? null,
|
|
1593
|
+
evidence_refs: [
|
|
1594
|
+
...(request.evidence_refs ?? []),
|
|
1595
|
+
...captured.captures.map((capture) => ({
|
|
1596
|
+
kind: "artifact_capture",
|
|
1597
|
+
capture,
|
|
1598
|
+
})),
|
|
1599
|
+
...verifierEvidence,
|
|
1600
|
+
...(executionAuthority === null ? [] : [{
|
|
1601
|
+
kind: "worker_stopped",
|
|
1602
|
+
receipt: executionAuthority.stop_receipt,
|
|
1603
|
+
}]),
|
|
1604
|
+
{ kind: "submission", submission_key_digest: submissionKeyDigest, submission_digest: submissionDigest },
|
|
1605
|
+
],
|
|
1606
|
+
};
|
|
1607
|
+
let published;
|
|
1608
|
+
try {
|
|
1609
|
+
published = await this.resultStore.publish(result);
|
|
1610
|
+
}
|
|
1611
|
+
catch (error) {
|
|
1612
|
+
if (error instanceof ResultStoreError && error.code === "immutable_conflict") {
|
|
1613
|
+
const winner = await this.resultStore.loadPublished(resultId);
|
|
1614
|
+
if (winner) {
|
|
1615
|
+
const submission = this.submissionEvidence(winner.result);
|
|
1616
|
+
if (submission.submission_key_digest === submissionKeyDigest
|
|
1617
|
+
&& submission.submission_digest === submissionDigest) {
|
|
1618
|
+
return {
|
|
1619
|
+
kind: "reconcile_published_winner",
|
|
1620
|
+
result: winner.result,
|
|
1621
|
+
};
|
|
1622
|
+
}
|
|
1623
|
+
}
|
|
1624
|
+
throw new CanonicalRunServiceError("submission_conflict", `attempt ${initial.attempt.attempt_id} already has a conflicting published result`, { cause: error });
|
|
1625
|
+
}
|
|
1626
|
+
throw error;
|
|
1627
|
+
}
|
|
1628
|
+
await this.hooks.after_result_publish?.(published.result);
|
|
1629
|
+
for (;;) {
|
|
1630
|
+
const located = await this.locateLease(request.lease_token, request.owner_id, true, true);
|
|
1631
|
+
const replayAfterPublish = await this.findReplay(located.snapshot, located.plan, located.attempt.attempt_id, resultId, submissionKeyDigest, submissionDigest);
|
|
1632
|
+
if (replayAfterPublish) {
|
|
1633
|
+
const drained = await this.drainReadySystemNodes(located.snapshot, located.plan);
|
|
1634
|
+
return {
|
|
1635
|
+
kind: "response",
|
|
1636
|
+
response: { ...replayAfterPublish, run: this.toView(drained, located.plan) },
|
|
1637
|
+
};
|
|
1638
|
+
}
|
|
1639
|
+
if (located.snapshot.run.status === "cancelling") {
|
|
1640
|
+
throw new CanonicalRunServiceError("run_terminal", "run cancellation has been requested");
|
|
1641
|
+
}
|
|
1642
|
+
if (located.attempt.phase !== "running" || located.attempt.executor === null) {
|
|
1643
|
+
throw new CanonicalRunServiceError("stale_lease", "attempt is no longer authoritative");
|
|
1644
|
+
}
|
|
1645
|
+
await captured.revalidate();
|
|
1646
|
+
await this.revalidateWorkspaceObservations(published.result, allocation);
|
|
1647
|
+
const committedAt = this.now();
|
|
1648
|
+
const drained = await this.prepareFailurePeerStops(located.snapshot, located.plan, located.attempt, published.result.terminal_outcome, committedAt);
|
|
1649
|
+
if (drained.record_version !== located.snapshot.record_version)
|
|
1650
|
+
continue;
|
|
1651
|
+
const decisionOutputs = await this.loadDecisionOutputs(located.snapshot, located.plan, published.result);
|
|
1652
|
+
const update = await this.buildSubmissionUpdate(located.snapshot, located.plan, located.activation, located.attempt, published.result, published.reference, captured.captures, verification, submissionKeyDigest, submissionDigest, committedAt, decisionOutputs, {
|
|
1653
|
+
worker_stop_receipt_digest: executionAuthority?.stop_receipt.receipt_digest ?? null,
|
|
1654
|
+
worker_stop_receipt: executionAuthority?.stop_receipt ?? null,
|
|
1655
|
+
workspace_authority: workspaceAuthority,
|
|
1656
|
+
purpose: "finalize_result",
|
|
1657
|
+
});
|
|
1658
|
+
if (update.run?.terminal !== null) {
|
|
1659
|
+
const peerDrained = await this.prepareTerminalPeerStops(located.snapshot, located.attempt, committedAt);
|
|
1660
|
+
if (peerDrained.record_version !== located.snapshot.record_version)
|
|
1661
|
+
continue;
|
|
1662
|
+
}
|
|
1663
|
+
try {
|
|
1664
|
+
const snapshot = (await this.runStore.updateCanonicalRun(update)).snapshot;
|
|
1665
|
+
const drained = await this.drainReadySystemNodes(snapshot, located.plan);
|
|
1666
|
+
return {
|
|
1667
|
+
kind: "response",
|
|
1668
|
+
response: {
|
|
1669
|
+
result: published.result,
|
|
1670
|
+
replayed: !published.published,
|
|
1671
|
+
run: this.toView(drained, located.plan),
|
|
1672
|
+
},
|
|
1673
|
+
};
|
|
1674
|
+
}
|
|
1675
|
+
catch (error) {
|
|
1676
|
+
if (this.isVersionConflict(error))
|
|
1677
|
+
continue;
|
|
1678
|
+
throw error;
|
|
1679
|
+
}
|
|
1680
|
+
}
|
|
1681
|
+
}));
|
|
1682
|
+
if (finalized.kind === "response")
|
|
1683
|
+
return finalized.response;
|
|
1684
|
+
const current = await this.requireSnapshot(initial.snapshot.run.run_id);
|
|
1685
|
+
const replayAfterAuthorityRelease = await this.findReplay(current, initial.plan, initial.attempt.attempt_id, resultId, submissionKeyDigest, submissionDigest);
|
|
1686
|
+
if (replayAfterAuthorityRelease) {
|
|
1687
|
+
const drained = await this.drainReadySystemNodes(current, initial.plan);
|
|
1688
|
+
return { ...replayAfterAuthorityRelease, run: this.toView(drained, initial.plan) };
|
|
1689
|
+
}
|
|
1690
|
+
const reconciled = await this.reconcilePublishedResult(current, initial.plan, finalized.result);
|
|
1691
|
+
const drained = await this.drainReadySystemNodes(reconciled, initial.plan);
|
|
1692
|
+
return {
|
|
1693
|
+
result: finalized.result,
|
|
1694
|
+
replayed: true,
|
|
1695
|
+
run: this.toView(drained, initial.plan),
|
|
1696
|
+
};
|
|
1697
|
+
}
|
|
1698
|
+
async failAttempt(request) {
|
|
1699
|
+
const located = await this.locateLease(request.lease_token, request.owner_id, true);
|
|
1700
|
+
if (located.attempt.executor === null || located.attempt.started_at === null) {
|
|
1701
|
+
throw new CanonicalRunServiceError("attempt_state_conflict", "attempt must be attached before failure");
|
|
1702
|
+
}
|
|
1703
|
+
const allocation = this.requireCommittedWorkspace(located.snapshot, located.attempt, nodeById(located.plan.plan, located.activation.step_id));
|
|
1704
|
+
const completedAt = request.completed_at ?? this.now();
|
|
1705
|
+
return this.submitAttempt({
|
|
1706
|
+
lease_token: request.lease_token,
|
|
1707
|
+
owner_id: request.owner_id,
|
|
1708
|
+
submission_key: request.submission_key,
|
|
1709
|
+
execution: {
|
|
1710
|
+
executor: located.attempt.executor,
|
|
1711
|
+
started_at: located.attempt.started_at,
|
|
1712
|
+
completed_at: completedAt,
|
|
1713
|
+
outcome: request.outcome,
|
|
1714
|
+
outputs: [],
|
|
1715
|
+
workspace: {
|
|
1716
|
+
root: allocation.working_directory,
|
|
1717
|
+
workspace_id: allocation.workspace_id,
|
|
1718
|
+
},
|
|
1719
|
+
},
|
|
1720
|
+
evidence_refs: request.evidence_refs,
|
|
1721
|
+
});
|
|
1722
|
+
}
|
|
1723
|
+
async settleDriverFailure(request) {
|
|
1724
|
+
requireNonEmpty(request.failure_key, "failure_key");
|
|
1725
|
+
requireNonEmpty(request.reason_code, "reason_code");
|
|
1726
|
+
const located = await this.locateLease(request.lease_token, request.owner_id, true);
|
|
1727
|
+
if (located.attempt.executor?.kind !== "worker" || located.attempt.started_at === null) {
|
|
1728
|
+
throw new CanonicalRunServiceError("attempt_state_conflict", "driver failure settlement requires an attached worker attempt");
|
|
1729
|
+
}
|
|
1730
|
+
const failureDigest = digest({
|
|
1731
|
+
failure_key: request.failure_key,
|
|
1732
|
+
reason_code: request.reason_code,
|
|
1733
|
+
message: request.message,
|
|
1734
|
+
});
|
|
1735
|
+
if (located.attempt.phase === "terminal") {
|
|
1736
|
+
this.assertDriverFailureReplay(located.snapshot, located.attempt, failureDigest, request);
|
|
1737
|
+
return this.toView(located.snapshot, located.plan);
|
|
1738
|
+
}
|
|
1739
|
+
if (this.executionAuthority?.attestNeverLaunched === undefined) {
|
|
1740
|
+
throw new CanonicalRunServiceError("unsupported", "nonlaunch execution authority is unavailable");
|
|
1741
|
+
}
|
|
1742
|
+
const recovery = this.attemptRecovery(located.snapshot, located.attempt);
|
|
1743
|
+
const now = this.now();
|
|
1744
|
+
const authorityRequest = {
|
|
1745
|
+
run_id: located.attempt.run_id,
|
|
1746
|
+
step_id: located.attempt.step_id,
|
|
1747
|
+
activation_id: located.attempt.activation_id,
|
|
1748
|
+
attempt_id: located.attempt.attempt_id,
|
|
1749
|
+
execution_claim_epoch: recovery.execution_claim_epoch,
|
|
1750
|
+
executor: located.attempt.executor,
|
|
1751
|
+
started_at: located.attempt.started_at,
|
|
1752
|
+
failure_idempotency_key: failureDigest,
|
|
1753
|
+
requested_at: now,
|
|
1754
|
+
};
|
|
1755
|
+
let attestation;
|
|
1756
|
+
try {
|
|
1757
|
+
attestation = await this.executionAuthority.attestNeverLaunched(authorityRequest);
|
|
1758
|
+
}
|
|
1759
|
+
catch (error) {
|
|
1760
|
+
const authorityCode = typeof error === "object"
|
|
1761
|
+
&& error !== null
|
|
1762
|
+
&& "code" in error
|
|
1763
|
+
? String(error.code)
|
|
1764
|
+
: null;
|
|
1765
|
+
if (authorityCode === "identity_conflict") {
|
|
1766
|
+
throw new CanonicalRunServiceError("submission_conflict", "execution authority has already recorded a different launch or nonlaunch fact", { cause: error });
|
|
1767
|
+
}
|
|
1768
|
+
throw new CanonicalRunServiceError("unsupported", "execution authority cannot attest that the worker was never launched", { cause: error });
|
|
1769
|
+
}
|
|
1770
|
+
if (!workerNonlaunchAttestationMatches(attestation, authorityRequest)) {
|
|
1771
|
+
throw new CanonicalRunServiceError("submission_conflict", "nonlaunch attestation does not match the driver failure");
|
|
1772
|
+
}
|
|
1773
|
+
const outcome = {
|
|
1774
|
+
kind: "worker_process",
|
|
1775
|
+
status: "failed",
|
|
1776
|
+
reason_code: request.reason_code,
|
|
1777
|
+
message: request.message,
|
|
1778
|
+
process: { kind: "launch_error", exit_code: null, signal: null, error: request.message ?? request.reason_code },
|
|
1779
|
+
};
|
|
1780
|
+
for (;;) {
|
|
1781
|
+
const snapshot = await this.requireSnapshot(located.snapshot.run.run_id);
|
|
1782
|
+
this.assertCurrentClaim(snapshot, located.attempt, located.claim, request);
|
|
1783
|
+
const attempt = snapshot.attempts.find((candidate) => candidate.attempt_id === located.attempt.attempt_id);
|
|
1784
|
+
const activation = snapshot.activations.find((candidate) => candidate.activation_id === located.activation.activation_id);
|
|
1785
|
+
if (!attempt || !activation)
|
|
1786
|
+
throw new CanonicalRunServiceError("attempt_state_conflict", "driver failure attempt disappeared");
|
|
1787
|
+
if (attempt.phase === "terminal") {
|
|
1788
|
+
this.assertDriverFailureReplay(snapshot, attempt, failureDigest, request);
|
|
1789
|
+
return this.toView(snapshot, located.plan);
|
|
1790
|
+
}
|
|
1791
|
+
const drained = await this.prepareFailurePeerStops(snapshot, located.plan, attempt, outcome, attestation.payload.attested_at);
|
|
1792
|
+
if (drained.record_version !== snapshot.record_version)
|
|
1793
|
+
continue;
|
|
1794
|
+
const update = await this.buildNonlaunchFailureUpdate(snapshot, located.plan, activation, attempt, outcome, failureDigest, attestation);
|
|
1795
|
+
try {
|
|
1796
|
+
const updated = (await this.runStore.updateCanonicalRun(update)).snapshot;
|
|
1797
|
+
return this.toView(updated, located.plan);
|
|
1798
|
+
}
|
|
1799
|
+
catch (error) {
|
|
1800
|
+
if (this.isVersionConflict(error))
|
|
1801
|
+
continue;
|
|
1802
|
+
throw error;
|
|
1803
|
+
}
|
|
1804
|
+
}
|
|
1805
|
+
}
|
|
1806
|
+
assertDriverFailureReplay(snapshot, attempt, failureDigest, request) {
|
|
1807
|
+
const event = snapshot.events.find((candidate) => candidate.type === "attempt_terminal" && candidate.attempt_id === attempt.attempt_id);
|
|
1808
|
+
const attestation = event?.payload.worker_nonlaunch_attestation ?? null;
|
|
1809
|
+
if (attestation?.payload.failure_idempotency_key === failureDigest
|
|
1810
|
+
&& attempt.terminal_outcome?.kind === "worker_process"
|
|
1811
|
+
&& attempt.terminal_outcome.reason_code === request.reason_code
|
|
1812
|
+
&& attempt.terminal_outcome.message === request.message)
|
|
1813
|
+
return;
|
|
1814
|
+
throw new CanonicalRunServiceError("submission_conflict", "attempt has a different committed driver failure");
|
|
1815
|
+
}
|
|
1816
|
+
async inspect(runId) {
|
|
1817
|
+
const snapshot = await this.requireSnapshot(runId);
|
|
1818
|
+
return this.toView(snapshot, await this.requirePinnedPlan(snapshot));
|
|
1819
|
+
}
|
|
1820
|
+
async inspectAudit(runId) {
|
|
1821
|
+
const snapshot = await this.requireSnapshot(runId);
|
|
1822
|
+
const view = this.toView(snapshot, await this.requirePinnedPlan(snapshot));
|
|
1823
|
+
return {
|
|
1824
|
+
...view,
|
|
1825
|
+
events: snapshot.events.map(publicAuditEvent),
|
|
1826
|
+
ledger: {
|
|
1827
|
+
created_at: snapshot.created_at,
|
|
1828
|
+
updated_at: snapshot.updated_at,
|
|
1829
|
+
record_version: snapshot.record_version,
|
|
1830
|
+
},
|
|
1831
|
+
};
|
|
1832
|
+
}
|
|
1833
|
+
async resume(runId) {
|
|
1834
|
+
let snapshot = await this.requireSnapshot(runId);
|
|
1835
|
+
const plan = await this.requirePinnedPlan(snapshot);
|
|
1836
|
+
this.assertVerifierExecutionSupported(plan.plan);
|
|
1837
|
+
snapshot = await this.continueDurableStops(snapshot);
|
|
1838
|
+
snapshot = await this.drainReadySystemNodes(snapshot, plan);
|
|
1839
|
+
return this.toView(snapshot, plan);
|
|
1840
|
+
}
|
|
1841
|
+
async cancel(runIdOrRequest, reason) {
|
|
1842
|
+
const request = typeof runIdOrRequest === "string"
|
|
1843
|
+
? { run_id: runIdOrRequest, reason: reason ?? null }
|
|
1844
|
+
: { run_id: runIdOrRequest.run_id, reason: runIdOrRequest.reason ?? null };
|
|
1845
|
+
let snapshot;
|
|
1846
|
+
let plan;
|
|
1847
|
+
for (;;) {
|
|
1848
|
+
snapshot = await this.requireSnapshot(request.run_id);
|
|
1849
|
+
plan = await this.requirePinnedPlan(snapshot);
|
|
1850
|
+
if (snapshot.run.terminal !== null)
|
|
1851
|
+
return this.toView(snapshot, plan);
|
|
1852
|
+
if (snapshot.run.status === "cancelling")
|
|
1853
|
+
break;
|
|
1854
|
+
const now = this.now();
|
|
1855
|
+
const cancellation = { requested_at: now, reason: request.reason };
|
|
1856
|
+
const cancellingRun = {
|
|
1857
|
+
...snapshot.run,
|
|
1858
|
+
status: "cancelling",
|
|
1859
|
+
cancellation,
|
|
1860
|
+
updated_at: now,
|
|
1861
|
+
};
|
|
1862
|
+
const event = this.event(snapshot, 0, {
|
|
1863
|
+
type: "run_cancel_requested", step_id: null, activation_id: null,
|
|
1864
|
+
attempt_id: null, result_id: null, key: `run:${request.run_id}:cancel-requested`, at: now,
|
|
1865
|
+
payload: { reason: request.reason },
|
|
1866
|
+
});
|
|
1867
|
+
try {
|
|
1868
|
+
snapshot = (await this.runStore.updateCanonicalRun({
|
|
1869
|
+
run_id: request.run_id,
|
|
1870
|
+
expected_record_version: snapshot.record_version,
|
|
1871
|
+
run: cancellingRun,
|
|
1872
|
+
events: [event],
|
|
1873
|
+
})).snapshot;
|
|
1874
|
+
break;
|
|
1875
|
+
}
|
|
1876
|
+
catch (error) {
|
|
1877
|
+
if (this.isVersionConflict(error))
|
|
1878
|
+
continue;
|
|
1879
|
+
throw error;
|
|
1880
|
+
}
|
|
1881
|
+
}
|
|
1882
|
+
snapshot = await this.persistStopIntents(snapshot, this.cancellationStopSpecifications(snapshot));
|
|
1883
|
+
snapshot = await this.continueDurableStops(snapshot);
|
|
1884
|
+
plan = await this.requirePinnedPlan(snapshot);
|
|
1885
|
+
return this.toView(snapshot, plan);
|
|
1886
|
+
}
|
|
1887
|
+
systemInput(snapshot, name) {
|
|
1888
|
+
const value = snapshot.run.run_binding.binding.validated_inputs[name];
|
|
1889
|
+
if (typeof value !== "string" || value.trim() === "") {
|
|
1890
|
+
throw new CanonicalRunServiceError("invalid_request", `system input ${name} must be a non-empty string`);
|
|
1891
|
+
}
|
|
1892
|
+
return value;
|
|
1893
|
+
}
|
|
1894
|
+
async requireOriginResult(current, planId, resultId) {
|
|
1895
|
+
if (!SHA256_DIGEST_PATTERN.test(planId)) {
|
|
1896
|
+
throw new CanonicalRunServiceError("invalid_request", "origin plan id must be a lowercase sha256 digest");
|
|
1897
|
+
}
|
|
1898
|
+
const published = await this.resultStore.loadPublished(resultId);
|
|
1899
|
+
if (!published) {
|
|
1900
|
+
throw new CanonicalRunServiceError("invalid_request", `origin result ${resultId} was not found`);
|
|
1901
|
+
}
|
|
1902
|
+
if (published.result.run_id === current.run.run_id) {
|
|
1903
|
+
throw new CanonicalRunServiceError("invalid_request", "canonical import must reference a previous run");
|
|
1904
|
+
}
|
|
1905
|
+
const snapshot = await this.runStore.loadCanonicalRun(published.result.run_id);
|
|
1906
|
+
if (!snapshot || snapshot.run.plan_id !== planId) {
|
|
1907
|
+
throw new CanonicalRunServiceError("submission_conflict", "origin result does not belong to the declared plan");
|
|
1908
|
+
}
|
|
1909
|
+
const reference = snapshot.result_refs.find((candidate) => candidate.result_id === resultId);
|
|
1910
|
+
if (!reference) {
|
|
1911
|
+
throw new CanonicalRunServiceError("submission_conflict", "origin result is not committed in its run ledger");
|
|
1912
|
+
}
|
|
1913
|
+
const result = await this.resultStore.require(reference);
|
|
1914
|
+
if (canonicalizeJson(result) !== canonicalizeJson(published.result)
|
|
1915
|
+
|| canonicalAttemptOutcomeStatus(result.terminal_outcome) !== "succeeded") {
|
|
1916
|
+
throw new CanonicalRunServiceError("submission_conflict", "origin result is not an immutable successful result");
|
|
1917
|
+
}
|
|
1918
|
+
return { snapshot, result, reference };
|
|
1919
|
+
}
|
|
1920
|
+
async captureImportedOutput(snapshot, activation, attempt, source, sourceOutput, outputName) {
|
|
1921
|
+
if (sourceOutput.kind === "text") {
|
|
1922
|
+
return { output: { ...sourceOutput, name: outputName }, capture: null };
|
|
1923
|
+
}
|
|
1924
|
+
if (sourceOutput.kind === "structured") {
|
|
1925
|
+
return { output: { ...sourceOutput, name: outputName }, capture: null };
|
|
1926
|
+
}
|
|
1927
|
+
const captured = await this.artifactStore.captureExisting({
|
|
1928
|
+
artifact_id: sourceOutput.capture.artifact_id,
|
|
1929
|
+
artifact_kind: sourceOutput.capture.artifact_kind,
|
|
1930
|
+
run_id: snapshot.run.run_id,
|
|
1931
|
+
activation_id: activation.activation_id,
|
|
1932
|
+
attempt_id: attempt.attempt_id,
|
|
1933
|
+
output_name: outputName,
|
|
1934
|
+
source_workspace_id: `canonical:${source.run_id}`,
|
|
1935
|
+
source_path: sourceOutput.capture.source_path,
|
|
1936
|
+
});
|
|
1937
|
+
return {
|
|
1938
|
+
output: { name: outputName, kind: "artifact", capture: captured.capture },
|
|
1939
|
+
capture: captured.capture,
|
|
1940
|
+
};
|
|
1941
|
+
}
|
|
1942
|
+
validateSystemOutputs(plan, node, outputs) {
|
|
1943
|
+
const contracts = new Map((node.output_contract ?? []).map((contract) => [contract.name, contract]));
|
|
1944
|
+
const seen = new Set();
|
|
1945
|
+
for (const output of outputs) {
|
|
1946
|
+
if (seen.has(output.name)) {
|
|
1947
|
+
throw new CanonicalRunServiceError("verification_failed", `system output ${output.name} is duplicated`);
|
|
1948
|
+
}
|
|
1949
|
+
seen.add(output.name);
|
|
1950
|
+
const contract = contracts.get(output.name);
|
|
1951
|
+
if (!contract) {
|
|
1952
|
+
throw new CanonicalRunServiceError("verification_failed", `system output ${output.name} is undeclared`);
|
|
1953
|
+
}
|
|
1954
|
+
const kind = output.kind === "artifact" ? "reference" : output.kind;
|
|
1955
|
+
if (contract.kind !== kind) {
|
|
1956
|
+
throw new CanonicalRunServiceError("verification_failed", `system output ${output.name} kind does not match its contract`);
|
|
1957
|
+
}
|
|
1958
|
+
if (output.kind === "artifact" && contract.artifact_kind !== output.capture.artifact_kind) {
|
|
1959
|
+
throw new CanonicalRunServiceError("verification_failed", `system output ${output.name} artifact kind does not match its contract`);
|
|
1960
|
+
}
|
|
1961
|
+
if (output.kind === "structured") {
|
|
1962
|
+
const validation = validatePinnedStructuredOutput({
|
|
1963
|
+
plan: plan.plan,
|
|
1964
|
+
node_id: node.node_id,
|
|
1965
|
+
output_name: output.name,
|
|
1966
|
+
schema_ref: output.schema_ref,
|
|
1967
|
+
value: output.value,
|
|
1968
|
+
}, this.structuredOutputSchemaRegistry);
|
|
1969
|
+
if (!validation.valid) {
|
|
1970
|
+
throw new CanonicalRunServiceError("verification_failed", `system output ${output.name} failed structured validation at ${validation.path}: ${validation.message}`);
|
|
1971
|
+
}
|
|
1972
|
+
}
|
|
1973
|
+
}
|
|
1974
|
+
for (const contract of contracts.values()) {
|
|
1975
|
+
if (contract.required && !seen.has(contract.name)) {
|
|
1976
|
+
throw new CanonicalRunServiceError("verification_failed", `required system output ${contract.name} is missing`);
|
|
1977
|
+
}
|
|
1978
|
+
}
|
|
1979
|
+
}
|
|
1980
|
+
currentRunResult(snapshot, nodeId) {
|
|
1981
|
+
const activation = this.latestActivationForStep(snapshot.activations, nodeId);
|
|
1982
|
+
if (!activation?.committed_result_id || activation.state !== "succeeded") {
|
|
1983
|
+
throw new CanonicalRunServiceError("attempt_state_conflict", `system source node ${nodeId} has no successful committed result`);
|
|
1984
|
+
}
|
|
1985
|
+
const reference = snapshot.result_refs.find((candidate) => candidate.result_id === activation.committed_result_id);
|
|
1986
|
+
if (!reference) {
|
|
1987
|
+
throw new CanonicalRunServiceError("submission_conflict", `system source result ${activation.committed_result_id} is missing from the run ledger`);
|
|
1988
|
+
}
|
|
1989
|
+
return { resultId: activation.committed_result_id, reference };
|
|
1990
|
+
}
|
|
1991
|
+
systemRequirementSatisfied(snapshot, plan, nodeId) {
|
|
1992
|
+
const node = nodeById(plan.plan, nodeId);
|
|
1993
|
+
if (node.kind === "gate") {
|
|
1994
|
+
return snapshot.events.some((event) => (event.type === "gate_evaluated"
|
|
1995
|
+
&& event.payload.gate_id === nodeId
|
|
1996
|
+
&& event.payload.passed));
|
|
1997
|
+
}
|
|
1998
|
+
if (ATTEMPT_NODE_KINDS.has(node.kind)) {
|
|
1999
|
+
const activation = this.latestActivationForStep(snapshot.activations, nodeId);
|
|
2000
|
+
return activation?.state === "succeeded" && activation.committed_result_id !== null;
|
|
2001
|
+
}
|
|
2002
|
+
return snapshot.events.some((event) => ((event.type === "decision_evaluated" && event.payload.decision_id === nodeId)
|
|
2003
|
+
|| (event.type === "run_terminal" && node.kind === "terminal")));
|
|
2004
|
+
}
|
|
2005
|
+
async executeSystemOperation(snapshot, plan, activation, attempt, node) {
|
|
2006
|
+
const operation = node.operation;
|
|
2007
|
+
if (operation.kind === "resolve_diagnosis_source") {
|
|
2008
|
+
const mode = this.systemInput(snapshot, operation.mode_input);
|
|
2009
|
+
if (!new Set(["fresh", "legacy", "canonical"]).has(mode)) {
|
|
2010
|
+
throw new CanonicalRunServiceError("invalid_request", `unsupported diagnosis source mode ${mode}`);
|
|
2011
|
+
}
|
|
2012
|
+
return {
|
|
2013
|
+
outputs: [{
|
|
2014
|
+
name: operation.output_name,
|
|
2015
|
+
kind: "structured",
|
|
2016
|
+
value: { mode },
|
|
2017
|
+
schema_ref: node.output_contract?.find((output) => output.name === operation.output_name)?.schema_ref ?? null,
|
|
2018
|
+
}],
|
|
2019
|
+
captures: [],
|
|
2020
|
+
evidence_refs: [],
|
|
2021
|
+
system_origin: { kind: "run_input", input_names: [operation.mode_input] },
|
|
2022
|
+
};
|
|
2023
|
+
}
|
|
2024
|
+
if (operation.kind === "adopt_artifact" || operation.kind === "adopt_artifact_bundle") {
|
|
2025
|
+
const artifacts = operation.kind === "adopt_artifact"
|
|
2026
|
+
? [{
|
|
2027
|
+
source_reference_input: operation.source_reference_input,
|
|
2028
|
+
expected_digest_input: operation.expected_digest_input,
|
|
2029
|
+
output_name: operation.output_name,
|
|
2030
|
+
artifact_kind: "file",
|
|
2031
|
+
}]
|
|
2032
|
+
: operation.artifacts;
|
|
2033
|
+
const outputs = [];
|
|
2034
|
+
const captures = [];
|
|
2035
|
+
const sources = [];
|
|
2036
|
+
for (const artifact of artifacts) {
|
|
2037
|
+
const sourcePath = this.systemInput(snapshot, artifact.source_reference_input);
|
|
2038
|
+
const expectedDigest = this.systemInput(snapshot, artifact.expected_digest_input);
|
|
2039
|
+
if (!SHA256_DIGEST_PATTERN.test(expectedDigest)) {
|
|
2040
|
+
throw new CanonicalRunServiceError("invalid_request", `system input ${artifact.expected_digest_input} must be a lowercase sha256 digest`);
|
|
2041
|
+
}
|
|
2042
|
+
const captured = await this.artifactStore.capture({
|
|
2043
|
+
kind: artifact.artifact_kind,
|
|
2044
|
+
workspace_root: snapshot.run.run_binding.binding.project_root,
|
|
2045
|
+
source_path: sourcePath,
|
|
2046
|
+
run_id: snapshot.run.run_id,
|
|
2047
|
+
activation_id: activation.activation_id,
|
|
2048
|
+
attempt_id: attempt.attempt_id,
|
|
2049
|
+
output_name: artifact.output_name,
|
|
2050
|
+
source_workspace_id: `project:${textDigest(snapshot.run.run_binding.binding.project_root)}`,
|
|
2051
|
+
});
|
|
2052
|
+
if (captured.object.artifact_id !== expectedDigest) {
|
|
2053
|
+
throw new CanonicalRunServiceError("source_drift", `legacy artifact ${sourcePath} does not match its expected digest`);
|
|
2054
|
+
}
|
|
2055
|
+
outputs.push({ name: artifact.output_name, kind: "artifact", capture: captured.capture });
|
|
2056
|
+
captures.push(captured.capture);
|
|
2057
|
+
sources.push({ source_path: sourcePath, expected_digest: expectedDigest, artifact_id: captured.object.artifact_id });
|
|
2058
|
+
}
|
|
2059
|
+
return {
|
|
2060
|
+
outputs,
|
|
2061
|
+
captures,
|
|
2062
|
+
evidence_refs: [],
|
|
2063
|
+
system_origin: { kind: "legacy_external", sources },
|
|
2064
|
+
};
|
|
2065
|
+
}
|
|
2066
|
+
if (operation.kind === "import_result" || operation.kind === "import_result_bundle") {
|
|
2067
|
+
const origin = await this.requireOriginResult(snapshot, this.systemInput(snapshot, operation.source_plan_input), this.systemInput(snapshot, operation.source_result_input));
|
|
2068
|
+
const mappings = operation.kind === "import_result"
|
|
2069
|
+
? [{ source_output_name: operation.output_name, output_name: operation.output_name }]
|
|
2070
|
+
: operation.required_outputs;
|
|
2071
|
+
const outputs = [];
|
|
2072
|
+
const captures = [];
|
|
2073
|
+
for (const mapping of mappings) {
|
|
2074
|
+
const sourceOutput = origin.result.outputs.find((output) => output.name === mapping.source_output_name);
|
|
2075
|
+
if (!sourceOutput) {
|
|
2076
|
+
throw new CanonicalRunServiceError("submission_conflict", `origin result lacks output ${mapping.source_output_name}`);
|
|
2077
|
+
}
|
|
2078
|
+
const copied = await this.captureImportedOutput(snapshot, activation, attempt, origin.result, sourceOutput, mapping.output_name);
|
|
2079
|
+
outputs.push(copied.output);
|
|
2080
|
+
if (copied.capture)
|
|
2081
|
+
captures.push(copied.capture);
|
|
2082
|
+
}
|
|
2083
|
+
return {
|
|
2084
|
+
outputs,
|
|
2085
|
+
captures,
|
|
2086
|
+
evidence_refs: [{ kind: "result", result: origin.reference }],
|
|
2087
|
+
system_origin: {
|
|
2088
|
+
kind: "canonical_import",
|
|
2089
|
+
origin_plan_id: origin.snapshot.run.plan_id,
|
|
2090
|
+
origin_run_ids: [origin.result.run_id],
|
|
2091
|
+
origin_result_ids: [origin.result.result_id],
|
|
2092
|
+
origin_artifact_ids: origin.result.outputs.flatMap((output) => output.kind === "artifact" ? [output.capture.artifact_id] : []),
|
|
2093
|
+
policy_id: null,
|
|
2094
|
+
},
|
|
2095
|
+
};
|
|
2096
|
+
}
|
|
2097
|
+
if (operation.kind === "import_review") {
|
|
2098
|
+
const sourcePlanId = this.systemInput(snapshot, operation.source_plan_input);
|
|
2099
|
+
const diagnosis = await this.requireOriginResult(snapshot, sourcePlanId, this.systemInput(snapshot, operation.source_diagnosis_result_input));
|
|
2100
|
+
const review = await this.requireOriginResult(snapshot, sourcePlanId, this.systemInput(snapshot, operation.source_review_result_input));
|
|
2101
|
+
const diagnosisArtifactNames = diagnosis.result.outputs.flatMap((output) => (output.kind === "artifact" ? [output.name] : []));
|
|
2102
|
+
const boundOutputNames = new Set(review.snapshot.events.flatMap((event) => (event.type === "handoff_resolved"
|
|
2103
|
+
&& event.activation_id === review.result.activation_id
|
|
2104
|
+
&& event.payload.source_result_id === diagnosis.result.result_id
|
|
2105
|
+
? [event.payload.output_name]
|
|
2106
|
+
: [])));
|
|
2107
|
+
if (diagnosisArtifactNames.length === 0
|
|
2108
|
+
|| diagnosisArtifactNames.some((name) => !boundOutputNames.has(name))) {
|
|
2109
|
+
throw new CanonicalRunServiceError("submission_conflict", "origin review is not bound to the declared diagnosis result artifacts");
|
|
2110
|
+
}
|
|
2111
|
+
const verdictMapping = operation.required_outputs.find((mapping) => (node.output_contract?.find((contract) => contract.name === mapping.output_name)?.kind === "structured"));
|
|
2112
|
+
const verdict = review.result.outputs.find((output) => (output.kind === "structured" && output.name === verdictMapping?.source_output_name));
|
|
2113
|
+
const verdictValue = verdict?.kind === "structured" ? verdict.value : null;
|
|
2114
|
+
const verdictRecord = verdictValue !== null
|
|
2115
|
+
&& typeof verdictValue === "object"
|
|
2116
|
+
&& !Array.isArray(verdictValue)
|
|
2117
|
+
? verdictValue
|
|
2118
|
+
: null;
|
|
2119
|
+
if (verdict?.kind !== "structured"
|
|
2120
|
+
|| verdictRecord === null
|
|
2121
|
+
|| verdictRecord.verdict !== "APPROVE"
|
|
2122
|
+
|| verdictRecord.contract_valid !== true) {
|
|
2123
|
+
throw new CanonicalRunServiceError("submission_conflict", "origin diagnosis review is not an approved typed verdict");
|
|
2124
|
+
}
|
|
2125
|
+
const outputs = [];
|
|
2126
|
+
const captures = [];
|
|
2127
|
+
for (const mapping of operation.required_outputs) {
|
|
2128
|
+
const sourceOutput = review.result.outputs.find((output) => output.name === mapping.source_output_name);
|
|
2129
|
+
if (!sourceOutput) {
|
|
2130
|
+
throw new CanonicalRunServiceError("submission_conflict", `origin review lacks output ${mapping.source_output_name}`);
|
|
2131
|
+
}
|
|
2132
|
+
const copied = await this.captureImportedOutput(snapshot, activation, attempt, review.result, sourceOutput, mapping.output_name);
|
|
2133
|
+
outputs.push(copied.output);
|
|
2134
|
+
if (copied.capture)
|
|
2135
|
+
captures.push(copied.capture);
|
|
2136
|
+
}
|
|
2137
|
+
const artifactIds = [...diagnosis.result.outputs, ...review.result.outputs].flatMap((output) => (output.kind === "artifact" ? [output.capture.artifact_id] : []));
|
|
2138
|
+
return {
|
|
2139
|
+
outputs,
|
|
2140
|
+
captures,
|
|
2141
|
+
evidence_refs: [
|
|
2142
|
+
{ kind: "result", result: diagnosis.reference },
|
|
2143
|
+
{ kind: "result", result: review.reference },
|
|
2144
|
+
],
|
|
2145
|
+
system_origin: {
|
|
2146
|
+
kind: "canonical_import",
|
|
2147
|
+
origin_plan_id: diagnosis.snapshot.run.plan_id,
|
|
2148
|
+
origin_run_ids: [...new Set([diagnosis.result.run_id, review.result.run_id])],
|
|
2149
|
+
origin_result_ids: [diagnosis.result.result_id, review.result.result_id],
|
|
2150
|
+
origin_artifact_ids: [...new Set(artifactIds)],
|
|
2151
|
+
policy_id: operation.policy_id,
|
|
2152
|
+
},
|
|
2153
|
+
};
|
|
2154
|
+
}
|
|
2155
|
+
if (operation.kind === "seal_result_bundle") {
|
|
2156
|
+
const candidates = operation.candidates.filter((candidate) => (candidate.required_node_ids.every((nodeId) => this.systemRequirementSatisfied(snapshot, plan, nodeId))));
|
|
2157
|
+
if (candidates.length !== 1) {
|
|
2158
|
+
throw new CanonicalRunServiceError("attempt_state_conflict", `system seal requires exactly one satisfied candidate, found ${candidates.length}`);
|
|
2159
|
+
}
|
|
2160
|
+
const selected = candidates[0];
|
|
2161
|
+
const sourceByNode = new Map();
|
|
2162
|
+
for (const mapping of selected.outputs) {
|
|
2163
|
+
if (sourceByNode.has(mapping.source_node_id))
|
|
2164
|
+
continue;
|
|
2165
|
+
const current = this.currentRunResult(snapshot, mapping.source_node_id);
|
|
2166
|
+
sourceByNode.set(mapping.source_node_id, {
|
|
2167
|
+
result: await this.resultStore.require(current.reference),
|
|
2168
|
+
reference: current.reference,
|
|
2169
|
+
});
|
|
2170
|
+
}
|
|
2171
|
+
const outputs = [];
|
|
2172
|
+
const captures = [];
|
|
2173
|
+
for (const mapping of selected.outputs) {
|
|
2174
|
+
const source = sourceByNode.get(mapping.source_node_id);
|
|
2175
|
+
const sourceOutput = source.result.outputs.find((output) => output.name === mapping.source_output_name);
|
|
2176
|
+
if (!sourceOutput) {
|
|
2177
|
+
throw new CanonicalRunServiceError("submission_conflict", `seal source ${mapping.source_node_id} lacks output ${mapping.source_output_name}`);
|
|
2178
|
+
}
|
|
2179
|
+
const copied = await this.captureImportedOutput(snapshot, activation, attempt, source.result, sourceOutput, mapping.output_name);
|
|
2180
|
+
outputs.push(copied.output);
|
|
2181
|
+
if (copied.capture)
|
|
2182
|
+
captures.push(copied.capture);
|
|
2183
|
+
}
|
|
2184
|
+
const sourceResults = [...sourceByNode.values()];
|
|
2185
|
+
if (operation.origin_output_name !== undefined) {
|
|
2186
|
+
const primary = sourceResults[0]?.result;
|
|
2187
|
+
const mode = primary?.executor.kind === "worker"
|
|
2188
|
+
? "fresh"
|
|
2189
|
+
: primary?.system_origin?.kind === "legacy_external"
|
|
2190
|
+
? "legacy"
|
|
2191
|
+
: primary?.system_origin?.kind === "canonical_import"
|
|
2192
|
+
? "canonical"
|
|
2193
|
+
: null;
|
|
2194
|
+
if (!primary || mode === null) {
|
|
2195
|
+
throw new CanonicalRunServiceError("submission_conflict", "diagnosis seal cannot determine its typed origin");
|
|
2196
|
+
}
|
|
2197
|
+
outputs.push({
|
|
2198
|
+
name: operation.origin_output_name,
|
|
2199
|
+
kind: "structured",
|
|
2200
|
+
value: { mode, source_node_id: primary.step_id, source_result_id: primary.result_id },
|
|
2201
|
+
schema_ref: node.output_contract?.find((output) => output.name === operation.origin_output_name)?.schema_ref ?? null,
|
|
2202
|
+
});
|
|
2203
|
+
}
|
|
2204
|
+
return {
|
|
2205
|
+
outputs,
|
|
2206
|
+
captures,
|
|
2207
|
+
evidence_refs: sourceResults.map((source) => ({ kind: "result", result: source.reference })),
|
|
2208
|
+
system_origin: {
|
|
2209
|
+
kind: "current_run_results",
|
|
2210
|
+
source_result_ids: sourceResults.map((source) => source.result.result_id),
|
|
2211
|
+
},
|
|
2212
|
+
};
|
|
2213
|
+
}
|
|
2214
|
+
if (operation.kind === "materialize_handoff") {
|
|
2215
|
+
const handoff = plan.plan.handoff_plan.find((candidate) => candidate.handoff_id === operation.handoff_id);
|
|
2216
|
+
if (!handoff) {
|
|
2217
|
+
throw new CanonicalRunServiceError("invalid_request", `handoff ${operation.handoff_id} is unavailable`);
|
|
2218
|
+
}
|
|
2219
|
+
const source = this.currentRunResult(snapshot, handoff.from_node_id);
|
|
2220
|
+
return {
|
|
2221
|
+
outputs: [],
|
|
2222
|
+
captures: [],
|
|
2223
|
+
evidence_refs: [{ kind: "result", result: source.reference }],
|
|
2224
|
+
system_origin: { kind: "current_run_results", source_result_ids: [source.resultId] },
|
|
2225
|
+
};
|
|
2226
|
+
}
|
|
2227
|
+
const exhaustive = operation;
|
|
2228
|
+
throw new CanonicalRunServiceError("unsupported", `unsupported system operation ${exhaustive.kind}`);
|
|
2229
|
+
}
|
|
2230
|
+
async beginSystemAttempt(snapshot, plan, activation) {
|
|
2231
|
+
const node = nodeById(plan.plan, activation.step_id);
|
|
2232
|
+
if (node.kind !== "system" || activation.state !== "ready")
|
|
2233
|
+
return snapshot;
|
|
2234
|
+
const now = this.now();
|
|
2235
|
+
const attemptNumber = activation.attempt_count + 1;
|
|
2236
|
+
const attemptId = this.identities.create("attempt", `${activation.activation_id}\0${attemptNumber}`);
|
|
2237
|
+
const attempt = {
|
|
2238
|
+
run_id: snapshot.run.run_id,
|
|
2239
|
+
step_id: activation.step_id,
|
|
2240
|
+
activation_id: activation.activation_id,
|
|
2241
|
+
attempt_id: attemptId,
|
|
2242
|
+
attempt_number: attemptNumber,
|
|
2243
|
+
phase: "claimed",
|
|
2244
|
+
executor: null,
|
|
2245
|
+
started_at: null,
|
|
2246
|
+
completed_at: null,
|
|
2247
|
+
terminal_outcome: null,
|
|
2248
|
+
};
|
|
2249
|
+
const claim = this.createClaim(snapshot, plan, attempt, `system:${this.serviceInstanceId}`, now, 1);
|
|
2250
|
+
const claimedActivation = {
|
|
2251
|
+
...activation,
|
|
2252
|
+
state: "claimed",
|
|
2253
|
+
attempt_count: attemptNumber,
|
|
2254
|
+
current_attempt_id: attemptId,
|
|
2255
|
+
updated_at: now,
|
|
2256
|
+
};
|
|
2257
|
+
const events = [
|
|
2258
|
+
this.activationEvent(snapshot, 0, activation, claimedActivation, "system_claimed", now),
|
|
2259
|
+
this.event(snapshot, 1, {
|
|
2260
|
+
type: "attempt_claimed",
|
|
2261
|
+
step_id: attempt.step_id,
|
|
2262
|
+
activation_id: attempt.activation_id,
|
|
2263
|
+
attempt_id: attempt.attempt_id,
|
|
2264
|
+
result_id: null,
|
|
2265
|
+
key: `attempt:${attempt.attempt_id}:claim:1`,
|
|
2266
|
+
at: now,
|
|
2267
|
+
payload: {
|
|
2268
|
+
claim_epoch: 1,
|
|
2269
|
+
owner_id: claim.owner_id,
|
|
2270
|
+
claim_expires_at: claim.driver_claim_lease.expires_at,
|
|
2271
|
+
execution_deadline_at: claim.execution_deadline_at,
|
|
2272
|
+
},
|
|
2273
|
+
}),
|
|
2274
|
+
];
|
|
2275
|
+
try {
|
|
2276
|
+
return (await this.runStore.updateCanonicalRun({
|
|
2277
|
+
run_id: snapshot.run.run_id,
|
|
2278
|
+
expected_record_version: snapshot.record_version,
|
|
2279
|
+
activations: [claimedActivation],
|
|
2280
|
+
attempts: [attempt],
|
|
2281
|
+
claims: [claim],
|
|
2282
|
+
events,
|
|
2283
|
+
})).snapshot;
|
|
2284
|
+
}
|
|
2285
|
+
catch (error) {
|
|
2286
|
+
if (this.isVersionConflict(error))
|
|
2287
|
+
return this.requireSnapshot(snapshot.run.run_id);
|
|
2288
|
+
throw error;
|
|
2289
|
+
}
|
|
2290
|
+
}
|
|
2291
|
+
async attachSystemAttempt(snapshot, plan, attempt) {
|
|
2292
|
+
if (attempt.phase !== "claimed")
|
|
2293
|
+
return snapshot;
|
|
2294
|
+
const activation = snapshot.activations.find((candidate) => candidate.activation_id === attempt.activation_id);
|
|
2295
|
+
const node = activation ? nodeById(plan.plan, activation.step_id) : null;
|
|
2296
|
+
if (!activation || activation.state !== "claimed" || node?.kind !== "system")
|
|
2297
|
+
return snapshot;
|
|
2298
|
+
const now = this.now();
|
|
2299
|
+
const executor = {
|
|
2300
|
+
kind: "system",
|
|
2301
|
+
service_instance_id: this.serviceInstanceId,
|
|
2302
|
+
operation: node.operation.kind,
|
|
2303
|
+
};
|
|
2304
|
+
const runningAttempt = {
|
|
2305
|
+
...attempt,
|
|
2306
|
+
phase: "running",
|
|
2307
|
+
executor,
|
|
2308
|
+
started_at: now,
|
|
2309
|
+
};
|
|
2310
|
+
const runningActivation = { ...activation, state: "running", updated_at: now };
|
|
2311
|
+
const claim = this.claimsFor(snapshot, attempt.attempt_id).at(-1);
|
|
2312
|
+
if (!claim) {
|
|
2313
|
+
throw new CanonicalRunServiceError("submission_conflict", "claimed system attempt has no durable claim");
|
|
2314
|
+
}
|
|
2315
|
+
try {
|
|
2316
|
+
return (await this.runStore.updateCanonicalRun({
|
|
2317
|
+
run_id: snapshot.run.run_id,
|
|
2318
|
+
expected_record_version: snapshot.record_version,
|
|
2319
|
+
activations: [runningActivation],
|
|
2320
|
+
attempts: [runningAttempt],
|
|
2321
|
+
events: [
|
|
2322
|
+
this.activationEvent(snapshot, 0, activation, runningActivation, "system_started", now),
|
|
2323
|
+
this.event(snapshot, 1, {
|
|
2324
|
+
type: "attempt_attached",
|
|
2325
|
+
step_id: attempt.step_id,
|
|
2326
|
+
activation_id: attempt.activation_id,
|
|
2327
|
+
attempt_id: attempt.attempt_id,
|
|
2328
|
+
result_id: null,
|
|
2329
|
+
key: `attempt:${attempt.attempt_id}:attached`,
|
|
2330
|
+
at: now,
|
|
2331
|
+
payload: { claim_epoch: claim.claim_epoch, executor },
|
|
2332
|
+
}),
|
|
2333
|
+
this.event(snapshot, 2, {
|
|
2334
|
+
type: "attempt_started",
|
|
2335
|
+
step_id: attempt.step_id,
|
|
2336
|
+
activation_id: attempt.activation_id,
|
|
2337
|
+
attempt_id: attempt.attempt_id,
|
|
2338
|
+
result_id: null,
|
|
2339
|
+
key: `attempt:${attempt.attempt_id}:started`,
|
|
2340
|
+
at: now,
|
|
2341
|
+
payload: { executor },
|
|
2342
|
+
}),
|
|
2343
|
+
],
|
|
2344
|
+
})).snapshot;
|
|
2345
|
+
}
|
|
2346
|
+
catch (error) {
|
|
2347
|
+
if (this.isVersionConflict(error))
|
|
2348
|
+
return this.requireSnapshot(snapshot.run.run_id);
|
|
2349
|
+
throw error;
|
|
2350
|
+
}
|
|
2351
|
+
}
|
|
2352
|
+
async reconcilePublishedSystemResult(initial, plan, result) {
|
|
2353
|
+
for (;;) {
|
|
2354
|
+
const snapshot = await this.requireSnapshot(initial.run.run_id);
|
|
2355
|
+
if (snapshot.result_refs.some((reference) => reference.result_id === result.result_id))
|
|
2356
|
+
return snapshot;
|
|
2357
|
+
const attempt = snapshot.attempts.find((candidate) => candidate.attempt_id === result.attempt_id);
|
|
2358
|
+
const activation = snapshot.activations.find((candidate) => candidate.activation_id === result.activation_id);
|
|
2359
|
+
const node = activation ? nodeById(plan.plan, activation.step_id) : null;
|
|
2360
|
+
if (!attempt
|
|
2361
|
+
|| !activation
|
|
2362
|
+
|| node?.kind !== "system"
|
|
2363
|
+
|| attempt.phase !== "running"
|
|
2364
|
+
|| attempt.executor?.kind !== "system"
|
|
2365
|
+
|| result.executor.kind !== "system"
|
|
2366
|
+
|| canonicalizeJson(attempt.executor) !== canonicalizeJson(result.executor)) {
|
|
2367
|
+
throw new CanonicalRunServiceError("submission_conflict", `published system result ${result.result_id} cannot reconcile to its attempt`);
|
|
2368
|
+
}
|
|
2369
|
+
const published = await this.resultStore.loadPublished(result.result_id);
|
|
2370
|
+
if (!published || canonicalizeJson(published.result) !== canonicalizeJson(result)) {
|
|
2371
|
+
throw new CanonicalRunServiceError("submission_conflict", "published system result changed during reconciliation");
|
|
2372
|
+
}
|
|
2373
|
+
if (canonicalAttemptOutcomeStatus(result.terminal_outcome) === "succeeded") {
|
|
2374
|
+
this.validateSystemOutputs(plan, node, result.outputs);
|
|
2375
|
+
this.validateDecisionOutputs(plan.plan, node.node_id, result.outputs);
|
|
2376
|
+
}
|
|
2377
|
+
const submission = this.submissionEvidence(result);
|
|
2378
|
+
const decisionOutputs = await this.loadDecisionOutputs(snapshot, plan, result);
|
|
2379
|
+
const update = await this.buildSubmissionUpdate(snapshot, plan, activation, attempt, result, published.reference, result.outputs.flatMap((output) => output.kind === "artifact" ? [output.capture] : []), [], submission.submission_key_digest, submission.submission_digest, this.now(), decisionOutputs, {
|
|
2380
|
+
worker_stop_receipt_digest: null,
|
|
2381
|
+
worker_stop_receipt: null,
|
|
2382
|
+
workspace_authority: null,
|
|
2383
|
+
purpose: "system_operation",
|
|
2384
|
+
});
|
|
2385
|
+
try {
|
|
2386
|
+
return (await this.runStore.updateCanonicalRun(update)).snapshot;
|
|
2387
|
+
}
|
|
2388
|
+
catch (error) {
|
|
2389
|
+
if (this.isVersionConflict(error))
|
|
2390
|
+
continue;
|
|
2391
|
+
throw error;
|
|
2392
|
+
}
|
|
2393
|
+
}
|
|
2394
|
+
}
|
|
2395
|
+
async completeSystemAttempt(snapshot, plan, attempt) {
|
|
2396
|
+
const activation = snapshot.activations.find((candidate) => candidate.activation_id === attempt.activation_id);
|
|
2397
|
+
const node = activation ? nodeById(plan.plan, activation.step_id) : null;
|
|
2398
|
+
if (!activation || node?.kind !== "system" || attempt.phase !== "running" || attempt.executor?.kind !== "system") {
|
|
2399
|
+
return snapshot;
|
|
2400
|
+
}
|
|
2401
|
+
const resultId = resultIdFor(this.identities, attempt.attempt_id);
|
|
2402
|
+
const existing = await this.resultStore.loadPublished(resultId);
|
|
2403
|
+
if (existing)
|
|
2404
|
+
return this.reconcilePublishedSystemResult(snapshot, plan, existing.result);
|
|
2405
|
+
let executed = null;
|
|
2406
|
+
let outcome = { kind: "succeeded" };
|
|
2407
|
+
try {
|
|
2408
|
+
executed = await this.executeSystemOperation(snapshot, plan, activation, attempt, node);
|
|
2409
|
+
this.validateSystemOutputs(plan, node, executed.outputs);
|
|
2410
|
+
this.validateDecisionOutputs(plan.plan, node.node_id, executed.outputs);
|
|
2411
|
+
}
|
|
2412
|
+
catch (error) {
|
|
2413
|
+
const code = error instanceof CanonicalRunServiceError ? error.code : "system_operation_failed";
|
|
2414
|
+
outcome = {
|
|
2415
|
+
kind: "failed",
|
|
2416
|
+
reason_code: `system_${code}`,
|
|
2417
|
+
message: error instanceof Error ? error.message : String(error),
|
|
2418
|
+
};
|
|
2419
|
+
}
|
|
2420
|
+
const completedAt = this.now();
|
|
2421
|
+
const submissionKeyDigest = digest({
|
|
2422
|
+
kind: "system_operation",
|
|
2423
|
+
run_id: snapshot.run.run_id,
|
|
2424
|
+
attempt_id: attempt.attempt_id,
|
|
2425
|
+
operation: node.operation.kind,
|
|
2426
|
+
});
|
|
2427
|
+
const submissionDigest = digest({
|
|
2428
|
+
operation: node.operation,
|
|
2429
|
+
outcome,
|
|
2430
|
+
outputs: executed?.outputs ?? [],
|
|
2431
|
+
system_origin: executed?.system_origin ?? null,
|
|
2432
|
+
});
|
|
2433
|
+
const result = {
|
|
2434
|
+
schema_version: CANONICAL_RESULT_SCHEMA_VERSION,
|
|
2435
|
+
run_id: snapshot.run.run_id,
|
|
2436
|
+
step_id: attempt.step_id,
|
|
2437
|
+
activation_id: attempt.activation_id,
|
|
2438
|
+
attempt_id: attempt.attempt_id,
|
|
2439
|
+
result_id: resultId,
|
|
2440
|
+
executor: attempt.executor,
|
|
2441
|
+
terminal_outcome: outcome,
|
|
2442
|
+
outputs: executed?.outputs ?? [],
|
|
2443
|
+
verification_results: [],
|
|
2444
|
+
started_at: attempt.started_at,
|
|
2445
|
+
completed_at: completedAt,
|
|
2446
|
+
committed_at: completedAt,
|
|
2447
|
+
stdout_reference: null,
|
|
2448
|
+
stderr_reference: null,
|
|
2449
|
+
evidence_refs: [
|
|
2450
|
+
...(executed?.evidence_refs ?? []),
|
|
2451
|
+
...(executed?.captures ?? []).map((capture) => ({ kind: "artifact_capture", capture })),
|
|
2452
|
+
{ kind: "submission", submission_key_digest: submissionKeyDigest, submission_digest: submissionDigest },
|
|
2453
|
+
],
|
|
2454
|
+
system_origin: executed?.system_origin ?? null,
|
|
2455
|
+
};
|
|
2456
|
+
try {
|
|
2457
|
+
const published = await this.resultStore.publish(result);
|
|
2458
|
+
if (published.published)
|
|
2459
|
+
await this.hooks.after_result_publish?.(published.result);
|
|
2460
|
+
return this.reconcilePublishedSystemResult(snapshot, plan, published.result);
|
|
2461
|
+
}
|
|
2462
|
+
catch (error) {
|
|
2463
|
+
if (error instanceof ResultStoreError && error.code === "immutable_conflict") {
|
|
2464
|
+
const winner = await this.resultStore.loadPublished(resultId);
|
|
2465
|
+
if (winner)
|
|
2466
|
+
return this.reconcilePublishedSystemResult(snapshot, plan, winner.result);
|
|
2467
|
+
}
|
|
2468
|
+
throw error;
|
|
2469
|
+
}
|
|
2470
|
+
}
|
|
2471
|
+
async drainReadySystemNodes(initial, plan) {
|
|
2472
|
+
let snapshot = initial;
|
|
2473
|
+
for (;;) {
|
|
2474
|
+
snapshot = await this.reconcilePublishedResults(snapshot, plan);
|
|
2475
|
+
snapshot = await this.reduceWithoutResult(snapshot, plan);
|
|
2476
|
+
if (snapshot.run.terminal !== null || snapshot.run.status === "cancelling")
|
|
2477
|
+
return snapshot;
|
|
2478
|
+
const running = snapshot.attempts.find((attempt) => (attempt.phase !== "terminal"
|
|
2479
|
+
&& nodeById(plan.plan, attempt.step_id).kind === "system"));
|
|
2480
|
+
if (running) {
|
|
2481
|
+
snapshot = running.phase === "claimed"
|
|
2482
|
+
? await this.attachSystemAttempt(snapshot, plan, running)
|
|
2483
|
+
: await this.completeSystemAttempt(snapshot, plan, running);
|
|
2484
|
+
continue;
|
|
2485
|
+
}
|
|
2486
|
+
const ready = snapshot.activations.find((activation) => (activation.state === "ready"
|
|
2487
|
+
&& nodeById(plan.plan, activation.step_id).kind === "system"));
|
|
2488
|
+
if (!ready)
|
|
2489
|
+
return snapshot;
|
|
2490
|
+
snapshot = await this.beginSystemAttempt(snapshot, plan, ready);
|
|
2491
|
+
}
|
|
2492
|
+
}
|
|
2493
|
+
now() {
|
|
2494
|
+
return this.clock.now().toISOString();
|
|
2495
|
+
}
|
|
2496
|
+
activationId(runId, nodeId) {
|
|
2497
|
+
return this.identities.create("activation", `${runId}\0${nodeId}\0${0}`);
|
|
2498
|
+
}
|
|
2499
|
+
activationRoundId(runId, nodeId, round) {
|
|
2500
|
+
return this.identities.create("activation", `${runId}\0${nodeId}\0${round - 1}`);
|
|
2501
|
+
}
|
|
2502
|
+
async requireSnapshot(runId) {
|
|
2503
|
+
try {
|
|
2504
|
+
return await this.runStore.requireCanonicalRun(runId);
|
|
2505
|
+
}
|
|
2506
|
+
catch (error) {
|
|
2507
|
+
if (error instanceof RunStoreError && error.code === "run_not_found") {
|
|
2508
|
+
throw new CanonicalRunServiceError("run_not_found", `run ${runId} was not found`, { cause: error });
|
|
2509
|
+
}
|
|
2510
|
+
throw error;
|
|
2511
|
+
}
|
|
2512
|
+
}
|
|
2513
|
+
async requirePinnedPlan(snapshot) {
|
|
2514
|
+
const plan = await this.planStore.requirePlan(snapshot.run.plan_id);
|
|
2515
|
+
if (plan.plan_digest !== snapshot.run.plan_digest) {
|
|
2516
|
+
throw new CanonicalRunServiceError("invalid_request", "pinned plan digest disagrees with run ledger");
|
|
2517
|
+
}
|
|
2518
|
+
return plan;
|
|
2519
|
+
}
|
|
2520
|
+
toView(snapshot, plan) {
|
|
2521
|
+
return {
|
|
2522
|
+
run: snapshot.run,
|
|
2523
|
+
plan,
|
|
2524
|
+
activations: snapshot.activations,
|
|
2525
|
+
attempts: snapshot.attempts,
|
|
2526
|
+
claims: snapshot.claims.map(({ claim_token: _claimToken, ...claim }) => claim),
|
|
2527
|
+
results: snapshot.result_refs,
|
|
2528
|
+
record_version: snapshot.record_version,
|
|
2529
|
+
};
|
|
2530
|
+
}
|
|
2531
|
+
claimsFor(snapshot, attemptId) {
|
|
2532
|
+
return snapshot.claims
|
|
2533
|
+
.filter((claim) => claim.attempt_id === attemptId)
|
|
2534
|
+
.sort((left, right) => left.claim_epoch - right.claim_epoch);
|
|
2535
|
+
}
|
|
2536
|
+
assertCurrentClaim(snapshot, attempt, claim, request) {
|
|
2537
|
+
const activation = snapshot.activations.find((candidate) => candidate.activation_id === attempt.activation_id);
|
|
2538
|
+
const latest = this.claimsFor(snapshot, attempt.attempt_id).at(-1);
|
|
2539
|
+
if (activation?.current_attempt_id !== attempt.attempt_id
|
|
2540
|
+
|| latest === undefined
|
|
2541
|
+
|| latest.claim_token !== request.lease_token
|
|
2542
|
+
|| latest.owner_id !== request.owner_id
|
|
2543
|
+
|| latest.claim_epoch !== claim.claim_epoch) {
|
|
2544
|
+
throw new CanonicalRunServiceError("stale_lease", "lease epoch changed while obtaining execution authority");
|
|
2545
|
+
}
|
|
2546
|
+
}
|
|
2547
|
+
attemptRecovery(snapshot, attempt) {
|
|
2548
|
+
if (attempt.executor?.kind !== "worker" || attempt.started_at === null) {
|
|
2549
|
+
throw new CanonicalRunServiceError("attempt_state_conflict", `running attempt ${attempt.attempt_id} lacks a durable worker execution identity`);
|
|
2550
|
+
}
|
|
2551
|
+
const attached = snapshot.events.find((event) => event.type === "attempt_attached" && event.attempt_id === attempt.attempt_id);
|
|
2552
|
+
if (!attached) {
|
|
2553
|
+
throw new CanonicalRunServiceError("attempt_state_conflict", `running attempt ${attempt.attempt_id} lacks a durable attach event`);
|
|
2554
|
+
}
|
|
2555
|
+
return {
|
|
2556
|
+
worker_session_id: attempt.executor.worker_session_id,
|
|
2557
|
+
execution_claim_epoch: attached.payload.claim_epoch,
|
|
2558
|
+
};
|
|
2559
|
+
}
|
|
2560
|
+
async runningAttemptIsRecoverable(snapshot, attempt, latestClaim, now, requestingDriverId) {
|
|
2561
|
+
if (Date.parse(latestClaim.driver_claim_lease.expires_at) <= Date.parse(now))
|
|
2562
|
+
return true;
|
|
2563
|
+
if (latestClaim.owner_id === requestingDriverId)
|
|
2564
|
+
return false;
|
|
2565
|
+
if (this.executionAuthority?.probeRecovery === undefined)
|
|
2566
|
+
return false;
|
|
2567
|
+
const recovery = this.attemptRecovery(snapshot, attempt);
|
|
2568
|
+
try {
|
|
2569
|
+
const state = await this.executionAuthority.probeRecovery({
|
|
2570
|
+
run_id: attempt.run_id,
|
|
2571
|
+
step_id: attempt.step_id,
|
|
2572
|
+
activation_id: attempt.activation_id,
|
|
2573
|
+
attempt_id: attempt.attempt_id,
|
|
2574
|
+
execution_claim_epoch: recovery.execution_claim_epoch,
|
|
2575
|
+
executor: { kind: "worker", worker_session_id: recovery.worker_session_id },
|
|
2576
|
+
started_at: attempt.started_at,
|
|
2577
|
+
});
|
|
2578
|
+
return state.raw_completion_durable;
|
|
2579
|
+
}
|
|
2580
|
+
catch (error) {
|
|
2581
|
+
throw new CanonicalRunServiceError("unsupported", `running attempt ${attempt.attempt_id} recovery authority is unavailable`, { cause: error });
|
|
2582
|
+
}
|
|
2583
|
+
}
|
|
2584
|
+
createClaim(snapshot, plan, attempt, ownerId, now, epoch) {
|
|
2585
|
+
const node = attempt.step_id;
|
|
2586
|
+
const definition = nodeById(plan.plan, node);
|
|
2587
|
+
const planTimeout = definition.kind === "worker"
|
|
2588
|
+
? definition.attempt_policy.timeout_ms ?? this.defaultExecutionTimeoutMs
|
|
2589
|
+
: this.defaultExecutionTimeoutMs;
|
|
2590
|
+
const deadline = epoch === 1
|
|
2591
|
+
? addMilliseconds(now, planTimeout)
|
|
2592
|
+
: this.claimsFor(snapshot, attempt.attempt_id)[0].execution_deadline_at;
|
|
2593
|
+
const expires = new Date(Math.min(Date.parse(deadline), Date.parse(now) + this.leaseDurationMs)).toISOString();
|
|
2594
|
+
return {
|
|
2595
|
+
run_id: snapshot.run.run_id,
|
|
2596
|
+
step_id: node,
|
|
2597
|
+
activation_id: attempt.activation_id,
|
|
2598
|
+
attempt_id: attempt.attempt_id,
|
|
2599
|
+
claim_token: randomUUID(),
|
|
2600
|
+
claim_epoch: epoch,
|
|
2601
|
+
owner_id: ownerId,
|
|
2602
|
+
claimed_record_version: snapshot.record_version,
|
|
2603
|
+
driver_claim_lease: { acquired_at: now, heartbeat_at: now, expires_at: expires },
|
|
2604
|
+
execution_deadline_at: deadline,
|
|
2605
|
+
};
|
|
2606
|
+
}
|
|
2607
|
+
toLease(_snapshot, attempt, claim) {
|
|
2608
|
+
return {
|
|
2609
|
+
run_id: attempt.run_id,
|
|
2610
|
+
step_id: attempt.step_id,
|
|
2611
|
+
activation_id: attempt.activation_id,
|
|
2612
|
+
attempt_id: attempt.attempt_id,
|
|
2613
|
+
attempt_number: attempt.attempt_number,
|
|
2614
|
+
claim_token: claim.claim_token,
|
|
2615
|
+
claim_epoch: claim.claim_epoch,
|
|
2616
|
+
owner_id: claim.owner_id,
|
|
2617
|
+
claim_expires_at: claim.driver_claim_lease.expires_at,
|
|
2618
|
+
execution_deadline_at: claim.execution_deadline_at,
|
|
2619
|
+
};
|
|
2620
|
+
}
|
|
2621
|
+
toLeaseView(snapshot, attempt, claim) {
|
|
2622
|
+
const { claim_token: _claimToken, ...lease } = this.toLease(snapshot, attempt, claim);
|
|
2623
|
+
return lease;
|
|
2624
|
+
}
|
|
2625
|
+
async recoverExpiredAttempt(snapshot, plan, attempt, now) {
|
|
2626
|
+
const activation = snapshot.activations.find((candidate) => candidate.activation_id === attempt.activation_id);
|
|
2627
|
+
const node = nodeById(plan.plan, attempt.step_id);
|
|
2628
|
+
const retryable = node.kind === "worker"
|
|
2629
|
+
&& attempt.attempt_number < node.attempt_policy.max_attempts
|
|
2630
|
+
&& node.attempt_policy.retryable_outcomes.includes("timeout");
|
|
2631
|
+
const executor = attempt.executor ?? {
|
|
2632
|
+
kind: "system",
|
|
2633
|
+
service_instance_id: this.serviceInstanceId,
|
|
2634
|
+
operation: "expire_unattached_attempt",
|
|
2635
|
+
};
|
|
2636
|
+
const timeoutMs = Math.max(1, Date.parse(this.claimsFor(snapshot, attempt.attempt_id).at(-1).execution_deadline_at)
|
|
2637
|
+
- Date.parse(attempt.started_at ?? this.claimsFor(snapshot, attempt.attempt_id).at(-1).driver_claim_lease.acquired_at));
|
|
2638
|
+
const outcome = executor.kind === "worker"
|
|
2639
|
+
? {
|
|
2640
|
+
kind: "worker_process",
|
|
2641
|
+
status: "failed",
|
|
2642
|
+
reason_code: "execution_deadline_elapsed",
|
|
2643
|
+
message: "worker execution deadline elapsed",
|
|
2644
|
+
process: {
|
|
2645
|
+
kind: "timeout",
|
|
2646
|
+
exit_code: null,
|
|
2647
|
+
signal: null,
|
|
2648
|
+
timeout_ms: timeoutMs,
|
|
2649
|
+
},
|
|
2650
|
+
}
|
|
2651
|
+
: retryable
|
|
2652
|
+
? { kind: "failed", reason_code: "execution_deadline_elapsed", message: null }
|
|
2653
|
+
: { kind: "cancelled", reason: "execution deadline elapsed before attach" };
|
|
2654
|
+
if (executor.kind === "worker") {
|
|
2655
|
+
const specifications = snapshot.attempts.flatMap((candidate) => {
|
|
2656
|
+
if (candidate.phase === "terminal" || candidate.executor?.kind !== "worker")
|
|
2657
|
+
return [];
|
|
2658
|
+
const source = candidate.attempt_id === attempt.attempt_id;
|
|
2659
|
+
const peerOutcome = source ? outcome : {
|
|
2660
|
+
kind: "worker_process",
|
|
2661
|
+
status: "cancelled",
|
|
2662
|
+
reason_code: "execution_deadline_elapsed",
|
|
2663
|
+
message: null,
|
|
2664
|
+
process: {
|
|
2665
|
+
kind: "cancelled",
|
|
2666
|
+
exit_code: null,
|
|
2667
|
+
signal: null,
|
|
2668
|
+
reason: "execution_deadline_elapsed",
|
|
2669
|
+
},
|
|
2670
|
+
};
|
|
2671
|
+
return [{
|
|
2672
|
+
attempt: candidate,
|
|
2673
|
+
purpose: source ? "execution_deadline" : "peer_terminal",
|
|
2674
|
+
requestedAt: now,
|
|
2675
|
+
outcome: peerOutcome,
|
|
2676
|
+
}];
|
|
2677
|
+
});
|
|
2678
|
+
snapshot = await this.persistStopIntents(snapshot, specifications);
|
|
2679
|
+
return this.continueDurableStops(snapshot);
|
|
2680
|
+
}
|
|
2681
|
+
const terminalAttempt = {
|
|
2682
|
+
...attempt,
|
|
2683
|
+
phase: "terminal",
|
|
2684
|
+
executor,
|
|
2685
|
+
started_at: attempt.started_at ?? now,
|
|
2686
|
+
completed_at: now,
|
|
2687
|
+
terminal_outcome: outcome,
|
|
2688
|
+
};
|
|
2689
|
+
const nextState = retryable
|
|
2690
|
+
? "ready"
|
|
2691
|
+
: attempt.phase === "running" ? "failed" : "cancelled";
|
|
2692
|
+
const nextActivation = {
|
|
2693
|
+
...activation,
|
|
2694
|
+
state: nextState,
|
|
2695
|
+
updated_at: now,
|
|
2696
|
+
};
|
|
2697
|
+
const events = [];
|
|
2698
|
+
let offset = 0;
|
|
2699
|
+
if (attempt.started_at === null) {
|
|
2700
|
+
events.push(this.event(snapshot, offset++, {
|
|
2701
|
+
type: "attempt_started", step_id: attempt.step_id,
|
|
2702
|
+
activation_id: attempt.activation_id, attempt_id: attempt.attempt_id,
|
|
2703
|
+
result_id: null, key: `attempt:${attempt.attempt_id}:deadline-started`, at: now,
|
|
2704
|
+
payload: { executor },
|
|
2705
|
+
}));
|
|
2706
|
+
}
|
|
2707
|
+
events.push(this.event(snapshot, offset++, {
|
|
2708
|
+
type: "attempt_terminal", step_id: attempt.step_id,
|
|
2709
|
+
activation_id: attempt.activation_id, attempt_id: attempt.attempt_id,
|
|
2710
|
+
result_id: null, key: `attempt:${attempt.attempt_id}:deadline-terminal`, at: now,
|
|
2711
|
+
payload: { executor, outcome, worker_stop_receipt: null, worker_nonlaunch_attestation: null },
|
|
2712
|
+
}));
|
|
2713
|
+
events.push(this.activationEvent(snapshot, offset++, activation, nextActivation, retryable ? "execution_deadline_retry" : "execution_deadline_exhausted", now));
|
|
2714
|
+
let run = snapshot.run;
|
|
2715
|
+
const activations = [nextActivation];
|
|
2716
|
+
const attempts = [terminalAttempt];
|
|
2717
|
+
if (!retryable) {
|
|
2718
|
+
run = this.terminalRun(snapshot.run, "failed", "execution_deadline_elapsed", now);
|
|
2719
|
+
const cancelledAttempts = await this.cancelConcurrentAttempts(snapshot, activation.activation_id, now, "execution_deadline_elapsed", events.length);
|
|
2720
|
+
attempts.push(...cancelledAttempts.attempts);
|
|
2721
|
+
events.push(...cancelledAttempts.events);
|
|
2722
|
+
offset = events.length;
|
|
2723
|
+
for (const other of snapshot.activations) {
|
|
2724
|
+
if (other.activation_id === activation.activation_id || TERMINAL_ACTIVATION_STATES.has(other.state))
|
|
2725
|
+
continue;
|
|
2726
|
+
const cancelled = { ...other, state: "cancelled", updated_at: now };
|
|
2727
|
+
activations.push(cancelled);
|
|
2728
|
+
events.push(this.activationEvent(snapshot, offset++, other, cancelled, "run_failed", now));
|
|
2729
|
+
}
|
|
2730
|
+
events.push(this.runTerminalEvent(snapshot, offset, run, now));
|
|
2731
|
+
}
|
|
2732
|
+
try {
|
|
2733
|
+
return (await this.runStore.updateCanonicalRun({
|
|
2734
|
+
run_id: snapshot.run.run_id,
|
|
2735
|
+
expected_record_version: snapshot.record_version,
|
|
2736
|
+
run,
|
|
2737
|
+
activations,
|
|
2738
|
+
attempts,
|
|
2739
|
+
events,
|
|
2740
|
+
})).snapshot;
|
|
2741
|
+
}
|
|
2742
|
+
catch (error) {
|
|
2743
|
+
if (this.isVersionConflict(error))
|
|
2744
|
+
return this.requireSnapshot(snapshot.run.run_id);
|
|
2745
|
+
throw error;
|
|
2746
|
+
}
|
|
2747
|
+
}
|
|
2748
|
+
async locateLease(token, ownerId, allowExpired, allowTerminal = false) {
|
|
2749
|
+
requireNonEmpty(token, "lease_token");
|
|
2750
|
+
const snapshots = await this.runStore.loadAllAnyRuns();
|
|
2751
|
+
const snapshot = snapshots.find((candidate) => (candidate.schema_version === 2 && candidate.claims.some((claim) => claim.claim_token === token)));
|
|
2752
|
+
if (!snapshot)
|
|
2753
|
+
throw new CanonicalRunServiceError("lease_not_found", "lease token was not found");
|
|
2754
|
+
const claim = snapshot.claims.find((candidate) => candidate.claim_token === token);
|
|
2755
|
+
requireNonEmpty(ownerId, "owner_id");
|
|
2756
|
+
if (claim.owner_id !== ownerId) {
|
|
2757
|
+
throw new CanonicalRunServiceError("stale_lease", "lease is owned by a different driver");
|
|
2758
|
+
}
|
|
2759
|
+
const attempt = snapshot.attempts.find((candidate) => candidate.attempt_id === claim.attempt_id);
|
|
2760
|
+
const activation = snapshot.activations.find((candidate) => candidate.activation_id === claim.activation_id);
|
|
2761
|
+
if (!attempt || !activation) {
|
|
2762
|
+
throw new CanonicalRunServiceError("lease_not_found", "lease target no longer exists");
|
|
2763
|
+
}
|
|
2764
|
+
if (activation.current_attempt_id !== attempt.attempt_id) {
|
|
2765
|
+
throw new CanonicalRunServiceError("stale_lease", "attempt has been replaced");
|
|
2766
|
+
}
|
|
2767
|
+
const latest = this.claimsFor(snapshot, attempt.attempt_id).at(-1);
|
|
2768
|
+
if (latest.claim_token !== token) {
|
|
2769
|
+
throw new CanonicalRunServiceError("stale_lease", "lease epoch has been replaced");
|
|
2770
|
+
}
|
|
2771
|
+
if (snapshot.run.terminal !== null && !allowTerminal) {
|
|
2772
|
+
throw new CanonicalRunServiceError("run_terminal", "run is terminal");
|
|
2773
|
+
}
|
|
2774
|
+
if (snapshot.run.terminal === null
|
|
2775
|
+
&& snapshot.run.status !== "cancelling"
|
|
2776
|
+
&& this.stopIntentFor(snapshot, attempt.attempt_id) !== null) {
|
|
2777
|
+
throw new CanonicalRunServiceError("stale_lease", "attempt stop has been requested");
|
|
2778
|
+
}
|
|
2779
|
+
if (!allowExpired && Date.parse(claim.driver_claim_lease.expires_at) <= this.clock.now().getTime()) {
|
|
2780
|
+
throw new CanonicalRunServiceError("lease_expired", "driver claim lease has expired");
|
|
2781
|
+
}
|
|
2782
|
+
return { snapshot, plan: await this.requirePinnedPlan(snapshot), claim, attempt, activation };
|
|
2783
|
+
}
|
|
2784
|
+
assertRunAttachable(snapshot) {
|
|
2785
|
+
if (snapshot.run.terminal !== null || snapshot.run.status === "cancelling") {
|
|
2786
|
+
throw new CanonicalRunServiceError("run_terminal", snapshot.run.status === "cancelling"
|
|
2787
|
+
? "run cancellation has been requested"
|
|
2788
|
+
: "run is terminal");
|
|
2789
|
+
}
|
|
2790
|
+
}
|
|
2791
|
+
assertCapabilities(required, provided) {
|
|
2792
|
+
for (const [name, needed] of Object.entries(required)) {
|
|
2793
|
+
if (needed && provided[name] !== true) {
|
|
2794
|
+
throw new CanonicalRunServiceError("unsupported", `driver capability ${name} is required`);
|
|
2795
|
+
}
|
|
2796
|
+
}
|
|
2797
|
+
}
|
|
2798
|
+
commandVerifiers(plan) {
|
|
2799
|
+
return plan.verifier_plan.filter((verifier) => ((verifier.kind === "command" || verifier.kind === "checkpoint") && verifier.command !== null));
|
|
2800
|
+
}
|
|
2801
|
+
assertVerifierExecutionSupported(plan) {
|
|
2802
|
+
const commandVerifiers = this.commandVerifiers(plan);
|
|
2803
|
+
if (commandVerifiers.length > 0
|
|
2804
|
+
&& !this.verifierAuthorityDescriptor.process_tree_containment) {
|
|
2805
|
+
throw new CanonicalRunServiceError("unsupported", "command verifiers require a configured process-tree containment authority");
|
|
2806
|
+
}
|
|
2807
|
+
if (!plan.workspace_policy.allow_network
|
|
2808
|
+
&& commandVerifiers.length > 0
|
|
2809
|
+
&& !this.verifierAuthorityDescriptor.network_policies.includes("deny_all")) {
|
|
2810
|
+
throw new CanonicalRunServiceError("unsupported", "command verifiers require a configured deny_all execution port");
|
|
2811
|
+
}
|
|
2812
|
+
}
|
|
2813
|
+
async validateCommandVerifierReceipts(request, results) {
|
|
2814
|
+
for (const verifier of this.commandVerifiers(request.plan.plan).filter((item) => (item.subject_node_id === request.activation.step_id))) {
|
|
2815
|
+
const matchingResults = results.filter((result) => result.verifier_id === verifier.verifier_id);
|
|
2816
|
+
const receipts = matchingResults.flatMap((result) => ((result.evidence_refs ?? []).flatMap((evidence) => (evidence.kind === "verifier_evidence" && evidence.execution_receipt !== null
|
|
2817
|
+
? [evidence.execution_receipt]
|
|
2818
|
+
: []))));
|
|
2819
|
+
if (matchingResults.length !== 1 || receipts.length !== 1) {
|
|
2820
|
+
throw new CanonicalRunServiceError("verification_failed", `verifier ${verifier.verifier_id} requires exactly one service-owned execution receipt`);
|
|
2821
|
+
}
|
|
2822
|
+
const receipt = receipts[0];
|
|
2823
|
+
const relativeWorkingDirectory = verifier.kind === "command" ? verifier.relative_working_directory : ".";
|
|
2824
|
+
const resolved = resolveVerifierCommand(verifier.command, request.handoff_bindings, request.run.run_binding.binding.validated_inputs, request.plan.plan.input_contract.inputs);
|
|
2825
|
+
const environment = canonicalVerifierEnvironment(resolved.environment);
|
|
2826
|
+
const environmentBindingDigest = digest(Object.fromEntries(Object.entries(environment)
|
|
2827
|
+
.sort(([left], [right]) => left.localeCompare(right))
|
|
2828
|
+
.map(([name, value]) => [name, textDigest(value)])));
|
|
2829
|
+
const networkPolicy = request.plan.plan.workspace_policy.allow_network ? "allow" : "deny_all";
|
|
2830
|
+
const workingDirectory = await canonicalWorkspaceDirectory(request.execution.workspace.root, relativeWorkingDirectory, `verifier ${verifier.verifier_id} working_directory`);
|
|
2831
|
+
const authorityDescriptorDigest = this.verifierAuthorityDescriptorDigest;
|
|
2832
|
+
const cwdIdentity = {
|
|
2833
|
+
workspace_id: request.execution.workspace.workspace_id,
|
|
2834
|
+
workspace_allocation_digest: digest(request.workspace_allocation),
|
|
2835
|
+
relative_working_directory: relativeWorkingDirectory,
|
|
2836
|
+
absolute_path_digest: textDigest(workingDirectory),
|
|
2837
|
+
};
|
|
2838
|
+
const requestDescriptor = {
|
|
2839
|
+
schema_version: "intentdna.verifier_execution_request.v1",
|
|
2840
|
+
plan_id: request.plan.plan_id,
|
|
2841
|
+
verifier_id: verifier.verifier_id,
|
|
2842
|
+
verifier_definition_digest: digest(verifier),
|
|
2843
|
+
raw_command_digest: textDigest(resolved.command),
|
|
2844
|
+
cwd: cwdIdentity,
|
|
2845
|
+
supplied_environment: {
|
|
2846
|
+
binding_digest: environmentBindingDigest,
|
|
2847
|
+
variable_names: Object.keys(environment).sort(),
|
|
2848
|
+
inherit_parent: false,
|
|
2849
|
+
startup_files: false,
|
|
2850
|
+
},
|
|
2851
|
+
shell: { executable: "bash", arguments: ["--noprofile", "--norc", "-c"] },
|
|
2852
|
+
timeout_ms: 30_000,
|
|
2853
|
+
requested_at: receipt.requested_at,
|
|
2854
|
+
network_mode: networkPolicy,
|
|
2855
|
+
expected_authority_descriptor_digest: authorityDescriptorDigest,
|
|
2856
|
+
};
|
|
2857
|
+
if (receipt.plan_id !== request.plan.plan_id
|
|
2858
|
+
|| receipt.verifier_id !== verifier.verifier_id
|
|
2859
|
+
|| receipt.verifier_definition_digest !== digest(verifier)
|
|
2860
|
+
|| receipt.resolved_command_digest !== textDigest(resolved.command)
|
|
2861
|
+
|| receipt.environment_binding_digest !== environmentBindingDigest
|
|
2862
|
+
|| canonicalizeJson(receipt.cwd_identity) !== canonicalizeJson(cwdIdentity)
|
|
2863
|
+
|| receipt.request_identity.request_digest !== digest(requestDescriptor)
|
|
2864
|
+
|| receipt.request_identity.command_digest !== receipt.resolved_command_digest
|
|
2865
|
+
|| receipt.request_identity.environment_digest !== receipt.environment_binding_digest
|
|
2866
|
+
|| receipt.request_identity.authority_descriptor_digest !== authorityDescriptorDigest
|
|
2867
|
+
|| receipt.request_identity.launch_descriptor_digest !== receipt.request_identity.request_digest
|
|
2868
|
+
|| receipt.supplied_environment.inherit_parent !== false
|
|
2869
|
+
|| receipt.supplied_environment.startup_files !== false
|
|
2870
|
+
|| canonicalizeJson(receipt.supplied_environment.variable_names) !== canonicalizeJson(Object.keys(environment).sort())
|
|
2871
|
+
|| receipt.shell_policy.executable !== "bash"
|
|
2872
|
+
|| canonicalizeJson(receipt.shell_policy.arguments) !== canonicalizeJson(["--noprofile", "--norc", "-c"])
|
|
2873
|
+
|| receipt.network_policy.mode !== networkPolicy
|
|
2874
|
+
|| (networkPolicy === "deny_all" && (receipt.network_policy.enforced !== true
|
|
2875
|
+
|| receipt.network_policy.enforcement_kind !== "configured_adapter"
|
|
2876
|
+
|| receipt.network_policy.enforcement_digest === null
|
|
2877
|
+
|| receipt.network_policy.enforcement_descriptor === null
|
|
2878
|
+
|| receipt.network_policy.enforcement_digest !== digest(receipt.network_policy.enforcement_descriptor)
|
|
2879
|
+
|| receipt.network_policy.enforcement_descriptor.mode !== "deny_all"
|
|
2880
|
+
|| receipt.network_policy.enforcement_descriptor.adapter_kind !== receipt.execution_authority.adapter_kind
|
|
2881
|
+
|| receipt.network_policy.enforcement_descriptor.adapter_version !== receipt.execution_authority.adapter_version
|
|
2882
|
+
|| receipt.network_policy.enforcement_descriptor.configuration_digest !== receipt.execution_authority.configuration_digest
|
|
2883
|
+
|| receipt.network_policy.enforcement_descriptor.authority_descriptor_digest !== authorityDescriptorDigest
|
|
2884
|
+
|| receipt.network_policy.enforcement_descriptor.request_digest !== receipt.request_identity.request_digest))
|
|
2885
|
+
|| (networkPolicy === "allow" && (receipt.network_policy.enforced !== false
|
|
2886
|
+
|| receipt.network_policy.enforcement_kind !== "not_required"
|
|
2887
|
+
|| receipt.network_policy.enforcement_digest !== null
|
|
2888
|
+
|| receipt.network_policy.enforcement_descriptor !== null))
|
|
2889
|
+
|| receipt.execution_authority.authority_descriptor_digest !== authorityDescriptorDigest
|
|
2890
|
+
|| receipt.execution_authority.authority_instance_id !== this.verifierAuthorityDescriptor.authority_instance_id
|
|
2891
|
+
|| receipt.execution_authority.adapter_kind !== this.verifierAuthorityDescriptor.adapter_kind
|
|
2892
|
+
|| receipt.execution_authority.adapter_version !== this.verifierAuthorityDescriptor.adapter_version
|
|
2893
|
+
|| receipt.execution_authority.configuration_digest !== this.verifierAuthorityDescriptor.configuration_digest
|
|
2894
|
+
|| receipt.execution_authority.process_tree_drained !== true) {
|
|
2895
|
+
const mismatch = {
|
|
2896
|
+
plan: receipt.plan_id !== request.plan.plan_id,
|
|
2897
|
+
verifier: receipt.verifier_definition_digest !== digest(verifier),
|
|
2898
|
+
command: receipt.resolved_command_digest !== textDigest(resolved.command),
|
|
2899
|
+
environment: receipt.environment_binding_digest !== environmentBindingDigest,
|
|
2900
|
+
cwd: canonicalizeJson(receipt.cwd_identity) !== canonicalizeJson(cwdIdentity),
|
|
2901
|
+
request: receipt.request_identity.request_digest !== digest(requestDescriptor),
|
|
2902
|
+
variables: canonicalizeJson(receipt.supplied_environment.variable_names) !== canonicalizeJson(Object.keys(environment).sort()),
|
|
2903
|
+
network: receipt.network_policy.mode !== networkPolicy,
|
|
2904
|
+
};
|
|
2905
|
+
throw new CanonicalRunServiceError("verification_failed", `verifier ${verifier.verifier_id} execution receipt does not match its pinned request: ${canonicalizeJson(mismatch)}`);
|
|
2906
|
+
}
|
|
2907
|
+
}
|
|
2908
|
+
}
|
|
2909
|
+
serviceVerifierIds(request) {
|
|
2910
|
+
const node = nodeById(request.plan.plan, request.activation.step_id);
|
|
2911
|
+
return new Set([
|
|
2912
|
+
...request.plan.plan.verifier_plan
|
|
2913
|
+
.filter((verifier) => verifier.subject_node_id === request.activation.step_id)
|
|
2914
|
+
.map((verifier) => verifier.verifier_id),
|
|
2915
|
+
...(node.kind === "worker"
|
|
2916
|
+
? node.packet_template.output_contract.flatMap((output) => [
|
|
2917
|
+
`output_contract:${output.name}`,
|
|
2918
|
+
...(output.artifact_kind === "git_commit"
|
|
2919
|
+
? [`output_contract:${output.name}:git_commit_provenance`]
|
|
2920
|
+
: []),
|
|
2921
|
+
])
|
|
2922
|
+
: []),
|
|
2923
|
+
]);
|
|
2924
|
+
}
|
|
2925
|
+
validateVerificationLanes(request, serviceResults, additionalResults) {
|
|
2926
|
+
const expected = this.serviceVerifierIds(request);
|
|
2927
|
+
const actualServiceIds = new Set();
|
|
2928
|
+
for (const result of serviceResults) {
|
|
2929
|
+
requireNonEmpty(result.verifier_id, "service verifier_id");
|
|
2930
|
+
if (!expected.has(result.verifier_id)) {
|
|
2931
|
+
throw new CanonicalRunServiceError("verification_failed", `unexpected service verifier result ${result.verifier_id}`);
|
|
2932
|
+
}
|
|
2933
|
+
if (actualServiceIds.has(result.verifier_id)) {
|
|
2934
|
+
throw new CanonicalRunServiceError("verification_failed", `mandatory verifier ${result.verifier_id} requires exactly one result`);
|
|
2935
|
+
}
|
|
2936
|
+
actualServiceIds.add(result.verifier_id);
|
|
2937
|
+
}
|
|
2938
|
+
for (const verifierId of expected) {
|
|
2939
|
+
if (!actualServiceIds.has(verifierId)) {
|
|
2940
|
+
throw new CanonicalRunServiceError("verification_failed", `mandatory verifier ${verifierId} requires exactly one result`);
|
|
2941
|
+
}
|
|
2942
|
+
}
|
|
2943
|
+
const reserved = new Set([
|
|
2944
|
+
...request.plan.plan.verifier_plan.map((verifier) => verifier.verifier_id),
|
|
2945
|
+
...request.plan.plan.nodes.flatMap((node) => node.kind === "worker"
|
|
2946
|
+
? node.packet_template.output_contract.flatMap((output) => [
|
|
2947
|
+
`output_contract:${output.name}`,
|
|
2948
|
+
...(output.artifact_kind === "git_commit"
|
|
2949
|
+
? [`output_contract:${output.name}:git_commit_provenance`]
|
|
2950
|
+
: []),
|
|
2951
|
+
])
|
|
2952
|
+
: []),
|
|
2953
|
+
]);
|
|
2954
|
+
const additionalIds = new Set();
|
|
2955
|
+
for (const result of additionalResults) {
|
|
2956
|
+
requireNonEmpty(result.verifier_id, "additional verifier_id");
|
|
2957
|
+
if (reserved.has(result.verifier_id)) {
|
|
2958
|
+
throw new CanonicalRunServiceError("verification_failed", `additional verifier ${result.verifier_id} collides with service authority`);
|
|
2959
|
+
}
|
|
2960
|
+
if (additionalIds.has(result.verifier_id)) {
|
|
2961
|
+
throw new CanonicalRunServiceError("verification_failed", `additional verifier ${result.verifier_id} requires exactly one result`);
|
|
2962
|
+
}
|
|
2963
|
+
additionalIds.add(result.verifier_id);
|
|
2964
|
+
}
|
|
2965
|
+
}
|
|
2966
|
+
validateSubmittedOutputs(snapshot, plan, activation, execution) {
|
|
2967
|
+
const node = nodeById(plan.plan, activation.step_id);
|
|
2968
|
+
if (node.kind !== "worker") {
|
|
2969
|
+
if (execution.outputs.length > 0) {
|
|
2970
|
+
throw new CanonicalRunServiceError("invalid_request", `${node.kind} attempt cannot submit worker outputs`);
|
|
2971
|
+
}
|
|
2972
|
+
return;
|
|
2973
|
+
}
|
|
2974
|
+
const contracts = new Map(node.packet_template.output_contract.map((item) => [item.name, item]));
|
|
2975
|
+
const seen = new Set();
|
|
2976
|
+
for (const output of execution.outputs) {
|
|
2977
|
+
if (seen.has(output.name)) {
|
|
2978
|
+
throw new CanonicalRunServiceError("invalid_request", `duplicate output ${output.name}`);
|
|
2979
|
+
}
|
|
2980
|
+
seen.add(output.name);
|
|
2981
|
+
const contract = contracts.get(output.name);
|
|
2982
|
+
if (!contract) {
|
|
2983
|
+
throw new CanonicalRunServiceError("invalid_request", `undeclared output ${output.name}`);
|
|
2984
|
+
}
|
|
2985
|
+
if (contract.kind !== output.kind) {
|
|
2986
|
+
throw new CanonicalRunServiceError("invalid_request", `output ${output.name} kind does not match its contract`);
|
|
2987
|
+
}
|
|
2988
|
+
if (output.kind === "reference") {
|
|
2989
|
+
if (contract.artifact_kind !== output.artifact_kind) {
|
|
2990
|
+
throw new CanonicalRunServiceError("invalid_request", `output ${output.name} artifact kind does not match its contract`);
|
|
2991
|
+
}
|
|
2992
|
+
const runtimeValues = new Map(Object.entries(snapshot.run.run_binding.binding.validated_inputs)
|
|
2993
|
+
.map(([name, value]) => [name, runtimeTemplateValue(value)]));
|
|
2994
|
+
const expectedPath = contract.relative_path_template === null
|
|
2995
|
+
? null
|
|
2996
|
+
: renderPacketPrompt(contract.relative_path_template, runtimeValues);
|
|
2997
|
+
if (expectedPath !== null
|
|
2998
|
+
&& resolve(execution.workspace.root, output.source_path)
|
|
2999
|
+
!== resolve(execution.workspace.root, expectedPath)) {
|
|
3000
|
+
throw new CanonicalRunServiceError("invalid_request", `output ${output.name} path does not match its contract`);
|
|
3001
|
+
}
|
|
3002
|
+
}
|
|
3003
|
+
if (output.kind === "structured") {
|
|
3004
|
+
const validation = validatePinnedStructuredOutput({
|
|
3005
|
+
plan: plan.plan,
|
|
3006
|
+
node_id: activation.step_id,
|
|
3007
|
+
output_name: output.name,
|
|
3008
|
+
schema_ref: output.schema_ref,
|
|
3009
|
+
value: output.value,
|
|
3010
|
+
}, this.structuredOutputSchemaRegistry);
|
|
3011
|
+
if (!validation.valid) {
|
|
3012
|
+
throw new CanonicalRunServiceError("verification_failed", `output ${output.name} failed structured validation at ${validation.path}: ${validation.message}`);
|
|
3013
|
+
}
|
|
3014
|
+
}
|
|
3015
|
+
}
|
|
3016
|
+
const succeeded = execution.outcome.kind === "worker_process"
|
|
3017
|
+
&& execution.outcome.status === "succeeded";
|
|
3018
|
+
for (const contract of contracts.values()) {
|
|
3019
|
+
if (succeeded && contract.required && !seen.has(contract.name)) {
|
|
3020
|
+
throw new CanonicalRunServiceError("invalid_request", `required output ${contract.name} is missing`);
|
|
3021
|
+
}
|
|
3022
|
+
}
|
|
3023
|
+
}
|
|
3024
|
+
async validateCallerAttestationIndependence(snapshot, plan, activation, evidenceRefs) {
|
|
3025
|
+
if (!plan.plan.required_capabilities.caller_attestation)
|
|
3026
|
+
return;
|
|
3027
|
+
const attestations = evidenceRefs.filter((evidence) => evidence.kind === "caller_attestation");
|
|
3028
|
+
if (attestations.length !== 1 || attestations[0].payload === undefined) {
|
|
3029
|
+
throw new CanonicalRunServiceError("unsupported", "run requires one trusted caller attestation identity payload");
|
|
3030
|
+
}
|
|
3031
|
+
const current = attestations[0].payload;
|
|
3032
|
+
const sourceResultIds = new Set();
|
|
3033
|
+
for (const handoff of plan.plan.handoff_plan.filter((item) => item.to_node_id === activation.step_id)) {
|
|
3034
|
+
const sourceActivation = snapshot.activations
|
|
3035
|
+
.filter((item) => item.step_id === handoff.from_node_id && item.committed_result_id !== null)
|
|
3036
|
+
.at(-1);
|
|
3037
|
+
if (sourceActivation?.committed_result_id !== null && sourceActivation?.committed_result_id !== undefined) {
|
|
3038
|
+
sourceResultIds.add(sourceActivation.committed_result_id);
|
|
3039
|
+
}
|
|
3040
|
+
}
|
|
3041
|
+
const visited = new Set();
|
|
3042
|
+
const sourceIdentities = [];
|
|
3043
|
+
const collectSourceIdentities = async (resultId) => {
|
|
3044
|
+
if (visited.has(resultId))
|
|
3045
|
+
return;
|
|
3046
|
+
visited.add(resultId);
|
|
3047
|
+
const published = await this.resultStore.loadPublished(resultId);
|
|
3048
|
+
if (published === null) {
|
|
3049
|
+
throw new CanonicalRunServiceError("submission_conflict", `source result ${resultId} is missing from immutable result storage`);
|
|
3050
|
+
}
|
|
3051
|
+
const sourceResult = published.result;
|
|
3052
|
+
const sourceAttestations = sourceResult.evidence_refs.filter((evidence) => evidence.kind === "caller_attestation" && evidence.payload !== undefined);
|
|
3053
|
+
if (sourceResult.executor.kind === "worker") {
|
|
3054
|
+
if (sourceAttestations.length !== 1) {
|
|
3055
|
+
throw new CanonicalRunServiceError("submission_conflict", `source result ${resultId} lacks one trusted caller attestation identity`);
|
|
3056
|
+
}
|
|
3057
|
+
sourceIdentities.push(sourceAttestations[0].payload);
|
|
3058
|
+
return;
|
|
3059
|
+
}
|
|
3060
|
+
const origin = sourceResult.system_origin;
|
|
3061
|
+
if (sourceAttestations.length !== 0 || origin === null || origin === undefined) {
|
|
3062
|
+
throw new CanonicalRunServiceError("submission_conflict", `system source result ${resultId} has invalid caller identity provenance`);
|
|
3063
|
+
}
|
|
3064
|
+
if (origin.kind === "current_run_results") {
|
|
3065
|
+
for (const originResultId of origin.source_result_ids) {
|
|
3066
|
+
await collectSourceIdentities(originResultId);
|
|
3067
|
+
}
|
|
3068
|
+
}
|
|
3069
|
+
else if (origin.kind === "canonical_import") {
|
|
3070
|
+
for (const originResultId of origin.origin_result_ids) {
|
|
3071
|
+
await collectSourceIdentities(originResultId);
|
|
3072
|
+
}
|
|
3073
|
+
}
|
|
3074
|
+
};
|
|
3075
|
+
for (const resultId of sourceResultIds) {
|
|
3076
|
+
await collectSourceIdentities(resultId);
|
|
3077
|
+
}
|
|
3078
|
+
for (const source of sourceIdentities) {
|
|
3079
|
+
const sameAuthority = source.authority_id === current.authority_id;
|
|
3080
|
+
if (source.worker_session_id === current.worker_session_id
|
|
3081
|
+
|| sameAuthority && (source.agent_id === current.agent_id
|
|
3082
|
+
|| source.session_id === current.session_id)) {
|
|
3083
|
+
throw new CanonicalRunServiceError("submission_conflict", "independent submission reuses the attested identity from its source result chain");
|
|
3084
|
+
}
|
|
3085
|
+
}
|
|
3086
|
+
}
|
|
3087
|
+
workspaceAllocation(snapshot, attemptId) {
|
|
3088
|
+
const event = snapshot.events.find((candidate) => candidate.type === "workspace_allocated" && candidate.attempt_id === attemptId);
|
|
3089
|
+
return event?.payload.allocation ?? null;
|
|
3090
|
+
}
|
|
3091
|
+
workspacePlan(snapshot, attemptId) {
|
|
3092
|
+
const event = snapshot.events.find((candidate) => candidate.type === "workspace_planned" && candidate.attempt_id === attemptId);
|
|
3093
|
+
return event?.payload.plan ?? null;
|
|
3094
|
+
}
|
|
3095
|
+
validateWorkspaceAllocation(allocation, attempt, node) {
|
|
3096
|
+
if (allocation.workspace_id !== attempt.attempt_id) {
|
|
3097
|
+
throw new CanonicalRunServiceError("unsupported", "workspace allocation identity does not match the attempt");
|
|
3098
|
+
}
|
|
3099
|
+
const expectedKind = node.kind === "worker" ? node.workspace.isolation : "none";
|
|
3100
|
+
if (allocation.kind !== expectedKind) {
|
|
3101
|
+
throw new CanonicalRunServiceError("unsupported", `workspace allocation kind ${allocation.kind} does not satisfy ${expectedKind}`);
|
|
3102
|
+
}
|
|
3103
|
+
}
|
|
3104
|
+
requireCommittedWorkspace(snapshot, attempt, node) {
|
|
3105
|
+
const allocation = this.workspaceAllocation(snapshot, attempt.attempt_id);
|
|
3106
|
+
if (allocation === null) {
|
|
3107
|
+
throw new CanonicalRunServiceError("unsupported", `attempt ${attempt.attempt_id} has no committed workspace allocation`);
|
|
3108
|
+
}
|
|
3109
|
+
this.validateWorkspaceAllocation(allocation, attempt, node);
|
|
3110
|
+
return allocation;
|
|
3111
|
+
}
|
|
3112
|
+
async planWorkspace(plan, run, activation, attempt) {
|
|
3113
|
+
if (this.workspaceAuthority === null) {
|
|
3114
|
+
throw new CanonicalRunServiceError("unsupported", "attempt workspace authority is unavailable");
|
|
3115
|
+
}
|
|
3116
|
+
try {
|
|
3117
|
+
return await this.workspaceAuthority.plan({ plan, run, activation, attempt });
|
|
3118
|
+
}
|
|
3119
|
+
catch (error) {
|
|
3120
|
+
throw this.workspaceError(error, "attempt workspace planning is unavailable");
|
|
3121
|
+
}
|
|
3122
|
+
}
|
|
3123
|
+
async withWorkspaceAuthority(allocation, purpose, requireCleanWorktree, operation) {
|
|
3124
|
+
if (this.workspaceAuthority === null) {
|
|
3125
|
+
throw new CanonicalRunServiceError("unsupported", "attempt workspace authority is unavailable");
|
|
3126
|
+
}
|
|
3127
|
+
let entered = false;
|
|
3128
|
+
try {
|
|
3129
|
+
return await this.workspaceAuthority.withExclusiveMutationAuthority({ allocation, purpose, require_clean_worktree: requireCleanWorktree }, async (authority) => {
|
|
3130
|
+
entered = true;
|
|
3131
|
+
if (authority.workspace_id !== allocation.workspace_id
|
|
3132
|
+
|| authority.workspace_allocation_digest !== digest(allocation)
|
|
3133
|
+
|| !authority.authority_instance_id
|
|
3134
|
+
|| !Number.isSafeInteger(authority.fencing_epoch)
|
|
3135
|
+
|| authority.fencing_epoch <= 0) {
|
|
3136
|
+
throw new CanonicalRunServiceError("workspace_conflict", "workspace mutation authority does not match the committed allocation");
|
|
3137
|
+
}
|
|
3138
|
+
return operation(authority);
|
|
3139
|
+
});
|
|
3140
|
+
}
|
|
3141
|
+
catch (error) {
|
|
3142
|
+
if (entered)
|
|
3143
|
+
throw error;
|
|
3144
|
+
throw this.workspaceError(error, "exclusive attempt workspace authority is unavailable");
|
|
3145
|
+
}
|
|
3146
|
+
}
|
|
3147
|
+
workspaceError(error, message) {
|
|
3148
|
+
if (error instanceof CanonicalRunServiceError)
|
|
3149
|
+
return error;
|
|
3150
|
+
const code = typeof error === "object" && error !== null && "code" in error
|
|
3151
|
+
? String(error.code)
|
|
3152
|
+
: "";
|
|
3153
|
+
return new CanonicalRunServiceError(["workspace_conflict", "workspace_mismatch", "path_conflict", "branch_conflict"].includes(code)
|
|
3154
|
+
? "workspace_conflict"
|
|
3155
|
+
: "unsupported", message, { cause: error });
|
|
3156
|
+
}
|
|
3157
|
+
stopIntentFor(snapshot, attemptId) {
|
|
3158
|
+
const event = snapshot.events.find((candidate) => (candidate.type === "attempt_stop_requested" && candidate.attempt_id === attemptId));
|
|
3159
|
+
return event?.type === "attempt_stop_requested" ? event.payload.intent : null;
|
|
3160
|
+
}
|
|
3161
|
+
stopIdentity(attempt, executionClaimEpoch) {
|
|
3162
|
+
return digest({
|
|
3163
|
+
run_id: attempt.run_id,
|
|
3164
|
+
step_id: attempt.step_id,
|
|
3165
|
+
activation_id: attempt.activation_id,
|
|
3166
|
+
attempt_id: attempt.attempt_id,
|
|
3167
|
+
claim_epoch: executionClaimEpoch,
|
|
3168
|
+
worker_session_id: attempt.executor?.kind === "worker"
|
|
3169
|
+
? attempt.executor.worker_session_id
|
|
3170
|
+
: null,
|
|
3171
|
+
});
|
|
3172
|
+
}
|
|
3173
|
+
async persistStopIntents(initial, specifications) {
|
|
3174
|
+
let snapshot = initial;
|
|
3175
|
+
for (;;) {
|
|
3176
|
+
const events = [];
|
|
3177
|
+
for (const specification of specifications) {
|
|
3178
|
+
const attempt = snapshot.attempts.find((candidate) => candidate.attempt_id === specification.attempt.attempt_id);
|
|
3179
|
+
if (!attempt || attempt.phase === "terminal" || this.stopIntentFor(snapshot, attempt.attempt_id))
|
|
3180
|
+
continue;
|
|
3181
|
+
if (attempt.executor?.kind !== "worker" || attempt.started_at === null)
|
|
3182
|
+
continue;
|
|
3183
|
+
const executionClaimEpoch = this.attemptRecovery(snapshot, attempt).execution_claim_epoch;
|
|
3184
|
+
const intent = {
|
|
3185
|
+
schema_version: "intentdna.attempt_stop_intent.v1",
|
|
3186
|
+
run_id: attempt.run_id,
|
|
3187
|
+
step_id: attempt.step_id,
|
|
3188
|
+
activation_id: attempt.activation_id,
|
|
3189
|
+
attempt_id: attempt.attempt_id,
|
|
3190
|
+
claim_epoch: executionClaimEpoch,
|
|
3191
|
+
worker_session_id: attempt.executor.worker_session_id,
|
|
3192
|
+
stop_idempotency_key: this.stopIdentity(attempt, executionClaimEpoch),
|
|
3193
|
+
purpose: specification.purpose,
|
|
3194
|
+
requested_at: specification.requestedAt,
|
|
3195
|
+
started_at: attempt.started_at,
|
|
3196
|
+
requested_outcome: specification.outcome,
|
|
3197
|
+
};
|
|
3198
|
+
events.push(this.event(snapshot, events.length, {
|
|
3199
|
+
type: "attempt_stop_requested",
|
|
3200
|
+
step_id: attempt.step_id,
|
|
3201
|
+
activation_id: attempt.activation_id,
|
|
3202
|
+
attempt_id: attempt.attempt_id,
|
|
3203
|
+
result_id: null,
|
|
3204
|
+
key: `attempt:${attempt.attempt_id}:stop-requested`,
|
|
3205
|
+
at: specification.requestedAt,
|
|
3206
|
+
payload: { intent },
|
|
3207
|
+
}));
|
|
3208
|
+
}
|
|
3209
|
+
if (events.length === 0)
|
|
3210
|
+
return snapshot;
|
|
3211
|
+
try {
|
|
3212
|
+
return (await this.runStore.updateCanonicalRun({
|
|
3213
|
+
run_id: snapshot.run.run_id,
|
|
3214
|
+
expected_record_version: snapshot.record_version,
|
|
3215
|
+
events,
|
|
3216
|
+
})).snapshot;
|
|
3217
|
+
}
|
|
3218
|
+
catch (error) {
|
|
3219
|
+
if (!this.isVersionConflict(error))
|
|
3220
|
+
throw error;
|
|
3221
|
+
snapshot = await this.requireSnapshot(snapshot.run.run_id);
|
|
3222
|
+
}
|
|
3223
|
+
}
|
|
3224
|
+
}
|
|
3225
|
+
async prepareFailurePeerStops(snapshot, plan, sourceAttempt, outcome, requestedAt) {
|
|
3226
|
+
if (canonicalAttemptOutcomeStatus(outcome) === "succeeded")
|
|
3227
|
+
return snapshot;
|
|
3228
|
+
const node = nodeById(plan.plan, sourceAttempt.step_id);
|
|
3229
|
+
const retryOutcome = outcome.kind === "worker_process" && outcome.process.kind !== "success"
|
|
3230
|
+
? outcome.process.kind
|
|
3231
|
+
: canonicalAttemptOutcomeStatus(outcome) === "cancelled" ? "cancelled" : "malformed_result";
|
|
3232
|
+
const retryable = node.kind === "worker"
|
|
3233
|
+
&& sourceAttempt.attempt_number < node.attempt_policy.max_attempts
|
|
3234
|
+
&& node.attempt_policy.retryable_outcomes.includes(retryOutcome);
|
|
3235
|
+
if (retryable)
|
|
3236
|
+
return snapshot;
|
|
3237
|
+
const peers = snapshot.attempts.flatMap((attempt) => {
|
|
3238
|
+
if (attempt.attempt_id === sourceAttempt.attempt_id || attempt.phase === "terminal" || attempt.executor?.kind !== "worker")
|
|
3239
|
+
return [];
|
|
3240
|
+
const peerOutcome = {
|
|
3241
|
+
kind: "worker_process",
|
|
3242
|
+
status: "cancelled",
|
|
3243
|
+
reason_code: "peer_attempt_failed",
|
|
3244
|
+
message: null,
|
|
3245
|
+
process: { kind: "cancelled", exit_code: null, signal: null, reason: "peer_attempt_failed" },
|
|
3246
|
+
};
|
|
3247
|
+
return [{ attempt, purpose: "peer_terminal", requestedAt, outcome: peerOutcome }];
|
|
3248
|
+
});
|
|
3249
|
+
if (peers.length === 0)
|
|
3250
|
+
return snapshot;
|
|
3251
|
+
const withIntents = await this.persistStopIntents(snapshot, peers);
|
|
3252
|
+
return this.continueDurableStops(withIntents);
|
|
3253
|
+
}
|
|
3254
|
+
async prepareTerminalPeerStops(snapshot, sourceAttempt, requestedAt) {
|
|
3255
|
+
const peers = snapshot.attempts.flatMap((attempt) => {
|
|
3256
|
+
if (attempt.attempt_id === sourceAttempt.attempt_id || attempt.phase === "terminal" || attempt.executor?.kind !== "worker")
|
|
3257
|
+
return [];
|
|
3258
|
+
const outcome = {
|
|
3259
|
+
kind: "worker_process",
|
|
3260
|
+
status: "cancelled",
|
|
3261
|
+
reason_code: "run_terminal",
|
|
3262
|
+
message: null,
|
|
3263
|
+
process: { kind: "cancelled", exit_code: null, signal: null, reason: "run_terminal" },
|
|
3264
|
+
};
|
|
3265
|
+
return [{ attempt, purpose: "peer_terminal", requestedAt, outcome }];
|
|
3266
|
+
});
|
|
3267
|
+
if (peers.length === 0)
|
|
3268
|
+
return snapshot;
|
|
3269
|
+
const withIntents = await this.persistStopIntents(snapshot, peers);
|
|
3270
|
+
return this.continueDurableStops(withIntents);
|
|
3271
|
+
}
|
|
3272
|
+
async continueDurableStops(initial) {
|
|
3273
|
+
let snapshot = initial;
|
|
3274
|
+
if (snapshot.run.status === "cancelling") {
|
|
3275
|
+
snapshot = await this.persistStopIntents(snapshot, this.cancellationStopSpecifications(snapshot));
|
|
3276
|
+
}
|
|
3277
|
+
const pending = snapshot.events.filter((event) => event.type === "attempt_stop_requested");
|
|
3278
|
+
for (const event of pending) {
|
|
3279
|
+
if (event.type !== "attempt_stop_requested")
|
|
3280
|
+
continue;
|
|
3281
|
+
let expectedReceipt = null;
|
|
3282
|
+
for (;;) {
|
|
3283
|
+
const attempt = snapshot.attempts.find((candidate) => candidate.attempt_id === event.attempt_id);
|
|
3284
|
+
if (!attempt || attempt.phase === "terminal")
|
|
3285
|
+
break;
|
|
3286
|
+
const resolution = await this.requestWorkerStop(snapshot, attempt, event.payload.intent, expectedReceipt);
|
|
3287
|
+
try {
|
|
3288
|
+
snapshot = resolution.kind === "stopped_by_request"
|
|
3289
|
+
? await this.confirmDurableStop(snapshot, attempt, event.payload.intent, resolution.authority.stop_receipt)
|
|
3290
|
+
: resolution.kind === "already_stopped"
|
|
3291
|
+
? await this.confirmAlreadyDurableStop(snapshot, attempt, event.payload.intent, resolution.drain_evidence)
|
|
3292
|
+
: await this.confirmNeverLaunchedStop(snapshot, attempt, event.payload.intent, resolution.nonlaunch_attestation);
|
|
3293
|
+
break;
|
|
3294
|
+
}
|
|
3295
|
+
catch (error) {
|
|
3296
|
+
if (!this.isVersionConflict(error))
|
|
3297
|
+
throw error;
|
|
3298
|
+
snapshot = await this.requireSnapshot(snapshot.run.run_id);
|
|
3299
|
+
const current = snapshot.attempts.find((candidate) => candidate.attempt_id === attempt.attempt_id);
|
|
3300
|
+
if (!current || current.phase === "terminal")
|
|
3301
|
+
break;
|
|
3302
|
+
expectedReceipt = resolution.kind === "stopped_by_request"
|
|
3303
|
+
? resolution.authority.stop_receipt
|
|
3304
|
+
: null;
|
|
3305
|
+
}
|
|
3306
|
+
}
|
|
3307
|
+
}
|
|
3308
|
+
return this.aggregateDurableStops(snapshot);
|
|
3309
|
+
}
|
|
3310
|
+
cancellationStopSpecifications(snapshot) {
|
|
3311
|
+
const cancellation = snapshot.run.cancellation;
|
|
3312
|
+
if (snapshot.run.status !== "cancelling" || cancellation === null)
|
|
3313
|
+
return [];
|
|
3314
|
+
return snapshot.attempts.flatMap((attempt) => {
|
|
3315
|
+
if (attempt.phase === "terminal" || attempt.executor?.kind !== "worker")
|
|
3316
|
+
return [];
|
|
3317
|
+
const outcome = {
|
|
3318
|
+
kind: "worker_process",
|
|
3319
|
+
status: "cancelled",
|
|
3320
|
+
reason_code: "run_cancelled",
|
|
3321
|
+
message: cancellation.reason,
|
|
3322
|
+
process: {
|
|
3323
|
+
kind: "cancelled",
|
|
3324
|
+
exit_code: null,
|
|
3325
|
+
signal: null,
|
|
3326
|
+
reason: cancellation.reason,
|
|
3327
|
+
},
|
|
3328
|
+
};
|
|
3329
|
+
return [{
|
|
3330
|
+
attempt,
|
|
3331
|
+
purpose: "cancel",
|
|
3332
|
+
requestedAt: cancellation.requested_at,
|
|
3333
|
+
outcome,
|
|
3334
|
+
}];
|
|
3335
|
+
});
|
|
3336
|
+
}
|
|
3337
|
+
async confirmDurableStop(snapshot, attempt, intent, receipt) {
|
|
3338
|
+
const activation = snapshot.activations.find((candidate) => candidate.activation_id === attempt.activation_id);
|
|
3339
|
+
const now = receipt.payload.stopped_at;
|
|
3340
|
+
const terminalAttempt = {
|
|
3341
|
+
...attempt,
|
|
3342
|
+
phase: "terminal",
|
|
3343
|
+
completed_at: now,
|
|
3344
|
+
terminal_outcome: receipt.payload.terminal_outcome,
|
|
3345
|
+
};
|
|
3346
|
+
const plan = await this.requirePinnedPlan(snapshot);
|
|
3347
|
+
const node = nodeById(plan.plan, attempt.step_id);
|
|
3348
|
+
const retryable = intent.purpose === "execution_deadline"
|
|
3349
|
+
&& node.kind === "worker"
|
|
3350
|
+
&& attempt.attempt_number < node.attempt_policy.max_attempts
|
|
3351
|
+
&& node.attempt_policy.retryable_outcomes.includes("timeout");
|
|
3352
|
+
const state = retryable ? "ready"
|
|
3353
|
+
: intent.purpose === "execution_deadline" ? "failed" : "cancelled";
|
|
3354
|
+
const terminalActivation = { ...activation, state, updated_at: now };
|
|
3355
|
+
const events = [
|
|
3356
|
+
this.event(snapshot, 0, {
|
|
3357
|
+
type: "attempt_terminal",
|
|
3358
|
+
step_id: attempt.step_id,
|
|
3359
|
+
activation_id: attempt.activation_id,
|
|
3360
|
+
attempt_id: attempt.attempt_id,
|
|
3361
|
+
result_id: null,
|
|
3362
|
+
key: `attempt:${attempt.attempt_id}:stopped`,
|
|
3363
|
+
at: now,
|
|
3364
|
+
payload: { executor: attempt.executor, outcome: receipt.payload.terminal_outcome, worker_stop_receipt: receipt, worker_nonlaunch_attestation: null },
|
|
3365
|
+
}),
|
|
3366
|
+
this.activationEvent(snapshot, 1, activation, terminalActivation, intent.purpose, now),
|
|
3367
|
+
];
|
|
3368
|
+
return (await this.runStore.updateCanonicalRun({
|
|
3369
|
+
run_id: snapshot.run.run_id,
|
|
3370
|
+
expected_record_version: snapshot.record_version,
|
|
3371
|
+
attempts: [terminalAttempt],
|
|
3372
|
+
activations: [terminalActivation],
|
|
3373
|
+
events,
|
|
3374
|
+
})).snapshot;
|
|
3375
|
+
}
|
|
3376
|
+
async confirmAlreadyDurableStop(snapshot, attempt, intent, evidence) {
|
|
3377
|
+
const activation = snapshot.activations.find((candidate) => candidate.activation_id === attempt.activation_id);
|
|
3378
|
+
const now = evidence.payload.completed_at;
|
|
3379
|
+
const terminalAttempt = {
|
|
3380
|
+
...attempt,
|
|
3381
|
+
phase: "terminal",
|
|
3382
|
+
completed_at: now,
|
|
3383
|
+
terminal_outcome: intent.requested_outcome,
|
|
3384
|
+
};
|
|
3385
|
+
const plan = await this.requirePinnedPlan(snapshot);
|
|
3386
|
+
const node = nodeById(plan.plan, attempt.step_id);
|
|
3387
|
+
const retryable = intent.purpose === "execution_deadline"
|
|
3388
|
+
&& node.kind === "worker"
|
|
3389
|
+
&& attempt.attempt_number < node.attempt_policy.max_attempts
|
|
3390
|
+
&& node.attempt_policy.retryable_outcomes.includes("timeout");
|
|
3391
|
+
const terminalActivation = {
|
|
3392
|
+
...activation,
|
|
3393
|
+
state: retryable ? "ready" : intent.purpose === "execution_deadline" ? "failed" : "cancelled",
|
|
3394
|
+
updated_at: now,
|
|
3395
|
+
};
|
|
3396
|
+
const events = [
|
|
3397
|
+
this.event(snapshot, 0, {
|
|
3398
|
+
type: "attempt_terminal",
|
|
3399
|
+
step_id: attempt.step_id,
|
|
3400
|
+
activation_id: attempt.activation_id,
|
|
3401
|
+
attempt_id: attempt.attempt_id,
|
|
3402
|
+
result_id: null,
|
|
3403
|
+
key: `attempt:${attempt.attempt_id}:already-stopped`,
|
|
3404
|
+
at: now,
|
|
3405
|
+
payload: {
|
|
3406
|
+
executor: attempt.executor,
|
|
3407
|
+
outcome: intent.requested_outcome,
|
|
3408
|
+
worker_stop_receipt: null,
|
|
3409
|
+
worker_nonlaunch_attestation: null,
|
|
3410
|
+
worker_already_stopped_evidence: evidence,
|
|
3411
|
+
},
|
|
3412
|
+
}),
|
|
3413
|
+
this.activationEvent(snapshot, 1, activation, terminalActivation, intent.purpose, now),
|
|
3414
|
+
];
|
|
3415
|
+
return (await this.runStore.updateCanonicalRun({
|
|
3416
|
+
run_id: snapshot.run.run_id,
|
|
3417
|
+
expected_record_version: snapshot.record_version,
|
|
3418
|
+
attempts: [terminalAttempt],
|
|
3419
|
+
activations: [terminalActivation],
|
|
3420
|
+
events,
|
|
3421
|
+
})).snapshot;
|
|
3422
|
+
}
|
|
3423
|
+
async confirmNeverLaunchedStop(snapshot, attempt, intent, attestation) {
|
|
3424
|
+
const activation = snapshot.activations.find((candidate) => candidate.activation_id === attempt.activation_id);
|
|
3425
|
+
const now = attestation.payload.attested_at;
|
|
3426
|
+
const terminalAttempt = {
|
|
3427
|
+
...attempt,
|
|
3428
|
+
phase: "terminal",
|
|
3429
|
+
completed_at: now,
|
|
3430
|
+
terminal_outcome: intent.requested_outcome,
|
|
3431
|
+
};
|
|
3432
|
+
const plan = await this.requirePinnedPlan(snapshot);
|
|
3433
|
+
const node = nodeById(plan.plan, attempt.step_id);
|
|
3434
|
+
const retryable = intent.purpose === "execution_deadline"
|
|
3435
|
+
&& node.kind === "worker"
|
|
3436
|
+
&& attempt.attempt_number < node.attempt_policy.max_attempts
|
|
3437
|
+
&& node.attempt_policy.retryable_outcomes.includes("timeout");
|
|
3438
|
+
const terminalActivation = {
|
|
3439
|
+
...activation,
|
|
3440
|
+
state: retryable ? "ready" : intent.purpose === "execution_deadline" ? "failed" : "cancelled",
|
|
3441
|
+
updated_at: now,
|
|
3442
|
+
};
|
|
3443
|
+
const events = [
|
|
3444
|
+
this.event(snapshot, 0, {
|
|
3445
|
+
type: "attempt_terminal",
|
|
3446
|
+
step_id: attempt.step_id,
|
|
3447
|
+
activation_id: attempt.activation_id,
|
|
3448
|
+
attempt_id: attempt.attempt_id,
|
|
3449
|
+
result_id: null,
|
|
3450
|
+
key: `attempt:${attempt.attempt_id}:never-launched`,
|
|
3451
|
+
at: now,
|
|
3452
|
+
payload: {
|
|
3453
|
+
executor: attempt.executor,
|
|
3454
|
+
outcome: intent.requested_outcome,
|
|
3455
|
+
worker_stop_receipt: null,
|
|
3456
|
+
worker_nonlaunch_attestation: attestation,
|
|
3457
|
+
},
|
|
3458
|
+
}),
|
|
3459
|
+
this.activationEvent(snapshot, 1, activation, terminalActivation, intent.purpose, now),
|
|
3460
|
+
];
|
|
3461
|
+
return (await this.runStore.updateCanonicalRun({
|
|
3462
|
+
run_id: snapshot.run.run_id,
|
|
3463
|
+
expected_record_version: snapshot.record_version,
|
|
3464
|
+
attempts: [terminalAttempt],
|
|
3465
|
+
activations: [terminalActivation],
|
|
3466
|
+
events,
|
|
3467
|
+
})).snapshot;
|
|
3468
|
+
}
|
|
3469
|
+
async aggregateDurableStops(initial) {
|
|
3470
|
+
let snapshot = initial;
|
|
3471
|
+
for (;;) {
|
|
3472
|
+
if (snapshot.run.terminal !== null)
|
|
3473
|
+
return snapshot;
|
|
3474
|
+
const intents = snapshot.events.filter((event) => event.type === "attempt_stop_requested");
|
|
3475
|
+
const cancelling = snapshot.run.status === "cancelling";
|
|
3476
|
+
const currentDeadlineIntents = intents.filter((event) => {
|
|
3477
|
+
if (event.type !== "attempt_stop_requested" || event.payload.intent.purpose !== "execution_deadline")
|
|
3478
|
+
return false;
|
|
3479
|
+
const attempt = snapshot.attempts.find((candidate) => candidate.attempt_id === event.attempt_id);
|
|
3480
|
+
const activation = attempt === undefined ? undefined : snapshot.activations.find((candidate) => candidate.activation_id === attempt.activation_id);
|
|
3481
|
+
return activation?.current_attempt_id === event.attempt_id;
|
|
3482
|
+
});
|
|
3483
|
+
const drainIntents = cancelling ? intents : currentDeadlineIntents;
|
|
3484
|
+
const activeWorkerExecution = snapshot.attempts.some((attempt) => (attempt.phase !== "terminal"
|
|
3485
|
+
&& attempt.executor?.kind === "worker"
|
|
3486
|
+
&& attempt.started_at !== null));
|
|
3487
|
+
if ((cancelling && activeWorkerExecution)
|
|
3488
|
+
|| (!cancelling && drainIntents.length === 0)
|
|
3489
|
+
|| drainIntents.some((event) => snapshot.attempts.find((attempt) => attempt.attempt_id === event.attempt_id)?.phase !== "terminal")) {
|
|
3490
|
+
return snapshot;
|
|
3491
|
+
}
|
|
3492
|
+
const now = this.now();
|
|
3493
|
+
const deadline = currentDeadlineIntents.length > 0;
|
|
3494
|
+
if (deadline && snapshot.activations.some((activation) => activation.state === "ready"))
|
|
3495
|
+
return snapshot;
|
|
3496
|
+
if (!cancelling && !deadline)
|
|
3497
|
+
return snapshot;
|
|
3498
|
+
const attempts = [];
|
|
3499
|
+
const activations = [];
|
|
3500
|
+
const events = [];
|
|
3501
|
+
for (const activation of snapshot.activations) {
|
|
3502
|
+
if (TERMINAL_ACTIVATION_STATES.has(activation.state))
|
|
3503
|
+
continue;
|
|
3504
|
+
const attempt = activation.current_attempt_id === null ? null : snapshot.attempts.find((candidate) => candidate.attempt_id === activation.current_attempt_id) ?? null;
|
|
3505
|
+
if (attempt && attempt.phase !== "terminal") {
|
|
3506
|
+
const executor = attempt.started_at === null
|
|
3507
|
+
? { kind: "system", service_instance_id: this.serviceInstanceId, operation: "aggregate_stop" }
|
|
3508
|
+
: attempt.executor ?? { kind: "system", service_instance_id: this.serviceInstanceId, operation: "aggregate_stop" };
|
|
3509
|
+
const outcome = { kind: "cancelled", reason: cancelling ? snapshot.run.cancellation?.reason ?? null : "execution_deadline_elapsed" };
|
|
3510
|
+
attempts.push({ ...attempt, phase: "terminal", executor, started_at: attempt.started_at ?? now, completed_at: now, terminal_outcome: outcome });
|
|
3511
|
+
if (attempt.started_at === null)
|
|
3512
|
+
events.push(this.event(snapshot, events.length, { type: "attempt_started", step_id: attempt.step_id, activation_id: attempt.activation_id, attempt_id: attempt.attempt_id, result_id: null, key: `attempt:${attempt.attempt_id}:aggregate-started`, at: now, payload: { executor } }));
|
|
3513
|
+
events.push(this.event(snapshot, events.length, { type: "attempt_terminal", step_id: attempt.step_id, activation_id: attempt.activation_id, attempt_id: attempt.attempt_id, result_id: null, key: `attempt:${attempt.attempt_id}:aggregate-terminal`, at: now, payload: { executor, outcome, worker_stop_receipt: null, worker_nonlaunch_attestation: null } }));
|
|
3514
|
+
}
|
|
3515
|
+
const terminal = { ...activation, state: "cancelled", updated_at: now };
|
|
3516
|
+
activations.push(terminal);
|
|
3517
|
+
events.push(this.activationEvent(snapshot, events.length, activation, terminal, cancelling ? "run_cancelled" : "run_failed", now));
|
|
3518
|
+
}
|
|
3519
|
+
const run = cancelling
|
|
3520
|
+
? this.terminalRun(snapshot.run, "cancelled", "run_cancelled", now)
|
|
3521
|
+
: this.terminalRun(snapshot.run, "failed", "execution_deadline_elapsed", now);
|
|
3522
|
+
events.push(this.runTerminalEvent(snapshot, events.length, run, now));
|
|
3523
|
+
try {
|
|
3524
|
+
return (await this.runStore.updateCanonicalRun({ run_id: snapshot.run.run_id, expected_record_version: snapshot.record_version, run, attempts, activations, events })).snapshot;
|
|
3525
|
+
}
|
|
3526
|
+
catch (error) {
|
|
3527
|
+
if (!this.isVersionConflict(error))
|
|
3528
|
+
throw error;
|
|
3529
|
+
snapshot = await this.requireSnapshot(snapshot.run.run_id);
|
|
3530
|
+
}
|
|
3531
|
+
}
|
|
3532
|
+
}
|
|
3533
|
+
async requestWorkerStop(snapshot, attempt, intent, expectedReceipt) {
|
|
3534
|
+
if (attempt.executor?.kind !== "worker" || attempt.started_at === null) {
|
|
3535
|
+
throw new CanonicalRunServiceError("attempt_state_conflict", "worker stop requires a running worker attempt");
|
|
3536
|
+
}
|
|
3537
|
+
if (this.executionAuthority?.requestStop === undefined) {
|
|
3538
|
+
throw new CanonicalRunServiceError("unsupported", "worker stop authority is unavailable");
|
|
3539
|
+
}
|
|
3540
|
+
const executionClaimEpoch = this.attemptRecovery(snapshot, attempt).execution_claim_epoch;
|
|
3541
|
+
const stopRequest = {
|
|
3542
|
+
run_id: attempt.run_id,
|
|
3543
|
+
step_id: attempt.step_id,
|
|
3544
|
+
activation_id: attempt.activation_id,
|
|
3545
|
+
attempt_id: attempt.attempt_id,
|
|
3546
|
+
claim_epoch: executionClaimEpoch,
|
|
3547
|
+
executor: attempt.executor,
|
|
3548
|
+
purpose: intent.purpose,
|
|
3549
|
+
requested_at: intent.requested_at,
|
|
3550
|
+
started_at: intent.started_at,
|
|
3551
|
+
requested_outcome: intent.requested_outcome,
|
|
3552
|
+
stop_idempotency_key: intent.stop_idempotency_key,
|
|
3553
|
+
expected_receipt: expectedReceipt,
|
|
3554
|
+
};
|
|
3555
|
+
let resolution;
|
|
3556
|
+
try {
|
|
3557
|
+
resolution = await this.executionAuthority.requestStop(stopRequest);
|
|
3558
|
+
}
|
|
3559
|
+
catch (error) {
|
|
3560
|
+
throw new CanonicalRunServiceError("unsupported", "worker stop authority is unavailable", { cause: error });
|
|
3561
|
+
}
|
|
3562
|
+
const matches = resolution?.kind === "stopped_by_request"
|
|
3563
|
+
? resolution.authority !== undefined
|
|
3564
|
+
&& workerStopReceiptMatchesStop(resolution.authority.stop_receipt, stopRequest)
|
|
3565
|
+
&& resolution.authority.authority_instance_id === resolution.authority.stop_receipt.payload.authority_instance_id
|
|
3566
|
+
&& resolution.authority.fencing_epoch === resolution.authority.stop_receipt.payload.fencing_epoch
|
|
3567
|
+
&& resolution.authority.fencing_epoch > 0
|
|
3568
|
+
: resolution?.kind === "already_stopped"
|
|
3569
|
+
&& resolution.drain_evidence !== undefined
|
|
3570
|
+
&& workerAlreadyStoppedEvidenceMatchesStop(resolution.drain_evidence, stopRequest)
|
|
3571
|
+
|| resolution?.kind === "never_launched"
|
|
3572
|
+
&& resolution.nonlaunch_attestation !== undefined
|
|
3573
|
+
&& workerNonlaunchAttestationMatches(resolution.nonlaunch_attestation, {
|
|
3574
|
+
run_id: stopRequest.run_id,
|
|
3575
|
+
step_id: stopRequest.step_id,
|
|
3576
|
+
activation_id: stopRequest.activation_id,
|
|
3577
|
+
attempt_id: stopRequest.attempt_id,
|
|
3578
|
+
execution_claim_epoch: stopRequest.claim_epoch,
|
|
3579
|
+
executor: stopRequest.executor,
|
|
3580
|
+
started_at: stopRequest.started_at,
|
|
3581
|
+
failure_idempotency_key: stopRequest.stop_idempotency_key,
|
|
3582
|
+
requested_at: stopRequest.requested_at,
|
|
3583
|
+
});
|
|
3584
|
+
if (!matches) {
|
|
3585
|
+
throw new CanonicalRunServiceError("submission_conflict", "worker stop authority does not match the durable attempt");
|
|
3586
|
+
}
|
|
3587
|
+
return resolution;
|
|
3588
|
+
}
|
|
3589
|
+
async withStoppedExecution(snapshot, _claim, attempt, execution, purpose, expectedReceipt, operation) {
|
|
3590
|
+
if (attempt.executor?.kind !== "worker")
|
|
3591
|
+
return operation(null);
|
|
3592
|
+
if (this.executionAuthority === null) {
|
|
3593
|
+
throw new CanonicalRunServiceError("unsupported", "worker finalization requires stopped-execution authority");
|
|
3594
|
+
}
|
|
3595
|
+
const authorityRequest = {
|
|
3596
|
+
run_id: attempt.run_id,
|
|
3597
|
+
step_id: attempt.step_id,
|
|
3598
|
+
activation_id: attempt.activation_id,
|
|
3599
|
+
attempt_id: attempt.attempt_id,
|
|
3600
|
+
claim_epoch: this.attemptRecovery(snapshot, attempt).execution_claim_epoch,
|
|
3601
|
+
executor: attempt.executor,
|
|
3602
|
+
purpose,
|
|
3603
|
+
reported_execution: {
|
|
3604
|
+
started_at: execution.started_at,
|
|
3605
|
+
completed_at: execution.completed_at,
|
|
3606
|
+
outcome: execution.outcome,
|
|
3607
|
+
},
|
|
3608
|
+
expected_receipt: expectedReceipt,
|
|
3609
|
+
};
|
|
3610
|
+
let entered = false;
|
|
3611
|
+
try {
|
|
3612
|
+
return await this.executionAuthority.withStoppedExecution(authorityRequest, async (authority) => {
|
|
3613
|
+
entered = true;
|
|
3614
|
+
if (!workerStopReceiptMatchesExecution(authority.stop_receipt, authorityRequest)
|
|
3615
|
+
|| authority.authority_instance_id
|
|
3616
|
+
!== authority.stop_receipt.payload.authority_instance_id
|
|
3617
|
+
|| authority.fencing_epoch !== authority.stop_receipt.payload.fencing_epoch
|
|
3618
|
+
|| authority.fencing_epoch <= 0
|
|
3619
|
+
|| (expectedReceipt !== null
|
|
3620
|
+
&& canonicalizeJson(authority.stop_receipt) !== canonicalizeJson(expectedReceipt))) {
|
|
3621
|
+
throw new CanonicalRunServiceError("submission_conflict", "stopped-execution authority does not match the durable attempt");
|
|
3622
|
+
}
|
|
3623
|
+
return operation(authority);
|
|
3624
|
+
});
|
|
3625
|
+
}
|
|
3626
|
+
catch (error) {
|
|
3627
|
+
if (entered)
|
|
3628
|
+
throw error;
|
|
3629
|
+
throw new CanonicalRunServiceError("unsupported", "stopped-execution authority is unavailable", { cause: error });
|
|
3630
|
+
}
|
|
3631
|
+
}
|
|
3632
|
+
async materializePacket(snapshot, plan, activation, attempt, executor, allocation) {
|
|
3633
|
+
const node = nodeById(plan.plan, activation.step_id);
|
|
3634
|
+
if (node.kind !== "worker" || executor.kind !== "worker")
|
|
3635
|
+
return null;
|
|
3636
|
+
if (allocation === null) {
|
|
3637
|
+
throw new CanonicalRunServiceError("unsupported", "worker attempt has no workspace allocation");
|
|
3638
|
+
}
|
|
3639
|
+
const handoffs = plan.plan.handoff_plan.filter((item) => item.to_node_id === node.node_id);
|
|
3640
|
+
const bindings = [];
|
|
3641
|
+
for (const handoff of handoffs) {
|
|
3642
|
+
const sourceActivation = snapshot.activations
|
|
3643
|
+
.filter((item) => item.step_id === handoff.from_node_id && item.committed_result_id !== null)
|
|
3644
|
+
.at(-1);
|
|
3645
|
+
if (!sourceActivation?.committed_result_id) {
|
|
3646
|
+
if (handoff.required) {
|
|
3647
|
+
throw new CanonicalRunServiceError("attempt_state_conflict", `required handoff ${handoff.handoff_id} is unavailable`);
|
|
3648
|
+
}
|
|
3649
|
+
bindings.push({
|
|
3650
|
+
binding_id: handoff.handoff_id, input_name: handoff.input_name,
|
|
3651
|
+
required: false, description: handoff.handoff_id, mode: "none", reason: "source_unavailable",
|
|
3652
|
+
});
|
|
3653
|
+
continue;
|
|
3654
|
+
}
|
|
3655
|
+
const reference = snapshot.result_refs.find((item) => item.result_id === sourceActivation.committed_result_id);
|
|
3656
|
+
const result = await this.resultStore.require(reference);
|
|
3657
|
+
const output = result.outputs.find((item) => item.name === handoff.output_name);
|
|
3658
|
+
if (!output) {
|
|
3659
|
+
if (handoff.required) {
|
|
3660
|
+
throw new CanonicalRunServiceError("attempt_state_conflict", `handoff output ${handoff.output_name} is unavailable`);
|
|
3661
|
+
}
|
|
3662
|
+
bindings.push({
|
|
3663
|
+
binding_id: handoff.handoff_id, input_name: handoff.input_name,
|
|
3664
|
+
required: false, description: handoff.handoff_id, mode: "none", reason: "output_unavailable",
|
|
3665
|
+
});
|
|
3666
|
+
continue;
|
|
3667
|
+
}
|
|
3668
|
+
const source = { result_id: result.result_id, step_id: result.step_id, output_name: output.name };
|
|
3669
|
+
if (handoff.mode === "reference") {
|
|
3670
|
+
if (output.kind !== "artifact") {
|
|
3671
|
+
throw new CanonicalRunServiceError("attempt_state_conflict", `handoff output ${handoff.output_name} is not a captured artifact`);
|
|
3672
|
+
}
|
|
3673
|
+
let deliveredReference;
|
|
3674
|
+
if (output.capture.artifact_kind === "git_commit") {
|
|
3675
|
+
const commit = await this.artifactStore.verifyGitCommitInWorkspace({
|
|
3676
|
+
artifact_id: output.capture.artifact_id,
|
|
3677
|
+
artifact_kind: "git_commit",
|
|
3678
|
+
}, allocation.working_directory);
|
|
3679
|
+
deliveredReference = {
|
|
3680
|
+
kind: "artifact",
|
|
3681
|
+
value: commit.commit_oid,
|
|
3682
|
+
artifact_id: output.capture.artifact_id,
|
|
3683
|
+
};
|
|
3684
|
+
}
|
|
3685
|
+
else {
|
|
3686
|
+
const materialized = await this.artifactMaterializer.materializeReplaySafe({
|
|
3687
|
+
artifact_id: output.capture.artifact_id,
|
|
3688
|
+
artifact_kind: output.capture.artifact_kind,
|
|
3689
|
+
destination_root: resolve(this.handoffRootDirectory, createHash("sha256").update(snapshot.run.run_id).digest("hex"), createHash("sha256").update(attempt.attempt_id).digest("hex")),
|
|
3690
|
+
binding_id: handoff.handoff_id,
|
|
3691
|
+
});
|
|
3692
|
+
deliveredReference = {
|
|
3693
|
+
kind: output.capture.artifact_kind === "file" ? "file" : "directory",
|
|
3694
|
+
value: materialized.materialized_path,
|
|
3695
|
+
artifact_id: output.capture.artifact_id,
|
|
3696
|
+
};
|
|
3697
|
+
}
|
|
3698
|
+
bindings.push({
|
|
3699
|
+
binding_id: handoff.handoff_id, input_name: handoff.input_name,
|
|
3700
|
+
required: handoff.required, description: handoff.handoff_id, mode: "reference", source,
|
|
3701
|
+
reference: deliveredReference,
|
|
3702
|
+
});
|
|
3703
|
+
}
|
|
3704
|
+
else if (handoff.mode === "quote") {
|
|
3705
|
+
if (output.kind !== "text") {
|
|
3706
|
+
throw new CanonicalRunServiceError("attempt_state_conflict", `handoff output ${handoff.output_name} is not exact text`);
|
|
3707
|
+
}
|
|
3708
|
+
bindings.push({
|
|
3709
|
+
binding_id: handoff.handoff_id, input_name: handoff.input_name,
|
|
3710
|
+
required: handoff.required, description: handoff.handoff_id, mode: "quote", source,
|
|
3711
|
+
exact_text: output.text, selection: { start: null, end: null, selector: null },
|
|
3712
|
+
});
|
|
3713
|
+
}
|
|
3714
|
+
else {
|
|
3715
|
+
if (output.kind !== "structured" || output.schema_ref !== handoff.schema_ref) {
|
|
3716
|
+
throw new CanonicalRunServiceError("attempt_state_conflict", `handoff output ${handoff.output_name} does not match its structured contract`);
|
|
3717
|
+
}
|
|
3718
|
+
bindings.push({
|
|
3719
|
+
binding_id: handoff.handoff_id, input_name: handoff.input_name,
|
|
3720
|
+
required: handoff.required, description: handoff.handoff_id, mode: "structured", source,
|
|
3721
|
+
value: output.value, schema_ref: output.schema_ref,
|
|
3722
|
+
});
|
|
3723
|
+
}
|
|
3724
|
+
}
|
|
3725
|
+
const workerNode = node;
|
|
3726
|
+
const promptValues = new Map();
|
|
3727
|
+
for (const inputName of workerNode.packet_template.runtime_input_names) {
|
|
3728
|
+
const value = snapshot.run.run_binding.binding.validated_inputs[inputName];
|
|
3729
|
+
if (value === undefined) {
|
|
3730
|
+
throw new CanonicalRunServiceError("invalid_request", `runtime input ${inputName} is missing from the pinned run binding`);
|
|
3731
|
+
}
|
|
3732
|
+
promptValues.set(inputName, runtimeTemplateValue(value));
|
|
3733
|
+
}
|
|
3734
|
+
const bindingById = new Map(bindings.map((binding) => [binding.binding_id, binding]));
|
|
3735
|
+
for (const handoff of handoffs) {
|
|
3736
|
+
const binding = bindingById.get(handoff.handoff_id);
|
|
3737
|
+
if (binding === undefined) {
|
|
3738
|
+
throw new CanonicalRunServiceError("attempt_state_conflict", `handoff ${handoff.handoff_id} was not resolved`);
|
|
3739
|
+
}
|
|
3740
|
+
if (binding.mode === "none") {
|
|
3741
|
+
const suffixes = handoff.mode === "reference"
|
|
3742
|
+
? handoff.artifact_kind === "git_commit"
|
|
3743
|
+
? ["commit_oid", "artifact_id"]
|
|
3744
|
+
: ["materialized_path", "artifact_id"]
|
|
3745
|
+
: handoff.mode === "quote" ? ["exact_text"] : ["json"];
|
|
3746
|
+
for (const suffix of suffixes) {
|
|
3747
|
+
promptValues.set(`handoff.${binding.binding_id}.${suffix}`, "null");
|
|
3748
|
+
}
|
|
3749
|
+
}
|
|
3750
|
+
else if (binding.mode === "reference" && handoff.mode === "reference") {
|
|
3751
|
+
promptValues.set(`handoff.${binding.binding_id}.${handoff.artifact_kind === "git_commit" ? "commit_oid" : "materialized_path"}`, binding.reference.value);
|
|
3752
|
+
if (binding.reference.artifact_id === null) {
|
|
3753
|
+
throw new CanonicalRunServiceError("attempt_state_conflict", `handoff ${binding.binding_id} has no immutable artifact identity`);
|
|
3754
|
+
}
|
|
3755
|
+
promptValues.set(`handoff.${binding.binding_id}.artifact_id`, binding.reference.artifact_id);
|
|
3756
|
+
}
|
|
3757
|
+
else if (binding.mode === "quote" && handoff.mode === "quote") {
|
|
3758
|
+
promptValues.set(`handoff.${binding.binding_id}.exact_text`, binding.exact_text);
|
|
3759
|
+
}
|
|
3760
|
+
else if (binding.mode === "structured" && handoff.mode === "structured") {
|
|
3761
|
+
promptValues.set(`handoff.${binding.binding_id}.json`, canonicalizeJson(binding.value));
|
|
3762
|
+
}
|
|
3763
|
+
else {
|
|
3764
|
+
throw new CanonicalRunServiceError("attempt_state_conflict", `handoff ${binding.binding_id} resolved with the wrong delivery mode`);
|
|
3765
|
+
}
|
|
3766
|
+
}
|
|
3767
|
+
const standalonePrompt = renderPacketPrompt(workerNode.packet_template.standalone_prompt, promptValues);
|
|
3768
|
+
return {
|
|
3769
|
+
packet_version: 1,
|
|
3770
|
+
run_id: snapshot.run.run_id,
|
|
3771
|
+
step_id: activation.step_id,
|
|
3772
|
+
attempt_id: attempt.attempt_id,
|
|
3773
|
+
worker_session_id: executor.worker_session_id,
|
|
3774
|
+
attempt_number: attempt.attempt_number,
|
|
3775
|
+
workflow_round: activation.workflow_round,
|
|
3776
|
+
role: workerNode.role,
|
|
3777
|
+
constraints: workerNode.packet_template.constraints,
|
|
3778
|
+
standalone_prompt: standalonePrompt,
|
|
3779
|
+
input_bindings: bindings,
|
|
3780
|
+
output_contract: workerNode.packet_template.output_contract.map((output) => ({
|
|
3781
|
+
name: output.name, kind: output.kind, required: output.required,
|
|
3782
|
+
description: output.description, schema_ref: output.schema_ref,
|
|
3783
|
+
})),
|
|
3784
|
+
workspace: {
|
|
3785
|
+
isolation: workerNode.workspace.isolation,
|
|
3786
|
+
working_directory: allocation.working_directory,
|
|
3787
|
+
},
|
|
3788
|
+
execution: {
|
|
3789
|
+
timeout_ms: workerNode.attempt_policy.timeout_ms,
|
|
3790
|
+
cancellation_grace_ms: workerNode.attempt_policy.cancellation_grace_ms,
|
|
3791
|
+
retry: {
|
|
3792
|
+
max_attempts: workerNode.attempt_policy.max_attempts,
|
|
3793
|
+
backoff: workerNode.attempt_policy.backoff,
|
|
3794
|
+
initial_delay_ms: workerNode.attempt_policy.initial_delay_ms,
|
|
3795
|
+
max_delay_ms: workerNode.attempt_policy.max_delay_ms,
|
|
3796
|
+
retryable_outcomes: workerNode.attempt_policy.retryable_outcomes,
|
|
3797
|
+
},
|
|
3798
|
+
},
|
|
3799
|
+
created_at: attempt.started_at ?? this.now(),
|
|
3800
|
+
};
|
|
3801
|
+
}
|
|
3802
|
+
async verifierHandoffBindings(snapshot, plan, activation, attempt, allocation) {
|
|
3803
|
+
if (attempt.executor?.kind !== "worker")
|
|
3804
|
+
return [];
|
|
3805
|
+
const packet = await this.materializePacket(snapshot, plan, activation, attempt, attempt.executor, allocation);
|
|
3806
|
+
const bindings = packet?.input_bindings ?? [];
|
|
3807
|
+
for (const binding of bindings) {
|
|
3808
|
+
if (binding.mode === "none")
|
|
3809
|
+
continue;
|
|
3810
|
+
const resolved = snapshot.events.some((event) => (event.type === "handoff_resolved"
|
|
3811
|
+
&& event.attempt_id === attempt.attempt_id
|
|
3812
|
+
&& event.payload.binding_id === binding.binding_id
|
|
3813
|
+
&& event.payload.source_result_id === binding.source.result_id
|
|
3814
|
+
&& event.payload.output_name === binding.source.output_name));
|
|
3815
|
+
if (!resolved) {
|
|
3816
|
+
throw new CanonicalRunServiceError("attempt_state_conflict", `handoff ${binding.binding_id} was not committed for attempt ${attempt.attempt_id}`);
|
|
3817
|
+
}
|
|
3818
|
+
}
|
|
3819
|
+
return bindings;
|
|
3820
|
+
}
|
|
3821
|
+
handoffResolutionEvents(snapshot, attempt, packet, initialOffset, at) {
|
|
3822
|
+
const bindings = packet?.input_bindings.filter((binding) => binding.mode !== "none") ?? [];
|
|
3823
|
+
return bindings.map((binding, offset) => this.event(snapshot, initialOffset + offset, {
|
|
3824
|
+
type: "handoff_resolved",
|
|
3825
|
+
step_id: attempt.step_id,
|
|
3826
|
+
activation_id: attempt.activation_id,
|
|
3827
|
+
attempt_id: attempt.attempt_id,
|
|
3828
|
+
result_id: null,
|
|
3829
|
+
key: `attempt:${attempt.attempt_id}:handoff:${binding.binding_id}`,
|
|
3830
|
+
at,
|
|
3831
|
+
payload: {
|
|
3832
|
+
binding_id: binding.binding_id,
|
|
3833
|
+
source_result_id: binding.source.result_id,
|
|
3834
|
+
output_name: binding.source.output_name,
|
|
3835
|
+
},
|
|
3836
|
+
}));
|
|
3837
|
+
}
|
|
3838
|
+
async findReplay(snapshot, plan, attemptId, resultId, keyDigest, submissionDigest) {
|
|
3839
|
+
const event = snapshot.events.find((candidate) => (candidate.type === "result_committed" && candidate.attempt_id === attemptId));
|
|
3840
|
+
if (!event)
|
|
3841
|
+
return null;
|
|
3842
|
+
if (event.payload.submission_key_digest !== keyDigest
|
|
3843
|
+
|| event.payload.submission_digest !== submissionDigest
|
|
3844
|
+
|| event.result_id !== resultId) {
|
|
3845
|
+
throw new CanonicalRunServiceError("submission_conflict", "attempt already has a different committed submission");
|
|
3846
|
+
}
|
|
3847
|
+
const reference = snapshot.result_refs.find((item) => item.result_id === resultId);
|
|
3848
|
+
if (!reference)
|
|
3849
|
+
throw new CanonicalRunServiceError("submission_conflict", "committed result reference is missing");
|
|
3850
|
+
return { result: await this.resultStore.require(reference), replayed: true, run: this.toView(snapshot, plan) };
|
|
3851
|
+
}
|
|
3852
|
+
submissionEvidence(result) {
|
|
3853
|
+
const evidence = result.evidence_refs.filter((item) => item.kind === "submission");
|
|
3854
|
+
if (evidence.length !== 1) {
|
|
3855
|
+
throw new CanonicalRunServiceError("submission_conflict", `published result ${result.result_id} lacks unique submission evidence`);
|
|
3856
|
+
}
|
|
3857
|
+
return evidence[0];
|
|
3858
|
+
}
|
|
3859
|
+
workerStopReceipt(result) {
|
|
3860
|
+
const evidence = result.evidence_refs.filter((item) => item.kind === "worker_stopped");
|
|
3861
|
+
if (result.executor.kind !== "worker") {
|
|
3862
|
+
if (evidence.length !== 0) {
|
|
3863
|
+
throw new CanonicalRunServiceError("submission_conflict", `non-worker result ${result.result_id} carries worker stop evidence`);
|
|
3864
|
+
}
|
|
3865
|
+
return null;
|
|
3866
|
+
}
|
|
3867
|
+
if (evidence.length !== 1) {
|
|
3868
|
+
throw new CanonicalRunServiceError("submission_conflict", `worker result ${result.result_id} lacks unique stop evidence`);
|
|
3869
|
+
}
|
|
3870
|
+
return evidence[0].receipt;
|
|
3871
|
+
}
|
|
3872
|
+
outputsPreserveAuthority(expected, actual) {
|
|
3873
|
+
if (expected.length !== actual.length)
|
|
3874
|
+
return false;
|
|
3875
|
+
const byName = new Map(actual.map((output) => [output.name, output]));
|
|
3876
|
+
return expected.every((output) => {
|
|
3877
|
+
const candidate = byName.get(output.name);
|
|
3878
|
+
if (!candidate || candidate.kind !== output.kind)
|
|
3879
|
+
return false;
|
|
3880
|
+
if (output.kind === "artifact" && candidate.kind === "artifact") {
|
|
3881
|
+
return output.capture.artifact_id === candidate.capture.artifact_id
|
|
3882
|
+
&& output.capture.artifact_kind === candidate.capture.artifact_kind;
|
|
3883
|
+
}
|
|
3884
|
+
return canonicalizeJson(output) === canonicalizeJson(candidate);
|
|
3885
|
+
});
|
|
3886
|
+
}
|
|
3887
|
+
async revalidateWorkspaceObservations(result, allocation) {
|
|
3888
|
+
const observations = result.verification_results.flatMap((verification) => (verification.evidence_refs.flatMap((evidence) => (evidence.workspace_observation === null ? [] : [evidence.workspace_observation]))));
|
|
3889
|
+
if (observations.length === 0)
|
|
3890
|
+
return;
|
|
3891
|
+
for (const expected of observations) {
|
|
3892
|
+
if (expected.workspace_id !== allocation.workspace_id) {
|
|
3893
|
+
throw new CanonicalRunServiceError("submission_conflict", `published result ${result.result_id} observes a different workspace allocation`);
|
|
3894
|
+
}
|
|
3895
|
+
const actual = await observeAttemptWorkspace(allocation.working_directory, allocation.workspace_id);
|
|
3896
|
+
if (canonicalizeJson(actual) !== canonicalizeJson(expected)) {
|
|
3897
|
+
throw new CanonicalRunServiceError("source_drift", `published result ${result.result_id} workspace changed after verification`);
|
|
3898
|
+
}
|
|
3899
|
+
}
|
|
3900
|
+
}
|
|
3901
|
+
async reconcilePublishedResults(initial, plan) {
|
|
3902
|
+
let snapshot = initial;
|
|
3903
|
+
for (;;) {
|
|
3904
|
+
const activeAttempts = snapshot.attempts.filter((candidate) => candidate.phase !== "terminal");
|
|
3905
|
+
let reconciled = false;
|
|
3906
|
+
for (const attempt of activeAttempts) {
|
|
3907
|
+
const published = await this.resultStore.loadPublished(resultIdFor(this.identities, attempt.attempt_id));
|
|
3908
|
+
if (!published)
|
|
3909
|
+
continue;
|
|
3910
|
+
snapshot = await this.reconcilePublishedResult(snapshot, plan, published.result);
|
|
3911
|
+
reconciled = true;
|
|
3912
|
+
break;
|
|
3913
|
+
}
|
|
3914
|
+
if (!reconciled)
|
|
3915
|
+
return snapshot;
|
|
3916
|
+
}
|
|
3917
|
+
}
|
|
3918
|
+
async reconcilePublishedResult(initial, plan, result) {
|
|
3919
|
+
if (result.executor.kind === "system") {
|
|
3920
|
+
return this.reconcilePublishedSystemResult(initial, plan, result);
|
|
3921
|
+
}
|
|
3922
|
+
const initialAttempt = initial.attempts.find((candidate) => candidate.attempt_id === result.attempt_id);
|
|
3923
|
+
const initialActivation = initial.activations.find((candidate) => candidate.activation_id === result.activation_id);
|
|
3924
|
+
if (!initialAttempt || !initialActivation) {
|
|
3925
|
+
throw new CanonicalRunServiceError("submission_conflict", `published result ${result.result_id} has no durable attempt allocation`);
|
|
3926
|
+
}
|
|
3927
|
+
const allocation = this.requireCommittedWorkspace(initial, initialAttempt, nodeById(plan.plan, initialActivation.step_id));
|
|
3928
|
+
const claim = this.claimsFor(initial, initialAttempt.attempt_id).at(-1);
|
|
3929
|
+
if (!claim) {
|
|
3930
|
+
throw new CanonicalRunServiceError("submission_conflict", `published result ${result.result_id} has no durable claim`);
|
|
3931
|
+
}
|
|
3932
|
+
const execution = {
|
|
3933
|
+
executor: result.executor,
|
|
3934
|
+
started_at: result.started_at,
|
|
3935
|
+
completed_at: result.completed_at,
|
|
3936
|
+
outcome: result.terminal_outcome,
|
|
3937
|
+
outputs: result.outputs.map((output) => {
|
|
3938
|
+
if (output.kind === "text")
|
|
3939
|
+
return output;
|
|
3940
|
+
if (output.kind === "structured")
|
|
3941
|
+
return output;
|
|
3942
|
+
return {
|
|
3943
|
+
name: output.name,
|
|
3944
|
+
kind: "reference",
|
|
3945
|
+
artifact_kind: output.capture.artifact_kind,
|
|
3946
|
+
source_path: output.capture.source_path,
|
|
3947
|
+
};
|
|
3948
|
+
}),
|
|
3949
|
+
workspace: {
|
|
3950
|
+
root: allocation.working_directory,
|
|
3951
|
+
workspace_id: allocation.workspace_id,
|
|
3952
|
+
},
|
|
3953
|
+
stdout_reference: result.stdout_reference,
|
|
3954
|
+
stderr_reference: result.stderr_reference,
|
|
3955
|
+
};
|
|
3956
|
+
const expectedReceipt = this.workerStopReceipt(result);
|
|
3957
|
+
const handoffBindings = await this.verifierHandoffBindings(initial, plan, initialActivation, initialAttempt, allocation);
|
|
3958
|
+
const replayVerificationRequest = {
|
|
3959
|
+
plan,
|
|
3960
|
+
run: initial.run,
|
|
3961
|
+
activation: initialActivation,
|
|
3962
|
+
attempt: initialAttempt,
|
|
3963
|
+
execution,
|
|
3964
|
+
workspace_allocation: allocation,
|
|
3965
|
+
captured_outputs: result.outputs,
|
|
3966
|
+
handoff_bindings: handoffBindings,
|
|
3967
|
+
};
|
|
3968
|
+
const serviceVerification = result.verification_results.filter((item) => item.authority === "service");
|
|
3969
|
+
const additionalVerification = result.verification_results.filter((item) => item.authority === "additional");
|
|
3970
|
+
this.validateVerificationLanes(replayVerificationRequest, serviceVerification, additionalVerification);
|
|
3971
|
+
await this.validateCommandVerifierReceipts(replayVerificationRequest, serviceVerification);
|
|
3972
|
+
return this.withStoppedExecution(initial, claim, initialAttempt, execution, "reconcile_published_result", expectedReceipt, async (executionAuthority) => this.withWorkspaceAuthority(allocation, "reconcile_published_result", false, async (workspaceAuthority) => {
|
|
3973
|
+
for (;;) {
|
|
3974
|
+
const snapshot = await this.requireSnapshot(initial.run.run_id);
|
|
3975
|
+
const existing = snapshot.result_refs.find((item) => item.result_id === result.result_id);
|
|
3976
|
+
if (existing)
|
|
3977
|
+
return snapshot;
|
|
3978
|
+
if (snapshot.run.status === "cancelling") {
|
|
3979
|
+
throw new CanonicalRunServiceError("run_terminal", "run cancellation has been requested");
|
|
3980
|
+
}
|
|
3981
|
+
const attempt = snapshot.attempts.find((candidate) => candidate.attempt_id === result.attempt_id);
|
|
3982
|
+
const activation = snapshot.activations.find((candidate) => candidate.activation_id === result.activation_id);
|
|
3983
|
+
if (!attempt
|
|
3984
|
+
|| !activation
|
|
3985
|
+
|| attempt.phase !== "running"
|
|
3986
|
+
|| attempt.executor === null
|
|
3987
|
+
|| result.run_id !== snapshot.run.run_id
|
|
3988
|
+
|| result.step_id !== attempt.step_id
|
|
3989
|
+
|| canonicalizeJson(result.executor) !== canonicalizeJson(attempt.executor)) {
|
|
3990
|
+
throw new CanonicalRunServiceError("submission_conflict", `published result ${result.result_id} cannot reconcile to its attempt`);
|
|
3991
|
+
}
|
|
3992
|
+
const published = await this.resultStore.loadPublished(result.result_id);
|
|
3993
|
+
if (!published || canonicalizeJson(published.result) !== canonicalizeJson(result)) {
|
|
3994
|
+
throw new CanonicalRunServiceError("submission_conflict", "published result changed during reconciliation");
|
|
3995
|
+
}
|
|
3996
|
+
const submission = this.submissionEvidence(result);
|
|
3997
|
+
const recaptured = await this.outputCapture.capture({
|
|
3998
|
+
run_id: snapshot.run.run_id,
|
|
3999
|
+
activation_id: activation.activation_id,
|
|
4000
|
+
attempt_id: attempt.attempt_id,
|
|
4001
|
+
execution,
|
|
4002
|
+
});
|
|
4003
|
+
await recaptured.revalidate();
|
|
4004
|
+
if (!this.outputsPreserveAuthority(result.outputs, recaptured.outputs)) {
|
|
4005
|
+
throw new CanonicalRunServiceError("source_drift", `published result ${result.result_id} source no longer matches its immutable capture`);
|
|
4006
|
+
}
|
|
4007
|
+
await this.revalidateWorkspaceObservations(result, allocation);
|
|
4008
|
+
const drained = await this.prepareFailurePeerStops(snapshot, plan, attempt, result.terminal_outcome, this.now());
|
|
4009
|
+
if (drained.record_version !== snapshot.record_version)
|
|
4010
|
+
continue;
|
|
4011
|
+
const verification = result.verification_results;
|
|
4012
|
+
const captures = result.outputs.flatMap((output) => output.kind === "artifact" ? [output.capture] : []);
|
|
4013
|
+
const decisionOutputs = await this.loadDecisionOutputs(snapshot, plan, result);
|
|
4014
|
+
const update = await this.buildSubmissionUpdate(snapshot, plan, activation, attempt, result, published.reference, captures, verification, submission.submission_key_digest, submission.submission_digest, this.now(), decisionOutputs, {
|
|
4015
|
+
worker_stop_receipt_digest: executionAuthority?.stop_receipt.receipt_digest ?? null,
|
|
4016
|
+
worker_stop_receipt: executionAuthority?.stop_receipt ?? null,
|
|
4017
|
+
workspace_authority: workspaceAuthority,
|
|
4018
|
+
purpose: "reconcile_published_result",
|
|
4019
|
+
});
|
|
4020
|
+
if (update.run?.terminal !== null) {
|
|
4021
|
+
const peerDrained = await this.prepareTerminalPeerStops(snapshot, attempt, this.now());
|
|
4022
|
+
if (peerDrained.record_version !== snapshot.record_version)
|
|
4023
|
+
continue;
|
|
4024
|
+
}
|
|
4025
|
+
try {
|
|
4026
|
+
return (await this.runStore.updateCanonicalRun(update)).snapshot;
|
|
4027
|
+
}
|
|
4028
|
+
catch (error) {
|
|
4029
|
+
if (this.isVersionConflict(error))
|
|
4030
|
+
continue;
|
|
4031
|
+
throw error;
|
|
4032
|
+
}
|
|
4033
|
+
}
|
|
4034
|
+
}));
|
|
4035
|
+
}
|
|
4036
|
+
async loadDecisionOutputs(snapshot, plan, currentResult) {
|
|
4037
|
+
const outputs = new Map();
|
|
4038
|
+
for (const decision of plan.plan.nodes) {
|
|
4039
|
+
if (decision.kind !== "decision")
|
|
4040
|
+
continue;
|
|
4041
|
+
if (decision.source.node_id === currentResult.step_id) {
|
|
4042
|
+
outputs.set(decision.node_id, currentResult.outputs);
|
|
4043
|
+
continue;
|
|
4044
|
+
}
|
|
4045
|
+
const activation = this.latestActivationForStep(snapshot.activations, decision.source.node_id);
|
|
4046
|
+
if (!activation?.committed_result_id)
|
|
4047
|
+
continue;
|
|
4048
|
+
const reference = snapshot.result_refs.find((candidate) => candidate.result_id === activation.committed_result_id);
|
|
4049
|
+
if (!reference)
|
|
4050
|
+
continue;
|
|
4051
|
+
outputs.set(decision.node_id, (await this.resultStore.require(reference)).outputs);
|
|
4052
|
+
}
|
|
4053
|
+
return outputs;
|
|
4054
|
+
}
|
|
4055
|
+
async buildSubmissionUpdate(snapshot, plan, activation, attempt, result, reference, captures, verification, keyDigest, submissionDigest, now, decisionOutputs, finalizationAuthority) {
|
|
4056
|
+
const outcomeStatus = canonicalAttemptOutcomeStatus(result.terminal_outcome);
|
|
4057
|
+
const node = nodeById(plan.plan, activation.step_id);
|
|
4058
|
+
const retryOutcome = result.terminal_outcome.kind === "worker_process"
|
|
4059
|
+
&& result.terminal_outcome.process.kind !== "success"
|
|
4060
|
+
? result.terminal_outcome.process.kind
|
|
4061
|
+
: outcomeStatus === "cancelled" ? "cancelled" : "malformed_result";
|
|
4062
|
+
const retryable = node.kind === "worker"
|
|
4063
|
+
&& outcomeStatus !== "succeeded"
|
|
4064
|
+
&& attempt.attempt_number < node.attempt_policy.max_attempts
|
|
4065
|
+
&& node.attempt_policy.retryable_outcomes.includes(retryOutcome);
|
|
4066
|
+
const retrySegment = node.kind === "worker" && node.attempt_policy.retry_segment_id !== null
|
|
4067
|
+
? plan.plan.retry_segments.find((segment) => (segment.retry_segment_id === node.attempt_policy.retry_segment_id)) ?? null
|
|
4068
|
+
: null;
|
|
4069
|
+
const fromRound = retrySegment === null ? 0 : activation.workflow_round;
|
|
4070
|
+
const segmentRetryable = !retryable
|
|
4071
|
+
&& outcomeStatus !== "succeeded"
|
|
4072
|
+
&& retrySegment !== null
|
|
4073
|
+
&& retrySegment.retryable_outcomes.includes(retryOutcome)
|
|
4074
|
+
&& fromRound < retrySegment.max_rounds;
|
|
4075
|
+
const retrySegmentExhausted = !retryable
|
|
4076
|
+
&& outcomeStatus !== "succeeded"
|
|
4077
|
+
&& retrySegment !== null
|
|
4078
|
+
&& retrySegment.retryable_outcomes.includes(retryOutcome)
|
|
4079
|
+
&& fromRound >= retrySegment.max_rounds;
|
|
4080
|
+
const terminalAttempt = {
|
|
4081
|
+
...attempt,
|
|
4082
|
+
phase: "terminal",
|
|
4083
|
+
completed_at: result.completed_at,
|
|
4084
|
+
terminal_outcome: result.terminal_outcome,
|
|
4085
|
+
};
|
|
4086
|
+
const completedActivation = {
|
|
4087
|
+
...activation,
|
|
4088
|
+
state: retryable ? "ready" : outcomeStatus,
|
|
4089
|
+
committed_result_id: outcomeStatus === "succeeded" ? result.result_id : activation.committed_result_id,
|
|
4090
|
+
updated_at: now,
|
|
4091
|
+
};
|
|
4092
|
+
const activations = [completedActivation];
|
|
4093
|
+
const attempts = [terminalAttempt];
|
|
4094
|
+
const events = [];
|
|
4095
|
+
let offset = 0;
|
|
4096
|
+
events.push(this.activationEvent(snapshot, offset++, activation, completedActivation, retryable ? "retryable_attempt_failed" : "attempt_completed", now));
|
|
4097
|
+
events.push(this.event(snapshot, offset++, {
|
|
4098
|
+
type: "attempt_terminal", step_id: attempt.step_id, activation_id: attempt.activation_id,
|
|
4099
|
+
attempt_id: attempt.attempt_id, result_id: null, key: `attempt:${attempt.attempt_id}:terminal`, at: now,
|
|
4100
|
+
payload: {
|
|
4101
|
+
executor: terminalAttempt.executor,
|
|
4102
|
+
outcome: terminalAttempt.terminal_outcome,
|
|
4103
|
+
worker_stop_receipt: finalizationAuthority.worker_stop_receipt,
|
|
4104
|
+
worker_nonlaunch_attestation: null,
|
|
4105
|
+
},
|
|
4106
|
+
}));
|
|
4107
|
+
for (const capture of captures) {
|
|
4108
|
+
events.push(this.event(snapshot, offset++, {
|
|
4109
|
+
type: "artifact_captured", step_id: attempt.step_id, activation_id: attempt.activation_id,
|
|
4110
|
+
attempt_id: attempt.attempt_id, result_id: null,
|
|
4111
|
+
key: `capture:${capture.capture_id}`, at: now, payload: { capture },
|
|
4112
|
+
}));
|
|
4113
|
+
}
|
|
4114
|
+
events.push(this.event(snapshot, offset++, {
|
|
4115
|
+
type: "result_committed", step_id: attempt.step_id, activation_id: attempt.activation_id,
|
|
4116
|
+
attempt_id: attempt.attempt_id, result_id: result.result_id,
|
|
4117
|
+
key: `result:${result.result_id}:committed`, at: now,
|
|
4118
|
+
payload: {
|
|
4119
|
+
result_digest: reference.result_digest,
|
|
4120
|
+
submission_key_digest: keyDigest,
|
|
4121
|
+
submission_digest: submissionDigest,
|
|
4122
|
+
worker_stop_receipt_digest: finalizationAuthority.worker_stop_receipt_digest,
|
|
4123
|
+
workspace_allocation_digest: finalizationAuthority.workspace_authority?.workspace_allocation_digest ?? null,
|
|
4124
|
+
mutation_authority_instance_id: finalizationAuthority.workspace_authority?.authority_instance_id ?? null,
|
|
4125
|
+
mutation_fencing_epoch: finalizationAuthority.workspace_authority?.fencing_epoch ?? null,
|
|
4126
|
+
finalization_purpose: finalizationAuthority.purpose,
|
|
4127
|
+
},
|
|
4128
|
+
}));
|
|
4129
|
+
if (segmentRetryable && retrySegment !== null) {
|
|
4130
|
+
const toRound = fromRound + 1;
|
|
4131
|
+
const priorActivations = retrySegment.node_ids.map((stepId) => (stepId === activation.step_id
|
|
4132
|
+
? completedActivation
|
|
4133
|
+
: snapshot.activations.filter((candidate) => candidate.step_id === stepId).at(-1)));
|
|
4134
|
+
if (priorActivations.some((candidate) => (candidate === undefined || !TERMINAL_ACTIVATION_STATES.has(candidate.state)))) {
|
|
4135
|
+
throw new CanonicalRunServiceError("attempt_state_conflict", `retry segment ${retrySegment.retry_segment_id} has non-terminal prior activations`);
|
|
4136
|
+
}
|
|
4137
|
+
const replacements = retrySegment.node_ids.map((stepId) => ({
|
|
4138
|
+
run_id: snapshot.run.run_id,
|
|
4139
|
+
step_id: stepId,
|
|
4140
|
+
activation_id: this.activationRoundId(snapshot.run.run_id, stepId, toRound),
|
|
4141
|
+
workflow_round: toRound,
|
|
4142
|
+
state: stepId === retrySegment.retry_from_node_id ? "ready" : "dependency_blocked",
|
|
4143
|
+
attempt_count: 0,
|
|
4144
|
+
current_attempt_id: null,
|
|
4145
|
+
committed_result_id: null,
|
|
4146
|
+
created_at: now,
|
|
4147
|
+
updated_at: now,
|
|
4148
|
+
}));
|
|
4149
|
+
const delayMs = this.retrySegmentDelay(retrySegment, fromRound);
|
|
4150
|
+
activations.push(...replacements);
|
|
4151
|
+
events.push(this.event(snapshot, offset++, {
|
|
4152
|
+
type: "retry_segment_scheduled",
|
|
4153
|
+
step_id: activation.step_id,
|
|
4154
|
+
activation_id: activation.activation_id,
|
|
4155
|
+
attempt_id: null,
|
|
4156
|
+
result_id: result.result_id,
|
|
4157
|
+
key: `retry-segment:${retrySegment.retry_segment_id}:round:${toRound}`,
|
|
4158
|
+
at: now,
|
|
4159
|
+
payload: {
|
|
4160
|
+
retry_segment_id: retrySegment.retry_segment_id,
|
|
4161
|
+
from_round: fromRound,
|
|
4162
|
+
to_round: toRound,
|
|
4163
|
+
prior_activation_ids: priorActivations.map((candidate) => candidate.activation_id),
|
|
4164
|
+
replacement_activation_ids: replacements.map((candidate) => candidate.activation_id),
|
|
4165
|
+
retry_from_activation_id: replacements.find((candidate) => (candidate.step_id === retrySegment.retry_from_node_id)).activation_id,
|
|
4166
|
+
triggering_result_id: result.result_id,
|
|
4167
|
+
delay_ms: delayMs,
|
|
4168
|
+
ready_at: addMilliseconds(now, delayMs),
|
|
4169
|
+
reason_code: retryOutcome,
|
|
4170
|
+
},
|
|
4171
|
+
}));
|
|
4172
|
+
}
|
|
4173
|
+
if (node.kind === "interaction" && terminalAttempt.executor?.kind === "human") {
|
|
4174
|
+
events.push(this.event(snapshot, offset++, {
|
|
4175
|
+
type: "interaction_submitted",
|
|
4176
|
+
step_id: attempt.step_id,
|
|
4177
|
+
activation_id: attempt.activation_id,
|
|
4178
|
+
attempt_id: attempt.attempt_id,
|
|
4179
|
+
result_id: null,
|
|
4180
|
+
key: `interaction:${node.interaction_id}:submitted:${attempt.attempt_id}`,
|
|
4181
|
+
at: now,
|
|
4182
|
+
payload: {
|
|
4183
|
+
interaction_id: node.interaction_id,
|
|
4184
|
+
principal_ref: terminalAttempt.executor.principal_ref,
|
|
4185
|
+
attestation_ref: terminalAttempt.executor.attestation_ref,
|
|
4186
|
+
},
|
|
4187
|
+
}));
|
|
4188
|
+
}
|
|
4189
|
+
let run = snapshot.run;
|
|
4190
|
+
if (!retryable && !segmentRetryable && outcomeStatus === "succeeded") {
|
|
4191
|
+
const reduction = this.reduceAfterSuccess(snapshot, plan, completedActivation, reference.result_id, verification, now, offset, decisionOutputs);
|
|
4192
|
+
activations.push(...reduction.activations);
|
|
4193
|
+
events.push(...reduction.events);
|
|
4194
|
+
run = reduction.run;
|
|
4195
|
+
if (run.terminal !== null) {
|
|
4196
|
+
const cancelledAttempts = await this.cancelConcurrentAttempts(snapshot, activation.activation_id, now, "run_terminal", events.length);
|
|
4197
|
+
attempts.push(...cancelledAttempts.attempts);
|
|
4198
|
+
events.push(...cancelledAttempts.events);
|
|
4199
|
+
}
|
|
4200
|
+
}
|
|
4201
|
+
else if (!retryable && !segmentRetryable) {
|
|
4202
|
+
run = this.terminalRun(snapshot.run, "failed", retrySegmentExhausted
|
|
4203
|
+
? "retry_segment_max_rounds_exhausted"
|
|
4204
|
+
: "attempt_failed", now);
|
|
4205
|
+
const cancelledAttempts = await this.cancelConcurrentAttempts(snapshot, activation.activation_id, now, "peer_attempt_failed", events.length);
|
|
4206
|
+
attempts.push(...cancelledAttempts.attempts);
|
|
4207
|
+
events.push(...cancelledAttempts.events);
|
|
4208
|
+
for (const other of snapshot.activations) {
|
|
4209
|
+
if (other.activation_id === activation.activation_id || TERMINAL_ACTIVATION_STATES.has(other.state))
|
|
4210
|
+
continue;
|
|
4211
|
+
const cancelled = { ...other, state: "cancelled", updated_at: now };
|
|
4212
|
+
activations.push(cancelled);
|
|
4213
|
+
events.push(this.activationEvent(snapshot, events.length, other, cancelled, "run_failed", now));
|
|
4214
|
+
}
|
|
4215
|
+
events.push(this.runTerminalEvent(snapshot, events.length, run, now));
|
|
4216
|
+
}
|
|
4217
|
+
return {
|
|
4218
|
+
run_id: snapshot.run.run_id,
|
|
4219
|
+
expected_record_version: snapshot.record_version,
|
|
4220
|
+
run,
|
|
4221
|
+
activations,
|
|
4222
|
+
attempts,
|
|
4223
|
+
events,
|
|
4224
|
+
result_refs: [reference],
|
|
4225
|
+
};
|
|
4226
|
+
}
|
|
4227
|
+
async buildNonlaunchFailureUpdate(snapshot, plan, activation, attempt, outcome, failureDigest, attestation) {
|
|
4228
|
+
const now = attestation.payload.attested_at;
|
|
4229
|
+
const node = nodeById(plan.plan, activation.step_id);
|
|
4230
|
+
const retryable = node.kind === "worker"
|
|
4231
|
+
&& attempt.attempt_number < node.attempt_policy.max_attempts
|
|
4232
|
+
&& node.attempt_policy.retryable_outcomes.includes("launch_error");
|
|
4233
|
+
const retrySegment = node.kind === "worker" && node.attempt_policy.retry_segment_id !== null
|
|
4234
|
+
? plan.plan.retry_segments.find((segment) => segment.retry_segment_id === node.attempt_policy.retry_segment_id) ?? null
|
|
4235
|
+
: null;
|
|
4236
|
+
const fromRound = retrySegment === null ? 0 : activation.workflow_round;
|
|
4237
|
+
const segmentRetryable = !retryable
|
|
4238
|
+
&& retrySegment !== null
|
|
4239
|
+
&& retrySegment.retryable_outcomes.includes("launch_error")
|
|
4240
|
+
&& fromRound < retrySegment.max_rounds;
|
|
4241
|
+
const retrySegmentExhausted = !retryable
|
|
4242
|
+
&& retrySegment !== null
|
|
4243
|
+
&& retrySegment.retryable_outcomes.includes("launch_error")
|
|
4244
|
+
&& fromRound >= retrySegment.max_rounds;
|
|
4245
|
+
const terminalAttempt = {
|
|
4246
|
+
...attempt,
|
|
4247
|
+
phase: "terminal",
|
|
4248
|
+
completed_at: now,
|
|
4249
|
+
terminal_outcome: outcome,
|
|
4250
|
+
};
|
|
4251
|
+
const completedActivation = {
|
|
4252
|
+
...activation,
|
|
4253
|
+
state: retryable ? "ready" : "failed",
|
|
4254
|
+
updated_at: now,
|
|
4255
|
+
};
|
|
4256
|
+
const activations = [completedActivation];
|
|
4257
|
+
const attempts = [terminalAttempt];
|
|
4258
|
+
const events = [
|
|
4259
|
+
this.activationEvent(snapshot, 0, activation, completedActivation, retryable ? "retryable_attempt_failed" : "attempt_completed", now),
|
|
4260
|
+
this.event(snapshot, 1, {
|
|
4261
|
+
type: "attempt_terminal", step_id: attempt.step_id, activation_id: attempt.activation_id,
|
|
4262
|
+
attempt_id: attempt.attempt_id, result_id: null,
|
|
4263
|
+
key: `attempt:${attempt.attempt_id}:driver-failure:${failureDigest}`, at: now,
|
|
4264
|
+
payload: {
|
|
4265
|
+
executor: terminalAttempt.executor, outcome,
|
|
4266
|
+
worker_stop_receipt: null, worker_nonlaunch_attestation: attestation,
|
|
4267
|
+
},
|
|
4268
|
+
}),
|
|
4269
|
+
];
|
|
4270
|
+
if (segmentRetryable && retrySegment !== null) {
|
|
4271
|
+
const toRound = fromRound + 1;
|
|
4272
|
+
const priorActivations = retrySegment.node_ids.map((stepId) => (stepId === activation.step_id
|
|
4273
|
+
? completedActivation
|
|
4274
|
+
: snapshot.activations.filter((candidate) => candidate.step_id === stepId).at(-1)));
|
|
4275
|
+
if (priorActivations.some((candidate) => candidate === undefined || !TERMINAL_ACTIVATION_STATES.has(candidate.state))) {
|
|
4276
|
+
throw new CanonicalRunServiceError("attempt_state_conflict", `retry segment ${retrySegment.retry_segment_id} has non-terminal prior activations`);
|
|
4277
|
+
}
|
|
4278
|
+
const replacements = retrySegment.node_ids.map((stepId) => ({
|
|
4279
|
+
run_id: snapshot.run.run_id,
|
|
4280
|
+
step_id: stepId,
|
|
4281
|
+
activation_id: this.activationRoundId(snapshot.run.run_id, stepId, toRound),
|
|
4282
|
+
workflow_round: toRound,
|
|
4283
|
+
state: stepId === retrySegment.retry_from_node_id ? "ready" : "dependency_blocked",
|
|
4284
|
+
attempt_count: 0,
|
|
4285
|
+
current_attempt_id: null,
|
|
4286
|
+
committed_result_id: null,
|
|
4287
|
+
created_at: now,
|
|
4288
|
+
updated_at: now,
|
|
4289
|
+
}));
|
|
4290
|
+
const delayMs = this.retrySegmentDelay(retrySegment, fromRound);
|
|
4291
|
+
activations.push(...replacements);
|
|
4292
|
+
events.push(this.event(snapshot, events.length, {
|
|
4293
|
+
type: "retry_segment_scheduled", step_id: activation.step_id,
|
|
4294
|
+
activation_id: activation.activation_id, attempt_id: null, result_id: null,
|
|
4295
|
+
key: `retry-segment:${retrySegment.retry_segment_id}:round:${toRound}`, at: now,
|
|
4296
|
+
payload: {
|
|
4297
|
+
retry_segment_id: retrySegment.retry_segment_id,
|
|
4298
|
+
from_round: fromRound,
|
|
4299
|
+
to_round: toRound,
|
|
4300
|
+
prior_activation_ids: priorActivations.map((candidate) => candidate.activation_id),
|
|
4301
|
+
replacement_activation_ids: replacements.map((candidate) => candidate.activation_id),
|
|
4302
|
+
retry_from_activation_id: replacements.find((candidate) => candidate.step_id === retrySegment.retry_from_node_id).activation_id,
|
|
4303
|
+
triggering_result_id: null,
|
|
4304
|
+
delay_ms: delayMs,
|
|
4305
|
+
ready_at: addMilliseconds(now, delayMs),
|
|
4306
|
+
reason_code: "launch_error",
|
|
4307
|
+
},
|
|
4308
|
+
}));
|
|
4309
|
+
}
|
|
4310
|
+
let run = snapshot.run;
|
|
4311
|
+
if (!retryable && !segmentRetryable) {
|
|
4312
|
+
run = this.terminalRun(snapshot.run, "failed", retrySegmentExhausted ? "retry_segment_max_rounds_exhausted" : "attempt_failed", now);
|
|
4313
|
+
const cancelledAttempts = await this.cancelConcurrentAttempts(snapshot, activation.activation_id, now, "peer_attempt_failed", events.length);
|
|
4314
|
+
attempts.push(...cancelledAttempts.attempts);
|
|
4315
|
+
events.push(...cancelledAttempts.events);
|
|
4316
|
+
for (const other of snapshot.activations) {
|
|
4317
|
+
if (other.activation_id === activation.activation_id || TERMINAL_ACTIVATION_STATES.has(other.state))
|
|
4318
|
+
continue;
|
|
4319
|
+
const cancelled = { ...other, state: "cancelled", updated_at: now };
|
|
4320
|
+
activations.push(cancelled);
|
|
4321
|
+
events.push(this.activationEvent(snapshot, events.length, other, cancelled, "run_failed", now));
|
|
4322
|
+
}
|
|
4323
|
+
events.push(this.runTerminalEvent(snapshot, events.length, run, now));
|
|
4324
|
+
}
|
|
4325
|
+
return {
|
|
4326
|
+
run_id: snapshot.run.run_id,
|
|
4327
|
+
expected_record_version: snapshot.record_version,
|
|
4328
|
+
run,
|
|
4329
|
+
activations,
|
|
4330
|
+
attempts,
|
|
4331
|
+
events,
|
|
4332
|
+
};
|
|
4333
|
+
}
|
|
4334
|
+
reduceAfterSuccess(snapshot, stored, source, sourceResultId, verification, now, initialOffset, decisionOutputs) {
|
|
4335
|
+
const plan = stored.plan;
|
|
4336
|
+
const changed = new Map();
|
|
4337
|
+
const created = [];
|
|
4338
|
+
const events = [];
|
|
4339
|
+
let run = snapshot.run;
|
|
4340
|
+
const completedDeterministicNodes = new Map();
|
|
4341
|
+
const markDeterministicNodeCompleted = (nodeId, round) => {
|
|
4342
|
+
const rounds = completedDeterministicNodes.get(nodeId) ?? new Set();
|
|
4343
|
+
rounds.add(round);
|
|
4344
|
+
completedDeterministicNodes.set(nodeId, rounds);
|
|
4345
|
+
};
|
|
4346
|
+
const effectiveActivation = (stepId, round) => {
|
|
4347
|
+
const activation = this.latestActivationForStep(snapshot.activations, stepId, round);
|
|
4348
|
+
if (!activation)
|
|
4349
|
+
return null;
|
|
4350
|
+
if (activation.activation_id === source.activation_id)
|
|
4351
|
+
return source;
|
|
4352
|
+
return changed.get(activation.activation_id) ?? activation;
|
|
4353
|
+
};
|
|
4354
|
+
const dependencyCompleted = (nodeId, round) => {
|
|
4355
|
+
const node = nodeById(plan, nodeId);
|
|
4356
|
+
if (!ATTEMPT_NODE_KINDS.has(node.kind)) {
|
|
4357
|
+
if (completedDeterministicNodes.get(nodeId)?.has(round))
|
|
4358
|
+
return true;
|
|
4359
|
+
return snapshot.events.some((event) => {
|
|
4360
|
+
const sourceActivation = event.activation_id === null
|
|
4361
|
+
? null
|
|
4362
|
+
: snapshot.activations.find((candidate) => candidate.activation_id === event.activation_id) ?? null;
|
|
4363
|
+
if (sourceActivation?.workflow_round !== round)
|
|
4364
|
+
return false;
|
|
4365
|
+
if (node.kind === "gate") {
|
|
4366
|
+
return event.type === "gate_evaluated"
|
|
4367
|
+
&& event.payload.gate_id === nodeId
|
|
4368
|
+
&& event.payload.passed;
|
|
4369
|
+
}
|
|
4370
|
+
if (node.kind === "decision") {
|
|
4371
|
+
return event.type === "decision_evaluated"
|
|
4372
|
+
&& event.payload.decision_id === nodeId;
|
|
4373
|
+
}
|
|
4374
|
+
return false;
|
|
4375
|
+
});
|
|
4376
|
+
}
|
|
4377
|
+
return effectiveActivation(nodeId, round)?.state === "succeeded";
|
|
4378
|
+
};
|
|
4379
|
+
const visited = new Set();
|
|
4380
|
+
const visit = (fromNodeId) => {
|
|
4381
|
+
if (visited.has(fromNodeId))
|
|
4382
|
+
return;
|
|
4383
|
+
visited.add(fromNodeId);
|
|
4384
|
+
const from = nodeById(plan, fromNodeId);
|
|
4385
|
+
let selected = [];
|
|
4386
|
+
if (from.kind === "gate") {
|
|
4387
|
+
const relevant = verification.filter((item) => from.verifier_ids.includes(item.verifier_id));
|
|
4388
|
+
const passed = from.verifier_ids.every((id) => relevant.some((item) => item.verifier_id === id && item.passed));
|
|
4389
|
+
events.push(this.event(snapshot, initialOffset + events.length, {
|
|
4390
|
+
type: "gate_evaluated", step_id: source.step_id, activation_id: source.activation_id,
|
|
4391
|
+
attempt_id: null, result_id: sourceResultId,
|
|
4392
|
+
key: `gate:${source.activation_id}:${from.node_id}`, at: now,
|
|
4393
|
+
payload: { gate_id: from.node_id, predicate: { verifier_ids: from.verifier_ids }, source_result_id: sourceResultId, passed, reason_code: passed ? "all_verifiers_passed" : "verifier_failed" },
|
|
4394
|
+
}));
|
|
4395
|
+
selected = plan.routes.filter((route) => route.from_node_id === from.node_id && route.condition.kind === "gate" && route.condition.outcome === (passed ? "pass" : "fail"));
|
|
4396
|
+
markDeterministicNodeCompleted(from.node_id, source.workflow_round);
|
|
4397
|
+
}
|
|
4398
|
+
else if (from.kind === "decision") {
|
|
4399
|
+
const output = decisionOutputs.get(from.node_id)?.find((item) => item.name === from.source.output_name);
|
|
4400
|
+
const value = output?.kind === "structured" ? this.jsonPointer(output.value, from.source.json_pointer) : null;
|
|
4401
|
+
if (typeof value !== "string" || !from.allowed_values.includes(value)) {
|
|
4402
|
+
throw new CanonicalRunServiceError("verification_failed", `decision ${from.node_id} produced an undeclared string value`);
|
|
4403
|
+
}
|
|
4404
|
+
events.push(this.event(snapshot, initialOffset + events.length, {
|
|
4405
|
+
type: "decision_evaluated", step_id: source.step_id, activation_id: source.activation_id,
|
|
4406
|
+
attempt_id: null, result_id: sourceResultId,
|
|
4407
|
+
key: `decision:${source.activation_id}:${from.node_id}`, at: now,
|
|
4408
|
+
payload: { decision_id: from.node_id, value: value, source_result_id: sourceResultId, reason_code: "typed_decision_evaluated" },
|
|
4409
|
+
}));
|
|
4410
|
+
selected = plan.routes.filter((route) => route.from_node_id === from.node_id && route.condition.kind === "decision" && route.condition.value === value);
|
|
4411
|
+
markDeterministicNodeCompleted(from.node_id, source.workflow_round);
|
|
4412
|
+
}
|
|
4413
|
+
else if (from.kind === "terminal") {
|
|
4414
|
+
const terminal = plan.terminals.find((item) => item.terminal_id === from.terminal_id);
|
|
4415
|
+
run = this.terminalRun(snapshot.run, terminal.status === "unsupported" ? "failed" : terminal.status, terminal.reason_code, now);
|
|
4416
|
+
return;
|
|
4417
|
+
}
|
|
4418
|
+
else if (from.kind === "interaction") {
|
|
4419
|
+
selected = plan.routes.filter((route) => (route.from_node_id === from.node_id
|
|
4420
|
+
&& route.condition.kind === "interaction"
|
|
4421
|
+
&& route.condition.outcome === "submitted"));
|
|
4422
|
+
}
|
|
4423
|
+
else {
|
|
4424
|
+
selected = plan.routes.filter((route) => route.from_node_id === from.node_id && route.condition.kind === "always");
|
|
4425
|
+
}
|
|
4426
|
+
for (const route of selected) {
|
|
4427
|
+
const decisionRetry = plan.retry_segments.find((segment) => (segment.continue_route_id === route.route_id));
|
|
4428
|
+
if (decisionRetry !== undefined && source.workflow_round < decisionRetry.max_rounds) {
|
|
4429
|
+
const toRound = source.workflow_round + 1;
|
|
4430
|
+
const priorActivations = decisionRetry.node_ids.map((stepId) => (effectiveActivation(stepId, source.workflow_round)));
|
|
4431
|
+
if (priorActivations.some((candidate) => (candidate === null || !TERMINAL_ACTIVATION_STATES.has(candidate.state)))) {
|
|
4432
|
+
throw new CanonicalRunServiceError("attempt_state_conflict", `retry segment ${decisionRetry.retry_segment_id} has non-terminal prior activations`);
|
|
4433
|
+
}
|
|
4434
|
+
const replacements = decisionRetry.node_ids.map((stepId) => ({
|
|
4435
|
+
run_id: snapshot.run.run_id,
|
|
4436
|
+
step_id: stepId,
|
|
4437
|
+
activation_id: this.activationRoundId(snapshot.run.run_id, stepId, toRound),
|
|
4438
|
+
workflow_round: toRound,
|
|
4439
|
+
state: stepId === decisionRetry.retry_from_node_id ? "ready" : "dependency_blocked",
|
|
4440
|
+
attempt_count: 0,
|
|
4441
|
+
current_attempt_id: null,
|
|
4442
|
+
committed_result_id: null,
|
|
4443
|
+
created_at: now,
|
|
4444
|
+
updated_at: now,
|
|
4445
|
+
}));
|
|
4446
|
+
created.push(...replacements);
|
|
4447
|
+
const delayMs = this.retrySegmentDelay(decisionRetry, source.workflow_round);
|
|
4448
|
+
events.push(this.event(snapshot, initialOffset + events.length, {
|
|
4449
|
+
type: "retry_segment_scheduled",
|
|
4450
|
+
step_id: source.step_id,
|
|
4451
|
+
activation_id: source.activation_id,
|
|
4452
|
+
attempt_id: null,
|
|
4453
|
+
result_id: sourceResultId,
|
|
4454
|
+
key: `retry-segment:${decisionRetry.retry_segment_id}:round:${toRound}`,
|
|
4455
|
+
at: now,
|
|
4456
|
+
payload: {
|
|
4457
|
+
retry_segment_id: decisionRetry.retry_segment_id,
|
|
4458
|
+
from_round: source.workflow_round,
|
|
4459
|
+
to_round: toRound,
|
|
4460
|
+
prior_activation_ids: priorActivations.map((candidate) => candidate.activation_id),
|
|
4461
|
+
replacement_activation_ids: replacements.map((candidate) => candidate.activation_id),
|
|
4462
|
+
retry_from_activation_id: replacements.find((candidate) => (candidate.step_id === decisionRetry.retry_from_node_id)).activation_id,
|
|
4463
|
+
triggering_result_id: sourceResultId,
|
|
4464
|
+
delay_ms: delayMs,
|
|
4465
|
+
ready_at: addMilliseconds(now, delayMs),
|
|
4466
|
+
reason_code: "typed_continue",
|
|
4467
|
+
},
|
|
4468
|
+
}));
|
|
4469
|
+
events.push(this.event(snapshot, initialOffset + events.length, {
|
|
4470
|
+
type: "route_selected",
|
|
4471
|
+
step_id: source.step_id,
|
|
4472
|
+
activation_id: source.activation_id,
|
|
4473
|
+
attempt_id: null,
|
|
4474
|
+
result_id: sourceResultId,
|
|
4475
|
+
key: `route:${source.activation_id}:${route.route_id}`,
|
|
4476
|
+
at: now,
|
|
4477
|
+
payload: {
|
|
4478
|
+
route_id: route.route_id,
|
|
4479
|
+
source_activation_id: source.activation_id,
|
|
4480
|
+
source_result_id: sourceResultId,
|
|
4481
|
+
successor_activation_ids: [],
|
|
4482
|
+
reason_code: "typed_continue",
|
|
4483
|
+
},
|
|
4484
|
+
}));
|
|
4485
|
+
continue;
|
|
4486
|
+
}
|
|
4487
|
+
const target = nodeById(plan, route.to_node_id);
|
|
4488
|
+
const successorIds = [];
|
|
4489
|
+
if (ATTEMPT_NODE_KINDS.has(target.kind)) {
|
|
4490
|
+
const current = this.latestActivationForStep(snapshot.activations.map((item) => changed.get(item.activation_id) ?? item), target.node_id);
|
|
4491
|
+
if (current && current.state === "dormant") {
|
|
4492
|
+
const ready = target.depends_on.every((dependency) => (dependencyCompleted(dependency, current.workflow_round)));
|
|
4493
|
+
const next = { ...current, state: ready ? "ready" : "dependency_blocked", updated_at: now };
|
|
4494
|
+
changed.set(next.activation_id, next);
|
|
4495
|
+
successorIds.push(next.activation_id);
|
|
4496
|
+
}
|
|
4497
|
+
}
|
|
4498
|
+
events.push(this.event(snapshot, initialOffset + events.length, {
|
|
4499
|
+
type: "route_selected", step_id: source.step_id, activation_id: source.activation_id,
|
|
4500
|
+
attempt_id: null, result_id: sourceResultId,
|
|
4501
|
+
key: `route:${source.activation_id}:${route.route_id}`, at: now,
|
|
4502
|
+
payload: { route_id: route.route_id, source_activation_id: source.activation_id, source_result_id: sourceResultId, successor_activation_ids: successorIds, reason_code: "typed_route_selected" },
|
|
4503
|
+
}));
|
|
4504
|
+
if (!ATTEMPT_NODE_KINDS.has(target.kind)) {
|
|
4505
|
+
visit(target.node_id);
|
|
4506
|
+
}
|
|
4507
|
+
}
|
|
4508
|
+
const routedTargetIds = new Set(plan.routes.map((route) => route.to_node_id));
|
|
4509
|
+
for (const dependent of plan.nodes) {
|
|
4510
|
+
if (ATTEMPT_NODE_KINDS.has(dependent.kind)
|
|
4511
|
+
|| routedTargetIds.has(dependent.node_id)
|
|
4512
|
+
|| dependent.depends_on.length === 0
|
|
4513
|
+
|| !dependent.depends_on.includes(fromNodeId)
|
|
4514
|
+
|| !dependent.depends_on.every((dependency) => (dependencyCompleted(dependency, source.workflow_round))))
|
|
4515
|
+
continue;
|
|
4516
|
+
markDeterministicNodeCompleted(dependent.node_id, source.workflow_round);
|
|
4517
|
+
visit(dependent.node_id);
|
|
4518
|
+
}
|
|
4519
|
+
};
|
|
4520
|
+
visit(source.step_id);
|
|
4521
|
+
for (const activation of snapshot.activations) {
|
|
4522
|
+
if (activation.state !== "dependency_blocked" || changed.has(activation.activation_id))
|
|
4523
|
+
continue;
|
|
4524
|
+
const latest = this.latestActivationForStep(snapshot.activations, activation.step_id);
|
|
4525
|
+
if (latest?.activation_id !== activation.activation_id)
|
|
4526
|
+
continue;
|
|
4527
|
+
const node = nodeById(plan, activation.step_id);
|
|
4528
|
+
if (node.depends_on.every((dependency) => (dependencyCompleted(dependency, activation.workflow_round)))) {
|
|
4529
|
+
changed.set(activation.activation_id, { ...activation, state: "ready", updated_at: now });
|
|
4530
|
+
}
|
|
4531
|
+
}
|
|
4532
|
+
for (const next of changed.values()) {
|
|
4533
|
+
const prior = snapshot.activations.find((item) => item.activation_id === next.activation_id);
|
|
4534
|
+
events.push(this.activationEvent(snapshot, initialOffset + events.length, prior, next, "dependencies_satisfied", now));
|
|
4535
|
+
}
|
|
4536
|
+
if (run.terminal !== null) {
|
|
4537
|
+
for (const activation of snapshot.activations) {
|
|
4538
|
+
const current = changed.get(activation.activation_id) ?? activation;
|
|
4539
|
+
if (current.activation_id === source.activation_id || TERMINAL_ACTIVATION_STATES.has(current.state))
|
|
4540
|
+
continue;
|
|
4541
|
+
const currentAttempt = current.current_attempt_id === null
|
|
4542
|
+
? null
|
|
4543
|
+
: snapshot.attempts.find((attempt) => (attempt.attempt_id === current.current_attempt_id
|
|
4544
|
+
&& attempt.phase !== "terminal")) ?? null;
|
|
4545
|
+
const skipped = {
|
|
4546
|
+
...current,
|
|
4547
|
+
state: currentAttempt ? "cancelled" : "skipped",
|
|
4548
|
+
updated_at: now,
|
|
4549
|
+
};
|
|
4550
|
+
changed.set(skipped.activation_id, skipped);
|
|
4551
|
+
events.push(this.activationEvent(snapshot, initialOffset + events.length, current, skipped, currentAttempt ? "run_terminal" : "terminal_route_selected", now));
|
|
4552
|
+
}
|
|
4553
|
+
events.push(this.runTerminalEvent(snapshot, initialOffset + events.length, run, now));
|
|
4554
|
+
}
|
|
4555
|
+
return { activations: [...changed.values(), ...created], events, run };
|
|
4556
|
+
}
|
|
4557
|
+
latestActivationForStep(activations, stepId, maxRound = Number.POSITIVE_INFINITY) {
|
|
4558
|
+
let latest = null;
|
|
4559
|
+
for (const activation of activations) {
|
|
4560
|
+
if (activation.step_id !== stepId || activation.workflow_round > maxRound)
|
|
4561
|
+
continue;
|
|
4562
|
+
if (latest === null || activation.workflow_round > latest.workflow_round) {
|
|
4563
|
+
latest = activation;
|
|
4564
|
+
}
|
|
4565
|
+
}
|
|
4566
|
+
return latest;
|
|
4567
|
+
}
|
|
4568
|
+
async cancelConcurrentAttempts(snapshot, excludedActivationId, now, reasonCode, initialOffset) {
|
|
4569
|
+
const attempts = [];
|
|
4570
|
+
const events = [];
|
|
4571
|
+
for (const attempt of snapshot.attempts) {
|
|
4572
|
+
if (attempt.activation_id === excludedActivationId || attempt.phase === "terminal")
|
|
4573
|
+
continue;
|
|
4574
|
+
const executor = attempt.executor ?? {
|
|
4575
|
+
kind: "system",
|
|
4576
|
+
service_instance_id: this.serviceInstanceId,
|
|
4577
|
+
operation: "cancel_concurrent_attempt",
|
|
4578
|
+
};
|
|
4579
|
+
const outcome = executor.kind === "worker"
|
|
4580
|
+
? {
|
|
4581
|
+
kind: "worker_process",
|
|
4582
|
+
status: "cancelled",
|
|
4583
|
+
reason_code: reasonCode,
|
|
4584
|
+
message: null,
|
|
4585
|
+
process: {
|
|
4586
|
+
kind: "cancelled",
|
|
4587
|
+
exit_code: null,
|
|
4588
|
+
signal: null,
|
|
4589
|
+
reason: reasonCode,
|
|
4590
|
+
},
|
|
4591
|
+
}
|
|
4592
|
+
: { kind: "cancelled", reason: reasonCode };
|
|
4593
|
+
if (executor.kind === "worker")
|
|
4594
|
+
continue;
|
|
4595
|
+
const terminalAttempt = {
|
|
4596
|
+
...attempt,
|
|
4597
|
+
phase: "terminal",
|
|
4598
|
+
executor,
|
|
4599
|
+
started_at: attempt.started_at ?? now,
|
|
4600
|
+
completed_at: now,
|
|
4601
|
+
terminal_outcome: outcome,
|
|
4602
|
+
};
|
|
4603
|
+
attempts.push(terminalAttempt);
|
|
4604
|
+
if (attempt.started_at === null) {
|
|
4605
|
+
events.push(this.event(snapshot, initialOffset + events.length, {
|
|
4606
|
+
type: "attempt_started",
|
|
4607
|
+
step_id: attempt.step_id,
|
|
4608
|
+
activation_id: attempt.activation_id,
|
|
4609
|
+
attempt_id: attempt.attempt_id,
|
|
4610
|
+
result_id: null,
|
|
4611
|
+
key: `attempt:${attempt.attempt_id}:terminal-started`,
|
|
4612
|
+
at: now,
|
|
4613
|
+
payload: { executor },
|
|
4614
|
+
}));
|
|
4615
|
+
}
|
|
4616
|
+
events.push(this.event(snapshot, initialOffset + events.length, {
|
|
4617
|
+
type: "attempt_terminal",
|
|
4618
|
+
step_id: attempt.step_id,
|
|
4619
|
+
activation_id: attempt.activation_id,
|
|
4620
|
+
attempt_id: attempt.attempt_id,
|
|
4621
|
+
result_id: null,
|
|
4622
|
+
key: `attempt:${attempt.attempt_id}:terminal-cancelled`,
|
|
4623
|
+
at: now,
|
|
4624
|
+
payload: { executor, outcome, worker_stop_receipt: null, worker_nonlaunch_attestation: null },
|
|
4625
|
+
}));
|
|
4626
|
+
}
|
|
4627
|
+
return { attempts, events };
|
|
4628
|
+
}
|
|
4629
|
+
async reduceWithoutResult(snapshot, plan) {
|
|
4630
|
+
if (snapshot.run.terminal !== null || snapshot.activations.length > 0)
|
|
4631
|
+
return snapshot;
|
|
4632
|
+
const routedTargets = new Set(plan.plan.routes.map((route) => route.to_node_id));
|
|
4633
|
+
const rootTerminal = plan.plan.nodes.find((node) => (node.kind === "terminal"
|
|
4634
|
+
&& node.depends_on.length === 0
|
|
4635
|
+
&& !routedTargets.has(node.node_id)));
|
|
4636
|
+
if (!rootTerminal || rootTerminal.kind !== "terminal")
|
|
4637
|
+
return snapshot;
|
|
4638
|
+
const terminal = plan.plan.terminals.find((item) => item.terminal_id === rootTerminal.terminal_id);
|
|
4639
|
+
const now = this.now();
|
|
4640
|
+
const run = this.terminalRun(snapshot.run, terminal.status === "unsupported" ? "failed" : terminal.status, terminal.reason_code, now);
|
|
4641
|
+
const event = this.runTerminalEvent(snapshot, 0, run, now);
|
|
4642
|
+
try {
|
|
4643
|
+
return (await this.runStore.updateCanonicalRun({
|
|
4644
|
+
run_id: snapshot.run.run_id,
|
|
4645
|
+
expected_record_version: snapshot.record_version,
|
|
4646
|
+
run,
|
|
4647
|
+
events: [event],
|
|
4648
|
+
})).snapshot;
|
|
4649
|
+
}
|
|
4650
|
+
catch (error) {
|
|
4651
|
+
if (this.isVersionConflict(error))
|
|
4652
|
+
return this.requireSnapshot(snapshot.run.run_id);
|
|
4653
|
+
throw error;
|
|
4654
|
+
}
|
|
4655
|
+
}
|
|
4656
|
+
terminalRun(run, status, reasonCode, now) {
|
|
4657
|
+
return {
|
|
4658
|
+
...run,
|
|
4659
|
+
status,
|
|
4660
|
+
terminal: { status, reason_code: reasonCode, reason: null, completed_at: now },
|
|
4661
|
+
updated_at: now,
|
|
4662
|
+
};
|
|
4663
|
+
}
|
|
4664
|
+
retryDelay(node, priorAttemptNumber) {
|
|
4665
|
+
if (node.attempt_policy.backoff === "none")
|
|
4666
|
+
return 0;
|
|
4667
|
+
const multiplier = node.attempt_policy.backoff === "linear"
|
|
4668
|
+
? priorAttemptNumber
|
|
4669
|
+
: 2 ** Math.max(0, priorAttemptNumber - 1);
|
|
4670
|
+
const delay = node.attempt_policy.initial_delay_ms * multiplier;
|
|
4671
|
+
return node.attempt_policy.max_delay_ms === null
|
|
4672
|
+
? delay
|
|
4673
|
+
: Math.min(delay, node.attempt_policy.max_delay_ms);
|
|
4674
|
+
}
|
|
4675
|
+
retrySegmentDelay(segment, priorRound) {
|
|
4676
|
+
if (segment.backoff === "none")
|
|
4677
|
+
return 0;
|
|
4678
|
+
const multiplier = segment.backoff === "linear"
|
|
4679
|
+
? priorRound
|
|
4680
|
+
: 2 ** Math.max(0, priorRound - 1);
|
|
4681
|
+
const delay = segment.initial_delay_ms * multiplier;
|
|
4682
|
+
return segment.max_delay_ms === null
|
|
4683
|
+
? delay
|
|
4684
|
+
: Math.min(delay, segment.max_delay_ms);
|
|
4685
|
+
}
|
|
4686
|
+
jsonPointer(value, pointer) {
|
|
4687
|
+
if (pointer === "")
|
|
4688
|
+
return value;
|
|
4689
|
+
let current = value;
|
|
4690
|
+
for (const raw of pointer.split("/").slice(1)) {
|
|
4691
|
+
const key = raw.replace(/~1/gu, "/").replace(/~0/gu, "~");
|
|
4692
|
+
if (Array.isArray(current))
|
|
4693
|
+
current = current[Number(key)];
|
|
4694
|
+
else if (current !== null && typeof current === "object") {
|
|
4695
|
+
current = current[key];
|
|
4696
|
+
}
|
|
4697
|
+
else
|
|
4698
|
+
current = undefined;
|
|
4699
|
+
}
|
|
4700
|
+
if (current === undefined) {
|
|
4701
|
+
throw new CanonicalRunServiceError("verification_failed", `decision JSON pointer ${pointer} was not found`);
|
|
4702
|
+
}
|
|
4703
|
+
return current;
|
|
4704
|
+
}
|
|
4705
|
+
validateDecisionOutputs(plan, sourceNodeId, outputs) {
|
|
4706
|
+
for (const decision of plan.nodes) {
|
|
4707
|
+
if (decision.kind !== "decision" || decision.source.node_id !== sourceNodeId)
|
|
4708
|
+
continue;
|
|
4709
|
+
const output = outputs.find((candidate) => candidate.name === decision.source.output_name);
|
|
4710
|
+
const value = output?.kind === "structured"
|
|
4711
|
+
? this.jsonPointer(output.value, decision.source.json_pointer)
|
|
4712
|
+
: null;
|
|
4713
|
+
if (typeof value !== "string" || !decision.allowed_values.includes(value)) {
|
|
4714
|
+
throw new CanonicalRunServiceError("verification_failed", `decision ${decision.node_id} produced an undeclared string value`);
|
|
4715
|
+
}
|
|
4716
|
+
}
|
|
4717
|
+
}
|
|
4718
|
+
activationEvent(snapshot, offset, before, after, reasonCode, at) {
|
|
4719
|
+
return this.event(snapshot, offset, {
|
|
4720
|
+
type: "activation_state_changed",
|
|
4721
|
+
step_id: after.step_id,
|
|
4722
|
+
activation_id: after.activation_id,
|
|
4723
|
+
attempt_id: null,
|
|
4724
|
+
result_id: null,
|
|
4725
|
+
key: `activation:${after.activation_id}:${before.state}:${after.state}:${after.attempt_count}`,
|
|
4726
|
+
at,
|
|
4727
|
+
payload: { from: before.state, to: after.state, reason_code: reasonCode },
|
|
4728
|
+
});
|
|
4729
|
+
}
|
|
4730
|
+
runTerminalEvent(snapshot, offset, run, at) {
|
|
4731
|
+
return this.event(snapshot, offset, {
|
|
4732
|
+
type: "run_terminal", step_id: null, activation_id: null,
|
|
4733
|
+
attempt_id: null, result_id: null,
|
|
4734
|
+
key: `run:${run.run_id}:terminal:${run.status}`, at,
|
|
4735
|
+
payload: {
|
|
4736
|
+
status: run.terminal.status,
|
|
4737
|
+
reason_code: run.terminal.reason_code,
|
|
4738
|
+
reason: run.terminal.reason,
|
|
4739
|
+
},
|
|
4740
|
+
});
|
|
4741
|
+
}
|
|
4742
|
+
event(snapshot, offset, input) {
|
|
4743
|
+
return {
|
|
4744
|
+
event_id: this.identities.create("event", input.key),
|
|
4745
|
+
run_id: snapshot.run.run_id,
|
|
4746
|
+
step_id: input.step_id,
|
|
4747
|
+
activation_id: input.activation_id,
|
|
4748
|
+
attempt_id: input.attempt_id,
|
|
4749
|
+
result_id: input.result_id,
|
|
4750
|
+
sequence: snapshot.events.length + offset + 1,
|
|
4751
|
+
idempotency_key: input.key,
|
|
4752
|
+
type: input.type,
|
|
4753
|
+
occurred_at: input.at,
|
|
4754
|
+
payload: input.payload,
|
|
4755
|
+
};
|
|
4756
|
+
}
|
|
4757
|
+
isVersionConflict(error) {
|
|
4758
|
+
return error instanceof RunStoreError && error.code === "version_conflict";
|
|
4759
|
+
}
|
|
4760
|
+
}
|