intentdna 1.9.0 → 1.9.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +2 -2
- package/.claude-plugin/plugin.json +1 -1
- package/README.md +227 -87
- package/dist/cli/commands/run-lifecycle.d.ts +61 -11
- package/dist/cli/commands/run-lifecycle.js +184 -50
- package/dist/cli/commands/run-observer.d.ts +15 -0
- package/dist/cli/commands/run-observer.js +210 -0
- package/dist/cli/commands/run.d.ts +16 -20
- package/dist/cli/commands/run.js +461 -659
- package/dist/cli/commands/sync.js +16 -12
- package/dist/cli/index.js +14 -5
- package/dist/compiler/compile.d.ts +1 -0
- package/dist/compiler/compile.js +1 -1
- package/dist/compiler/controller.d.ts +16 -0
- package/dist/compiler/controller.js +693 -0
- package/dist/compiler/input-resolver.d.ts +2 -0
- package/dist/compiler/input-resolver.js +22 -6
- package/dist/hooks/cli.d.ts +45 -0
- package/dist/hooks/cli.js +263 -31
- package/dist/hooks/host-event.d.ts +57 -0
- package/dist/hooks/host-event.js +187 -0
- package/dist/hooks/index.d.ts +1 -0
- package/dist/hooks/index.js +1 -0
- package/dist/hooks/protocol.d.ts +7 -0
- package/dist/hooks/schema.d.ts +1 -0
- package/dist/hooks/schema.js +89 -1
- package/dist/hooks/state.d.ts +23 -1
- package/dist/hooks/state.js +9 -0
- package/dist/index.d.ts +1 -0
- package/dist/index.js +1 -0
- package/dist/mcp/index.js +2 -0
- package/dist/mcp/tools-run.d.ts +4 -0
- package/dist/mcp/tools-run.js +419 -0
- package/dist/mcp/tools-state.d.ts +1 -1
- package/dist/mcp/tools-state.js +13 -241
- package/dist/runtime/artifact-store.d.ts +146 -0
- package/dist/runtime/artifact-store.js +1436 -0
- package/dist/runtime/canonical-attempt-outcome.d.ts +12 -0
- package/dist/runtime/canonical-attempt-outcome.js +195 -0
- package/dist/runtime/canonical-json.d.ts +10 -0
- package/dist/runtime/canonical-json.js +105 -0
- package/dist/runtime/canonical-run-application.d.ts +93 -0
- package/dist/runtime/canonical-run-application.js +229 -0
- package/dist/runtime/canonical-run-service.d.ts +454 -0
- package/dist/runtime/canonical-run-service.js +4760 -0
- package/dist/runtime/canonical-runtime-composition.d.ts +22 -0
- package/dist/runtime/canonical-runtime-composition.js +34 -0
- package/dist/runtime/canonical-runtime-projection.d.ts +3 -0
- package/dist/runtime/canonical-runtime-projection.js +4 -0
- package/dist/runtime/canonical-target-compiler.d.ts +46 -0
- package/dist/runtime/canonical-target-compiler.js +514 -0
- package/dist/runtime/claude-agent-identity.d.ts +2 -0
- package/dist/runtime/claude-agent-identity.js +21 -0
- package/dist/runtime/executable-run-plan.d.ts +328 -0
- package/dist/runtime/executable-run-plan.js +1485 -0
- package/dist/runtime/execution-authority.d.ts +83 -0
- package/dist/runtime/execution-authority.js +98 -0
- package/dist/runtime/handoff-resolver.js +4 -1
- package/dist/runtime/harness-pull-adapter.d.ts +146 -0
- package/dist/runtime/harness-pull-adapter.js +315 -0
- package/dist/runtime/index.d.ts +39 -14
- package/dist/runtime/index.js +29 -7
- package/dist/runtime/local-execution-authority.d.ts +194 -0
- package/dist/runtime/local-execution-authority.js +2877 -0
- package/dist/runtime/local-execution-reconciliation.d.ts +20 -0
- package/dist/runtime/local-execution-reconciliation.js +107 -0
- package/dist/runtime/local-execution-supervisor-script.d.ts +7 -0
- package/dist/runtime/local-execution-supervisor-script.js +793 -0
- package/dist/runtime/local-provider-sandbox.d.ts +42 -0
- package/dist/runtime/local-provider-sandbox.js +154 -0
- package/dist/runtime/local-verifier-execution.d.ts +24 -0
- package/dist/runtime/local-verifier-execution.js +290 -0
- package/dist/runtime/local-windows-execution-supervisor-script.d.ts +6 -0
- package/dist/runtime/local-windows-execution-supervisor-script.js +771 -0
- package/dist/runtime/plan-store.d.ts +30 -0
- package/dist/runtime/plan-store.js +231 -0
- package/dist/runtime/process-tree.d.ts +9 -0
- package/dist/runtime/process-tree.js +97 -24
- package/dist/runtime/push-driver.d.ts +184 -0
- package/dist/runtime/push-driver.js +1002 -0
- package/dist/runtime/result-store.d.ts +34 -3
- package/dist/runtime/result-store.js +1073 -2
- package/dist/runtime/run-binding.d.ts +68 -0
- package/dist/runtime/run-binding.js +278 -0
- package/dist/runtime/run-contracts.d.ts +575 -0
- package/dist/runtime/run-contracts.js +58 -7
- package/dist/runtime/run-controller.d.ts +1 -6
- package/dist/runtime/run-controller.js +0 -41
- package/dist/runtime/run-store.d.ts +85 -7
- package/dist/runtime/run-store.js +2528 -186
- package/dist/runtime/skill-adapter.d.ts +18 -7
- package/dist/runtime/skill-adapter.js +100 -742
- package/dist/runtime/structured-output-validator.d.ts +44 -0
- package/dist/runtime/structured-output-validator.js +171 -0
- package/dist/runtime/verifier-command-binding.d.ts +22 -0
- package/dist/runtime/verifier-command-binding.js +162 -0
- package/dist/runtime/verifier.d.ts +46 -6
- package/dist/runtime/verifier.js +355 -53
- package/dist/runtime/windows-job-keeper.d.ts +47 -0
- package/dist/runtime/windows-job-keeper.js +231 -0
- package/dist/runtime/worker-executor.d.ts +17 -1
- package/dist/runtime/worker-executor.js +26 -5
- package/dist/runtime/workflow-plan-adapter.d.ts +19 -0
- package/dist/runtime/workflow-plan-adapter.js +502 -6
- package/dist/runtime/workflow-runtime-manifest.d.ts +1 -0
- package/dist/runtime/workflow-runtime-manifest.js +5 -0
- package/dist/runtime/workspace-isolation.d.ts +29 -2
- package/dist/runtime/workspace-isolation.js +488 -11
- package/dist/runtime/workspace-observation.d.ts +44 -0
- package/dist/runtime/workspace-observation.js +216 -0
- package/dist/schema/controller-registry.d.ts +0 -7
- package/dist/schema/controller-registry.js +0 -8
- package/dist/schema/types.d.ts +84 -10
- package/dist/schema/types.js +2 -0
- package/dist/schema/validate.js +151 -1
- package/dist/schema/validators/controllers.d.ts +1 -1
- package/dist/schema/validators/controllers.js +126 -41
- package/dist/schema/workflow-authoring-contract.js +2 -0
- package/dist/schema/yaml-parser.js +1 -1
- package/dist/templates/flutter-rewrite-new.dna.yaml +1 -0
- package/dist/templates/flutter-rewrite.dna.yaml +87 -60
- package/dist/templates/safe-refactoring.dna.yaml +1 -0
- package/native/windows-job-keeper/README.md +80 -0
- package/native/windows-job-keeper/bin/aarch64/intentdna-windows-job-keeper.exe +0 -0
- package/native/windows-job-keeper/bin/x86_64/intentdna-windows-job-keeper.exe +0 -0
- package/native/windows-job-keeper/keeper.c +979 -0
- package/native/windows-job-keeper/manifest.json +33 -0
- package/package.json +5 -1
- package/scripts/build-windows-job-keeper.mjs +172 -0
- package/scripts/verify-windows-job-keeper.mjs +184 -0
|
@@ -1,7 +1,13 @@
|
|
|
1
1
|
import { createHash, randomUUID } from "node:crypto";
|
|
2
|
-
import { mkdir, open, readFile, readdir, rename, rm,
|
|
3
|
-
import { dirname, join } from "node:path";
|
|
4
|
-
|
|
2
|
+
import { link, mkdir, open, readFile, readdir, rename, rm, unlink, } from "node:fs/promises";
|
|
3
|
+
import { dirname, isAbsolute, join, normalize, relative, sep } from "node:path";
|
|
4
|
+
import { CANONICAL_ACTIVATION_STATE_VALUES, CANONICAL_ACTIVATION_STATE_TRANSITIONS, CANONICAL_RUNTIME_EVENT_TYPE_VALUES, } from "./run-contracts.js";
|
|
5
|
+
import { canonicalizeJson } from "./canonical-json.js";
|
|
6
|
+
import { parseStoredRunBinding, } from "./run-binding.js";
|
|
7
|
+
import { canonicalAttemptOutcomeStatus, validateCanonicalAttemptOutcome, } from "./canonical-attempt-outcome.js";
|
|
8
|
+
export const LEGACY_RUN_STORE_SCHEMA_VERSION = 1;
|
|
9
|
+
export const RUN_STORE_SCHEMA_VERSION = LEGACY_RUN_STORE_SCHEMA_VERSION;
|
|
10
|
+
export const CANONICAL_RUN_STORE_SCHEMA_VERSION = 2;
|
|
5
11
|
export const RUN_STATUS_VALUES = [
|
|
6
12
|
"active",
|
|
7
13
|
"cancelling",
|
|
@@ -11,8 +17,8 @@ export const RUN_STATUS_VALUES = [
|
|
|
11
17
|
];
|
|
12
18
|
export class RunStoreError extends Error {
|
|
13
19
|
code;
|
|
14
|
-
constructor(code, message) {
|
|
15
|
-
super(message);
|
|
20
|
+
constructor(code, message, options) {
|
|
21
|
+
super(message, options);
|
|
16
22
|
this.name = "RunStoreError";
|
|
17
23
|
this.code = code;
|
|
18
24
|
}
|
|
@@ -21,10 +27,12 @@ const DEFAULT_LOCK_WAIT_MS = 5_000;
|
|
|
21
27
|
const DEFAULT_LOCK_STALE_MS = 5 * 60_000;
|
|
22
28
|
const DEFAULT_LOCK_RETRY_MS = 25;
|
|
23
29
|
const LEDGER_FILE_NAME = "ledger.json";
|
|
24
|
-
const
|
|
25
|
-
const
|
|
30
|
+
const VERSION_DIRECTORY_NAME = "versions";
|
|
31
|
+
const VERSION_FILE_PATTERN = /^(\d{16})\.json$/u;
|
|
26
32
|
const DIRECTORY_MODE = 0o700;
|
|
27
33
|
const FILE_MODE = 0o600;
|
|
34
|
+
const IMMUTABLE_FILE_MODE = 0o400;
|
|
35
|
+
const RFC3339_PATTERN = /^(\d{4})-(\d{2})-(\d{2})T(\d{2}):(\d{2}):(\d{2})(?:\.\d{1,9})?(?:Z|[+-](\d{2}):(\d{2}))$/u;
|
|
28
36
|
function assertPositiveInteger(name, value) {
|
|
29
37
|
if (!Number.isSafeInteger(value) || value <= 0) {
|
|
30
38
|
throw new RunStoreError("invalid_options", `${name} must be a positive safe integer`);
|
|
@@ -60,9 +68,6 @@ function isErrno(error, code) {
|
|
|
60
68
|
"code" in error &&
|
|
61
69
|
error.code === code);
|
|
62
70
|
}
|
|
63
|
-
function sleep(ms) {
|
|
64
|
-
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
65
|
-
}
|
|
66
71
|
function stableJson(value) {
|
|
67
72
|
if (Array.isArray(value)) {
|
|
68
73
|
return `[${value.map(stableJson).join(",")}]`;
|
|
@@ -78,11 +83,104 @@ function stableJson(value) {
|
|
|
78
83
|
function recordsEqual(left, right) {
|
|
79
84
|
return stableJson(left) === stableJson(right);
|
|
80
85
|
}
|
|
86
|
+
function legacyEvent(snapshot, offset, input) {
|
|
87
|
+
return {
|
|
88
|
+
...input,
|
|
89
|
+
event_id: randomUUID(),
|
|
90
|
+
sequence: snapshot.events.length + offset + 1,
|
|
91
|
+
};
|
|
92
|
+
}
|
|
81
93
|
function assertString(value, label) {
|
|
82
94
|
if (typeof value !== "string" || value.length === 0) {
|
|
83
95
|
throw new RunStoreError("invalid_record", `${label} must be a non-empty string`);
|
|
84
96
|
}
|
|
85
97
|
}
|
|
98
|
+
function assertCanonicalIdentifier(value, label) {
|
|
99
|
+
assertString(value, label);
|
|
100
|
+
if (value.trim() !== value || /[\u0000-\u001f\u007f]/u.test(value)) {
|
|
101
|
+
throw new RunStoreError("invalid_record", `${label} must be a printable identifier without surrounding whitespace`);
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
function assertCanonicalAbsolutePath(value, label) {
|
|
105
|
+
assertString(value, label);
|
|
106
|
+
if (!isAbsolute(value) || normalize(value) !== value || value.includes("\0")) {
|
|
107
|
+
throw new RunStoreError("invalid_record", `${label} must be a normalized absolute path`);
|
|
108
|
+
}
|
|
109
|
+
}
|
|
110
|
+
function isContainedPath(root, candidate) {
|
|
111
|
+
const child = relative(root, candidate);
|
|
112
|
+
return child === "" || (child !== ".."
|
|
113
|
+
&& !child.startsWith(`..${sep}`)
|
|
114
|
+
&& !isAbsolute(child));
|
|
115
|
+
}
|
|
116
|
+
function isValidGitBranchName(value) {
|
|
117
|
+
const segments = value.split("/");
|
|
118
|
+
return value !== "@"
|
|
119
|
+
&& !value.startsWith("/")
|
|
120
|
+
&& !value.endsWith("/")
|
|
121
|
+
&& !value.endsWith(".")
|
|
122
|
+
&& !value.includes("..")
|
|
123
|
+
&& !value.includes("//")
|
|
124
|
+
&& !value.includes("@{")
|
|
125
|
+
&& !/[\u0000-\u0020\u007f~^:?*\\\[]/u.test(value)
|
|
126
|
+
&& segments.every((segment) => (segment.length > 0
|
|
127
|
+
&& !segment.startsWith(".")
|
|
128
|
+
&& !segment.endsWith(".lock")));
|
|
129
|
+
}
|
|
130
|
+
function assertTimestamp(value, label) {
|
|
131
|
+
assertString(value, label);
|
|
132
|
+
const match = value.match(RFC3339_PATTERN);
|
|
133
|
+
const timestamp = Date.parse(value);
|
|
134
|
+
if (!match || !Number.isFinite(timestamp)) {
|
|
135
|
+
throw new RunStoreError("invalid_record", `${label} must be a valid RFC3339 timestamp`);
|
|
136
|
+
}
|
|
137
|
+
const [, yearText, monthText, dayText, hourText, minuteText, secondText, offsetHourText, offsetMinuteText,] = match;
|
|
138
|
+
const year = Number(yearText);
|
|
139
|
+
const month = Number(monthText);
|
|
140
|
+
const day = Number(dayText);
|
|
141
|
+
const hour = Number(hourText);
|
|
142
|
+
const minute = Number(minuteText);
|
|
143
|
+
const second = Number(secondText);
|
|
144
|
+
const offsetHour = Number(offsetHourText ?? 0);
|
|
145
|
+
const offsetMinute = Number(offsetMinuteText ?? 0);
|
|
146
|
+
const leap = year % 4 === 0 && (year % 100 !== 0 || year % 400 === 0);
|
|
147
|
+
const monthDays = [31, leap ? 29 : 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31];
|
|
148
|
+
if (month < 1
|
|
149
|
+
|| month > 12
|
|
150
|
+
|| day < 1
|
|
151
|
+
|| day > monthDays[month - 1]
|
|
152
|
+
|| hour > 23
|
|
153
|
+
|| minute > 59
|
|
154
|
+
|| second > 59
|
|
155
|
+
|| offsetHour > 23
|
|
156
|
+
|| offsetMinute > 59) {
|
|
157
|
+
throw new RunStoreError("invalid_record", `${label} must be a valid RFC3339 timestamp`);
|
|
158
|
+
}
|
|
159
|
+
return timestamp;
|
|
160
|
+
}
|
|
161
|
+
function assertNullableString(value, label) {
|
|
162
|
+
if (value !== null && typeof value !== "string") {
|
|
163
|
+
throw new RunStoreError("invalid_record", `${label} must be a string or null`);
|
|
164
|
+
}
|
|
165
|
+
}
|
|
166
|
+
function assertBoolean(value, label) {
|
|
167
|
+
if (typeof value !== "boolean") {
|
|
168
|
+
throw new RunStoreError("invalid_record", `${label} must be a boolean`);
|
|
169
|
+
}
|
|
170
|
+
}
|
|
171
|
+
function assertExactFields(value, label, fields) {
|
|
172
|
+
if (value === null || typeof value !== "object" || Array.isArray(value)) {
|
|
173
|
+
throw new RunStoreError("invalid_record", `${label} must be an object`);
|
|
174
|
+
}
|
|
175
|
+
const unknown = Object.keys(value).filter((key) => !fields.includes(key));
|
|
176
|
+
if (unknown.length > 0) {
|
|
177
|
+
throw new RunStoreError("invalid_record", `${label} contains unknown field ${unknown[0]}`);
|
|
178
|
+
}
|
|
179
|
+
const missing = fields.filter((field) => !Object.prototype.hasOwnProperty.call(value, field));
|
|
180
|
+
if (missing.length > 0) {
|
|
181
|
+
throw new RunStoreError("invalid_record", `${label} is missing required field ${missing[0]}`);
|
|
182
|
+
}
|
|
183
|
+
}
|
|
86
184
|
function assertRunRecord(record, expectedRunId) {
|
|
87
185
|
assertString(record.run_id, "run.run_id");
|
|
88
186
|
assertString(record.workflow_id, "run.workflow_id");
|
|
@@ -108,17 +206,193 @@ function assertRunRecord(record, expectedRunId) {
|
|
|
108
206
|
throw new RunStoreError("invalid_record", "run status and terminal status must agree");
|
|
109
207
|
}
|
|
110
208
|
}
|
|
209
|
+
function assertDigest(value, label) {
|
|
210
|
+
if (typeof value !== "string" || !/^sha256:[a-f0-9]{64}$/u.test(value)) {
|
|
211
|
+
throw new RunStoreError("invalid_record", `${label} must be a lowercase sha256 digest`);
|
|
212
|
+
}
|
|
213
|
+
}
|
|
214
|
+
function assertWorkspacePhysicalIdentity(value, label, attemptId, allocation) {
|
|
215
|
+
if (value === null || typeof value !== "object" || Array.isArray(value)) {
|
|
216
|
+
throw new RunStoreError("invalid_record", `${label} must be an object`);
|
|
217
|
+
}
|
|
218
|
+
const record = value;
|
|
219
|
+
const allocationFields = allocation
|
|
220
|
+
? ["workspace_plan_id", "observed_head", "observed_branch_ref"]
|
|
221
|
+
: [];
|
|
222
|
+
const common = ["kind", "workspace_id", "working_directory", "authority_key"];
|
|
223
|
+
if (record.kind === "none") {
|
|
224
|
+
assertExactFields(record, label, [...common, ...allocationFields]);
|
|
225
|
+
}
|
|
226
|
+
else if (record.kind === "worktree") {
|
|
227
|
+
assertExactFields(record, label, [
|
|
228
|
+
...common,
|
|
229
|
+
"repository_root", "worktree_root", "branch_name", "base_ref",
|
|
230
|
+
"base_commit", "ownership_key", ...allocationFields,
|
|
231
|
+
]);
|
|
232
|
+
}
|
|
233
|
+
else {
|
|
234
|
+
throw new RunStoreError("invalid_record", `${label}.kind must be none or worktree`);
|
|
235
|
+
}
|
|
236
|
+
assertCanonicalIdentifier(record.workspace_id, `${label}.workspace_id`);
|
|
237
|
+
assertCanonicalIdentifier(record.authority_key, `${label}.authority_key`);
|
|
238
|
+
assertCanonicalAbsolutePath(record.working_directory, `${label}.working_directory`);
|
|
239
|
+
if (record.workspace_id !== attemptId) {
|
|
240
|
+
throw new RunStoreError("identity_conflict", `${label}.workspace_id must match the event attempt`);
|
|
241
|
+
}
|
|
242
|
+
if (allocation) {
|
|
243
|
+
assertDigest(record.workspace_plan_id, `${label}.workspace_plan_id`);
|
|
244
|
+
if (record.kind === "none") {
|
|
245
|
+
const head = record.observed_head;
|
|
246
|
+
const branch = record.observed_branch_ref;
|
|
247
|
+
if ((head === null && branch !== null)
|
|
248
|
+
|| (head !== null && !/^[0-9a-f]{40}(?:[0-9a-f]{24})?$/u.test(head))
|
|
249
|
+
|| (branch !== null && (typeof branch !== "string"
|
|
250
|
+
|| !branch.startsWith("refs/heads/")
|
|
251
|
+
|| !isValidGitBranchName(branch.slice("refs/heads/".length))))) {
|
|
252
|
+
throw new RunStoreError("invalid_record", `${label} shared workspace Git observation is invalid`);
|
|
253
|
+
}
|
|
254
|
+
}
|
|
255
|
+
}
|
|
256
|
+
if (record.kind === "worktree") {
|
|
257
|
+
assertCanonicalAbsolutePath(record.repository_root, `${label}.repository_root`);
|
|
258
|
+
assertCanonicalAbsolutePath(record.worktree_root, `${label}.worktree_root`);
|
|
259
|
+
for (const field of ["branch_name", "base_ref", "ownership_key"]) {
|
|
260
|
+
assertCanonicalIdentifier(record[field], `${label}.${field}`);
|
|
261
|
+
}
|
|
262
|
+
assertCanonicalIdentifier(record.base_commit, `${label}.base_commit`);
|
|
263
|
+
const baseCommit = record.base_commit;
|
|
264
|
+
const branchName = record.branch_name;
|
|
265
|
+
if (!/^[0-9a-f]{40}(?:[0-9a-f]{24})?$/u.test(baseCommit)
|
|
266
|
+
|| !isContainedPath(record.worktree_root, record.working_directory)
|
|
267
|
+
|| !isValidGitBranchName(branchName)) {
|
|
268
|
+
throw new RunStoreError("invalid_record", `${label} contains invalid worktree identity or paths`);
|
|
269
|
+
}
|
|
270
|
+
if (allocation && (record.observed_head !== baseCommit
|
|
271
|
+
|| record.observed_branch_ref !== `refs/heads/${branchName}`)) {
|
|
272
|
+
throw new RunStoreError("invalid_record", `${label} observation does not match its planned base and branch`);
|
|
273
|
+
}
|
|
274
|
+
}
|
|
275
|
+
return record;
|
|
276
|
+
}
|
|
277
|
+
function assertCanonicalRunRecord(record, expectedRunId) {
|
|
278
|
+
assertExactFields(record, "run", [
|
|
279
|
+
"run_id",
|
|
280
|
+
"target",
|
|
281
|
+
"plan_id",
|
|
282
|
+
"plan_digest",
|
|
283
|
+
"run_binding",
|
|
284
|
+
"status",
|
|
285
|
+
"cancellation",
|
|
286
|
+
"terminal",
|
|
287
|
+
"metadata",
|
|
288
|
+
"created_at",
|
|
289
|
+
"updated_at",
|
|
290
|
+
]);
|
|
291
|
+
assertString(record.run_id, "run.run_id");
|
|
292
|
+
if (expectedRunId !== undefined && record.run_id !== expectedRunId) {
|
|
293
|
+
throw new RunStoreError("identity_conflict", `run record ${record.run_id} does not match ${expectedRunId}`);
|
|
294
|
+
}
|
|
295
|
+
if (!record.target
|
|
296
|
+
|| !["workflow", "controller"].includes(record.target.kind)) {
|
|
297
|
+
throw new RunStoreError("invalid_record", "run.target.kind must be workflow or controller");
|
|
298
|
+
}
|
|
299
|
+
assertExactFields(record.target, "run.target", ["kind", "key"]);
|
|
300
|
+
assertString(record.target.key, "run.target.key");
|
|
301
|
+
assertDigest(record.plan_id, "run.plan_id");
|
|
302
|
+
assertDigest(record.plan_digest, "run.plan_digest");
|
|
303
|
+
if (record.plan_id !== record.plan_digest) {
|
|
304
|
+
throw new RunStoreError("identity_conflict", "run.plan_id and run.plan_digest must match");
|
|
305
|
+
}
|
|
306
|
+
let binding;
|
|
307
|
+
try {
|
|
308
|
+
binding = parseStoredRunBinding(record.run_binding);
|
|
309
|
+
}
|
|
310
|
+
catch (error) {
|
|
311
|
+
throw new RunStoreError("invalid_record", "run.run_binding must be a valid stored binding", { cause: error });
|
|
312
|
+
}
|
|
313
|
+
if (binding.binding.plan_id !== record.plan_id) {
|
|
314
|
+
throw new RunStoreError("identity_conflict", "run binding must reference run.plan_id");
|
|
315
|
+
}
|
|
316
|
+
const createdAt = assertTimestamp(record.created_at, "run.created_at");
|
|
317
|
+
const updatedAt = assertTimestamp(record.updated_at, "run.updated_at");
|
|
318
|
+
if (updatedAt < createdAt) {
|
|
319
|
+
throw new RunStoreError("invalid_record", "run.updated_at cannot precede run.created_at");
|
|
320
|
+
}
|
|
321
|
+
if (!RUN_STATUS_VALUES.includes(record.status)) {
|
|
322
|
+
throw new RunStoreError("invalid_record", `unsupported run status: ${String(record.status)}`);
|
|
323
|
+
}
|
|
324
|
+
if (record.status === "active" && record.terminal !== null) {
|
|
325
|
+
throw new RunStoreError("invalid_record", "an active run cannot contain a terminal record");
|
|
326
|
+
}
|
|
327
|
+
if (record.status === "active" && record.cancellation !== null) {
|
|
328
|
+
throw new RunStoreError("invalid_record", "an active canonical run cannot contain cancellation intent");
|
|
329
|
+
}
|
|
330
|
+
if (["cancelling", "cancelled"].includes(record.status)
|
|
331
|
+
&& record.cancellation === null) {
|
|
332
|
+
throw new RunStoreError("invalid_record", `canonical run status ${record.status} requires cancellation intent`);
|
|
333
|
+
}
|
|
334
|
+
if (["succeeded", "failed"].includes(record.status)
|
|
335
|
+
&& record.cancellation !== null) {
|
|
336
|
+
throw new RunStoreError("invalid_record", `canonical run status ${record.status} cannot retain cancellation intent`);
|
|
337
|
+
}
|
|
338
|
+
if (["succeeded", "failed", "cancelled"].includes(record.status)
|
|
339
|
+
&& record.terminal === null) {
|
|
340
|
+
throw new RunStoreError("invalid_record", `terminal run status ${record.status} requires a terminal record`);
|
|
341
|
+
}
|
|
342
|
+
if (record.terminal !== null
|
|
343
|
+
&& (record.status !== record.terminal.status
|
|
344
|
+
|| !["succeeded", "failed", "cancelled"].includes(record.status))) {
|
|
345
|
+
throw new RunStoreError("invalid_record", "run status and terminal status must agree");
|
|
346
|
+
}
|
|
347
|
+
if (record.cancellation !== null) {
|
|
348
|
+
assertExactFields(record.cancellation, "run.cancellation", [
|
|
349
|
+
"requested_at",
|
|
350
|
+
"reason",
|
|
351
|
+
]);
|
|
352
|
+
const requestedAt = assertTimestamp(record.cancellation.requested_at, "run.cancellation.requested_at");
|
|
353
|
+
if (requestedAt < createdAt || updatedAt < requestedAt) {
|
|
354
|
+
throw new RunStoreError("invalid_record", "run cancellation timestamps are inconsistent");
|
|
355
|
+
}
|
|
356
|
+
if (record.cancellation.reason !== null
|
|
357
|
+
&& typeof record.cancellation.reason !== "string") {
|
|
358
|
+
throw new RunStoreError("invalid_record", "run.cancellation.reason must be a string or null");
|
|
359
|
+
}
|
|
360
|
+
}
|
|
361
|
+
if (record.terminal !== null) {
|
|
362
|
+
assertExactFields(record.terminal, "run.terminal", [
|
|
363
|
+
"status",
|
|
364
|
+
"reason_code",
|
|
365
|
+
"reason",
|
|
366
|
+
"completed_at",
|
|
367
|
+
]);
|
|
368
|
+
assertString(record.terminal.reason_code, "run.terminal.reason_code");
|
|
369
|
+
const completedAt = assertTimestamp(record.terminal.completed_at, "run.terminal.completed_at");
|
|
370
|
+
if (completedAt < createdAt || updatedAt < completedAt) {
|
|
371
|
+
throw new RunStoreError("invalid_record", "run terminal timestamps are inconsistent");
|
|
372
|
+
}
|
|
373
|
+
if (record.terminal.reason !== null
|
|
374
|
+
&& typeof record.terminal.reason !== "string") {
|
|
375
|
+
throw new RunStoreError("invalid_record", "run.terminal.reason must be a string or null");
|
|
376
|
+
}
|
|
377
|
+
}
|
|
378
|
+
try {
|
|
379
|
+
canonicalizeJson(record.metadata);
|
|
380
|
+
}
|
|
381
|
+
catch (error) {
|
|
382
|
+
throw new RunStoreError("invalid_record", "run.metadata must be canonical JSON", { cause: error });
|
|
383
|
+
}
|
|
384
|
+
}
|
|
111
385
|
function assertCorrelatedRunId(actual, expected, label) {
|
|
112
386
|
if (actual !== expected) {
|
|
113
387
|
throw new RunStoreError("identity_conflict", `${label} belongs to run ${actual}, expected ${expected}`);
|
|
114
388
|
}
|
|
115
389
|
}
|
|
116
|
-
function
|
|
390
|
+
function assertLegacySnapshot(value, expectedRunId) {
|
|
117
391
|
if (value === null || typeof value !== "object" || Array.isArray(value)) {
|
|
118
392
|
throw new RunStoreError("invalid_record", "run ledger must be a JSON object");
|
|
119
393
|
}
|
|
120
394
|
const candidate = value;
|
|
121
|
-
if (candidate.schema_version !==
|
|
395
|
+
if (candidate.schema_version !== LEGACY_RUN_STORE_SCHEMA_VERSION) {
|
|
122
396
|
throw new RunStoreError("unsupported_schema", `unsupported run-store schema version: ${String(candidate.schema_version)}`);
|
|
123
397
|
}
|
|
124
398
|
if (!Number.isSafeInteger(candidate.record_version) ||
|
|
@@ -234,6 +508,1277 @@ function validateCollections(snapshot) {
|
|
|
234
508
|
}
|
|
235
509
|
}
|
|
236
510
|
}
|
|
511
|
+
function assertNonNegativeSafeInteger(value, label) {
|
|
512
|
+
if (!Number.isSafeInteger(value) || value < 0) {
|
|
513
|
+
throw new RunStoreError("invalid_record", `${label} must be a non-negative safe integer`);
|
|
514
|
+
}
|
|
515
|
+
}
|
|
516
|
+
function assertPositiveSafeInteger(value, label) {
|
|
517
|
+
if (!Number.isSafeInteger(value) || value <= 0) {
|
|
518
|
+
throw new RunStoreError("invalid_record", `${label} must be a positive safe integer`);
|
|
519
|
+
}
|
|
520
|
+
}
|
|
521
|
+
function assertAttemptExecutor(value, label) {
|
|
522
|
+
if (value === null || typeof value !== "object" || Array.isArray(value)) {
|
|
523
|
+
throw new RunStoreError("invalid_record", `${label} must be an object`);
|
|
524
|
+
}
|
|
525
|
+
const executor = value;
|
|
526
|
+
if (executor.kind === "worker") {
|
|
527
|
+
assertExactFields(executor, label, ["kind", "worker_session_id"]);
|
|
528
|
+
assertString(executor.worker_session_id, `${label}.worker_session_id`);
|
|
529
|
+
return;
|
|
530
|
+
}
|
|
531
|
+
if (executor.kind === "system") {
|
|
532
|
+
assertExactFields(executor, label, [
|
|
533
|
+
"kind",
|
|
534
|
+
"service_instance_id",
|
|
535
|
+
"operation",
|
|
536
|
+
]);
|
|
537
|
+
assertString(executor.service_instance_id, `${label}.service_instance_id`);
|
|
538
|
+
assertString(executor.operation, `${label}.operation`);
|
|
539
|
+
return;
|
|
540
|
+
}
|
|
541
|
+
if (executor.kind === "human") {
|
|
542
|
+
assertExactFields(executor, label, [
|
|
543
|
+
"kind",
|
|
544
|
+
"principal_ref",
|
|
545
|
+
"attestation_ref",
|
|
546
|
+
]);
|
|
547
|
+
assertString(executor.principal_ref, `${label}.principal_ref`);
|
|
548
|
+
assertString(executor.attestation_ref, `${label}.attestation_ref`);
|
|
549
|
+
return;
|
|
550
|
+
}
|
|
551
|
+
throw new RunStoreError("invalid_record", `${label}.kind must be worker, system, or human`);
|
|
552
|
+
}
|
|
553
|
+
function assertRunScopedCanonicalEvent(event) {
|
|
554
|
+
if (event.step_id !== null
|
|
555
|
+
|| event.activation_id !== null
|
|
556
|
+
|| event.attempt_id !== null
|
|
557
|
+
|| event.result_id !== null) {
|
|
558
|
+
throw new RunStoreError("invalid_record", `event ${event.event_id} must be run-scoped`);
|
|
559
|
+
}
|
|
560
|
+
}
|
|
561
|
+
function requireEventAttempt(event, attempts) {
|
|
562
|
+
if (event.attempt_id === null) {
|
|
563
|
+
throw new RunStoreError("invalid_record", `event ${event.event_id} requires attempt identity`);
|
|
564
|
+
}
|
|
565
|
+
const attempt = attempts.get(event.attempt_id);
|
|
566
|
+
if (!attempt || attempt.activation_id !== event.activation_id) {
|
|
567
|
+
throw new RunStoreError("invalid_record", `event ${event.event_id} does not resolve its attempt`);
|
|
568
|
+
}
|
|
569
|
+
return attempt;
|
|
570
|
+
}
|
|
571
|
+
function requireAttemptScopedCanonicalEvent(event, attempts) {
|
|
572
|
+
const attempt = requireEventAttempt(event, attempts);
|
|
573
|
+
if (event.result_id !== null) {
|
|
574
|
+
throw new RunStoreError("invalid_record", `event ${event.event_id} must not claim result identity`);
|
|
575
|
+
}
|
|
576
|
+
return attempt;
|
|
577
|
+
}
|
|
578
|
+
function assertActivationScopedCanonicalEvent(event) {
|
|
579
|
+
if (event.activation_id === null
|
|
580
|
+
|| event.attempt_id !== null
|
|
581
|
+
|| event.result_id !== null) {
|
|
582
|
+
throw new RunStoreError("invalid_record", `event ${event.event_id} must be activation-scoped`);
|
|
583
|
+
}
|
|
584
|
+
}
|
|
585
|
+
function requirePayloadResult(resultId, label, resultRefs) {
|
|
586
|
+
assertString(resultId, label);
|
|
587
|
+
const reference = resultRefs.get(resultId);
|
|
588
|
+
if (!reference) {
|
|
589
|
+
throw new RunStoreError("invalid_record", `${label} does not resolve a canonical result`);
|
|
590
|
+
}
|
|
591
|
+
return reference;
|
|
592
|
+
}
|
|
593
|
+
function assertCanonicalEventPayload(event, snapshot, activations, attempts, resultRefs) {
|
|
594
|
+
assertExactFields(event.payload, `event ${event.event_id}.payload`, (() => {
|
|
595
|
+
switch (event.type) {
|
|
596
|
+
case "run_created": return ["plan_id", "run_binding_id"];
|
|
597
|
+
case "run_cancel_requested": return ["reason"];
|
|
598
|
+
case "run_closed": return ["reason_code", "reason"];
|
|
599
|
+
case "run_terminal": return ["status", "reason_code", "reason"];
|
|
600
|
+
case "activation_state_changed": return ["from", "to", "reason_code"];
|
|
601
|
+
case "attempt_claimed": return [
|
|
602
|
+
"claim_epoch",
|
|
603
|
+
"owner_id",
|
|
604
|
+
"claim_expires_at",
|
|
605
|
+
"execution_deadline_at",
|
|
606
|
+
];
|
|
607
|
+
case "attempt_attached": return ["claim_epoch", "executor"];
|
|
608
|
+
case "workspace_planned": return ["workspace_plan_id", "plan"];
|
|
609
|
+
case "workspace_allocated": return ["workspace_plan_id", "allocation"];
|
|
610
|
+
case "attempt_heartbeat": return [
|
|
611
|
+
"claim_epoch",
|
|
612
|
+
"heartbeat_at",
|
|
613
|
+
"claim_expires_at",
|
|
614
|
+
];
|
|
615
|
+
case "attempt_released": return ["claim_epoch", "owner_id", "reason_code"];
|
|
616
|
+
case "attempt_started": return ["executor"];
|
|
617
|
+
case "attempt_stop_requested": return ["intent"];
|
|
618
|
+
case "attempt_terminal": return "worker_already_stopped_evidence" in event.payload
|
|
619
|
+
? ["executor", "outcome", "worker_stop_receipt", "worker_nonlaunch_attestation", "worker_already_stopped_evidence"]
|
|
620
|
+
: "worker_nonlaunch_attestation" in event.payload
|
|
621
|
+
? ["executor", "outcome", "worker_stop_receipt", "worker_nonlaunch_attestation"]
|
|
622
|
+
: ["executor", "outcome", "worker_stop_receipt"];
|
|
623
|
+
case "handoff_resolved": return ["binding_id", "source_result_id", "output_name"];
|
|
624
|
+
case "artifact_captured": return ["capture"];
|
|
625
|
+
case "result_committed": return [
|
|
626
|
+
"result_digest",
|
|
627
|
+
"submission_key_digest",
|
|
628
|
+
"submission_digest",
|
|
629
|
+
"worker_stop_receipt_digest",
|
|
630
|
+
"workspace_allocation_digest",
|
|
631
|
+
"mutation_authority_instance_id",
|
|
632
|
+
"mutation_fencing_epoch",
|
|
633
|
+
"finalization_purpose",
|
|
634
|
+
];
|
|
635
|
+
case "gate_evaluated": return [
|
|
636
|
+
"gate_id",
|
|
637
|
+
"predicate",
|
|
638
|
+
"source_result_id",
|
|
639
|
+
"passed",
|
|
640
|
+
"reason_code",
|
|
641
|
+
];
|
|
642
|
+
case "decision_evaluated": return [
|
|
643
|
+
"decision_id",
|
|
644
|
+
"value",
|
|
645
|
+
"source_result_id",
|
|
646
|
+
"reason_code",
|
|
647
|
+
];
|
|
648
|
+
case "route_selected": return [
|
|
649
|
+
"route_id",
|
|
650
|
+
"source_activation_id",
|
|
651
|
+
"source_result_id",
|
|
652
|
+
"successor_activation_ids",
|
|
653
|
+
"reason_code",
|
|
654
|
+
];
|
|
655
|
+
case "retry_segment_scheduled": return [
|
|
656
|
+
"retry_segment_id",
|
|
657
|
+
"from_round",
|
|
658
|
+
"to_round",
|
|
659
|
+
"prior_activation_ids",
|
|
660
|
+
"replacement_activation_ids",
|
|
661
|
+
"retry_from_activation_id",
|
|
662
|
+
"triggering_result_id",
|
|
663
|
+
"delay_ms",
|
|
664
|
+
"ready_at",
|
|
665
|
+
"reason_code",
|
|
666
|
+
];
|
|
667
|
+
case "retry_scheduled": return [
|
|
668
|
+
"prior_attempt_id",
|
|
669
|
+
"replacement_attempt_id",
|
|
670
|
+
"next_attempt_number",
|
|
671
|
+
"delay_ms",
|
|
672
|
+
"reason_code",
|
|
673
|
+
];
|
|
674
|
+
case "interaction_requested": return [
|
|
675
|
+
"interaction_id",
|
|
676
|
+
"expected_principal_ref",
|
|
677
|
+
];
|
|
678
|
+
case "interaction_submitted": return [
|
|
679
|
+
"interaction_id",
|
|
680
|
+
"principal_ref",
|
|
681
|
+
"attestation_ref",
|
|
682
|
+
];
|
|
683
|
+
}
|
|
684
|
+
})());
|
|
685
|
+
const payload = event.payload;
|
|
686
|
+
switch (event.type) {
|
|
687
|
+
case "run_created": {
|
|
688
|
+
assertRunScopedCanonicalEvent(event);
|
|
689
|
+
assertDigest(payload.plan_id, `event ${event.event_id}.payload.plan_id`);
|
|
690
|
+
assertString(payload.run_binding_id, `event ${event.event_id}.payload.run_binding_id`);
|
|
691
|
+
if (payload.plan_id !== snapshot.run.plan_id
|
|
692
|
+
|| payload.run_binding_id !== snapshot.run.run_binding.binding_id) {
|
|
693
|
+
throw new RunStoreError("identity_conflict", `run_created event ${event.event_id} does not match the run binding`);
|
|
694
|
+
}
|
|
695
|
+
break;
|
|
696
|
+
}
|
|
697
|
+
case "run_cancel_requested": {
|
|
698
|
+
assertRunScopedCanonicalEvent(event);
|
|
699
|
+
assertNullableString(payload.reason, `event ${event.event_id}.payload.reason`);
|
|
700
|
+
if (snapshot.run.cancellation === null
|
|
701
|
+
|| snapshot.run.cancellation.reason !== payload.reason) {
|
|
702
|
+
throw new RunStoreError("invalid_record", `run_cancel_requested event ${event.event_id} does not match cancellation intent`);
|
|
703
|
+
}
|
|
704
|
+
break;
|
|
705
|
+
}
|
|
706
|
+
case "run_closed": {
|
|
707
|
+
assertRunScopedCanonicalEvent(event);
|
|
708
|
+
assertString(payload.reason_code, `event ${event.event_id}.payload.reason_code`);
|
|
709
|
+
assertNullableString(payload.reason, `event ${event.event_id}.payload.reason`);
|
|
710
|
+
break;
|
|
711
|
+
}
|
|
712
|
+
case "run_terminal": {
|
|
713
|
+
assertRunScopedCanonicalEvent(event);
|
|
714
|
+
assertString(payload.reason_code, `event ${event.event_id}.payload.reason_code`);
|
|
715
|
+
assertNullableString(payload.reason, `event ${event.event_id}.payload.reason`);
|
|
716
|
+
if (!["succeeded", "failed", "cancelled"].includes(payload.status)
|
|
717
|
+
|| snapshot.run.terminal === null
|
|
718
|
+
|| snapshot.run.terminal.status !== payload.status
|
|
719
|
+
|| snapshot.run.terminal.reason_code !== payload.reason_code
|
|
720
|
+
|| snapshot.run.terminal.reason !== payload.reason) {
|
|
721
|
+
throw new RunStoreError("invalid_record", `run_terminal event ${event.event_id} does not match terminal run state`);
|
|
722
|
+
}
|
|
723
|
+
break;
|
|
724
|
+
}
|
|
725
|
+
case "activation_state_changed": {
|
|
726
|
+
assertActivationScopedCanonicalEvent(event);
|
|
727
|
+
if (!activations.has(event.activation_id)) {
|
|
728
|
+
throw new RunStoreError("invalid_record", `activation event ${event.event_id} requires activation identity`);
|
|
729
|
+
}
|
|
730
|
+
if (!CANONICAL_ACTIVATION_STATE_VALUES.includes(payload.from)
|
|
731
|
+
|| !CANONICAL_ACTIVATION_STATE_VALUES.includes(payload.to)) {
|
|
732
|
+
throw new RunStoreError("invalid_record", `activation event ${event.event_id} has invalid states`);
|
|
733
|
+
}
|
|
734
|
+
assertString(payload.reason_code, `event ${event.event_id}.payload.reason_code`);
|
|
735
|
+
if (payload.from === payload.to
|
|
736
|
+
|| !CANONICAL_ACTIVATION_STATE_TRANSITIONS[payload.from]
|
|
737
|
+
.includes(payload.to)) {
|
|
738
|
+
throw new RunStoreError("invalid_record", `activation event ${event.event_id} has an invalid transition`);
|
|
739
|
+
}
|
|
740
|
+
break;
|
|
741
|
+
}
|
|
742
|
+
case "attempt_claimed": {
|
|
743
|
+
const attempt = requireAttemptScopedCanonicalEvent(event, attempts);
|
|
744
|
+
assertPositiveSafeInteger(payload.claim_epoch, `event ${event.event_id}.payload.claim_epoch`);
|
|
745
|
+
assertString(payload.owner_id, `event ${event.event_id}.payload.owner_id`);
|
|
746
|
+
const expiresAt = assertTimestamp(payload.claim_expires_at, `event ${event.event_id}.payload.claim_expires_at`);
|
|
747
|
+
const deadlineAt = assertTimestamp(payload.execution_deadline_at, `event ${event.event_id}.payload.execution_deadline_at`);
|
|
748
|
+
const claim = snapshot.claims.find((candidate) => (candidate.attempt_id === attempt.attempt_id
|
|
749
|
+
&& candidate.claim_epoch === payload.claim_epoch));
|
|
750
|
+
if (!claim
|
|
751
|
+
|| claim.owner_id !== payload.owner_id
|
|
752
|
+
|| claim.execution_deadline_at !== payload.execution_deadline_at
|
|
753
|
+
|| expiresAt > Date.parse(claim.driver_claim_lease.expires_at)
|
|
754
|
+
|| expiresAt > deadlineAt) {
|
|
755
|
+
throw new RunStoreError("invalid_record", `attempt_claimed event ${event.event_id} does not match a durable claim`);
|
|
756
|
+
}
|
|
757
|
+
break;
|
|
758
|
+
}
|
|
759
|
+
case "attempt_attached": {
|
|
760
|
+
const attempt = requireAttemptScopedCanonicalEvent(event, attempts);
|
|
761
|
+
assertPositiveSafeInteger(payload.claim_epoch, `event ${event.event_id}.payload.claim_epoch`);
|
|
762
|
+
assertAttemptExecutor(payload.executor, `event ${event.event_id}.payload.executor`);
|
|
763
|
+
if (attempt.executor === null
|
|
764
|
+
|| !recordsEqual(attempt.executor, payload.executor)
|
|
765
|
+
|| !snapshot.claims.some((claim) => (claim.attempt_id === attempt.attempt_id
|
|
766
|
+
&& claim.claim_epoch === payload.claim_epoch))) {
|
|
767
|
+
throw new RunStoreError("invalid_record", `attempt_attached event ${event.event_id} does not match its attempt`);
|
|
768
|
+
}
|
|
769
|
+
break;
|
|
770
|
+
}
|
|
771
|
+
case "workspace_planned": {
|
|
772
|
+
requireAttemptScopedCanonicalEvent(event, attempts);
|
|
773
|
+
if (event.result_id !== null) {
|
|
774
|
+
throw new RunStoreError("identity_conflict", `workspace_planned event ${event.event_id} must not carry result identity`);
|
|
775
|
+
}
|
|
776
|
+
assertDigest(payload.workspace_plan_id, `event ${event.event_id}.payload.workspace_plan_id`);
|
|
777
|
+
const plan = payload.plan;
|
|
778
|
+
assertExactFields(plan, `event ${event.event_id}.payload.plan`, ["workspace_plan_id", "payload"]);
|
|
779
|
+
if (plan.workspace_plan_id !== payload.workspace_plan_id) {
|
|
780
|
+
throw new RunStoreError("identity_conflict", `workspace_planned event ${event.event_id} has inconsistent plan identity`);
|
|
781
|
+
}
|
|
782
|
+
assertWorkspacePhysicalIdentity(plan.payload, `event ${event.event_id}.payload.plan.payload`, event.attempt_id, false);
|
|
783
|
+
const actualPlanId = `sha256:${createHash("sha256").update(canonicalizeJson(plan.payload)).digest("hex")}`;
|
|
784
|
+
if (actualPlanId !== payload.workspace_plan_id) {
|
|
785
|
+
throw new RunStoreError("identity_conflict", `workspace_planned event ${event.event_id} plan digest does not match its payload`);
|
|
786
|
+
}
|
|
787
|
+
break;
|
|
788
|
+
}
|
|
789
|
+
case "workspace_allocated": {
|
|
790
|
+
requireAttemptScopedCanonicalEvent(event, attempts);
|
|
791
|
+
if (event.result_id !== null) {
|
|
792
|
+
throw new RunStoreError("identity_conflict", `workspace_allocated event ${event.event_id} must not carry result identity`);
|
|
793
|
+
}
|
|
794
|
+
assertDigest(payload.workspace_plan_id, `event ${event.event_id}.payload.workspace_plan_id`);
|
|
795
|
+
const record = assertWorkspacePhysicalIdentity(payload.allocation, `event ${event.event_id}.payload.allocation`, event.attempt_id, true);
|
|
796
|
+
if (record.workspace_plan_id !== payload.workspace_plan_id) {
|
|
797
|
+
throw new RunStoreError("identity_conflict", `workspace_allocated event ${event.event_id} has inconsistent plan identity`);
|
|
798
|
+
}
|
|
799
|
+
break;
|
|
800
|
+
}
|
|
801
|
+
case "attempt_heartbeat": {
|
|
802
|
+
const attempt = requireAttemptScopedCanonicalEvent(event, attempts);
|
|
803
|
+
assertPositiveSafeInteger(payload.claim_epoch, `event ${event.event_id}.payload.claim_epoch`);
|
|
804
|
+
const heartbeatAt = assertTimestamp(payload.heartbeat_at, `event ${event.event_id}.payload.heartbeat_at`);
|
|
805
|
+
const expiresAt = assertTimestamp(payload.claim_expires_at, `event ${event.event_id}.payload.claim_expires_at`);
|
|
806
|
+
const claim = snapshot.claims.find((candidate) => (candidate.attempt_id === attempt.attempt_id
|
|
807
|
+
&& candidate.claim_epoch === payload.claim_epoch));
|
|
808
|
+
if (!claim
|
|
809
|
+
|| heartbeatAt > Date.parse(claim.driver_claim_lease.heartbeat_at)
|
|
810
|
+
|| expiresAt > Date.parse(claim.driver_claim_lease.expires_at)
|
|
811
|
+
|| heartbeatAt > expiresAt) {
|
|
812
|
+
throw new RunStoreError("invalid_record", `attempt_heartbeat event ${event.event_id} exceeds its durable lease`);
|
|
813
|
+
}
|
|
814
|
+
break;
|
|
815
|
+
}
|
|
816
|
+
case "attempt_released": {
|
|
817
|
+
const attempt = requireAttemptScopedCanonicalEvent(event, attempts);
|
|
818
|
+
assertPositiveSafeInteger(payload.claim_epoch, `event ${event.event_id}.payload.claim_epoch`);
|
|
819
|
+
assertString(payload.owner_id, `event ${event.event_id}.payload.owner_id`);
|
|
820
|
+
assertString(payload.reason_code, `event ${event.event_id}.payload.reason_code`);
|
|
821
|
+
if (!snapshot.claims.some((claim) => (claim.attempt_id === attempt.attempt_id
|
|
822
|
+
&& claim.claim_epoch === payload.claim_epoch
|
|
823
|
+
&& claim.owner_id === payload.owner_id))) {
|
|
824
|
+
throw new RunStoreError("invalid_record", `attempt_released event ${event.event_id} does not match a durable claim`);
|
|
825
|
+
}
|
|
826
|
+
break;
|
|
827
|
+
}
|
|
828
|
+
case "attempt_started": {
|
|
829
|
+
const attempt = requireAttemptScopedCanonicalEvent(event, attempts);
|
|
830
|
+
assertAttemptExecutor(payload.executor, `event ${event.event_id}.payload.executor`);
|
|
831
|
+
if (attempt.executor === null || !recordsEqual(attempt.executor, payload.executor)) {
|
|
832
|
+
throw new RunStoreError("invalid_record", `attempt_started event ${event.event_id} does not match its executor`);
|
|
833
|
+
}
|
|
834
|
+
break;
|
|
835
|
+
}
|
|
836
|
+
case "attempt_stop_requested": {
|
|
837
|
+
const attempt = requireAttemptScopedCanonicalEvent(event, attempts);
|
|
838
|
+
const intent = payload.intent;
|
|
839
|
+
assertExactFields(intent, `event ${event.event_id}.payload.intent`, [
|
|
840
|
+
"schema_version", "run_id", "step_id", "activation_id", "attempt_id",
|
|
841
|
+
"claim_epoch", "worker_session_id", "stop_idempotency_key", "purpose",
|
|
842
|
+
"requested_at", "started_at", "requested_outcome",
|
|
843
|
+
]);
|
|
844
|
+
if (intent.schema_version !== "intentdna.attempt_stop_intent.v1"
|
|
845
|
+
|| intent.run_id !== attempt.run_id
|
|
846
|
+
|| intent.step_id !== attempt.step_id
|
|
847
|
+
|| intent.activation_id !== attempt.activation_id
|
|
848
|
+
|| intent.attempt_id !== attempt.attempt_id
|
|
849
|
+
|| attempt.executor?.kind !== "worker"
|
|
850
|
+
|| intent.worker_session_id !== attempt.executor.worker_session_id
|
|
851
|
+
|| intent.started_at !== attempt.started_at
|
|
852
|
+
|| typeof intent.purpose !== "string"
|
|
853
|
+
|| !["cancel", "execution_deadline", "peer_terminal"].includes(intent.purpose)) {
|
|
854
|
+
throw new RunStoreError("invalid_record", `attempt stop intent ${event.event_id} does not match its worker attempt`);
|
|
855
|
+
}
|
|
856
|
+
assertPositiveSafeInteger(intent.claim_epoch, `event ${event.event_id}.payload.intent.claim_epoch`);
|
|
857
|
+
assertDigest(intent.stop_idempotency_key, `event ${event.event_id}.payload.intent.stop_idempotency_key`);
|
|
858
|
+
const expectedStopKey = `sha256:${createHash("sha256").update(canonicalizeJson({
|
|
859
|
+
run_id: attempt.run_id,
|
|
860
|
+
step_id: attempt.step_id,
|
|
861
|
+
activation_id: attempt.activation_id,
|
|
862
|
+
attempt_id: attempt.attempt_id,
|
|
863
|
+
claim_epoch: intent.claim_epoch,
|
|
864
|
+
worker_session_id: intent.worker_session_id,
|
|
865
|
+
})).digest("hex")}`;
|
|
866
|
+
if (intent.stop_idempotency_key !== expectedStopKey) {
|
|
867
|
+
throw new RunStoreError("identity_conflict", `attempt stop intent ${event.event_id} has an invalid execution identity digest`);
|
|
868
|
+
}
|
|
869
|
+
assertTimestamp(intent.requested_at, `event ${event.event_id}.payload.intent.requested_at`);
|
|
870
|
+
try {
|
|
871
|
+
validateCanonicalAttemptOutcome(attempt.executor, intent.requested_outcome);
|
|
872
|
+
}
|
|
873
|
+
catch (error) {
|
|
874
|
+
throw new RunStoreError("invalid_record", `attempt stop intent ${event.event_id} has invalid outcome`, { cause: error });
|
|
875
|
+
}
|
|
876
|
+
if (!snapshot.claims.some((claim) => claim.attempt_id === attempt.attempt_id && claim.claim_epoch === intent.claim_epoch)) {
|
|
877
|
+
throw new RunStoreError("invalid_record", `attempt stop intent ${event.event_id} does not match a durable claim`);
|
|
878
|
+
}
|
|
879
|
+
const duplicate = snapshot.events.filter((candidate) => candidate.type === "attempt_stop_requested" && candidate.attempt_id === attempt.attempt_id);
|
|
880
|
+
if (duplicate.length !== 1 || duplicate[0]?.event_id !== event.event_id) {
|
|
881
|
+
throw new RunStoreError("immutable_record", `attempt ${attempt.attempt_id} can have only one durable stop intent`);
|
|
882
|
+
}
|
|
883
|
+
break;
|
|
884
|
+
}
|
|
885
|
+
case "attempt_terminal": {
|
|
886
|
+
const attempt = requireAttemptScopedCanonicalEvent(event, attempts);
|
|
887
|
+
try {
|
|
888
|
+
validateCanonicalAttemptOutcome(payload.executor, payload.outcome);
|
|
889
|
+
}
|
|
890
|
+
catch (error) {
|
|
891
|
+
throw new RunStoreError("invalid_record", `attempt_terminal event ${event.event_id} has invalid outcome`, { cause: error });
|
|
892
|
+
}
|
|
893
|
+
if (attempt.phase !== "terminal"
|
|
894
|
+
|| attempt.executor === null
|
|
895
|
+
|| !recordsEqual(attempt.executor, payload.executor)
|
|
896
|
+
|| !recordsEqual(attempt.terminal_outcome, payload.outcome)) {
|
|
897
|
+
throw new RunStoreError("invalid_record", `attempt_terminal event ${event.event_id} does not match terminal attempt state`);
|
|
898
|
+
}
|
|
899
|
+
if (attempt.executor.kind === "worker") {
|
|
900
|
+
const receipt = payload.worker_stop_receipt;
|
|
901
|
+
const nonlaunch = payload.worker_nonlaunch_attestation ?? null;
|
|
902
|
+
const alreadyStopped = payload.worker_already_stopped_evidence ?? null;
|
|
903
|
+
if ([receipt, nonlaunch, alreadyStopped].filter((item) => item !== null).length !== 1) {
|
|
904
|
+
throw new RunStoreError("invalid_record", `worker attempt_terminal event ${event.event_id} requires exactly one execution authority evidence`);
|
|
905
|
+
}
|
|
906
|
+
if (alreadyStopped !== null) {
|
|
907
|
+
assertExactFields(alreadyStopped, `event ${event.event_id}.payload.worker_already_stopped_evidence`, [
|
|
908
|
+
"evidence_digest", "payload",
|
|
909
|
+
]);
|
|
910
|
+
assertDigest(alreadyStopped.evidence_digest, `event ${event.event_id}.payload.worker_already_stopped_evidence.evidence_digest`);
|
|
911
|
+
const fact = alreadyStopped.payload;
|
|
912
|
+
assertExactFields(fact, `event ${event.event_id}.payload.worker_already_stopped_evidence.payload`, [
|
|
913
|
+
"schema_version", "run_id", "step_id", "activation_id", "attempt_id",
|
|
914
|
+
"claim_epoch", "worker_session_id", "execution_instance_id", "authority_kind",
|
|
915
|
+
"authority_instance_id", "fencing_epoch", "started_at", "completed_at",
|
|
916
|
+
"observed_outcome", "process_tree_drained", "stop_idempotency_key",
|
|
917
|
+
"stop_requested_at", "stop_purpose", "stop_requested_outcome", "source_evidence_digest",
|
|
918
|
+
]);
|
|
919
|
+
const stopIntent = snapshot.events.find((candidate) => (candidate.type === "attempt_stop_requested" && candidate.attempt_id === attempt.attempt_id));
|
|
920
|
+
if (fact.schema_version !== "intentdna.worker_already_stopped_evidence.v1"
|
|
921
|
+
|| fact.run_id !== attempt.run_id
|
|
922
|
+
|| fact.step_id !== attempt.step_id
|
|
923
|
+
|| fact.activation_id !== attempt.activation_id
|
|
924
|
+
|| fact.attempt_id !== attempt.attempt_id
|
|
925
|
+
|| fact.worker_session_id !== attempt.executor.worker_session_id
|
|
926
|
+
|| fact.started_at !== attempt.started_at
|
|
927
|
+
|| fact.completed_at !== attempt.completed_at
|
|
928
|
+
|| fact.process_tree_drained !== true
|
|
929
|
+
|| !["local_process", "host_attested"].includes(fact.authority_kind)
|
|
930
|
+
|| stopIntent?.type !== "attempt_stop_requested"
|
|
931
|
+
|| fact.claim_epoch !== stopIntent.payload.intent.claim_epoch
|
|
932
|
+
|| fact.stop_idempotency_key !== stopIntent.payload.intent.stop_idempotency_key
|
|
933
|
+
|| fact.stop_requested_at !== stopIntent.payload.intent.requested_at
|
|
934
|
+
|| fact.stop_purpose !== stopIntent.payload.intent.purpose
|
|
935
|
+
|| !recordsEqual(fact.stop_requested_outcome, stopIntent.payload.intent.requested_outcome)) {
|
|
936
|
+
throw new RunStoreError("invalid_record", `worker already-stopped evidence on event ${event.event_id} does not match its durable stop intent`);
|
|
937
|
+
}
|
|
938
|
+
assertPositiveSafeInteger(fact.claim_epoch, `event ${event.event_id}.payload.worker_already_stopped_evidence.payload.claim_epoch`);
|
|
939
|
+
assertPositiveSafeInteger(fact.fencing_epoch, `event ${event.event_id}.payload.worker_already_stopped_evidence.payload.fencing_epoch`);
|
|
940
|
+
assertTimestamp(fact.completed_at, `event ${event.event_id}.payload.worker_already_stopped_evidence.payload.completed_at`);
|
|
941
|
+
assertTimestamp(fact.stop_requested_at, `event ${event.event_id}.payload.worker_already_stopped_evidence.payload.stop_requested_at`);
|
|
942
|
+
assertString(fact.execution_instance_id, `event ${event.event_id}.payload.worker_already_stopped_evidence.payload.execution_instance_id`);
|
|
943
|
+
assertString(fact.authority_instance_id, `event ${event.event_id}.payload.worker_already_stopped_evidence.payload.authority_instance_id`);
|
|
944
|
+
assertDigest(fact.stop_idempotency_key, `event ${event.event_id}.payload.worker_already_stopped_evidence.payload.stop_idempotency_key`);
|
|
945
|
+
assertDigest(fact.source_evidence_digest, `event ${event.event_id}.payload.worker_already_stopped_evidence.payload.source_evidence_digest`);
|
|
946
|
+
try {
|
|
947
|
+
validateCanonicalAttemptOutcome(attempt.executor, fact.observed_outcome);
|
|
948
|
+
validateCanonicalAttemptOutcome(attempt.executor, fact.stop_requested_outcome);
|
|
949
|
+
}
|
|
950
|
+
catch (error) {
|
|
951
|
+
throw new RunStoreError("invalid_record", `worker already-stopped evidence on event ${event.event_id} has invalid observed outcome`, { cause: error });
|
|
952
|
+
}
|
|
953
|
+
const actual = `sha256:${createHash("sha256").update(canonicalizeJson(fact), "utf8").digest("hex")}`;
|
|
954
|
+
if (alreadyStopped.evidence_digest !== actual) {
|
|
955
|
+
throw new RunStoreError("invalid_record", `worker already-stopped evidence on event ${event.event_id} has an invalid digest`);
|
|
956
|
+
}
|
|
957
|
+
break;
|
|
958
|
+
}
|
|
959
|
+
if (nonlaunch !== null) {
|
|
960
|
+
assertExactFields(nonlaunch, `event ${event.event_id}.payload.worker_nonlaunch_attestation`, [
|
|
961
|
+
"attestation_digest", "payload",
|
|
962
|
+
]);
|
|
963
|
+
assertDigest(nonlaunch.attestation_digest, `event ${event.event_id}.payload.worker_nonlaunch_attestation.attestation_digest`);
|
|
964
|
+
const fact = nonlaunch.payload;
|
|
965
|
+
assertExactFields(fact, `event ${event.event_id}.payload.worker_nonlaunch_attestation.payload`, [
|
|
966
|
+
"schema_version", "run_id", "step_id", "activation_id", "attempt_id",
|
|
967
|
+
"claim_epoch", "worker_session_id", "execution_instance_id", "authority_kind",
|
|
968
|
+
"authority_instance_id", "fencing_epoch", "started_at", "attested_at",
|
|
969
|
+
"failure_idempotency_key", "launch_state", "source_evidence_digest",
|
|
970
|
+
]);
|
|
971
|
+
if (fact.schema_version !== "intentdna.worker_nonlaunch_attestation.v1"
|
|
972
|
+
|| fact.run_id !== attempt.run_id
|
|
973
|
+
|| fact.step_id !== attempt.step_id
|
|
974
|
+
|| fact.activation_id !== attempt.activation_id
|
|
975
|
+
|| fact.attempt_id !== attempt.attempt_id
|
|
976
|
+
|| fact.worker_session_id !== attempt.executor.worker_session_id
|
|
977
|
+
|| fact.started_at !== attempt.started_at
|
|
978
|
+
|| fact.launch_state !== "never_launched"
|
|
979
|
+
|| fact.authority_kind !== "local_process" && fact.authority_kind !== "host_attested") {
|
|
980
|
+
throw new RunStoreError("invalid_record", `worker nonlaunch attestation on event ${event.event_id} does not match its attempt`);
|
|
981
|
+
}
|
|
982
|
+
assertPositiveSafeInteger(fact.claim_epoch, `event ${event.event_id}.payload.worker_nonlaunch_attestation.payload.claim_epoch`);
|
|
983
|
+
assertPositiveSafeInteger(fact.fencing_epoch, `event ${event.event_id}.payload.worker_nonlaunch_attestation.payload.fencing_epoch`);
|
|
984
|
+
assertTimestamp(fact.attested_at, `event ${event.event_id}.payload.worker_nonlaunch_attestation.payload.attested_at`);
|
|
985
|
+
assertString(fact.execution_instance_id, `event ${event.event_id}.payload.worker_nonlaunch_attestation.payload.execution_instance_id`);
|
|
986
|
+
assertString(fact.authority_instance_id, `event ${event.event_id}.payload.worker_nonlaunch_attestation.payload.authority_instance_id`);
|
|
987
|
+
assertDigest(fact.failure_idempotency_key, `event ${event.event_id}.payload.worker_nonlaunch_attestation.payload.failure_idempotency_key`);
|
|
988
|
+
assertDigest(fact.source_evidence_digest, `event ${event.event_id}.payload.worker_nonlaunch_attestation.payload.source_evidence_digest`);
|
|
989
|
+
const actual = `sha256:${createHash("sha256").update(canonicalizeJson(fact), "utf8").digest("hex")}`;
|
|
990
|
+
if (nonlaunch.attestation_digest !== actual) {
|
|
991
|
+
throw new RunStoreError("invalid_record", `worker nonlaunch attestation on event ${event.event_id} has an invalid digest`);
|
|
992
|
+
}
|
|
993
|
+
const attached = snapshot.events.find((candidate) => (candidate.type === "attempt_attached"
|
|
994
|
+
&& candidate.attempt_id === attempt.attempt_id
|
|
995
|
+
&& candidate.payload.claim_epoch === fact.claim_epoch));
|
|
996
|
+
const claim = snapshot.claims.find((candidate) => (candidate.attempt_id === attempt.attempt_id
|
|
997
|
+
&& candidate.claim_epoch === fact.claim_epoch));
|
|
998
|
+
const started = snapshot.events.find((candidate) => (candidate.type === "attempt_started"
|
|
999
|
+
&& candidate.attempt_id === attempt.attempt_id));
|
|
1000
|
+
const stopIntent = snapshot.events.find((candidate) => (candidate.type === "attempt_stop_requested"
|
|
1001
|
+
&& candidate.attempt_id === attempt.attempt_id));
|
|
1002
|
+
if (attached?.type !== "attempt_attached"
|
|
1003
|
+
|| !recordsEqual(attached.payload.executor, attempt.executor)
|
|
1004
|
+
|| claim === undefined
|
|
1005
|
+
|| started?.type !== "attempt_started"
|
|
1006
|
+
|| !recordsEqual(started.payload.executor, attempt.executor)
|
|
1007
|
+
|| started.occurred_at !== fact.started_at) {
|
|
1008
|
+
throw new RunStoreError("invalid_record", `worker nonlaunch attestation on event ${event.event_id} does not match its durable attach identity`);
|
|
1009
|
+
}
|
|
1010
|
+
if (stopIntent?.type === "attempt_stop_requested" && (fact.failure_idempotency_key !== stopIntent.payload.intent.stop_idempotency_key
|
|
1011
|
+
|| fact.claim_epoch !== stopIntent.payload.intent.claim_epoch
|
|
1012
|
+
|| fact.worker_session_id !== stopIntent.payload.intent.worker_session_id
|
|
1013
|
+
|| fact.started_at !== stopIntent.payload.intent.started_at
|
|
1014
|
+
|| Date.parse(String(fact.attested_at))
|
|
1015
|
+
< Date.parse(String(stopIntent.payload.intent.requested_at)))) {
|
|
1016
|
+
throw new RunStoreError("invalid_record", `worker nonlaunch attestation on event ${event.event_id} does not confirm its durable stop intent`);
|
|
1017
|
+
}
|
|
1018
|
+
break;
|
|
1019
|
+
}
|
|
1020
|
+
if (receipt === null || typeof receipt !== "object" || Array.isArray(receipt)) {
|
|
1021
|
+
throw new RunStoreError("invalid_record", `worker attempt_terminal event ${event.event_id} lacks stopped-execution evidence`);
|
|
1022
|
+
}
|
|
1023
|
+
assertExactFields(receipt, `event ${event.event_id}.payload.worker_stop_receipt`, [
|
|
1024
|
+
"receipt_digest",
|
|
1025
|
+
"payload",
|
|
1026
|
+
]);
|
|
1027
|
+
assertDigest(receipt.receipt_digest, `event ${event.event_id}.payload.worker_stop_receipt.receipt_digest`);
|
|
1028
|
+
const stop = receipt.payload;
|
|
1029
|
+
assertExactFields(stop, `event ${event.event_id}.payload.worker_stop_receipt.payload`, [
|
|
1030
|
+
"schema_version", "run_id", "step_id", "activation_id", "attempt_id",
|
|
1031
|
+
"claim_epoch", "worker_session_id", "execution_instance_id",
|
|
1032
|
+
"authority_kind", "authority_instance_id", "fencing_epoch", "started_at",
|
|
1033
|
+
"stopped_at", "terminal_outcome", "process_tree_drained",
|
|
1034
|
+
"source_evidence_digest",
|
|
1035
|
+
]);
|
|
1036
|
+
if (stop.schema_version !== "intentdna.worker_stop_receipt.v1"
|
|
1037
|
+
|| stop.run_id !== attempt.run_id
|
|
1038
|
+
|| stop.step_id !== attempt.step_id
|
|
1039
|
+
|| stop.activation_id !== attempt.activation_id
|
|
1040
|
+
|| stop.attempt_id !== attempt.attempt_id
|
|
1041
|
+
|| stop.worker_session_id !== attempt.executor.worker_session_id
|
|
1042
|
+
|| stop.started_at !== attempt.started_at
|
|
1043
|
+
|| stop.stopped_at !== attempt.completed_at
|
|
1044
|
+
|| stop.process_tree_drained !== true
|
|
1045
|
+
|| !recordsEqual(stop.terminal_outcome, attempt.terminal_outcome)) {
|
|
1046
|
+
throw new RunStoreError("invalid_record", `worker stop receipt on event ${event.event_id} does not match its terminal attempt`);
|
|
1047
|
+
}
|
|
1048
|
+
assertPositiveSafeInteger(stop.claim_epoch, `event ${event.event_id}.payload.worker_stop_receipt.payload.claim_epoch`);
|
|
1049
|
+
assertPositiveSafeInteger(stop.fencing_epoch, `event ${event.event_id}.payload.worker_stop_receipt.payload.fencing_epoch`);
|
|
1050
|
+
assertString(stop.execution_instance_id, `event ${event.event_id}.payload.worker_stop_receipt.payload.execution_instance_id`);
|
|
1051
|
+
assertString(stop.authority_instance_id, `event ${event.event_id}.payload.worker_stop_receipt.payload.authority_instance_id`);
|
|
1052
|
+
if (!["local_process", "host_attested"].includes(stop.authority_kind)) {
|
|
1053
|
+
throw new RunStoreError("invalid_record", `event ${event.event_id}.payload.worker_stop_receipt.payload.authority_kind is unsupported`);
|
|
1054
|
+
}
|
|
1055
|
+
assertDigest(stop.source_evidence_digest, `event ${event.event_id}.payload.worker_stop_receipt.payload.source_evidence_digest`);
|
|
1056
|
+
const actualReceiptDigest = `sha256:${createHash("sha256")
|
|
1057
|
+
.update(canonicalizeJson(stop), "utf8")
|
|
1058
|
+
.digest("hex")}`;
|
|
1059
|
+
if (receipt.receipt_digest !== actualReceiptDigest) {
|
|
1060
|
+
throw new RunStoreError("invalid_record", `worker stop receipt on event ${event.event_id} has an invalid digest`);
|
|
1061
|
+
}
|
|
1062
|
+
const stopIntent = snapshot.events.find((candidate) => (candidate.type === "attempt_stop_requested" && candidate.attempt_id === attempt.attempt_id));
|
|
1063
|
+
if (stopIntent?.type === "attempt_stop_requested") {
|
|
1064
|
+
const intent = stopIntent.payload.intent;
|
|
1065
|
+
if (stop.claim_epoch !== intent.claim_epoch
|
|
1066
|
+
|| stop.worker_session_id !== intent.worker_session_id
|
|
1067
|
+
|| stop.started_at !== intent.started_at
|
|
1068
|
+
|| typeof stop.stopped_at !== "string"
|
|
1069
|
+
|| Date.parse(stop.stopped_at) < Date.parse(intent.requested_at)
|
|
1070
|
+
|| !recordsEqual(stop.terminal_outcome, intent.requested_outcome)) {
|
|
1071
|
+
throw new RunStoreError("invalid_record", `worker stop receipt on event ${event.event_id} does not confirm its durable stop intent`);
|
|
1072
|
+
}
|
|
1073
|
+
}
|
|
1074
|
+
}
|
|
1075
|
+
else if (payload.worker_stop_receipt !== null || (payload.worker_nonlaunch_attestation ?? null) !== null || (payload.worker_already_stopped_evidence ?? null) !== null) {
|
|
1076
|
+
throw new RunStoreError("invalid_record", `non-worker attempt_terminal event ${event.event_id} cannot carry worker stop evidence`);
|
|
1077
|
+
}
|
|
1078
|
+
break;
|
|
1079
|
+
}
|
|
1080
|
+
case "handoff_resolved": {
|
|
1081
|
+
requireAttemptScopedCanonicalEvent(event, attempts);
|
|
1082
|
+
assertString(payload.binding_id, `event ${event.event_id}.payload.binding_id`);
|
|
1083
|
+
requirePayloadResult(payload.source_result_id, `event ${event.event_id}.payload.source_result_id`, resultRefs);
|
|
1084
|
+
assertString(payload.output_name, `event ${event.event_id}.payload.output_name`);
|
|
1085
|
+
break;
|
|
1086
|
+
}
|
|
1087
|
+
case "artifact_captured": {
|
|
1088
|
+
requireAttemptScopedCanonicalEvent(event, attempts);
|
|
1089
|
+
assertExactFields(payload.capture, `event ${event.event_id}.payload.capture`, [
|
|
1090
|
+
"capture_id",
|
|
1091
|
+
"artifact_id",
|
|
1092
|
+
"artifact_kind",
|
|
1093
|
+
"run_id",
|
|
1094
|
+
"activation_id",
|
|
1095
|
+
"attempt_id",
|
|
1096
|
+
"output_name",
|
|
1097
|
+
"source_workspace_id",
|
|
1098
|
+
"source_path",
|
|
1099
|
+
"captured_at",
|
|
1100
|
+
]);
|
|
1101
|
+
const capture = payload.capture;
|
|
1102
|
+
for (const field of [
|
|
1103
|
+
"capture_id",
|
|
1104
|
+
"run_id",
|
|
1105
|
+
"activation_id",
|
|
1106
|
+
"attempt_id",
|
|
1107
|
+
"output_name",
|
|
1108
|
+
"source_workspace_id",
|
|
1109
|
+
"source_path",
|
|
1110
|
+
]) {
|
|
1111
|
+
assertString(capture[field], `event ${event.event_id}.payload.capture.${field}`);
|
|
1112
|
+
}
|
|
1113
|
+
assertDigest(capture.artifact_id, `event ${event.event_id}.payload.capture.artifact_id`);
|
|
1114
|
+
assertTimestamp(capture.captured_at, `event ${event.event_id}.payload.capture.captured_at`);
|
|
1115
|
+
if (!["file", "directory", "git_commit"].includes(capture.artifact_kind)
|
|
1116
|
+
|| capture.run_id !== event.run_id
|
|
1117
|
+
|| capture.activation_id !== event.activation_id
|
|
1118
|
+
|| capture.attempt_id !== event.attempt_id) {
|
|
1119
|
+
throw new RunStoreError("identity_conflict", `artifact_captured event ${event.event_id} has mismatched capture identity`);
|
|
1120
|
+
}
|
|
1121
|
+
break;
|
|
1122
|
+
}
|
|
1123
|
+
case "result_committed": {
|
|
1124
|
+
const attempt = requireEventAttempt(event, attempts);
|
|
1125
|
+
if (event.result_id === null) {
|
|
1126
|
+
throw new RunStoreError("invalid_record", `result_committed event ${event.event_id} requires result identity`);
|
|
1127
|
+
}
|
|
1128
|
+
assertDigest(payload.result_digest, `event ${event.event_id}.payload.result_digest`);
|
|
1129
|
+
assertDigest(payload.submission_key_digest, `event ${event.event_id}.payload.submission_key_digest`);
|
|
1130
|
+
assertDigest(payload.submission_digest, `event ${event.event_id}.payload.submission_digest`);
|
|
1131
|
+
if (payload.worker_stop_receipt_digest !== null) {
|
|
1132
|
+
assertDigest(payload.worker_stop_receipt_digest, `event ${event.event_id}.payload.worker_stop_receipt_digest`);
|
|
1133
|
+
}
|
|
1134
|
+
if ((attempt.executor?.kind === "worker") !== (payload.worker_stop_receipt_digest !== null)) {
|
|
1135
|
+
throw new RunStoreError("invalid_record", `result_committed event ${event.event_id} has inconsistent worker stop authority`);
|
|
1136
|
+
}
|
|
1137
|
+
const finalizationPurpose = payload.finalization_purpose;
|
|
1138
|
+
if (typeof finalizationPurpose !== "string" || !new Set(["finalize_result", "reconcile_published_result", "system_operation"]).has(finalizationPurpose)) {
|
|
1139
|
+
throw new RunStoreError("invalid_record", `event ${event.event_id}.payload.finalization_purpose is invalid`);
|
|
1140
|
+
}
|
|
1141
|
+
if (finalizationPurpose === "system_operation") {
|
|
1142
|
+
if (attempt.executor?.kind !== "system"
|
|
1143
|
+
|| payload.workspace_allocation_digest !== null
|
|
1144
|
+
|| payload.mutation_authority_instance_id !== null
|
|
1145
|
+
|| payload.mutation_fencing_epoch !== null) {
|
|
1146
|
+
throw new RunStoreError("invalid_record", `result_committed event ${event.event_id} has invalid system finalization authority`);
|
|
1147
|
+
}
|
|
1148
|
+
}
|
|
1149
|
+
else {
|
|
1150
|
+
assertDigest(payload.workspace_allocation_digest, `event ${event.event_id}.payload.workspace_allocation_digest`);
|
|
1151
|
+
assertCanonicalIdentifier(payload.mutation_authority_instance_id, `event ${event.event_id}.payload.mutation_authority_instance_id`);
|
|
1152
|
+
const mutationFencingEpoch = payload.mutation_fencing_epoch;
|
|
1153
|
+
if (typeof mutationFencingEpoch !== "number" || !Number.isSafeInteger(mutationFencingEpoch) || mutationFencingEpoch <= 0) {
|
|
1154
|
+
throw new RunStoreError("invalid_record", `event ${event.event_id}.payload.mutation_fencing_epoch must be a positive safe integer`);
|
|
1155
|
+
}
|
|
1156
|
+
}
|
|
1157
|
+
const reference = resultRefs.get(event.result_id);
|
|
1158
|
+
if (!reference
|
|
1159
|
+
|| reference.attempt_id !== attempt.attempt_id
|
|
1160
|
+
|| reference.step_id !== event.step_id
|
|
1161
|
+
|| reference.result_digest !== payload.result_digest) {
|
|
1162
|
+
throw new RunStoreError("invalid_record", `result_committed event ${event.event_id} does not match its result reference`);
|
|
1163
|
+
}
|
|
1164
|
+
break;
|
|
1165
|
+
}
|
|
1166
|
+
case "gate_evaluated":
|
|
1167
|
+
case "decision_evaluated": {
|
|
1168
|
+
if (event.activation_id === null || event.attempt_id !== null) {
|
|
1169
|
+
throw new RunStoreError("invalid_record", `${event.type} event ${event.event_id} requires activation identity`);
|
|
1170
|
+
}
|
|
1171
|
+
const idField = event.type === "gate_evaluated" ? "gate_id" : "decision_id";
|
|
1172
|
+
assertString(payload[idField], `event ${event.event_id}.payload.${idField}`);
|
|
1173
|
+
assertString(payload.reason_code, `event ${event.event_id}.payload.reason_code`);
|
|
1174
|
+
if (payload.source_result_id !== null) {
|
|
1175
|
+
requirePayloadResult(payload.source_result_id, `event ${event.event_id}.payload.source_result_id`, resultRefs);
|
|
1176
|
+
}
|
|
1177
|
+
if (event.result_id !== payload.source_result_id) {
|
|
1178
|
+
throw new RunStoreError("identity_conflict", `${event.type} event ${event.event_id} result identity does not match its source`);
|
|
1179
|
+
}
|
|
1180
|
+
if (event.type === "gate_evaluated") {
|
|
1181
|
+
assertBoolean(payload.passed, `event ${event.event_id}.payload.passed`);
|
|
1182
|
+
}
|
|
1183
|
+
break;
|
|
1184
|
+
}
|
|
1185
|
+
case "route_selected": {
|
|
1186
|
+
if (event.activation_id === null || event.attempt_id !== null) {
|
|
1187
|
+
throw new RunStoreError("invalid_record", `route_selected event ${event.event_id} requires activation identity`);
|
|
1188
|
+
}
|
|
1189
|
+
assertString(payload.route_id, `event ${event.event_id}.payload.route_id`);
|
|
1190
|
+
assertString(payload.source_activation_id, `event ${event.event_id}.payload.source_activation_id`);
|
|
1191
|
+
assertString(payload.reason_code, `event ${event.event_id}.payload.reason_code`);
|
|
1192
|
+
if (payload.source_activation_id !== event.activation_id) {
|
|
1193
|
+
throw new RunStoreError("identity_conflict", `route_selected event ${event.event_id} source activation does not match envelope`);
|
|
1194
|
+
}
|
|
1195
|
+
if (payload.source_result_id !== null) {
|
|
1196
|
+
const source = requirePayloadResult(payload.source_result_id, `event ${event.event_id}.payload.source_result_id`, resultRefs);
|
|
1197
|
+
if (source.activation_id !== event.activation_id) {
|
|
1198
|
+
throw new RunStoreError("identity_conflict", `route_selected event ${event.event_id} source result does not match activation`);
|
|
1199
|
+
}
|
|
1200
|
+
}
|
|
1201
|
+
if (event.result_id !== payload.source_result_id) {
|
|
1202
|
+
throw new RunStoreError("identity_conflict", `route_selected event ${event.event_id} result identity does not match its source`);
|
|
1203
|
+
}
|
|
1204
|
+
if (!Array.isArray(payload.successor_activation_ids)) {
|
|
1205
|
+
throw new RunStoreError("invalid_record", `event ${event.event_id}.payload.successor_activation_ids must be an array`);
|
|
1206
|
+
}
|
|
1207
|
+
const successors = new Set();
|
|
1208
|
+
for (const successorId of payload.successor_activation_ids) {
|
|
1209
|
+
assertString(successorId, `event ${event.event_id}.payload.successor_activation_ids[]`);
|
|
1210
|
+
if (successorId === event.activation_id
|
|
1211
|
+
|| successors.has(successorId)
|
|
1212
|
+
|| !activations.has(successorId)) {
|
|
1213
|
+
throw new RunStoreError("identity_conflict", `route_selected event ${event.event_id} has invalid successor ${successorId}`);
|
|
1214
|
+
}
|
|
1215
|
+
successors.add(successorId);
|
|
1216
|
+
}
|
|
1217
|
+
break;
|
|
1218
|
+
}
|
|
1219
|
+
case "retry_segment_scheduled": {
|
|
1220
|
+
if (event.activation_id === null || event.attempt_id !== null) {
|
|
1221
|
+
throw new RunStoreError("invalid_record", `retry_segment_scheduled event ${event.event_id} requires activation identity`);
|
|
1222
|
+
}
|
|
1223
|
+
const label = `event ${event.event_id}.payload`;
|
|
1224
|
+
assertCanonicalIdentifier(payload.retry_segment_id, `${label}.retry_segment_id`);
|
|
1225
|
+
assertPositiveSafeInteger(payload.from_round, `${label}.from_round`);
|
|
1226
|
+
assertPositiveSafeInteger(payload.to_round, `${label}.to_round`);
|
|
1227
|
+
assertNonNegativeSafeInteger(payload.delay_ms, `${label}.delay_ms`);
|
|
1228
|
+
const readyAt = assertTimestamp(payload.ready_at, `${label}.ready_at`);
|
|
1229
|
+
assertString(payload.reason_code, `${label}.reason_code`);
|
|
1230
|
+
assertCanonicalIdentifier(payload.retry_from_activation_id, `${label}.retry_from_activation_id`);
|
|
1231
|
+
if (payload.to_round !== payload.from_round + 1
|
|
1232
|
+
|| readyAt !== Date.parse(event.occurred_at) + payload.delay_ms) {
|
|
1233
|
+
throw new RunStoreError("invalid_record", `retry_segment_scheduled event ${event.event_id} has invalid round or delay timing`);
|
|
1234
|
+
}
|
|
1235
|
+
if (!Array.isArray(payload.prior_activation_ids)
|
|
1236
|
+
|| !Array.isArray(payload.replacement_activation_ids)
|
|
1237
|
+
|| payload.prior_activation_ids.length === 0
|
|
1238
|
+
|| payload.prior_activation_ids.length !== payload.replacement_activation_ids.length) {
|
|
1239
|
+
throw new RunStoreError("invalid_record", `retry_segment_scheduled event ${event.event_id} requires paired activation arrays`);
|
|
1240
|
+
}
|
|
1241
|
+
const priorIds = new Set();
|
|
1242
|
+
const replacementIds = new Set();
|
|
1243
|
+
for (let index = 0; index < payload.prior_activation_ids.length; index += 1) {
|
|
1244
|
+
const priorId = payload.prior_activation_ids[index];
|
|
1245
|
+
const replacementId = payload.replacement_activation_ids[index];
|
|
1246
|
+
assertCanonicalIdentifier(priorId, `${label}.prior_activation_ids[${index}]`);
|
|
1247
|
+
assertCanonicalIdentifier(replacementId, `${label}.replacement_activation_ids[${index}]`);
|
|
1248
|
+
const prior = activations.get(priorId);
|
|
1249
|
+
const replacement = activations.get(replacementId);
|
|
1250
|
+
if (priorIds.has(priorId)
|
|
1251
|
+
|| replacementIds.has(replacementId)
|
|
1252
|
+
|| priorId === replacementId
|
|
1253
|
+
|| !prior
|
|
1254
|
+
|| !replacement
|
|
1255
|
+
|| !["succeeded", "failed", "skipped", "cancelled"].includes(prior.state)
|
|
1256
|
+
|| prior.step_id !== replacement.step_id
|
|
1257
|
+
|| prior.workflow_round !== payload.from_round
|
|
1258
|
+
|| replacement.workflow_round !== payload.to_round) {
|
|
1259
|
+
throw new RunStoreError("identity_conflict", `retry_segment_scheduled event ${event.event_id} has invalid activation pair`);
|
|
1260
|
+
}
|
|
1261
|
+
priorIds.add(priorId);
|
|
1262
|
+
replacementIds.add(replacementId);
|
|
1263
|
+
}
|
|
1264
|
+
if (!priorIds.has(event.activation_id)
|
|
1265
|
+
|| !replacementIds.has(payload.retry_from_activation_id)) {
|
|
1266
|
+
throw new RunStoreError("identity_conflict", `retry_segment_scheduled event ${event.event_id} trigger or retry-from replacement is invalid`);
|
|
1267
|
+
}
|
|
1268
|
+
if (payload.triggering_result_id !== null) {
|
|
1269
|
+
const triggeringResult = requirePayloadResult(payload.triggering_result_id, `${label}.triggering_result_id`, resultRefs);
|
|
1270
|
+
if (triggeringResult.activation_id !== event.activation_id) {
|
|
1271
|
+
throw new RunStoreError("identity_conflict", `retry_segment_scheduled event ${event.event_id} triggering result does not match retry source`);
|
|
1272
|
+
}
|
|
1273
|
+
}
|
|
1274
|
+
if (event.result_id !== payload.triggering_result_id) {
|
|
1275
|
+
throw new RunStoreError("identity_conflict", `retry_segment_scheduled event ${event.event_id} result identity does not match trigger`);
|
|
1276
|
+
}
|
|
1277
|
+
break;
|
|
1278
|
+
}
|
|
1279
|
+
case "retry_scheduled": {
|
|
1280
|
+
const envelopeAttempt = requireAttemptScopedCanonicalEvent(event, attempts);
|
|
1281
|
+
assertString(payload.prior_attempt_id, `event ${event.event_id}.payload.prior_attempt_id`);
|
|
1282
|
+
assertString(payload.replacement_attempt_id, `event ${event.event_id}.payload.replacement_attempt_id`);
|
|
1283
|
+
assertPositiveSafeInteger(payload.next_attempt_number, `event ${event.event_id}.payload.next_attempt_number`);
|
|
1284
|
+
assertNonNegativeSafeInteger(payload.delay_ms, `event ${event.event_id}.payload.delay_ms`);
|
|
1285
|
+
assertString(payload.reason_code, `event ${event.event_id}.payload.reason_code`);
|
|
1286
|
+
const prior = attempts.get(payload.prior_attempt_id);
|
|
1287
|
+
const replacement = attempts.get(payload.replacement_attempt_id);
|
|
1288
|
+
if (!prior
|
|
1289
|
+
|| !replacement
|
|
1290
|
+
|| envelopeAttempt.attempt_id !== prior.attempt_id
|
|
1291
|
+
|| prior.activation_id !== replacement.activation_id
|
|
1292
|
+
|| replacement.attempt_number !== prior.attempt_number + 1
|
|
1293
|
+
|| replacement.attempt_number !== payload.next_attempt_number) {
|
|
1294
|
+
throw new RunStoreError("identity_conflict", `retry_scheduled event ${event.event_id} does not match attempt history`);
|
|
1295
|
+
}
|
|
1296
|
+
break;
|
|
1297
|
+
}
|
|
1298
|
+
case "interaction_requested": {
|
|
1299
|
+
requireAttemptScopedCanonicalEvent(event, attempts);
|
|
1300
|
+
assertString(payload.interaction_id, `event ${event.event_id}.payload.interaction_id`);
|
|
1301
|
+
assertNullableString(payload.expected_principal_ref, `event ${event.event_id}.payload.expected_principal_ref`);
|
|
1302
|
+
break;
|
|
1303
|
+
}
|
|
1304
|
+
case "interaction_submitted": {
|
|
1305
|
+
const attempt = requireAttemptScopedCanonicalEvent(event, attempts);
|
|
1306
|
+
assertString(payload.interaction_id, `event ${event.event_id}.payload.interaction_id`);
|
|
1307
|
+
assertString(payload.principal_ref, `event ${event.event_id}.payload.principal_ref`);
|
|
1308
|
+
assertString(payload.attestation_ref, `event ${event.event_id}.payload.attestation_ref`);
|
|
1309
|
+
if (attempt.executor?.kind !== "human"
|
|
1310
|
+
|| attempt.executor.principal_ref !== payload.principal_ref
|
|
1311
|
+
|| attempt.executor.attestation_ref !== payload.attestation_ref) {
|
|
1312
|
+
throw new RunStoreError("identity_conflict", `interaction_submitted event ${event.event_id} does not match human executor`);
|
|
1313
|
+
}
|
|
1314
|
+
break;
|
|
1315
|
+
}
|
|
1316
|
+
}
|
|
1317
|
+
}
|
|
1318
|
+
function validateCanonicalCollections(snapshot) {
|
|
1319
|
+
const runId = snapshot.run.run_id;
|
|
1320
|
+
const activations = new Map();
|
|
1321
|
+
const lastActivationRoundByStep = new Map();
|
|
1322
|
+
for (const activation of snapshot.activations) {
|
|
1323
|
+
assertExactFields(activation, "activation", [
|
|
1324
|
+
"run_id",
|
|
1325
|
+
"step_id",
|
|
1326
|
+
"activation_id",
|
|
1327
|
+
"workflow_round",
|
|
1328
|
+
"state",
|
|
1329
|
+
"attempt_count",
|
|
1330
|
+
"current_attempt_id",
|
|
1331
|
+
"committed_result_id",
|
|
1332
|
+
"created_at",
|
|
1333
|
+
"updated_at",
|
|
1334
|
+
]);
|
|
1335
|
+
assertCorrelatedRunId(activation.run_id, runId, "activation");
|
|
1336
|
+
assertString(activation.step_id, "activation.step_id");
|
|
1337
|
+
assertString(activation.activation_id, "activation.activation_id");
|
|
1338
|
+
assertPositiveSafeInteger(activation.workflow_round, `activation ${activation.activation_id}.workflow_round`);
|
|
1339
|
+
const previousRound = lastActivationRoundByStep.get(activation.step_id);
|
|
1340
|
+
if (previousRound !== undefined && activation.workflow_round <= previousRound) {
|
|
1341
|
+
throw new RunStoreError("identity_conflict", `step ${activation.step_id} has non-monotonic workflow round ${activation.workflow_round}`);
|
|
1342
|
+
}
|
|
1343
|
+
lastActivationRoundByStep.set(activation.step_id, activation.workflow_round);
|
|
1344
|
+
if (!CANONICAL_ACTIVATION_STATE_VALUES.includes(activation.state)) {
|
|
1345
|
+
throw new RunStoreError("invalid_record", `activation ${activation.activation_id} has unsupported state ${String(activation.state)}`);
|
|
1346
|
+
}
|
|
1347
|
+
assertNonNegativeSafeInteger(activation.attempt_count, `activation ${activation.activation_id}.attempt_count`);
|
|
1348
|
+
const activationCreatedAt = assertTimestamp(activation.created_at, `activation ${activation.activation_id}.created_at`);
|
|
1349
|
+
const activationUpdatedAt = assertTimestamp(activation.updated_at, `activation ${activation.activation_id}.updated_at`);
|
|
1350
|
+
if (activationUpdatedAt < activationCreatedAt) {
|
|
1351
|
+
throw new RunStoreError("invalid_record", `activation ${activation.activation_id} updated_at precedes created_at`);
|
|
1352
|
+
}
|
|
1353
|
+
if (activations.has(activation.activation_id)) {
|
|
1354
|
+
throw new RunStoreError("identity_conflict", `duplicate activation ${activation.activation_id}`);
|
|
1355
|
+
}
|
|
1356
|
+
activations.set(activation.activation_id, activation);
|
|
1357
|
+
}
|
|
1358
|
+
const attempts = new Map();
|
|
1359
|
+
const attemptNumbers = new Set();
|
|
1360
|
+
const workerSessions = new Set();
|
|
1361
|
+
for (const attempt of snapshot.attempts) {
|
|
1362
|
+
assertExactFields(attempt, "attempt", [
|
|
1363
|
+
"run_id",
|
|
1364
|
+
"step_id",
|
|
1365
|
+
"activation_id",
|
|
1366
|
+
"attempt_id",
|
|
1367
|
+
"attempt_number",
|
|
1368
|
+
"phase",
|
|
1369
|
+
"executor",
|
|
1370
|
+
"started_at",
|
|
1371
|
+
"completed_at",
|
|
1372
|
+
"terminal_outcome",
|
|
1373
|
+
]);
|
|
1374
|
+
assertCorrelatedRunId(attempt.run_id, runId, "attempt");
|
|
1375
|
+
assertString(attempt.step_id, "attempt.step_id");
|
|
1376
|
+
assertString(attempt.activation_id, "attempt.activation_id");
|
|
1377
|
+
assertString(attempt.attempt_id, "attempt.attempt_id");
|
|
1378
|
+
assertPositiveSafeInteger(attempt.attempt_number, `attempt ${attempt.attempt_id}.attempt_number`);
|
|
1379
|
+
const activation = activations.get(attempt.activation_id);
|
|
1380
|
+
if (!activation || activation.step_id !== attempt.step_id) {
|
|
1381
|
+
throw new RunStoreError("identity_conflict", `attempt ${attempt.attempt_id} does not match activation ${attempt.activation_id}`);
|
|
1382
|
+
}
|
|
1383
|
+
if (!["claimed", "running", "terminal"].includes(attempt.phase)) {
|
|
1384
|
+
throw new RunStoreError("invalid_record", `attempt ${attempt.attempt_id} has unsupported phase ${String(attempt.phase)}`);
|
|
1385
|
+
}
|
|
1386
|
+
if (attempt.phase === "claimed") {
|
|
1387
|
+
if (attempt.started_at !== null
|
|
1388
|
+
|| attempt.completed_at !== null
|
|
1389
|
+
|| attempt.terminal_outcome !== null) {
|
|
1390
|
+
throw new RunStoreError("invalid_record", `claimed attempt ${attempt.attempt_id} cannot have execution timestamps or outcome`);
|
|
1391
|
+
}
|
|
1392
|
+
}
|
|
1393
|
+
else {
|
|
1394
|
+
if (attempt.executor === null) {
|
|
1395
|
+
throw new RunStoreError("invalid_record", `${attempt.phase} attempt ${attempt.attempt_id} requires an executor`);
|
|
1396
|
+
}
|
|
1397
|
+
const startedAt = assertTimestamp(attempt.started_at, `attempt ${attempt.attempt_id}.started_at`);
|
|
1398
|
+
if (attempt.phase === "running" && (attempt.completed_at !== null
|
|
1399
|
+
|| attempt.terminal_outcome !== null)) {
|
|
1400
|
+
throw new RunStoreError("invalid_record", `running attempt ${attempt.attempt_id} cannot have completion data`);
|
|
1401
|
+
}
|
|
1402
|
+
if (attempt.phase === "terminal" && (attempt.completed_at === null
|
|
1403
|
+
|| attempt.terminal_outcome === null)) {
|
|
1404
|
+
throw new RunStoreError("invalid_record", `terminal attempt ${attempt.attempt_id} requires completion data`);
|
|
1405
|
+
}
|
|
1406
|
+
if (attempt.phase === "terminal") {
|
|
1407
|
+
const completedAt = assertTimestamp(attempt.completed_at, `attempt ${attempt.attempt_id}.completed_at`);
|
|
1408
|
+
if (completedAt < startedAt) {
|
|
1409
|
+
throw new RunStoreError("invalid_record", `attempt ${attempt.attempt_id} completed_at precedes started_at`);
|
|
1410
|
+
}
|
|
1411
|
+
}
|
|
1412
|
+
}
|
|
1413
|
+
if (attempt.executor !== null) {
|
|
1414
|
+
if (attempt.phase === "terminal") {
|
|
1415
|
+
try {
|
|
1416
|
+
validateCanonicalAttemptOutcome(attempt.executor, attempt.terminal_outcome);
|
|
1417
|
+
}
|
|
1418
|
+
catch (error) {
|
|
1419
|
+
throw new RunStoreError("invalid_record", `attempt ${attempt.attempt_id} has an invalid canonical outcome`, { cause: error });
|
|
1420
|
+
}
|
|
1421
|
+
}
|
|
1422
|
+
else {
|
|
1423
|
+
assertAttemptExecutor(attempt.executor, `attempt ${attempt.attempt_id}.executor`);
|
|
1424
|
+
}
|
|
1425
|
+
if (attempt.executor.kind === "worker") {
|
|
1426
|
+
if (workerSessions.has(attempt.executor.worker_session_id)) {
|
|
1427
|
+
throw new RunStoreError("identity_conflict", `worker session ${attempt.executor.worker_session_id} is bound to multiple attempts`);
|
|
1428
|
+
}
|
|
1429
|
+
workerSessions.add(attempt.executor.worker_session_id);
|
|
1430
|
+
}
|
|
1431
|
+
}
|
|
1432
|
+
if (attempts.has(attempt.attempt_id)) {
|
|
1433
|
+
throw new RunStoreError("identity_conflict", `duplicate attempt ${attempt.attempt_id}`);
|
|
1434
|
+
}
|
|
1435
|
+
const attemptNumberKey = `${attempt.activation_id}\0${attempt.attempt_number}`;
|
|
1436
|
+
if (attemptNumbers.has(attemptNumberKey)) {
|
|
1437
|
+
throw new RunStoreError("identity_conflict", `activation ${attempt.activation_id} has duplicate attempt number ${attempt.attempt_number}`);
|
|
1438
|
+
}
|
|
1439
|
+
attemptNumbers.add(attemptNumberKey);
|
|
1440
|
+
attempts.set(attempt.attempt_id, attempt);
|
|
1441
|
+
}
|
|
1442
|
+
const claimTokens = new Set();
|
|
1443
|
+
const claimEpochs = new Set();
|
|
1444
|
+
const claimsByAttempt = new Map();
|
|
1445
|
+
for (const claim of snapshot.claims) {
|
|
1446
|
+
assertExactFields(claim, "claim", [
|
|
1447
|
+
"run_id",
|
|
1448
|
+
"step_id",
|
|
1449
|
+
"activation_id",
|
|
1450
|
+
"attempt_id",
|
|
1451
|
+
"claim_token",
|
|
1452
|
+
"claim_epoch",
|
|
1453
|
+
"owner_id",
|
|
1454
|
+
"claimed_record_version",
|
|
1455
|
+
"driver_claim_lease",
|
|
1456
|
+
"execution_deadline_at",
|
|
1457
|
+
]);
|
|
1458
|
+
assertCorrelatedRunId(claim.run_id, runId, "claim");
|
|
1459
|
+
assertString(claim.claim_token, "claim.claim_token");
|
|
1460
|
+
assertString(claim.owner_id, "claim.owner_id");
|
|
1461
|
+
assertPositiveSafeInteger(claim.claim_epoch, "claim.claim_epoch");
|
|
1462
|
+
assertPositiveSafeInteger(claim.claimed_record_version, "claim.claimed_record_version");
|
|
1463
|
+
if (claim.claimed_record_version >= snapshot.record_version) {
|
|
1464
|
+
throw new RunStoreError("invalid_record", `claim ${claim.claim_token} must reference an earlier observed record version`);
|
|
1465
|
+
}
|
|
1466
|
+
const executionDeadlineAt = assertTimestamp(claim.execution_deadline_at, "claim.execution_deadline_at");
|
|
1467
|
+
assertExactFields(claim.driver_claim_lease, "claim.driver_claim_lease", [
|
|
1468
|
+
"acquired_at",
|
|
1469
|
+
"expires_at",
|
|
1470
|
+
"heartbeat_at",
|
|
1471
|
+
]);
|
|
1472
|
+
const acquiredAt = assertTimestamp(claim.driver_claim_lease.acquired_at, "claim.driver_claim_lease.acquired_at");
|
|
1473
|
+
const expiresAt = assertTimestamp(claim.driver_claim_lease.expires_at, "claim.driver_claim_lease.expires_at");
|
|
1474
|
+
const heartbeatAt = assertTimestamp(claim.driver_claim_lease.heartbeat_at, "claim.driver_claim_lease.heartbeat_at");
|
|
1475
|
+
if (acquiredAt > heartbeatAt
|
|
1476
|
+
|| heartbeatAt > expiresAt
|
|
1477
|
+
|| expiresAt > executionDeadlineAt) {
|
|
1478
|
+
throw new RunStoreError("invalid_record", `claim ${claim.claim_token} has inconsistent lease timestamps`);
|
|
1479
|
+
}
|
|
1480
|
+
const attempt = attempts.get(claim.attempt_id);
|
|
1481
|
+
if (!attempt
|
|
1482
|
+
|| attempt.activation_id !== claim.activation_id
|
|
1483
|
+
|| attempt.step_id !== claim.step_id) {
|
|
1484
|
+
throw new RunStoreError("invalid_record", `claim ${claim.claim_token} does not match its attempt`);
|
|
1485
|
+
}
|
|
1486
|
+
if (claimTokens.has(claim.claim_token)) {
|
|
1487
|
+
throw new RunStoreError("identity_conflict", `duplicate claim token ${claim.claim_token}`);
|
|
1488
|
+
}
|
|
1489
|
+
const epochKey = `${claim.attempt_id}\0${claim.claim_epoch}`;
|
|
1490
|
+
if (claimEpochs.has(epochKey)) {
|
|
1491
|
+
throw new RunStoreError("identity_conflict", `attempt ${claim.attempt_id} has duplicate claim epoch ${claim.claim_epoch}`);
|
|
1492
|
+
}
|
|
1493
|
+
claimTokens.add(claim.claim_token);
|
|
1494
|
+
claimEpochs.add(epochKey);
|
|
1495
|
+
const attemptClaims = claimsByAttempt.get(claim.attempt_id) ?? [];
|
|
1496
|
+
attemptClaims.push(claim);
|
|
1497
|
+
claimsByAttempt.set(claim.attempt_id, attemptClaims);
|
|
1498
|
+
}
|
|
1499
|
+
for (const attempt of snapshot.attempts) {
|
|
1500
|
+
if (attempt.phase !== "terminal"
|
|
1501
|
+
&& (claimsByAttempt.get(attempt.attempt_id)?.length ?? 0) === 0) {
|
|
1502
|
+
throw new RunStoreError("invalid_record", `active attempt ${attempt.attempt_id} requires a durable claim`);
|
|
1503
|
+
}
|
|
1504
|
+
}
|
|
1505
|
+
for (const [attemptId, attemptClaims] of claimsByAttempt) {
|
|
1506
|
+
const ordered = [...attemptClaims].sort((left, right) => left.claim_epoch - right.claim_epoch);
|
|
1507
|
+
const executionDeadline = ordered[0]?.execution_deadline_at;
|
|
1508
|
+
const invalidEpoch = ordered.find((claim, index) => claim.claim_epoch !== index + 1);
|
|
1509
|
+
if (invalidEpoch) {
|
|
1510
|
+
throw new RunStoreError("invalid_record", `attempt ${attemptId} claim epochs must be contiguous from 1`);
|
|
1511
|
+
}
|
|
1512
|
+
if (ordered.some((claim) => claim.execution_deadline_at !== executionDeadline)) {
|
|
1513
|
+
throw new RunStoreError("identity_conflict", `attempt ${attemptId} cannot change execution deadline across claim epochs`);
|
|
1514
|
+
}
|
|
1515
|
+
}
|
|
1516
|
+
const resultRefs = new Map();
|
|
1517
|
+
const resultByAttempt = new Map();
|
|
1518
|
+
for (const reference of snapshot.result_refs) {
|
|
1519
|
+
assertExactFields(reference, "result_ref", [
|
|
1520
|
+
"run_id",
|
|
1521
|
+
"step_id",
|
|
1522
|
+
"activation_id",
|
|
1523
|
+
"attempt_id",
|
|
1524
|
+
"result_id",
|
|
1525
|
+
"result_digest",
|
|
1526
|
+
]);
|
|
1527
|
+
assertCorrelatedRunId(reference.run_id, runId, "result_ref");
|
|
1528
|
+
assertString(reference.step_id, "result_ref.step_id");
|
|
1529
|
+
assertString(reference.activation_id, "result_ref.activation_id");
|
|
1530
|
+
assertString(reference.attempt_id, "result_ref.attempt_id");
|
|
1531
|
+
assertString(reference.result_id, "result_ref.result_id");
|
|
1532
|
+
assertDigest(reference.result_digest, "result_ref.result_digest");
|
|
1533
|
+
const attempt = attempts.get(reference.attempt_id);
|
|
1534
|
+
if (!attempt
|
|
1535
|
+
|| attempt.phase !== "terminal"
|
|
1536
|
+
|| attempt.activation_id !== reference.activation_id
|
|
1537
|
+
|| attempt.step_id !== reference.step_id) {
|
|
1538
|
+
throw new RunStoreError("invalid_record", `result ${reference.result_id} does not match a terminal attempt`);
|
|
1539
|
+
}
|
|
1540
|
+
if (resultRefs.has(reference.result_id)) {
|
|
1541
|
+
throw new RunStoreError("result_conflict", `duplicate canonical result ${reference.result_id}`);
|
|
1542
|
+
}
|
|
1543
|
+
if (resultByAttempt.has(reference.attempt_id)) {
|
|
1544
|
+
throw new RunStoreError("result_conflict", `attempt ${reference.attempt_id} has multiple canonical results`);
|
|
1545
|
+
}
|
|
1546
|
+
resultRefs.set(reference.result_id, reference);
|
|
1547
|
+
resultByAttempt.set(reference.attempt_id, reference);
|
|
1548
|
+
}
|
|
1549
|
+
for (const activation of snapshot.activations) {
|
|
1550
|
+
const activationAttempts = snapshot.attempts
|
|
1551
|
+
.filter((attempt) => attempt.activation_id === activation.activation_id)
|
|
1552
|
+
.sort((left, right) => left.attempt_number - right.attempt_number);
|
|
1553
|
+
if (activation.attempt_count !== activationAttempts.length) {
|
|
1554
|
+
throw new RunStoreError("invalid_record", `activation ${activation.activation_id} attempt_count does not match stored attempts`);
|
|
1555
|
+
}
|
|
1556
|
+
const nonContiguousAttempt = activationAttempts.find((attempt, index) => attempt.attempt_number !== index + 1);
|
|
1557
|
+
if (nonContiguousAttempt) {
|
|
1558
|
+
throw new RunStoreError("invalid_record", `activation ${activation.activation_id} attempt numbers must be contiguous from 1`);
|
|
1559
|
+
}
|
|
1560
|
+
if (activation.current_attempt_id !== null
|
|
1561
|
+
&& attempts.get(activation.current_attempt_id)?.activation_id
|
|
1562
|
+
!== activation.activation_id) {
|
|
1563
|
+
throw new RunStoreError("invalid_record", `activation ${activation.activation_id} references an unknown current attempt`);
|
|
1564
|
+
}
|
|
1565
|
+
const latestAttempt = activationAttempts.at(-1) ?? null;
|
|
1566
|
+
if ((latestAttempt === null && activation.current_attempt_id !== null)
|
|
1567
|
+
|| (latestAttempt !== null
|
|
1568
|
+
&& activation.current_attempt_id !== latestAttempt.attempt_id)) {
|
|
1569
|
+
throw new RunStoreError("invalid_record", `activation ${activation.activation_id} current_attempt_id is not its latest attempt`);
|
|
1570
|
+
}
|
|
1571
|
+
const activeAttempts = activationAttempts.filter((attempt) => attempt.phase !== "terminal");
|
|
1572
|
+
if (activeAttempts.length > 1) {
|
|
1573
|
+
throw new RunStoreError("invalid_record", `activation ${activation.activation_id} has multiple active attempts`);
|
|
1574
|
+
}
|
|
1575
|
+
if (activeAttempts.length === 1
|
|
1576
|
+
&& activeAttempts[0].attempt_id !== activation.current_attempt_id) {
|
|
1577
|
+
throw new RunStoreError("invalid_record", `activation ${activation.activation_id} active attempt is not current`);
|
|
1578
|
+
}
|
|
1579
|
+
const currentAttempt = activation.current_attempt_id === null
|
|
1580
|
+
? null
|
|
1581
|
+
: attempts.get(activation.current_attempt_id) ?? null;
|
|
1582
|
+
if (activation.state === "claimed"
|
|
1583
|
+
&& currentAttempt?.phase !== "claimed") {
|
|
1584
|
+
throw new RunStoreError("invalid_record", `claimed activation ${activation.activation_id} requires a claimed current attempt`);
|
|
1585
|
+
}
|
|
1586
|
+
if (activation.state === "running"
|
|
1587
|
+
&& currentAttempt?.phase !== "running") {
|
|
1588
|
+
throw new RunStoreError("invalid_record", `running activation ${activation.activation_id} requires a running current attempt`);
|
|
1589
|
+
}
|
|
1590
|
+
if (activation.state === "waiting_for_human"
|
|
1591
|
+
&& (currentAttempt === null
|
|
1592
|
+
|| !["claimed", "running"].includes(currentAttempt.phase)
|
|
1593
|
+
|| (currentAttempt.phase === "running"
|
|
1594
|
+
&& currentAttempt.executor?.kind !== "human"))) {
|
|
1595
|
+
throw new RunStoreError("invalid_record", `waiting_for_human activation ${activation.activation_id} requires a claimed or human-running attempt`);
|
|
1596
|
+
}
|
|
1597
|
+
if (["dormant", "dependency_blocked", "ready", "skipped"].includes(activation.state)
|
|
1598
|
+
&& activeAttempts.length !== 0) {
|
|
1599
|
+
throw new RunStoreError("invalid_record", `activation ${activation.activation_id} state ${activation.state} cannot retain an active attempt`);
|
|
1600
|
+
}
|
|
1601
|
+
if (["succeeded", "failed"].includes(activation.state)
|
|
1602
|
+
&& currentAttempt?.phase !== "terminal") {
|
|
1603
|
+
throw new RunStoreError("invalid_record", `${activation.state} activation ${activation.activation_id} requires a terminal current attempt`);
|
|
1604
|
+
}
|
|
1605
|
+
if (currentAttempt?.phase === "terminal"
|
|
1606
|
+
&& ["succeeded", "failed", "cancelled"].includes(activation.state)
|
|
1607
|
+
&& canonicalAttemptOutcomeStatus(currentAttempt.terminal_outcome)
|
|
1608
|
+
!== activation.state) {
|
|
1609
|
+
throw new RunStoreError("invalid_record", `activation ${activation.activation_id} state ${activation.state} disagrees with attempt ${currentAttempt.attempt_id} outcome`);
|
|
1610
|
+
}
|
|
1611
|
+
if (activation.state === "cancelled"
|
|
1612
|
+
&& activeAttempts.length !== 0) {
|
|
1613
|
+
throw new RunStoreError("invalid_record", `cancelled activation ${activation.activation_id} cannot retain an active attempt`);
|
|
1614
|
+
}
|
|
1615
|
+
if (activation.committed_result_id !== null
|
|
1616
|
+
&& (resultRefs.get(activation.committed_result_id)?.activation_id
|
|
1617
|
+
!== activation.activation_id
|
|
1618
|
+
|| resultRefs.get(activation.committed_result_id)?.attempt_id
|
|
1619
|
+
!== activation.current_attempt_id)) {
|
|
1620
|
+
throw new RunStoreError("invalid_record", `activation ${activation.activation_id} references an unknown committed result`);
|
|
1621
|
+
}
|
|
1622
|
+
if (activation.state === "succeeded"
|
|
1623
|
+
&& activation.committed_result_id === null) {
|
|
1624
|
+
throw new RunStoreError("invalid_record", `succeeded activation ${activation.activation_id} requires a committed result`);
|
|
1625
|
+
}
|
|
1626
|
+
}
|
|
1627
|
+
if (snapshot.run.terminal !== null) {
|
|
1628
|
+
const nonterminalActivation = snapshot.activations.find((activation) => (!["succeeded", "failed", "skipped", "cancelled"].includes(activation.state)));
|
|
1629
|
+
const nonterminalAttempt = snapshot.attempts.find((attempt) => attempt.phase !== "terminal");
|
|
1630
|
+
if (nonterminalActivation || nonterminalAttempt) {
|
|
1631
|
+
throw new RunStoreError("invalid_record", `terminal run ${runId} cannot retain active activations or attempts`);
|
|
1632
|
+
}
|
|
1633
|
+
}
|
|
1634
|
+
const eventIds = new Set();
|
|
1635
|
+
const eventKeys = new Set();
|
|
1636
|
+
const workspacePlans = new Map();
|
|
1637
|
+
const workspaceAttempts = new Set();
|
|
1638
|
+
let previousSequence = 0;
|
|
1639
|
+
for (const event of snapshot.events) {
|
|
1640
|
+
assertExactFields(event, "event", [
|
|
1641
|
+
"event_id",
|
|
1642
|
+
"run_id",
|
|
1643
|
+
"step_id",
|
|
1644
|
+
"activation_id",
|
|
1645
|
+
"attempt_id",
|
|
1646
|
+
"result_id",
|
|
1647
|
+
"sequence",
|
|
1648
|
+
"idempotency_key",
|
|
1649
|
+
"type",
|
|
1650
|
+
"occurred_at",
|
|
1651
|
+
"payload",
|
|
1652
|
+
]);
|
|
1653
|
+
assertCorrelatedRunId(event.run_id, runId, "event");
|
|
1654
|
+
assertString(event.event_id, "event.event_id");
|
|
1655
|
+
assertString(event.idempotency_key, "event.idempotency_key");
|
|
1656
|
+
assertTimestamp(event.occurred_at, "event.occurred_at");
|
|
1657
|
+
if (!CANONICAL_RUNTIME_EVENT_TYPE_VALUES.includes(event.type)) {
|
|
1658
|
+
throw new RunStoreError("invalid_record", `event ${event.event_id} has unsupported type ${String(event.type)}`);
|
|
1659
|
+
}
|
|
1660
|
+
if (!Number.isSafeInteger(event.sequence)
|
|
1661
|
+
|| event.sequence !== previousSequence + 1) {
|
|
1662
|
+
throw new RunStoreError("invalid_record", `event ${event.event_id} has non-contiguous sequence ${event.sequence}`);
|
|
1663
|
+
}
|
|
1664
|
+
if (eventIds.has(event.event_id) || eventKeys.has(event.idempotency_key)) {
|
|
1665
|
+
throw new RunStoreError("event_conflict", `duplicate stored event ${event.event_id}`);
|
|
1666
|
+
}
|
|
1667
|
+
if (event.activation_id === null) {
|
|
1668
|
+
if (event.step_id !== null || event.attempt_id !== null) {
|
|
1669
|
+
throw new RunStoreError("invalid_record", `event ${event.event_id} has partial activation identity`);
|
|
1670
|
+
}
|
|
1671
|
+
}
|
|
1672
|
+
else {
|
|
1673
|
+
const activation = activations.get(event.activation_id);
|
|
1674
|
+
if (!activation || activation.step_id !== event.step_id) {
|
|
1675
|
+
throw new RunStoreError("invalid_record", `event ${event.event_id} does not match its activation`);
|
|
1676
|
+
}
|
|
1677
|
+
}
|
|
1678
|
+
if (event.attempt_id !== null
|
|
1679
|
+
&& attempts.get(event.attempt_id)?.activation_id !== event.activation_id) {
|
|
1680
|
+
throw new RunStoreError("invalid_record", `event ${event.event_id} does not match its attempt`);
|
|
1681
|
+
}
|
|
1682
|
+
if (event.result_id !== null) {
|
|
1683
|
+
const eventResult = resultRefs.get(event.result_id);
|
|
1684
|
+
if (eventResult?.activation_id !== event.activation_id
|
|
1685
|
+
|| (event.attempt_id !== null
|
|
1686
|
+
&& eventResult.attempt_id !== event.attempt_id)) {
|
|
1687
|
+
throw new RunStoreError("invalid_record", `event ${event.event_id} does not match its result`);
|
|
1688
|
+
}
|
|
1689
|
+
}
|
|
1690
|
+
if (event.type === "result_committed" && event.result_id === null) {
|
|
1691
|
+
throw new RunStoreError("invalid_record", `result_committed event ${event.event_id} requires result_id`);
|
|
1692
|
+
}
|
|
1693
|
+
assertCanonicalEventPayload(event, snapshot, activations, attempts, resultRefs);
|
|
1694
|
+
if (event.type === "workspace_planned") {
|
|
1695
|
+
const attemptId = event.attempt_id;
|
|
1696
|
+
if (workspacePlans.has(attemptId)) {
|
|
1697
|
+
throw new RunStoreError("invalid_record", `attempt ${attemptId} has more than one workspace plan`);
|
|
1698
|
+
}
|
|
1699
|
+
workspacePlans.set(attemptId, event);
|
|
1700
|
+
}
|
|
1701
|
+
if (event.type === "workspace_allocated") {
|
|
1702
|
+
const attemptId = event.attempt_id;
|
|
1703
|
+
const planned = workspacePlans.get(attemptId);
|
|
1704
|
+
if (!planned) {
|
|
1705
|
+
throw new RunStoreError("invalid_record", `attempt ${attemptId} allocation has no prior workspace plan`);
|
|
1706
|
+
}
|
|
1707
|
+
const expected = {
|
|
1708
|
+
...planned.payload.plan.payload,
|
|
1709
|
+
workspace_plan_id: planned.payload.workspace_plan_id,
|
|
1710
|
+
};
|
|
1711
|
+
const actual = { ...event.payload.allocation };
|
|
1712
|
+
delete actual.observed_head;
|
|
1713
|
+
delete actual.observed_branch_ref;
|
|
1714
|
+
if (event.payload.workspace_plan_id !== planned.payload.workspace_plan_id
|
|
1715
|
+
|| !recordsEqual(actual, expected)) {
|
|
1716
|
+
throw new RunStoreError("identity_conflict", `attempt ${attemptId} allocation does not exactly match its committed workspace plan`);
|
|
1717
|
+
}
|
|
1718
|
+
if (workspaceAttempts.has(event.attempt_id)) {
|
|
1719
|
+
throw new RunStoreError("invalid_record", `attempt ${event.attempt_id} has more than one workspace allocation`);
|
|
1720
|
+
}
|
|
1721
|
+
workspaceAttempts.add(attemptId);
|
|
1722
|
+
}
|
|
1723
|
+
try {
|
|
1724
|
+
canonicalizeJson(event.payload);
|
|
1725
|
+
}
|
|
1726
|
+
catch (error) {
|
|
1727
|
+
throw new RunStoreError("invalid_record", `event ${event.event_id} payload must be canonical JSON`, { cause: error });
|
|
1728
|
+
}
|
|
1729
|
+
eventIds.add(event.event_id);
|
|
1730
|
+
eventKeys.add(event.idempotency_key);
|
|
1731
|
+
previousSequence = event.sequence;
|
|
1732
|
+
}
|
|
1733
|
+
}
|
|
1734
|
+
function assertCanonicalSnapshot(value, expectedRunId) {
|
|
1735
|
+
assertExactFields(value, "run ledger", [
|
|
1736
|
+
"schema_version",
|
|
1737
|
+
"record_version",
|
|
1738
|
+
"run",
|
|
1739
|
+
"activations",
|
|
1740
|
+
"attempts",
|
|
1741
|
+
"claims",
|
|
1742
|
+
"events",
|
|
1743
|
+
"result_refs",
|
|
1744
|
+
"created_at",
|
|
1745
|
+
"updated_at",
|
|
1746
|
+
]);
|
|
1747
|
+
const candidate = value;
|
|
1748
|
+
if (candidate.schema_version !== CANONICAL_RUN_STORE_SCHEMA_VERSION) {
|
|
1749
|
+
throw new RunStoreError("unsupported_schema", `unsupported run-store schema version: ${String(candidate.schema_version)}`);
|
|
1750
|
+
}
|
|
1751
|
+
assertPositiveSafeInteger(candidate.record_version, "record_version");
|
|
1752
|
+
if (!candidate.run
|
|
1753
|
+
|| !Array.isArray(candidate.activations)
|
|
1754
|
+
|| !Array.isArray(candidate.attempts)
|
|
1755
|
+
|| !Array.isArray(candidate.claims)
|
|
1756
|
+
|| !Array.isArray(candidate.events)
|
|
1757
|
+
|| !Array.isArray(candidate.result_refs)) {
|
|
1758
|
+
throw new RunStoreError("invalid_record", "canonical run ledger is missing a required record collection");
|
|
1759
|
+
}
|
|
1760
|
+
const createdAt = assertTimestamp(candidate.created_at, "ledger.created_at");
|
|
1761
|
+
const updatedAt = assertTimestamp(candidate.updated_at, "ledger.updated_at");
|
|
1762
|
+
if (updatedAt < createdAt) {
|
|
1763
|
+
throw new RunStoreError("invalid_record", "ledger.updated_at cannot precede ledger.created_at");
|
|
1764
|
+
}
|
|
1765
|
+
assertCanonicalRunRecord(candidate.run, expectedRunId);
|
|
1766
|
+
validateCanonicalCollections(candidate);
|
|
1767
|
+
return candidate;
|
|
1768
|
+
}
|
|
1769
|
+
function assertAnySnapshot(value, expectedRunId) {
|
|
1770
|
+
if (value === null || typeof value !== "object" || Array.isArray(value)) {
|
|
1771
|
+
throw new RunStoreError("invalid_record", "run ledger must be a JSON object");
|
|
1772
|
+
}
|
|
1773
|
+
const version = value.schema_version;
|
|
1774
|
+
if (version === LEGACY_RUN_STORE_SCHEMA_VERSION) {
|
|
1775
|
+
return assertLegacySnapshot(value, expectedRunId);
|
|
1776
|
+
}
|
|
1777
|
+
if (version === CANONICAL_RUN_STORE_SCHEMA_VERSION) {
|
|
1778
|
+
return assertCanonicalSnapshot(value, expectedRunId);
|
|
1779
|
+
}
|
|
1780
|
+
throw new RunStoreError("unsupported_schema", `unsupported run-store schema version: ${String(version)}`);
|
|
1781
|
+
}
|
|
237
1782
|
function eventReplayEquivalent(existing, incoming) {
|
|
238
1783
|
return recordsEqual({
|
|
239
1784
|
run_id: existing.run_id,
|
|
@@ -339,66 +1884,519 @@ function applyUpdate(current, update) {
|
|
|
339
1884
|
const eventByKey = new Map(events.map((event) => [event.idempotency_key, event]));
|
|
340
1885
|
const duplicateEventIds = [];
|
|
341
1886
|
for (const event of update.events ?? []) {
|
|
342
|
-
assertCorrelatedRunId(event.run_id, runId, "event");
|
|
1887
|
+
assertCorrelatedRunId(event.run_id, runId, "event");
|
|
1888
|
+
const byId = eventById.get(event.event_id);
|
|
1889
|
+
const byKey = eventByKey.get(event.idempotency_key);
|
|
1890
|
+
const existing = byId ?? byKey;
|
|
1891
|
+
if (existing) {
|
|
1892
|
+
if ((byId && byKey && byId.event_id !== byKey.event_id) ||
|
|
1893
|
+
!eventReplayEquivalent(existing, event)) {
|
|
1894
|
+
throw new RunStoreError("event_conflict", `event ${event.event_id} conflicts with a stored event identity`);
|
|
1895
|
+
}
|
|
1896
|
+
duplicateEventIds.push(existing.event_id);
|
|
1897
|
+
continue;
|
|
1898
|
+
}
|
|
1899
|
+
const expectedSequence = events.length + 1;
|
|
1900
|
+
if (event.sequence !== expectedSequence) {
|
|
1901
|
+
throw new RunStoreError("event_conflict", `event ${event.event_id} sequence ${event.sequence} must be ${expectedSequence}`);
|
|
1902
|
+
}
|
|
1903
|
+
events.push(event);
|
|
1904
|
+
eventById.set(event.event_id, event);
|
|
1905
|
+
eventByKey.set(event.idempotency_key, event);
|
|
1906
|
+
changed = true;
|
|
1907
|
+
}
|
|
1908
|
+
const results = [...current.results];
|
|
1909
|
+
const resultById = new Map(results.map((result) => [result.result_id, result]));
|
|
1910
|
+
const resultByAttempt = new Map(results.map((result) => [result.attempt_id, result]));
|
|
1911
|
+
const duplicateResultIds = [];
|
|
1912
|
+
for (const result of update.results ?? []) {
|
|
1913
|
+
assertCorrelatedRunId(result.run_id, runId, "result");
|
|
1914
|
+
const existing = resultById.get(result.result_id);
|
|
1915
|
+
if (existing) {
|
|
1916
|
+
if (!recordsEqual(existing, result)) {
|
|
1917
|
+
throw new RunStoreError("result_conflict", `committed result ${result.result_id} is immutable`);
|
|
1918
|
+
}
|
|
1919
|
+
duplicateResultIds.push(existing.result_id);
|
|
1920
|
+
continue;
|
|
1921
|
+
}
|
|
1922
|
+
const attemptResult = resultByAttempt.get(result.attempt_id);
|
|
1923
|
+
if (attemptResult) {
|
|
1924
|
+
throw new RunStoreError("result_conflict", `attempt ${result.attempt_id} already committed result ${attemptResult.result_id}`);
|
|
1925
|
+
}
|
|
1926
|
+
results.push(result);
|
|
1927
|
+
resultById.set(result.result_id, result);
|
|
1928
|
+
resultByAttempt.set(result.attempt_id, result);
|
|
1929
|
+
changed = true;
|
|
1930
|
+
}
|
|
1931
|
+
const now = new Date().toISOString();
|
|
1932
|
+
const candidate = {
|
|
1933
|
+
schema_version: LEGACY_RUN_STORE_SCHEMA_VERSION,
|
|
1934
|
+
record_version: changed
|
|
1935
|
+
? current.record_version + 1
|
|
1936
|
+
: current.record_version,
|
|
1937
|
+
run,
|
|
1938
|
+
tasks: tasks.records,
|
|
1939
|
+
attempts: attempts.records,
|
|
1940
|
+
claims: claims.records,
|
|
1941
|
+
events,
|
|
1942
|
+
results,
|
|
1943
|
+
created_at: current.created_at,
|
|
1944
|
+
updated_at: changed ? now : current.updated_at,
|
|
1945
|
+
};
|
|
1946
|
+
validateCollections(candidate);
|
|
1947
|
+
return {
|
|
1948
|
+
snapshot: candidate,
|
|
1949
|
+
changed,
|
|
1950
|
+
duplicate_event_ids: duplicateEventIds,
|
|
1951
|
+
duplicate_result_ids: duplicateResultIds,
|
|
1952
|
+
};
|
|
1953
|
+
}
|
|
1954
|
+
function canonicalEventReplayEquivalent(existing, incoming) {
|
|
1955
|
+
return recordsEqual({
|
|
1956
|
+
run_id: existing.run_id,
|
|
1957
|
+
step_id: existing.step_id,
|
|
1958
|
+
activation_id: existing.activation_id,
|
|
1959
|
+
attempt_id: existing.attempt_id,
|
|
1960
|
+
result_id: existing.result_id,
|
|
1961
|
+
idempotency_key: existing.idempotency_key,
|
|
1962
|
+
type: existing.type,
|
|
1963
|
+
payload: existing.payload,
|
|
1964
|
+
}, {
|
|
1965
|
+
run_id: incoming.run_id,
|
|
1966
|
+
step_id: incoming.step_id,
|
|
1967
|
+
activation_id: incoming.activation_id,
|
|
1968
|
+
attempt_id: incoming.attempt_id,
|
|
1969
|
+
result_id: incoming.result_id,
|
|
1970
|
+
idempotency_key: incoming.idempotency_key,
|
|
1971
|
+
type: incoming.type,
|
|
1972
|
+
payload: incoming.payload,
|
|
1973
|
+
});
|
|
1974
|
+
}
|
|
1975
|
+
function canonicalEventsOfType(events, type) {
|
|
1976
|
+
return events.filter((event) => event.type === type);
|
|
1977
|
+
}
|
|
1978
|
+
function assertOneToOneCanonicalEventCoupling(mutations, events, type, matches, describeMutation) {
|
|
1979
|
+
const typedEvents = canonicalEventsOfType(events, type);
|
|
1980
|
+
for (const mutation of mutations) {
|
|
1981
|
+
if (typedEvents.filter((event) => matches(mutation, event)).length !== 1) {
|
|
1982
|
+
throw new RunStoreError("invalid_record", `${describeMutation(mutation)} must commit exactly one ${type} event`);
|
|
1983
|
+
}
|
|
1984
|
+
}
|
|
1985
|
+
for (const event of typedEvents) {
|
|
1986
|
+
if (mutations.filter((mutation) => matches(mutation, event)).length !== 1) {
|
|
1987
|
+
throw new RunStoreError("invalid_record", `${type} event ${event.event_id} does not match exactly one transaction mutation`);
|
|
1988
|
+
}
|
|
1989
|
+
}
|
|
1990
|
+
}
|
|
1991
|
+
function applyCanonicalUpdate(current, update) {
|
|
1992
|
+
const runId = current.run.run_id;
|
|
1993
|
+
assertCorrelatedRunId(update.run_id, runId, "update");
|
|
1994
|
+
let changed = false;
|
|
1995
|
+
let run = current.run;
|
|
1996
|
+
if (update.run) {
|
|
1997
|
+
assertCanonicalRunRecord(update.run, runId);
|
|
1998
|
+
if (!recordsEqual(run, update.run)) {
|
|
1999
|
+
if (run.terminal !== null) {
|
|
2000
|
+
throw new RunStoreError("immutable_record", `terminal canonical run ${runId} cannot be changed`);
|
|
2001
|
+
}
|
|
2002
|
+
if (!recordsEqual(run.target, update.run.target)
|
|
2003
|
+
|| run.plan_id !== update.run.plan_id
|
|
2004
|
+
|| run.plan_digest !== update.run.plan_digest
|
|
2005
|
+
|| !recordsEqual(run.run_binding, update.run.run_binding)
|
|
2006
|
+
|| run.created_at !== update.run.created_at) {
|
|
2007
|
+
throw new RunStoreError("identity_conflict", `canonical run ${runId} cannot change target, plan, binding, or creation identity`);
|
|
2008
|
+
}
|
|
2009
|
+
const allowedStatuses = {
|
|
2010
|
+
active: ["active", "cancelling", "succeeded", "failed", "cancelled"],
|
|
2011
|
+
cancelling: ["cancelling", "cancelled"],
|
|
2012
|
+
succeeded: ["succeeded"],
|
|
2013
|
+
failed: ["failed"],
|
|
2014
|
+
cancelled: ["cancelled"],
|
|
2015
|
+
};
|
|
2016
|
+
if (!allowedStatuses[run.status].includes(update.run.status)) {
|
|
2017
|
+
throw new RunStoreError("immutable_record", `canonical run ${runId} cannot transition from ${run.status} to ${update.run.status}`);
|
|
2018
|
+
}
|
|
2019
|
+
if (run.cancellation !== null
|
|
2020
|
+
&& !recordsEqual(run.cancellation, update.run.cancellation)) {
|
|
2021
|
+
throw new RunStoreError("immutable_record", `canonical run ${runId} cannot change cancellation intent`);
|
|
2022
|
+
}
|
|
2023
|
+
if (Date.parse(update.run.updated_at) < Date.parse(run.updated_at)) {
|
|
2024
|
+
throw new RunStoreError("immutable_record", `canonical run ${runId} cannot move updated_at backward`);
|
|
2025
|
+
}
|
|
2026
|
+
run = update.run;
|
|
2027
|
+
changed = true;
|
|
2028
|
+
}
|
|
2029
|
+
}
|
|
2030
|
+
const activations = upsertRecords(current.activations, update.activations ?? [], (activation) => activation.activation_id, (existing, incoming) => {
|
|
2031
|
+
if (existing.run_id !== incoming.run_id
|
|
2032
|
+
|| existing.step_id !== incoming.step_id
|
|
2033
|
+
|| existing.workflow_round !== incoming.workflow_round
|
|
2034
|
+
|| existing.created_at !== incoming.created_at) {
|
|
2035
|
+
throw new RunStoreError("identity_conflict", `activation ${existing.activation_id} cannot change its identity`);
|
|
2036
|
+
}
|
|
2037
|
+
if (["succeeded", "failed", "skipped", "cancelled"].includes(existing.state)) {
|
|
2038
|
+
throw new RunStoreError("immutable_record", `terminal activation ${existing.activation_id} cannot be changed`);
|
|
2039
|
+
}
|
|
2040
|
+
if (incoming.state !== existing.state
|
|
2041
|
+
&& !CANONICAL_ACTIVATION_STATE_TRANSITIONS[existing.state]
|
|
2042
|
+
.includes(incoming.state)) {
|
|
2043
|
+
throw new RunStoreError("immutable_record", `activation ${existing.activation_id} cannot transition from ${existing.state} to ${incoming.state}`);
|
|
2044
|
+
}
|
|
2045
|
+
if (incoming.attempt_count < existing.attempt_count) {
|
|
2046
|
+
throw new RunStoreError("immutable_record", `activation ${existing.activation_id} cannot decrease attempt_count`);
|
|
2047
|
+
}
|
|
2048
|
+
if (Date.parse(incoming.updated_at) < Date.parse(existing.updated_at)) {
|
|
2049
|
+
throw new RunStoreError("immutable_record", `activation ${existing.activation_id} cannot move updated_at backward`);
|
|
2050
|
+
}
|
|
2051
|
+
if (existing.committed_result_id !== null
|
|
2052
|
+
&& incoming.committed_result_id !== existing.committed_result_id) {
|
|
2053
|
+
throw new RunStoreError("immutable_record", `activation ${existing.activation_id} cannot change its committed result`);
|
|
2054
|
+
}
|
|
2055
|
+
});
|
|
2056
|
+
changed ||= activations.changed;
|
|
2057
|
+
const activationMutations = [];
|
|
2058
|
+
const activationStateById = new Map(current.activations.map((activation) => [activation.activation_id, activation]));
|
|
2059
|
+
for (const incoming of update.activations ?? []) {
|
|
2060
|
+
const before = activationStateById.get(incoming.activation_id) ?? null;
|
|
2061
|
+
if (!before || !recordsEqual(before, incoming)) {
|
|
2062
|
+
activationMutations.push({ before, after: incoming });
|
|
2063
|
+
}
|
|
2064
|
+
activationStateById.set(incoming.activation_id, incoming);
|
|
2065
|
+
}
|
|
2066
|
+
const activationStateChanges = activationMutations.filter((mutation) => mutation.before !== null && mutation.before.state !== mutation.after.state);
|
|
2067
|
+
const conditionalActivationTargetMutations = activationMutations.filter(({ before, after }) => (["ready", "dependency_blocked"].includes(after.state)
|
|
2068
|
+
&& (before === null || before.state === "dormant")));
|
|
2069
|
+
const attempts = upsertRecords(current.attempts, update.attempts ?? [], (attempt) => attempt.attempt_id, (existing, incoming) => {
|
|
2070
|
+
if (existing.run_id !== incoming.run_id
|
|
2071
|
+
|| existing.step_id !== incoming.step_id
|
|
2072
|
+
|| existing.activation_id !== incoming.activation_id
|
|
2073
|
+
|| existing.attempt_number !== incoming.attempt_number) {
|
|
2074
|
+
throw new RunStoreError("identity_conflict", `canonical attempt ${existing.attempt_id} cannot change its identity`);
|
|
2075
|
+
}
|
|
2076
|
+
if (existing.executor !== null
|
|
2077
|
+
&& !recordsEqual(existing.executor, incoming.executor)) {
|
|
2078
|
+
throw new RunStoreError("identity_conflict", `canonical attempt ${existing.attempt_id} cannot change executor`);
|
|
2079
|
+
}
|
|
2080
|
+
if (existing.started_at !== null
|
|
2081
|
+
&& incoming.started_at !== existing.started_at) {
|
|
2082
|
+
throw new RunStoreError("immutable_record", `canonical attempt ${existing.attempt_id} cannot change started_at`);
|
|
2083
|
+
}
|
|
2084
|
+
if (existing.phase === "terminal") {
|
|
2085
|
+
throw new RunStoreError("immutable_record", `terminal canonical attempt ${existing.attempt_id} cannot be changed`);
|
|
2086
|
+
}
|
|
2087
|
+
if ((existing.phase === "running" && incoming.phase !== "terminal")
|
|
2088
|
+
|| (existing.phase === "claimed" && !["claimed", "running", "terminal"].includes(incoming.phase))) {
|
|
2089
|
+
throw new RunStoreError("immutable_record", `canonical attempt ${existing.attempt_id} has an invalid phase transition`);
|
|
2090
|
+
}
|
|
2091
|
+
});
|
|
2092
|
+
changed ||= attempts.changed;
|
|
2093
|
+
const attemptMutations = [];
|
|
2094
|
+
const attemptStateById = new Map(current.attempts.map((attempt) => [attempt.attempt_id, attempt]));
|
|
2095
|
+
for (const incoming of update.attempts ?? []) {
|
|
2096
|
+
const before = attemptStateById.get(incoming.attempt_id) ?? null;
|
|
2097
|
+
if (!before || !recordsEqual(before, incoming)) {
|
|
2098
|
+
attemptMutations.push({ before, after: incoming });
|
|
2099
|
+
}
|
|
2100
|
+
attemptStateById.set(incoming.attempt_id, incoming);
|
|
2101
|
+
}
|
|
2102
|
+
const newlyTerminalAttempts = attemptMutations.filter(({ before, after }) => (before?.phase !== "terminal" && after.phase === "terminal"));
|
|
2103
|
+
const newlyStartedAttempts = attemptMutations.filter(({ before, after }) => (before?.started_at == null && after.started_at !== null));
|
|
2104
|
+
const executorIntroductions = attemptMutations.filter(({ before, after }) => (before?.executor == null && after.executor !== null));
|
|
2105
|
+
const highestClaimEpoch = new Map();
|
|
2106
|
+
for (const claim of current.claims) {
|
|
2107
|
+
highestClaimEpoch.set(claim.attempt_id, Math.max(highestClaimEpoch.get(claim.attempt_id) ?? 0, claim.claim_epoch));
|
|
2108
|
+
}
|
|
2109
|
+
const storedClaimTokens = new Set(current.claims.map((claim) => claim.claim_token));
|
|
2110
|
+
const stopFencedAttemptIds = new Set(current.events.flatMap((event) => (event.type === "attempt_stop_requested" && event.attempt_id !== null
|
|
2111
|
+
? [event.attempt_id]
|
|
2112
|
+
: [])));
|
|
2113
|
+
const attemptsWithNewClaims = new Set();
|
|
2114
|
+
const newClaims = [];
|
|
2115
|
+
const renewedClaims = [];
|
|
2116
|
+
for (const claim of update.claims ?? []) {
|
|
2117
|
+
const existing = current.claims.find((candidate) => candidate.claim_token === claim.claim_token);
|
|
2118
|
+
if (stopFencedAttemptIds.has(claim.attempt_id)
|
|
2119
|
+
&& (existing === undefined || !recordsEqual(existing, claim))) {
|
|
2120
|
+
throw new RunStoreError("immutable_record", `attempt ${claim.attempt_id} cannot change claims after a durable stop intent`);
|
|
2121
|
+
}
|
|
2122
|
+
if (storedClaimTokens.has(claim.claim_token)) {
|
|
2123
|
+
if (existing && !recordsEqual(existing, claim))
|
|
2124
|
+
renewedClaims.push(claim);
|
|
2125
|
+
continue;
|
|
2126
|
+
}
|
|
2127
|
+
const currentAttempt = current.attempts.find((attempt) => attempt.attempt_id === claim.attempt_id);
|
|
2128
|
+
const incomingAttempt = update.attempts?.find((attempt) => attempt.attempt_id === claim.attempt_id);
|
|
2129
|
+
if (currentAttempt?.phase === "terminal"
|
|
2130
|
+
|| incomingAttempt?.phase === "terminal") {
|
|
2131
|
+
throw new RunStoreError("immutable_record", `terminal attempt ${claim.attempt_id} cannot receive a new claim`);
|
|
2132
|
+
}
|
|
2133
|
+
const existingDeadline = current.claims.find((candidate) => candidate.attempt_id === claim.attempt_id)?.execution_deadline_at;
|
|
2134
|
+
if (existingDeadline !== undefined
|
|
2135
|
+
&& claim.execution_deadline_at !== existingDeadline) {
|
|
2136
|
+
throw new RunStoreError("identity_conflict", `attempt ${claim.attempt_id} cannot change execution deadline across claim epochs`);
|
|
2137
|
+
}
|
|
2138
|
+
const highest = highestClaimEpoch.get(claim.attempt_id) ?? 0;
|
|
2139
|
+
if (attemptsWithNewClaims.has(claim.attempt_id)) {
|
|
2140
|
+
throw new RunStoreError("invalid_record", `canonical update cannot create multiple claim epochs for attempt ${claim.attempt_id}`);
|
|
2141
|
+
}
|
|
2142
|
+
if (claim.claim_epoch !== highest + 1) {
|
|
2143
|
+
throw new RunStoreError("immutable_record", `canonical claim epoch ${claim.claim_epoch} must follow epoch ${highest} for attempt ${claim.attempt_id}`);
|
|
2144
|
+
}
|
|
2145
|
+
if (claim.claimed_record_version !== current.record_version) {
|
|
2146
|
+
throw new RunStoreError("version_conflict", `canonical claim ${claim.claim_token} observed record version ${claim.claimed_record_version}, expected ${current.record_version}`);
|
|
2147
|
+
}
|
|
2148
|
+
highestClaimEpoch.set(claim.attempt_id, claim.claim_epoch);
|
|
2149
|
+
attemptsWithNewClaims.add(claim.attempt_id);
|
|
2150
|
+
newClaims.push(claim);
|
|
2151
|
+
}
|
|
2152
|
+
const claims = upsertRecords(current.claims, update.claims ?? [], (claim) => claim.claim_token, (existing, incoming) => {
|
|
2153
|
+
if (existing.run_id !== incoming.run_id
|
|
2154
|
+
|| existing.step_id !== incoming.step_id
|
|
2155
|
+
|| existing.activation_id !== incoming.activation_id
|
|
2156
|
+
|| existing.attempt_id !== incoming.attempt_id
|
|
2157
|
+
|| existing.claim_epoch !== incoming.claim_epoch
|
|
2158
|
+
|| existing.owner_id !== incoming.owner_id
|
|
2159
|
+
|| existing.claimed_record_version !== incoming.claimed_record_version
|
|
2160
|
+
|| existing.driver_claim_lease.acquired_at
|
|
2161
|
+
!== incoming.driver_claim_lease.acquired_at
|
|
2162
|
+
|| existing.execution_deadline_at !== incoming.execution_deadline_at) {
|
|
2163
|
+
throw new RunStoreError("identity_conflict", `canonical claim ${existing.claim_token} cannot change ownership identity`);
|
|
2164
|
+
}
|
|
2165
|
+
if (Date.parse(incoming.driver_claim_lease.heartbeat_at)
|
|
2166
|
+
< Date.parse(existing.driver_claim_lease.heartbeat_at)
|
|
2167
|
+
|| Date.parse(incoming.driver_claim_lease.expires_at)
|
|
2168
|
+
< Date.parse(existing.driver_claim_lease.expires_at)) {
|
|
2169
|
+
throw new RunStoreError("immutable_record", `canonical claim ${existing.claim_token} cannot move its lease clocks backward`);
|
|
2170
|
+
}
|
|
2171
|
+
});
|
|
2172
|
+
changed ||= claims.changed;
|
|
2173
|
+
const events = [...current.events];
|
|
2174
|
+
const eventById = new Map(events.map((event) => [event.event_id, event]));
|
|
2175
|
+
const eventByKey = new Map(events.map((event) => [event.idempotency_key, event]));
|
|
2176
|
+
const duplicateEventIds = [];
|
|
2177
|
+
const newEvents = [];
|
|
2178
|
+
for (const event of update.events ?? []) {
|
|
2179
|
+
assertCorrelatedRunId(event.run_id, runId, "canonical event");
|
|
343
2180
|
const byId = eventById.get(event.event_id);
|
|
344
2181
|
const byKey = eventByKey.get(event.idempotency_key);
|
|
345
2182
|
const existing = byId ?? byKey;
|
|
346
2183
|
if (existing) {
|
|
347
|
-
if ((byId && byKey && byId.event_id !== byKey.event_id)
|
|
348
|
-
!
|
|
349
|
-
throw new RunStoreError("event_conflict", `event ${event.event_id} conflicts with
|
|
2184
|
+
if ((byId && byKey && byId.event_id !== byKey.event_id)
|
|
2185
|
+
|| !canonicalEventReplayEquivalent(existing, event)) {
|
|
2186
|
+
throw new RunStoreError("event_conflict", `canonical event ${event.event_id} conflicts with stored identity`);
|
|
350
2187
|
}
|
|
351
2188
|
duplicateEventIds.push(existing.event_id);
|
|
352
2189
|
continue;
|
|
353
2190
|
}
|
|
354
2191
|
const expectedSequence = events.length + 1;
|
|
355
2192
|
if (event.sequence !== expectedSequence) {
|
|
356
|
-
throw new RunStoreError("event_conflict", `event ${event.event_id} sequence ${event.sequence} must be ${expectedSequence}`);
|
|
2193
|
+
throw new RunStoreError("event_conflict", `canonical event ${event.event_id} sequence ${event.sequence} must be ${expectedSequence}`);
|
|
357
2194
|
}
|
|
358
2195
|
events.push(event);
|
|
2196
|
+
newEvents.push(event);
|
|
359
2197
|
eventById.set(event.event_id, event);
|
|
360
2198
|
eventByKey.set(event.idempotency_key, event);
|
|
361
2199
|
changed = true;
|
|
362
2200
|
}
|
|
363
|
-
const
|
|
364
|
-
const resultById = new Map(
|
|
365
|
-
const resultByAttempt = new Map(
|
|
2201
|
+
const resultRefs = [...current.result_refs];
|
|
2202
|
+
const resultById = new Map(resultRefs.map((reference) => [reference.result_id, reference]));
|
|
2203
|
+
const resultByAttempt = new Map(resultRefs.map((reference) => [reference.attempt_id, reference]));
|
|
366
2204
|
const duplicateResultIds = [];
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
2205
|
+
const newResultRefs = [];
|
|
2206
|
+
for (const reference of update.result_refs ?? []) {
|
|
2207
|
+
assertCorrelatedRunId(reference.run_id, runId, "canonical result reference");
|
|
2208
|
+
const existing = resultById.get(reference.result_id);
|
|
370
2209
|
if (existing) {
|
|
371
|
-
if (!recordsEqual(existing,
|
|
372
|
-
throw new RunStoreError("result_conflict", `
|
|
2210
|
+
if (!recordsEqual(existing, reference)) {
|
|
2211
|
+
throw new RunStoreError("result_conflict", `canonical result ${reference.result_id} is immutable`);
|
|
373
2212
|
}
|
|
374
2213
|
duplicateResultIds.push(existing.result_id);
|
|
375
2214
|
continue;
|
|
376
2215
|
}
|
|
377
|
-
const attemptResult = resultByAttempt.get(
|
|
2216
|
+
const attemptResult = resultByAttempt.get(reference.attempt_id);
|
|
378
2217
|
if (attemptResult) {
|
|
379
|
-
throw new RunStoreError("result_conflict", `attempt ${
|
|
2218
|
+
throw new RunStoreError("result_conflict", `canonical attempt ${reference.attempt_id} already references result ${attemptResult.result_id}`);
|
|
380
2219
|
}
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
2220
|
+
resultRefs.push(reference);
|
|
2221
|
+
newResultRefs.push(reference);
|
|
2222
|
+
resultById.set(reference.result_id, reference);
|
|
2223
|
+
resultByAttempt.set(reference.attempt_id, reference);
|
|
384
2224
|
changed = true;
|
|
385
2225
|
}
|
|
2226
|
+
for (const attemptId of stopFencedAttemptIds) {
|
|
2227
|
+
const storedAttempt = current.attempts.find((attempt) => attempt.attempt_id === attemptId);
|
|
2228
|
+
if (!storedAttempt)
|
|
2229
|
+
continue;
|
|
2230
|
+
const stopPending = storedAttempt.phase !== "terminal";
|
|
2231
|
+
const attemptMutation = attemptMutations.find(({ after }) => after.attempt_id === attemptId);
|
|
2232
|
+
const terminalMutation = attemptMutation?.after.phase === "terminal" ? attemptMutation : null;
|
|
2233
|
+
if (attemptMutation && terminalMutation === null) {
|
|
2234
|
+
throw new RunStoreError("immutable_record", `attempt ${attemptId} can only confirm its durable stop intent`);
|
|
2235
|
+
}
|
|
2236
|
+
if (newResultRefs.some((reference) => reference.attempt_id === attemptId)) {
|
|
2237
|
+
throw new RunStoreError("immutable_record", `attempt ${attemptId} cannot commit a result after a durable stop intent`);
|
|
2238
|
+
}
|
|
2239
|
+
const activationMutation = stopPending
|
|
2240
|
+
? activationMutations.find(({ after }) => after.activation_id === storedAttempt.activation_id)
|
|
2241
|
+
: undefined;
|
|
2242
|
+
if (activationMutation
|
|
2243
|
+
&& (terminalMutation === null
|
|
2244
|
+
|| !["ready", "succeeded", "failed", "cancelled", "skipped"].includes(activationMutation.after.state)
|
|
2245
|
+
|| activationMutation.after.committed_result_id
|
|
2246
|
+
!== activationMutation.before?.committed_result_id)) {
|
|
2247
|
+
throw new RunStoreError("immutable_record", `activation ${storedAttempt.activation_id} can only transition with durable stop confirmation`);
|
|
2248
|
+
}
|
|
2249
|
+
for (const event of newEvents) {
|
|
2250
|
+
const scopesAttempt = stopPending && (event.attempt_id === attemptId
|
|
2251
|
+
|| event.activation_id === storedAttempt.activation_id);
|
|
2252
|
+
if (!scopesAttempt)
|
|
2253
|
+
continue;
|
|
2254
|
+
const allowedTerminal = terminalMutation !== null
|
|
2255
|
+
&& event.type === "attempt_terminal"
|
|
2256
|
+
&& event.attempt_id === attemptId;
|
|
2257
|
+
const allowedActivation = terminalMutation !== null
|
|
2258
|
+
&& activationMutation !== undefined
|
|
2259
|
+
&& event.type === "activation_state_changed"
|
|
2260
|
+
&& event.activation_id === storedAttempt.activation_id;
|
|
2261
|
+
if (!allowedTerminal && !allowedActivation) {
|
|
2262
|
+
throw new RunStoreError("immutable_record", `attempt ${attemptId} cannot append ${event.type} after a durable stop intent`);
|
|
2263
|
+
}
|
|
2264
|
+
}
|
|
2265
|
+
}
|
|
386
2266
|
const now = new Date().toISOString();
|
|
387
2267
|
const candidate = {
|
|
388
|
-
schema_version:
|
|
2268
|
+
schema_version: CANONICAL_RUN_STORE_SCHEMA_VERSION,
|
|
389
2269
|
record_version: changed
|
|
390
2270
|
? current.record_version + 1
|
|
391
2271
|
: current.record_version,
|
|
392
2272
|
run,
|
|
393
|
-
|
|
2273
|
+
activations: activations.records,
|
|
394
2274
|
attempts: attempts.records,
|
|
395
2275
|
claims: claims.records,
|
|
396
2276
|
events,
|
|
397
|
-
|
|
2277
|
+
result_refs: resultRefs,
|
|
398
2278
|
created_at: current.created_at,
|
|
399
2279
|
updated_at: changed ? now : current.updated_at,
|
|
400
2280
|
};
|
|
401
|
-
|
|
2281
|
+
if (current.run.terminal !== null && changed) {
|
|
2282
|
+
throw new RunStoreError("immutable_record", `terminal canonical run ${runId} cannot be changed`);
|
|
2283
|
+
}
|
|
2284
|
+
validateCanonicalCollections(candidate);
|
|
2285
|
+
const routeEvents = canonicalEventsOfType(newEvents, "route_selected");
|
|
2286
|
+
const retrySegmentEvents = canonicalEventsOfType(newEvents, "retry_segment_scheduled");
|
|
2287
|
+
for (const mutation of conditionalActivationTargetMutations) {
|
|
2288
|
+
const matchingRoutes = routeEvents.filter((event) => (event.payload.successor_activation_ids.includes(mutation.after.activation_id)));
|
|
2289
|
+
const matchingRetries = mutation.before === null
|
|
2290
|
+
? retrySegmentEvents.filter((event) => (event.payload.replacement_activation_ids.includes(mutation.after.activation_id)))
|
|
2291
|
+
: [];
|
|
2292
|
+
if (matchingRoutes.length + matchingRetries.length !== 1) {
|
|
2293
|
+
throw new RunStoreError("invalid_record", `conditional activation ${mutation.after.activation_id} must be authorized by exactly one route or retry segment event`);
|
|
2294
|
+
}
|
|
2295
|
+
}
|
|
2296
|
+
for (const event of routeEvents) {
|
|
2297
|
+
for (const successorId of event.payload.successor_activation_ids) {
|
|
2298
|
+
const matchingMutations = conditionalActivationTargetMutations.filter(({ after }) => after.activation_id === successorId);
|
|
2299
|
+
if (matchingMutations.length !== 1) {
|
|
2300
|
+
throw new RunStoreError("invalid_record", `route_selected event ${event.event_id} successor ${successorId} does not match exactly one readiness mutation`);
|
|
2301
|
+
}
|
|
2302
|
+
}
|
|
2303
|
+
if (event.payload.successor_activation_ids.length > 0
|
|
2304
|
+
&& event.payload.source_result_id !== null
|
|
2305
|
+
&& !newResultRefs.some((reference) => reference.result_id === event.payload.source_result_id)) {
|
|
2306
|
+
throw new RunStoreError("invalid_record", `route_selected event ${event.event_id} must commit its source result in the same transaction`);
|
|
2307
|
+
}
|
|
2308
|
+
}
|
|
2309
|
+
for (const event of retrySegmentEvents) {
|
|
2310
|
+
for (let index = 0; index < event.payload.replacement_activation_ids.length; index += 1) {
|
|
2311
|
+
const priorId = event.payload.prior_activation_ids[index];
|
|
2312
|
+
const replacementId = event.payload.replacement_activation_ids[index];
|
|
2313
|
+
const prior = candidate.activations.find((activation) => activation.activation_id === priorId);
|
|
2314
|
+
const matchingMutations = conditionalActivationTargetMutations.filter(({ before, after }) => before === null && after.activation_id === replacementId);
|
|
2315
|
+
if (!prior
|
|
2316
|
+
|| !["succeeded", "failed", "skipped", "cancelled"].includes(prior.state)
|
|
2317
|
+
|| matchingMutations.length !== 1
|
|
2318
|
+
|| matchingMutations[0].after.step_id !== prior.step_id
|
|
2319
|
+
|| prior.workflow_round !== event.payload.from_round
|
|
2320
|
+
|| matchingMutations[0].after.workflow_round !== event.payload.to_round
|
|
2321
|
+
|| matchingMutations[0].after.state !== (replacementId === event.payload.retry_from_activation_id
|
|
2322
|
+
? "ready"
|
|
2323
|
+
: "dependency_blocked")) {
|
|
2324
|
+
throw new RunStoreError("invalid_record", `retry_segment_scheduled event ${event.event_id} does not match one prior and replacement activation pair`);
|
|
2325
|
+
}
|
|
2326
|
+
}
|
|
2327
|
+
}
|
|
2328
|
+
assertOneToOneCanonicalEventCoupling(activationStateChanges, newEvents, "activation_state_changed", ({ before, after }, event) => (event.activation_id === after.activation_id
|
|
2329
|
+
&& event.payload.from === before.state
|
|
2330
|
+
&& event.payload.to === after.state), ({ after }) => `activation transition ${after.activation_id}`);
|
|
2331
|
+
assertOneToOneCanonicalEventCoupling(newClaims, newEvents, "attempt_claimed", (claim, event) => (event.attempt_id === claim.attempt_id
|
|
2332
|
+
&& event.payload.claim_epoch === claim.claim_epoch
|
|
2333
|
+
&& event.payload.owner_id === claim.owner_id
|
|
2334
|
+
&& event.payload.claim_expires_at === claim.driver_claim_lease.expires_at
|
|
2335
|
+
&& event.payload.execution_deadline_at === claim.execution_deadline_at), (claim) => `canonical claim ${claim.claim_token}`);
|
|
2336
|
+
assertOneToOneCanonicalEventCoupling(renewedClaims, newEvents, "attempt_heartbeat", (claim, event) => (event.attempt_id === claim.attempt_id
|
|
2337
|
+
&& event.payload.claim_epoch === claim.claim_epoch
|
|
2338
|
+
&& event.payload.heartbeat_at === claim.driver_claim_lease.heartbeat_at
|
|
2339
|
+
&& event.payload.claim_expires_at === claim.driver_claim_lease.expires_at), (claim) => `canonical claim renewal ${claim.claim_token}`);
|
|
2340
|
+
assertOneToOneCanonicalEventCoupling(newlyStartedAttempts, newEvents, "attempt_started", ({ after }, event) => (event.attempt_id === after.attempt_id
|
|
2341
|
+
&& recordsEqual(event.payload.executor, after.executor)), ({ after }) => `started attempt ${after.attempt_id}`);
|
|
2342
|
+
assertOneToOneCanonicalEventCoupling(newlyTerminalAttempts, newEvents, "attempt_terminal", ({ after }, event) => (event.attempt_id === after.attempt_id
|
|
2343
|
+
&& recordsEqual(event.payload.executor, after.executor)
|
|
2344
|
+
&& recordsEqual(event.payload.outcome, after.terminal_outcome)), ({ after }) => `terminal attempt ${after.attempt_id}`);
|
|
2345
|
+
const attachedEvents = canonicalEventsOfType(newEvents, "attempt_attached");
|
|
2346
|
+
for (const mutation of executorIntroductions) {
|
|
2347
|
+
const matchingAttached = attachedEvents.filter((event) => (event.attempt_id === mutation.after.attempt_id
|
|
2348
|
+
&& recordsEqual(event.payload.executor, mutation.after.executor)));
|
|
2349
|
+
if (matchingAttached.length > 1) {
|
|
2350
|
+
throw new RunStoreError("invalid_record", `attempt ${mutation.after.attempt_id} cannot attach its executor more than once per transaction`);
|
|
2351
|
+
}
|
|
2352
|
+
if (mutation.after.executor?.kind === "worker" && matchingAttached.length !== 1) {
|
|
2353
|
+
throw new RunStoreError("invalid_record", `attempt ${mutation.after.attempt_id} must commit exactly one attempt_attached event`);
|
|
2354
|
+
}
|
|
2355
|
+
}
|
|
2356
|
+
for (const event of attachedEvents) {
|
|
2357
|
+
if (executorIntroductions.filter(({ after }) => (event.attempt_id === after.attempt_id
|
|
2358
|
+
&& recordsEqual(event.payload.executor, after.executor))).length !== 1) {
|
|
2359
|
+
throw new RunStoreError("invalid_record", `attempt_attached event ${event.event_id} does not introduce an executor in this transaction`);
|
|
2360
|
+
}
|
|
2361
|
+
}
|
|
2362
|
+
const workerExecutorIntroductions = executorIntroductions.filter(({ after }) => after.executor?.kind === "worker");
|
|
2363
|
+
const currentAllocatedAttempts = new Set(current.events
|
|
2364
|
+
.filter((event) => event.type === "workspace_allocated")
|
|
2365
|
+
.map((event) => event.attempt_id));
|
|
2366
|
+
const currentPlannedAttempts = new Set(current.events
|
|
2367
|
+
.filter((event) => event.type === "workspace_planned")
|
|
2368
|
+
.map((event) => event.attempt_id));
|
|
2369
|
+
for (const event of canonicalEventsOfType(newEvents, "workspace_allocated")) {
|
|
2370
|
+
if (!currentPlannedAttempts.has(event.attempt_id)) {
|
|
2371
|
+
throw new RunStoreError("invalid_record", `workspace allocation for attempt ${event.attempt_id} must reference a plan committed in an earlier transaction`);
|
|
2372
|
+
}
|
|
2373
|
+
}
|
|
2374
|
+
for (const { after } of newlyStartedAttempts.filter(({ after }) => after.executor?.kind === "worker")) {
|
|
2375
|
+
if (!currentPlannedAttempts.has(after.attempt_id) || !currentAllocatedAttempts.has(after.attempt_id)) {
|
|
2376
|
+
throw new RunStoreError("invalid_record", `worker attempt ${after.attempt_id} must reference a workspace plan and allocation committed before its start transaction`);
|
|
2377
|
+
}
|
|
2378
|
+
}
|
|
2379
|
+
if (canonicalEventsOfType(newEvents, "workspace_allocated")
|
|
2380
|
+
.some((event) => newlyStartedAttempts.some(({ after }) => after.attempt_id === event.attempt_id))) {
|
|
2381
|
+
throw new RunStoreError("invalid_record", "workspace allocation and attempt start cannot share a transaction");
|
|
2382
|
+
}
|
|
2383
|
+
assertOneToOneCanonicalEventCoupling(newResultRefs, newEvents, "result_committed", (reference, event) => (event.result_id === reference.result_id
|
|
2384
|
+
&& event.attempt_id === reference.attempt_id
|
|
2385
|
+
&& event.activation_id === reference.activation_id
|
|
2386
|
+
&& event.payload.result_digest === reference.result_digest), (reference) => `canonical result ${reference.result_id}`);
|
|
2387
|
+
for (const reference of newResultRefs) {
|
|
2388
|
+
const terminalAttempt = newlyTerminalAttempts.find(({ after }) => after.attempt_id === reference.attempt_id);
|
|
2389
|
+
const activation = activationMutations.find(({ after }) => after.activation_id === reference.activation_id);
|
|
2390
|
+
if (!terminalAttempt || !activation) {
|
|
2391
|
+
throw new RunStoreError("invalid_record", `canonical result ${reference.result_id} must commit attempt and activation state in one update`);
|
|
2392
|
+
}
|
|
2393
|
+
}
|
|
2394
|
+
const cancellationMutations = (current.run.cancellation === null && run.cancellation !== null) ? [run.cancellation] : [];
|
|
2395
|
+
assertOneToOneCanonicalEventCoupling(cancellationMutations, newEvents, "run_cancel_requested", (cancellation, event) => event.payload.reason === cancellation.reason, () => `canonical run ${runId} cancellation intent`);
|
|
2396
|
+
const terminalRunMutations = (current.run.terminal === null && run.terminal !== null) ? [run.terminal] : [];
|
|
2397
|
+
assertOneToOneCanonicalEventCoupling(terminalRunMutations, newEvents, "run_terminal", (terminal, event) => (event.payload.status === terminal.status
|
|
2398
|
+
&& event.payload.reason_code === terminal.reason_code
|
|
2399
|
+
&& event.payload.reason === terminal.reason), () => `canonical run ${runId} terminal state`);
|
|
402
2400
|
return {
|
|
403
2401
|
snapshot: candidate,
|
|
404
2402
|
changed,
|
|
@@ -455,129 +2453,45 @@ async function writeAtomic(filePath, data) {
|
|
|
455
2453
|
throw error;
|
|
456
2454
|
}
|
|
457
2455
|
}
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
}
|
|
462
|
-
catch (error) {
|
|
463
|
-
if (isErrno(error, "ENOENT"))
|
|
464
|
-
return null;
|
|
465
|
-
throw error;
|
|
466
|
-
}
|
|
467
|
-
}
|
|
468
|
-
async function lockIsStale(lockDirectory, staleMs) {
|
|
469
|
-
try {
|
|
470
|
-
const lease = await stat(join(lockDirectory, LOCK_LEASE_FILE_NAME));
|
|
471
|
-
return Date.now() - lease.mtimeMs > staleMs;
|
|
472
|
-
}
|
|
473
|
-
catch (error) {
|
|
474
|
-
if (!isErrno(error, "ENOENT"))
|
|
475
|
-
return false;
|
|
476
|
-
}
|
|
477
|
-
try {
|
|
478
|
-
const lock = await stat(lockDirectory);
|
|
479
|
-
return Date.now() - lock.mtimeMs > staleMs;
|
|
480
|
-
}
|
|
481
|
-
catch {
|
|
482
|
-
return false;
|
|
2456
|
+
function versionFileName(recordVersion) {
|
|
2457
|
+
if (!Number.isSafeInteger(recordVersion) || recordVersion < 1) {
|
|
2458
|
+
throw new RunStoreError("invalid_record", "record_version must be a positive safe integer");
|
|
483
2459
|
}
|
|
2460
|
+
return `${String(recordVersion).padStart(16, "0")}.json`;
|
|
484
2461
|
}
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
|
|
2462
|
+
/**
|
|
2463
|
+
* Publishes a complete immutable snapshot without replacing an existing
|
|
2464
|
+
* version. The hard link is the compare-and-swap boundary.
|
|
2465
|
+
*/
|
|
2466
|
+
async function publishVersion(destination, data) {
|
|
2467
|
+
const directory = dirname(destination);
|
|
2468
|
+
await mkdir(directory, { recursive: true, mode: DIRECTORY_MODE });
|
|
2469
|
+
const temporaryPath = join(directory, `.tmp.${process.pid}.${randomUUID()}`);
|
|
2470
|
+
let handle;
|
|
489
2471
|
try {
|
|
490
|
-
await
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
}
|
|
497
|
-
await rm(quarantine, { recursive: true, force: true });
|
|
498
|
-
return true;
|
|
499
|
-
}
|
|
500
|
-
async function withRunLock(lockDirectory, options, operation) {
|
|
501
|
-
const ownerToken = `${process.pid}.${Date.now()}.${randomUUID()}`;
|
|
502
|
-
const ownerPath = join(lockDirectory, LOCK_OWNER_FILE_NAME);
|
|
503
|
-
const leasePath = join(lockDirectory, LOCK_LEASE_FILE_NAME);
|
|
504
|
-
const deadline = Date.now() + options.lock_wait_ms;
|
|
505
|
-
await mkdir(dirname(lockDirectory), {
|
|
506
|
-
recursive: true,
|
|
507
|
-
mode: DIRECTORY_MODE,
|
|
508
|
-
});
|
|
509
|
-
while (true) {
|
|
2472
|
+
handle = await open(temporaryPath, "wx", FILE_MODE);
|
|
2473
|
+
await handle.writeFile(data, "utf8");
|
|
2474
|
+
await handle.chmod(IMMUTABLE_FILE_MODE);
|
|
2475
|
+
await handle.sync();
|
|
2476
|
+
await handle.close();
|
|
2477
|
+
handle = undefined;
|
|
510
2478
|
try {
|
|
511
|
-
await
|
|
512
|
-
try {
|
|
513
|
-
const owner = await open(ownerPath, "wx", FILE_MODE);
|
|
514
|
-
await owner.writeFile(ownerToken, "utf8");
|
|
515
|
-
await owner.sync();
|
|
516
|
-
await owner.close();
|
|
517
|
-
const lease = await open(leasePath, "wx", FILE_MODE);
|
|
518
|
-
await lease.writeFile(ownerToken, "utf8");
|
|
519
|
-
await lease.sync();
|
|
520
|
-
await lease.close();
|
|
521
|
-
}
|
|
522
|
-
catch (error) {
|
|
523
|
-
await rm(lockDirectory, { recursive: true, force: true });
|
|
524
|
-
throw error;
|
|
525
|
-
}
|
|
526
|
-
break;
|
|
2479
|
+
await link(temporaryPath, destination);
|
|
527
2480
|
}
|
|
528
2481
|
catch (error) {
|
|
529
|
-
if (
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
continue;
|
|
533
|
-
if (Date.now() >= deadline) {
|
|
534
|
-
throw new RunStoreError("lock_timeout", `timed out acquiring run-store lock ${lockDirectory}`);
|
|
535
|
-
}
|
|
536
|
-
await sleep(options.lock_retry_ms);
|
|
537
|
-
}
|
|
538
|
-
}
|
|
539
|
-
let heartbeatRunning = false;
|
|
540
|
-
let lostOwnership = false;
|
|
541
|
-
const refresh = async () => {
|
|
542
|
-
if (heartbeatRunning || lostOwnership)
|
|
543
|
-
return;
|
|
544
|
-
heartbeatRunning = true;
|
|
545
|
-
try {
|
|
546
|
-
if ((await readLockOwner(ownerPath)) !== ownerToken) {
|
|
547
|
-
lostOwnership = true;
|
|
548
|
-
return;
|
|
549
|
-
}
|
|
550
|
-
const now = new Date();
|
|
551
|
-
await utimes(leasePath, now, now);
|
|
552
|
-
}
|
|
553
|
-
catch {
|
|
554
|
-
lostOwnership = true;
|
|
555
|
-
}
|
|
556
|
-
finally {
|
|
557
|
-
heartbeatRunning = false;
|
|
2482
|
+
if (isErrno(error, "EEXIST"))
|
|
2483
|
+
return false;
|
|
2484
|
+
throw error;
|
|
558
2485
|
}
|
|
559
|
-
|
|
560
|
-
|
|
561
|
-
timer.unref();
|
|
562
|
-
const lock = {
|
|
563
|
-
async assertOwned() {
|
|
564
|
-
if (lostOwnership ||
|
|
565
|
-
(await readLockOwner(ownerPath)) !== ownerToken) {
|
|
566
|
-
lostOwnership = true;
|
|
567
|
-
throw new RunStoreError("lock_lost", `run-store lock ownership was lost for ${lockDirectory}`);
|
|
568
|
-
}
|
|
569
|
-
},
|
|
570
|
-
};
|
|
571
|
-
try {
|
|
572
|
-
return await operation(lock);
|
|
2486
|
+
await syncParentDirectory(destination);
|
|
2487
|
+
return true;
|
|
573
2488
|
}
|
|
574
2489
|
finally {
|
|
575
|
-
|
|
576
|
-
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
}
|
|
2490
|
+
await handle?.close().catch(() => undefined);
|
|
2491
|
+
await unlink(temporaryPath).catch((error) => {
|
|
2492
|
+
if (!isErrno(error, "ENOENT"))
|
|
2493
|
+
throw error;
|
|
2494
|
+
});
|
|
581
2495
|
}
|
|
582
2496
|
}
|
|
583
2497
|
export class DurableRunStore {
|
|
@@ -585,54 +2499,198 @@ export class DurableRunStore {
|
|
|
585
2499
|
constructor(options) {
|
|
586
2500
|
this.options = resolveOptions(options);
|
|
587
2501
|
}
|
|
2502
|
+
serviceOwnedRuntimeDirectory(component) {
|
|
2503
|
+
return join(this.options.root_directory, "runtime", component);
|
|
2504
|
+
}
|
|
588
2505
|
runDirectory(runId) {
|
|
589
2506
|
return join(this.options.root_directory, "runs", runStorageKey(runId));
|
|
590
2507
|
}
|
|
591
2508
|
ledgerPath(runId) {
|
|
592
2509
|
return join(this.runDirectory(runId), LEDGER_FILE_NAME);
|
|
593
2510
|
}
|
|
594
|
-
|
|
595
|
-
return join(this.
|
|
2511
|
+
versionsDirectory(runId) {
|
|
2512
|
+
return join(this.runDirectory(runId), VERSION_DIRECTORY_NAME);
|
|
596
2513
|
}
|
|
597
|
-
|
|
2514
|
+
versionPath(runId, recordVersion) {
|
|
2515
|
+
return join(this.versionsDirectory(runId), versionFileName(recordVersion));
|
|
2516
|
+
}
|
|
2517
|
+
async readLedgerSnapshot(runId) {
|
|
598
2518
|
try {
|
|
599
2519
|
const raw = await readFile(this.ledgerPath(runId), "utf8");
|
|
600
|
-
return
|
|
2520
|
+
return assertAnySnapshot(JSON.parse(raw), runId);
|
|
2521
|
+
}
|
|
2522
|
+
catch (error) {
|
|
2523
|
+
if (isErrno(error, "ENOENT"))
|
|
2524
|
+
return null;
|
|
2525
|
+
if (error instanceof SyntaxError) {
|
|
2526
|
+
throw new RunStoreError("invalid_record", `run ${runId} contains invalid JSON`, { cause: error });
|
|
2527
|
+
}
|
|
2528
|
+
throw error;
|
|
2529
|
+
}
|
|
2530
|
+
}
|
|
2531
|
+
async readCommittedHeadFromDirectory(runDirectory, expectedRunId) {
|
|
2532
|
+
const versionsDirectory = join(runDirectory, VERSION_DIRECTORY_NAME);
|
|
2533
|
+
let entries;
|
|
2534
|
+
try {
|
|
2535
|
+
entries = await readdir(versionsDirectory, { withFileTypes: true });
|
|
601
2536
|
}
|
|
602
2537
|
catch (error) {
|
|
603
2538
|
if (isErrno(error, "ENOENT"))
|
|
604
2539
|
return null;
|
|
2540
|
+
throw error;
|
|
2541
|
+
}
|
|
2542
|
+
const versionNames = entries
|
|
2543
|
+
.map((entry) => entry.name)
|
|
2544
|
+
.filter((name) => VERSION_FILE_PATTERN.test(name))
|
|
2545
|
+
.sort();
|
|
2546
|
+
const headName = versionNames.at(-1);
|
|
2547
|
+
if (headName === undefined)
|
|
2548
|
+
return null;
|
|
2549
|
+
const match = headName.match(VERSION_FILE_PATTERN);
|
|
2550
|
+
const recordVersion = Number(match?.[1]);
|
|
2551
|
+
if (!Number.isSafeInteger(recordVersion) || recordVersion < 1) {
|
|
2552
|
+
throw new RunStoreError("invalid_record", `run version filename ${headName} is outside the supported range`);
|
|
2553
|
+
}
|
|
2554
|
+
const headPath = join(versionsDirectory, headName);
|
|
2555
|
+
let value;
|
|
2556
|
+
try {
|
|
2557
|
+
value = JSON.parse(await readFile(headPath, "utf8"));
|
|
2558
|
+
}
|
|
2559
|
+
catch (error) {
|
|
605
2560
|
if (error instanceof SyntaxError) {
|
|
606
|
-
throw new RunStoreError("invalid_record", `run ${
|
|
2561
|
+
throw new RunStoreError("invalid_record", `run version ${headPath} contains invalid JSON`, { cause: error });
|
|
607
2562
|
}
|
|
608
2563
|
throw error;
|
|
609
2564
|
}
|
|
2565
|
+
const snapshot = assertAnySnapshot(value, expectedRunId);
|
|
2566
|
+
if (snapshot.record_version !== recordVersion) {
|
|
2567
|
+
throw new RunStoreError("invalid_record", `run version ${headPath} contains record version ${snapshot.record_version}`);
|
|
2568
|
+
}
|
|
2569
|
+
return snapshot;
|
|
2570
|
+
}
|
|
2571
|
+
async readCommittedHead(runId) {
|
|
2572
|
+
return this.readCommittedHeadFromDirectory(this.runDirectory(runId), runId);
|
|
2573
|
+
}
|
|
2574
|
+
async readProjectionForRepair(runId) {
|
|
2575
|
+
try {
|
|
2576
|
+
return await this.readLedgerSnapshot(runId);
|
|
2577
|
+
}
|
|
2578
|
+
catch (error) {
|
|
2579
|
+
if (error instanceof RunStoreError)
|
|
2580
|
+
return null;
|
|
2581
|
+
throw error;
|
|
2582
|
+
}
|
|
2583
|
+
}
|
|
2584
|
+
async projectCommittedHead(runId) {
|
|
2585
|
+
while (true) {
|
|
2586
|
+
const head = await this.readCommittedHead(runId);
|
|
2587
|
+
if (head === null)
|
|
2588
|
+
return null;
|
|
2589
|
+
const projection = await this.readProjectionForRepair(runId);
|
|
2590
|
+
if (projection === null || !recordsEqual(projection, head)) {
|
|
2591
|
+
await writeAtomic(this.ledgerPath(runId), `${JSON.stringify(head, null, 2)}\n`);
|
|
2592
|
+
}
|
|
2593
|
+
const latest = await this.readCommittedHead(runId);
|
|
2594
|
+
const latestProjection = await this.readProjectionForRepair(runId);
|
|
2595
|
+
if (latest !== null
|
|
2596
|
+
&& recordsEqual(latest, head)
|
|
2597
|
+
&& latestProjection !== null
|
|
2598
|
+
&& recordsEqual(latestProjection, latest)) {
|
|
2599
|
+
return latest;
|
|
2600
|
+
}
|
|
2601
|
+
}
|
|
2602
|
+
}
|
|
2603
|
+
async beforeLedgerCommit(_snapshot) { }
|
|
2604
|
+
async afterLedgerCommit(_snapshot) { }
|
|
2605
|
+
async commitSnapshot(snapshot) {
|
|
2606
|
+
await this.beforeLedgerCommit(snapshot);
|
|
2607
|
+
const committed = await publishVersion(this.versionPath(snapshot.run.run_id, snapshot.record_version), `${JSON.stringify(snapshot, null, 2)}\n`);
|
|
2608
|
+
if (!committed)
|
|
2609
|
+
return false;
|
|
2610
|
+
await this.afterLedgerCommit(snapshot);
|
|
2611
|
+
await this.projectCommittedHead(snapshot.run.run_id);
|
|
2612
|
+
return true;
|
|
2613
|
+
}
|
|
2614
|
+
async readAnySnapshot(runId) {
|
|
2615
|
+
const committed = await this.readCommittedHead(runId);
|
|
2616
|
+
if (committed !== null) {
|
|
2617
|
+
return (await this.projectCommittedHead(runId)) ?? committed;
|
|
2618
|
+
}
|
|
2619
|
+
return this.readLedgerSnapshot(runId);
|
|
2620
|
+
}
|
|
2621
|
+
async readSnapshot(runId) {
|
|
2622
|
+
const snapshot = await this.readAnySnapshot(runId);
|
|
2623
|
+
if (snapshot !== null
|
|
2624
|
+
&& snapshot.schema_version !== LEGACY_RUN_STORE_SCHEMA_VERSION) {
|
|
2625
|
+
throw new RunStoreError("unsupported_schema", `canonical run ${runId} requires the canonical store API`);
|
|
2626
|
+
}
|
|
2627
|
+
return snapshot;
|
|
610
2628
|
}
|
|
611
2629
|
async createRun(input) {
|
|
612
2630
|
assertRunRecord(input.run);
|
|
613
2631
|
const runId = input.run.run_id;
|
|
614
|
-
|
|
615
|
-
|
|
616
|
-
|
|
617
|
-
|
|
618
|
-
|
|
619
|
-
|
|
620
|
-
|
|
621
|
-
|
|
622
|
-
|
|
623
|
-
|
|
624
|
-
|
|
625
|
-
|
|
626
|
-
|
|
627
|
-
|
|
628
|
-
|
|
629
|
-
|
|
630
|
-
|
|
631
|
-
|
|
632
|
-
|
|
633
|
-
|
|
634
|
-
|
|
635
|
-
|
|
2632
|
+
if (await this.readAnySnapshot(runId)) {
|
|
2633
|
+
throw new RunStoreError("run_exists", `run ${runId} already exists`);
|
|
2634
|
+
}
|
|
2635
|
+
const now = new Date().toISOString();
|
|
2636
|
+
const snapshot = {
|
|
2637
|
+
schema_version: LEGACY_RUN_STORE_SCHEMA_VERSION,
|
|
2638
|
+
record_version: 1,
|
|
2639
|
+
run: input.run,
|
|
2640
|
+
tasks: [...(input.tasks ?? [])],
|
|
2641
|
+
attempts: [],
|
|
2642
|
+
claims: [],
|
|
2643
|
+
events: [],
|
|
2644
|
+
results: [],
|
|
2645
|
+
created_at: now,
|
|
2646
|
+
updated_at: now,
|
|
2647
|
+
};
|
|
2648
|
+
validateCollections(snapshot);
|
|
2649
|
+
if (!await this.commitSnapshot(snapshot)) {
|
|
2650
|
+
throw new RunStoreError("run_exists", `run ${runId} already exists`);
|
|
2651
|
+
}
|
|
2652
|
+
return snapshot;
|
|
2653
|
+
}
|
|
2654
|
+
async createCanonicalRun(input) {
|
|
2655
|
+
assertCanonicalRunRecord(input.run);
|
|
2656
|
+
const runId = input.run.run_id;
|
|
2657
|
+
if (await this.readAnySnapshot(runId)) {
|
|
2658
|
+
throw new RunStoreError("run_exists", `run ${runId} already exists`);
|
|
2659
|
+
}
|
|
2660
|
+
const now = new Date().toISOString();
|
|
2661
|
+
const createdEvent = {
|
|
2662
|
+
event_id: randomUUID(),
|
|
2663
|
+
run_id: runId,
|
|
2664
|
+
step_id: null,
|
|
2665
|
+
activation_id: null,
|
|
2666
|
+
attempt_id: null,
|
|
2667
|
+
result_id: null,
|
|
2668
|
+
sequence: 1,
|
|
2669
|
+
idempotency_key: `run:${runId}:created`,
|
|
2670
|
+
type: "run_created",
|
|
2671
|
+
occurred_at: input.run.created_at,
|
|
2672
|
+
payload: {
|
|
2673
|
+
plan_id: input.run.plan_id,
|
|
2674
|
+
run_binding_id: input.run.run_binding.binding_id,
|
|
2675
|
+
},
|
|
2676
|
+
};
|
|
2677
|
+
const snapshot = {
|
|
2678
|
+
schema_version: CANONICAL_RUN_STORE_SCHEMA_VERSION,
|
|
2679
|
+
record_version: 1,
|
|
2680
|
+
run: input.run,
|
|
2681
|
+
activations: [...(input.activations ?? [])],
|
|
2682
|
+
attempts: [],
|
|
2683
|
+
claims: [],
|
|
2684
|
+
events: [createdEvent],
|
|
2685
|
+
result_refs: [],
|
|
2686
|
+
created_at: now,
|
|
2687
|
+
updated_at: now,
|
|
2688
|
+
};
|
|
2689
|
+
validateCanonicalCollections(snapshot);
|
|
2690
|
+
if (!await this.commitSnapshot(snapshot)) {
|
|
2691
|
+
throw new RunStoreError("run_exists", `run ${runId} already exists`);
|
|
2692
|
+
}
|
|
2693
|
+
return snapshot;
|
|
636
2694
|
}
|
|
637
2695
|
async loadRun(runId) {
|
|
638
2696
|
assertString(runId, "run_id");
|
|
@@ -645,13 +2703,238 @@ export class DurableRunStore {
|
|
|
645
2703
|
}
|
|
646
2704
|
return snapshot;
|
|
647
2705
|
}
|
|
2706
|
+
async cancelLegacyRun(runId, reason = null) {
|
|
2707
|
+
assertString(runId, "run_id");
|
|
2708
|
+
if (reason !== null && (typeof reason !== "string" || !reason.trim())) {
|
|
2709
|
+
throw new RunStoreError("invalid_record", "legacy cancellation reason must be a non-empty string or null");
|
|
2710
|
+
}
|
|
2711
|
+
while (true) {
|
|
2712
|
+
const snapshot = await this.requireRun(runId);
|
|
2713
|
+
if (snapshot.run.terminal !== null)
|
|
2714
|
+
return snapshot;
|
|
2715
|
+
const now = new Date().toISOString();
|
|
2716
|
+
const cancellation = snapshot.run.cancellation ?? {
|
|
2717
|
+
requested_at: now,
|
|
2718
|
+
reason,
|
|
2719
|
+
};
|
|
2720
|
+
const tasks = snapshot.tasks
|
|
2721
|
+
.filter((task) => [
|
|
2722
|
+
"dependency_blocked",
|
|
2723
|
+
"ready",
|
|
2724
|
+
"round_succeeded",
|
|
2725
|
+
].includes(task.state))
|
|
2726
|
+
.map((task) => ({
|
|
2727
|
+
...task,
|
|
2728
|
+
state: "cancelled",
|
|
2729
|
+
updated_at: now,
|
|
2730
|
+
}));
|
|
2731
|
+
const eventInputs = [];
|
|
2732
|
+
if (snapshot.run.status !== "cancelling") {
|
|
2733
|
+
eventInputs.push({
|
|
2734
|
+
run_id: runId,
|
|
2735
|
+
step_id: null,
|
|
2736
|
+
attempt_id: null,
|
|
2737
|
+
worker_session_id: null,
|
|
2738
|
+
result_id: null,
|
|
2739
|
+
idempotency_key: `run:${runId}:cancel-requested`,
|
|
2740
|
+
type: "run_cancel_requested",
|
|
2741
|
+
occurred_at: cancellation.requested_at,
|
|
2742
|
+
payload: { reason: cancellation.reason },
|
|
2743
|
+
});
|
|
2744
|
+
}
|
|
2745
|
+
for (const task of tasks) {
|
|
2746
|
+
const prior = snapshot.tasks.find((item) => item.step_id === task.step_id);
|
|
2747
|
+
eventInputs.push({
|
|
2748
|
+
run_id: runId,
|
|
2749
|
+
step_id: task.step_id,
|
|
2750
|
+
attempt_id: null,
|
|
2751
|
+
worker_session_id: null,
|
|
2752
|
+
result_id: null,
|
|
2753
|
+
idempotency_key: `task:${task.step_id}:cancelled`,
|
|
2754
|
+
type: "task_state_changed",
|
|
2755
|
+
occurred_at: now,
|
|
2756
|
+
payload: { from: prior.state, to: "cancelled" },
|
|
2757
|
+
});
|
|
2758
|
+
}
|
|
2759
|
+
const run = {
|
|
2760
|
+
...snapshot.run,
|
|
2761
|
+
status: "cancelling",
|
|
2762
|
+
cancellation,
|
|
2763
|
+
updated_at: now,
|
|
2764
|
+
};
|
|
2765
|
+
try {
|
|
2766
|
+
return (await this.updateRun({
|
|
2767
|
+
run_id: runId,
|
|
2768
|
+
expected_record_version: snapshot.record_version,
|
|
2769
|
+
run,
|
|
2770
|
+
tasks,
|
|
2771
|
+
events: eventInputs.map((input, index) => (legacyEvent(snapshot, index, input))),
|
|
2772
|
+
})).snapshot;
|
|
2773
|
+
}
|
|
2774
|
+
catch (error) {
|
|
2775
|
+
if (error instanceof RunStoreError && error.code === "version_conflict") {
|
|
2776
|
+
continue;
|
|
2777
|
+
}
|
|
2778
|
+
throw error;
|
|
2779
|
+
}
|
|
2780
|
+
}
|
|
2781
|
+
}
|
|
2782
|
+
async closeLegacyRun(runId, reason = null) {
|
|
2783
|
+
assertString(runId, "run_id");
|
|
2784
|
+
if (reason !== null && (typeof reason !== "string" || !reason.trim())) {
|
|
2785
|
+
throw new RunStoreError("invalid_record", "legacy close reason must be a non-empty string or null");
|
|
2786
|
+
}
|
|
2787
|
+
while (true) {
|
|
2788
|
+
const snapshot = await this.requireRun(runId);
|
|
2789
|
+
if (snapshot.run.terminal !== null)
|
|
2790
|
+
return snapshot;
|
|
2791
|
+
const now = new Date().toISOString();
|
|
2792
|
+
const terminalReason = reason
|
|
2793
|
+
?? snapshot.run.cancellation?.reason
|
|
2794
|
+
?? "legacy run closed";
|
|
2795
|
+
const cancellation = snapshot.run.cancellation ?? {
|
|
2796
|
+
requested_at: now,
|
|
2797
|
+
reason: terminalReason,
|
|
2798
|
+
};
|
|
2799
|
+
const tasks = snapshot.tasks
|
|
2800
|
+
.filter((task) => ![
|
|
2801
|
+
"succeeded",
|
|
2802
|
+
"failed",
|
|
2803
|
+
"skipped",
|
|
2804
|
+
"cancelled",
|
|
2805
|
+
].includes(task.state))
|
|
2806
|
+
.map((task) => ({
|
|
2807
|
+
...task,
|
|
2808
|
+
state: "cancelled",
|
|
2809
|
+
updated_at: now,
|
|
2810
|
+
}));
|
|
2811
|
+
const attempts = snapshot.attempts
|
|
2812
|
+
.filter((attempt) => attempt.phase !== "terminal")
|
|
2813
|
+
.map((attempt) => ({
|
|
2814
|
+
...attempt,
|
|
2815
|
+
phase: "terminal",
|
|
2816
|
+
completed_at: now,
|
|
2817
|
+
terminal_outcome: {
|
|
2818
|
+
kind: "cancelled",
|
|
2819
|
+
exit_code: null,
|
|
2820
|
+
signal: null,
|
|
2821
|
+
reason: terminalReason,
|
|
2822
|
+
},
|
|
2823
|
+
}));
|
|
2824
|
+
const eventInputs = [];
|
|
2825
|
+
if (snapshot.run.cancellation === null) {
|
|
2826
|
+
eventInputs.push({
|
|
2827
|
+
run_id: runId,
|
|
2828
|
+
step_id: null,
|
|
2829
|
+
attempt_id: null,
|
|
2830
|
+
worker_session_id: null,
|
|
2831
|
+
result_id: null,
|
|
2832
|
+
idempotency_key: `run:${runId}:cancel-requested`,
|
|
2833
|
+
type: "run_cancel_requested",
|
|
2834
|
+
occurred_at: cancellation.requested_at,
|
|
2835
|
+
payload: { reason: cancellation.reason },
|
|
2836
|
+
});
|
|
2837
|
+
}
|
|
2838
|
+
for (const task of tasks) {
|
|
2839
|
+
const prior = snapshot.tasks.find((item) => item.step_id === task.step_id);
|
|
2840
|
+
eventInputs.push({
|
|
2841
|
+
run_id: runId,
|
|
2842
|
+
step_id: task.step_id,
|
|
2843
|
+
attempt_id: null,
|
|
2844
|
+
worker_session_id: null,
|
|
2845
|
+
result_id: null,
|
|
2846
|
+
idempotency_key: `task:${task.step_id}:closed`,
|
|
2847
|
+
type: "task_state_changed",
|
|
2848
|
+
occurred_at: now,
|
|
2849
|
+
payload: { from: prior.state, to: "cancelled", close: true },
|
|
2850
|
+
});
|
|
2851
|
+
}
|
|
2852
|
+
for (const attempt of attempts) {
|
|
2853
|
+
eventInputs.push({
|
|
2854
|
+
run_id: runId,
|
|
2855
|
+
step_id: attempt.step_id,
|
|
2856
|
+
attempt_id: attempt.attempt_id,
|
|
2857
|
+
worker_session_id: attempt.worker_session_id,
|
|
2858
|
+
result_id: null,
|
|
2859
|
+
idempotency_key: `attempt:${attempt.attempt_id}:closed`,
|
|
2860
|
+
type: "attempt_terminal",
|
|
2861
|
+
occurred_at: now,
|
|
2862
|
+
payload: { outcome: "cancelled", reason: terminalReason, close: true },
|
|
2863
|
+
});
|
|
2864
|
+
}
|
|
2865
|
+
eventInputs.push({
|
|
2866
|
+
run_id: runId,
|
|
2867
|
+
step_id: null,
|
|
2868
|
+
attempt_id: null,
|
|
2869
|
+
worker_session_id: null,
|
|
2870
|
+
result_id: null,
|
|
2871
|
+
idempotency_key: `run:${runId}:terminal:cancelled`,
|
|
2872
|
+
type: "run_terminal",
|
|
2873
|
+
occurred_at: now,
|
|
2874
|
+
payload: { status: "cancelled", reason: terminalReason, close: true },
|
|
2875
|
+
});
|
|
2876
|
+
const run = {
|
|
2877
|
+
...snapshot.run,
|
|
2878
|
+
status: "cancelled",
|
|
2879
|
+
cancellation,
|
|
2880
|
+
terminal: {
|
|
2881
|
+
status: "cancelled",
|
|
2882
|
+
reason: terminalReason,
|
|
2883
|
+
completed_at: now,
|
|
2884
|
+
},
|
|
2885
|
+
updated_at: now,
|
|
2886
|
+
};
|
|
2887
|
+
try {
|
|
2888
|
+
return (await this.updateRun({
|
|
2889
|
+
run_id: runId,
|
|
2890
|
+
expected_record_version: snapshot.record_version,
|
|
2891
|
+
run,
|
|
2892
|
+
tasks,
|
|
2893
|
+
attempts,
|
|
2894
|
+
events: eventInputs.map((input, index) => (legacyEvent(snapshot, index, input))),
|
|
2895
|
+
})).snapshot;
|
|
2896
|
+
}
|
|
2897
|
+
catch (error) {
|
|
2898
|
+
if (error instanceof RunStoreError && error.code === "version_conflict") {
|
|
2899
|
+
continue;
|
|
2900
|
+
}
|
|
2901
|
+
throw error;
|
|
2902
|
+
}
|
|
2903
|
+
}
|
|
2904
|
+
}
|
|
2905
|
+
async loadAnyRun(runId) {
|
|
2906
|
+
assertString(runId, "run_id");
|
|
2907
|
+
return this.readAnySnapshot(runId);
|
|
2908
|
+
}
|
|
2909
|
+
async requireAnyRun(runId) {
|
|
2910
|
+
const snapshot = await this.loadAnyRun(runId);
|
|
2911
|
+
if (!snapshot) {
|
|
2912
|
+
throw new RunStoreError("run_not_found", `run ${runId} was not found`);
|
|
2913
|
+
}
|
|
2914
|
+
return snapshot;
|
|
2915
|
+
}
|
|
2916
|
+
async loadCanonicalRun(runId) {
|
|
2917
|
+
const snapshot = await this.loadAnyRun(runId);
|
|
2918
|
+
if (snapshot !== null
|
|
2919
|
+
&& snapshot.schema_version === LEGACY_RUN_STORE_SCHEMA_VERSION) {
|
|
2920
|
+
throw new RunStoreError("unsupported_legacy_run", `legacy run ${runId} cannot be loaded as a canonical run`);
|
|
2921
|
+
}
|
|
2922
|
+
return snapshot;
|
|
2923
|
+
}
|
|
2924
|
+
async requireCanonicalRun(runId) {
|
|
2925
|
+
const snapshot = await this.loadCanonicalRun(runId);
|
|
2926
|
+
if (!snapshot) {
|
|
2927
|
+
throw new RunStoreError("run_not_found", `run ${runId} was not found`);
|
|
2928
|
+
}
|
|
2929
|
+
return snapshot;
|
|
2930
|
+
}
|
|
648
2931
|
async updateRun(update) {
|
|
649
2932
|
assertString(update.run_id, "run_id");
|
|
650
2933
|
if (!Number.isSafeInteger(update.expected_record_version) ||
|
|
651
2934
|
update.expected_record_version < 1) {
|
|
652
2935
|
throw new RunStoreError("invalid_record", "expected_record_version must be a positive safe integer");
|
|
653
2936
|
}
|
|
654
|
-
|
|
2937
|
+
while (true) {
|
|
655
2938
|
const current = await this.readSnapshot(update.run_id);
|
|
656
2939
|
if (!current) {
|
|
657
2940
|
throw new RunStoreError("run_not_found", `run ${update.run_id} was not found`);
|
|
@@ -677,21 +2960,59 @@ export class DurableRunStore {
|
|
|
677
2960
|
if (current.record_version !== update.expected_record_version) {
|
|
678
2961
|
throw new RunStoreError("version_conflict", `run ${update.run_id} is at version ${current.record_version}, expected ${update.expected_record_version}`);
|
|
679
2962
|
}
|
|
680
|
-
await
|
|
681
|
-
|
|
2963
|
+
if (!await this.commitSnapshot(applied.snapshot))
|
|
2964
|
+
continue;
|
|
682
2965
|
return {
|
|
683
2966
|
snapshot: applied.snapshot,
|
|
684
2967
|
applied: true,
|
|
685
2968
|
duplicate_event_ids: applied.duplicate_event_ids,
|
|
686
2969
|
duplicate_result_ids: applied.duplicate_result_ids,
|
|
687
2970
|
};
|
|
688
|
-
}
|
|
2971
|
+
}
|
|
689
2972
|
}
|
|
690
|
-
|
|
691
|
-
|
|
692
|
-
|
|
693
|
-
|
|
694
|
-
|
|
2973
|
+
async updateCanonicalRun(update) {
|
|
2974
|
+
assertString(update.run_id, "run_id");
|
|
2975
|
+
assertPositiveSafeInteger(update.expected_record_version, "expected_record_version");
|
|
2976
|
+
while (true) {
|
|
2977
|
+
const anyCurrent = await this.readAnySnapshot(update.run_id);
|
|
2978
|
+
if (!anyCurrent) {
|
|
2979
|
+
throw new RunStoreError("run_not_found", `run ${update.run_id} was not found`);
|
|
2980
|
+
}
|
|
2981
|
+
if (anyCurrent.schema_version === LEGACY_RUN_STORE_SCHEMA_VERSION) {
|
|
2982
|
+
throw new RunStoreError("unsupported_legacy_run", `legacy run ${update.run_id} cannot be updated by the canonical store`);
|
|
2983
|
+
}
|
|
2984
|
+
let applied;
|
|
2985
|
+
try {
|
|
2986
|
+
applied = applyCanonicalUpdate(anyCurrent, update);
|
|
2987
|
+
}
|
|
2988
|
+
catch (error) {
|
|
2989
|
+
if (anyCurrent.record_version !== update.expected_record_version) {
|
|
2990
|
+
throw new RunStoreError("version_conflict", `run ${update.run_id} is at version ${anyCurrent.record_version}, expected ${update.expected_record_version}`);
|
|
2991
|
+
}
|
|
2992
|
+
throw error;
|
|
2993
|
+
}
|
|
2994
|
+
if (!applied.changed) {
|
|
2995
|
+
return {
|
|
2996
|
+
snapshot: anyCurrent,
|
|
2997
|
+
applied: false,
|
|
2998
|
+
duplicate_event_ids: applied.duplicate_event_ids,
|
|
2999
|
+
duplicate_result_ids: applied.duplicate_result_ids,
|
|
3000
|
+
};
|
|
3001
|
+
}
|
|
3002
|
+
if (anyCurrent.record_version !== update.expected_record_version) {
|
|
3003
|
+
throw new RunStoreError("version_conflict", `run ${update.run_id} is at version ${anyCurrent.record_version}, expected ${update.expected_record_version}`);
|
|
3004
|
+
}
|
|
3005
|
+
if (!await this.commitSnapshot(applied.snapshot))
|
|
3006
|
+
continue;
|
|
3007
|
+
return {
|
|
3008
|
+
snapshot: applied.snapshot,
|
|
3009
|
+
applied: true,
|
|
3010
|
+
duplicate_event_ids: applied.duplicate_event_ids,
|
|
3011
|
+
duplicate_result_ids: applied.duplicate_result_ids,
|
|
3012
|
+
};
|
|
3013
|
+
}
|
|
3014
|
+
}
|
|
3015
|
+
async loadAllAnyRuns() {
|
|
695
3016
|
const runsDirectory = join(this.options.root_directory, "runs");
|
|
696
3017
|
let entries;
|
|
697
3018
|
try {
|
|
@@ -706,10 +3027,21 @@ export class DurableRunStore {
|
|
|
706
3027
|
for (const entry of entries.sort((left, right) => left.name.localeCompare(right.name))) {
|
|
707
3028
|
if (!entry.isDirectory())
|
|
708
3029
|
continue;
|
|
709
|
-
const
|
|
3030
|
+
const runDirectory = join(runsDirectory, entry.name);
|
|
3031
|
+
const committed = await this.readCommittedHeadFromDirectory(runDirectory);
|
|
3032
|
+
if (committed !== null) {
|
|
3033
|
+
if (runStorageKey(committed.run.run_id) !== entry.name) {
|
|
3034
|
+
throw new RunStoreError("invalid_record", `run directory ${runDirectory} does not match run ${committed.run.run_id}`);
|
|
3035
|
+
}
|
|
3036
|
+
const projected = await this.projectCommittedHead(committed.run.run_id);
|
|
3037
|
+
if (projected !== null)
|
|
3038
|
+
snapshots.push(projected);
|
|
3039
|
+
continue;
|
|
3040
|
+
}
|
|
3041
|
+
const ledgerPath = join(runDirectory, LEDGER_FILE_NAME);
|
|
710
3042
|
try {
|
|
711
3043
|
const raw = await readFile(ledgerPath, "utf8");
|
|
712
|
-
snapshots.push(
|
|
3044
|
+
snapshots.push(assertAnySnapshot(JSON.parse(raw)));
|
|
713
3045
|
}
|
|
714
3046
|
catch (error) {
|
|
715
3047
|
if (isErrno(error, "ENOENT"))
|
|
@@ -722,4 +3054,14 @@ export class DurableRunStore {
|
|
|
722
3054
|
}
|
|
723
3055
|
return snapshots;
|
|
724
3056
|
}
|
|
3057
|
+
/**
|
|
3058
|
+
* Reads every valid legacy ledger after restart while allowing canonical
|
|
3059
|
+
* ledgers to coexist in the same durable root.
|
|
3060
|
+
*/
|
|
3061
|
+
async loadAllRuns() {
|
|
3062
|
+
return (await this.loadAllAnyRuns()).filter((snapshot) => (snapshot.schema_version === LEGACY_RUN_STORE_SCHEMA_VERSION));
|
|
3063
|
+
}
|
|
3064
|
+
async loadAllCanonicalRuns() {
|
|
3065
|
+
return (await this.loadAllAnyRuns()).filter((snapshot) => (snapshot.schema_version === CANONICAL_RUN_STORE_SCHEMA_VERSION));
|
|
3066
|
+
}
|
|
725
3067
|
}
|