intentdna 1.8.6 → 1.8.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +2 -2
- package/.claude-plugin/plugin.json +1 -1
- package/README.md +1 -0
- package/dist/cli/commands/run-lifecycle.d.ts +73 -0
- package/dist/cli/commands/run-lifecycle.js +240 -0
- package/dist/cli/commands/run.d.ts +22 -40
- package/dist/cli/commands/run.js +674 -392
- package/dist/cli/index.js +87 -2
- package/dist/compiler/workflow.js +3 -2
- package/dist/hooks/cli.d.ts +1 -2
- package/dist/hooks/cli.js +119 -80
- package/dist/hooks/enforce.d.ts +2 -0
- package/dist/hooks/enforce.js +56 -27
- package/dist/hooks/enforcement-boundary.d.ts +13 -0
- package/dist/hooks/enforcement-boundary.js +33 -0
- package/dist/hooks/index.d.ts +3 -2
- package/dist/hooks/index.js +3 -2
- package/dist/hooks/protocol.d.ts +12 -4
- package/dist/hooks/protocol.js +20 -14
- package/dist/hooks/schema.d.ts +2 -1
- package/dist/hooks/schema.js +6 -2
- package/dist/hooks/state-manager.d.ts +5 -5
- package/dist/hooks/state-manager.js +26 -24
- package/dist/hooks/state.d.ts +19 -3
- package/dist/hooks/state.js +327 -80
- package/dist/mcp/index.js +0 -0
- package/dist/runtime/diagnosis-contract-verifier.d.ts +11 -0
- package/dist/runtime/diagnosis-contract-verifier.js +417 -0
- package/dist/runtime/execution-provider.d.ts +40 -0
- package/dist/runtime/execution-provider.js +138 -0
- package/dist/runtime/handoff-resolver.d.ts +61 -0
- package/dist/runtime/handoff-resolver.js +167 -0
- package/dist/runtime/index.d.ts +24 -0
- package/dist/runtime/index.js +13 -0
- package/dist/runtime/process-tree.d.ts +47 -0
- package/dist/runtime/process-tree.js +402 -0
- package/dist/runtime/providers/claude.d.ts +9 -0
- package/dist/runtime/providers/claude.js +64 -0
- package/dist/runtime/providers/codex.d.ts +8 -0
- package/dist/runtime/providers/codex.js +72 -0
- package/dist/runtime/result-store.d.ts +32 -0
- package/dist/runtime/result-store.js +130 -0
- package/dist/runtime/run-contracts.d.ts +290 -0
- package/dist/runtime/run-contracts.js +58 -0
- package/dist/runtime/run-controller.d.ts +149 -0
- package/dist/runtime/run-controller.js +1108 -0
- package/dist/runtime/run-store.d.ts +96 -0
- package/dist/runtime/run-store.js +725 -0
- package/dist/runtime/worker-executor.d.ts +19 -0
- package/dist/runtime/worker-executor.js +194 -0
- package/dist/runtime/workflow-plan-adapter.d.ts +26 -0
- package/dist/runtime/workflow-plan-adapter.js +416 -0
- package/dist/runtime/workflow-runner.d.ts +15 -3
- package/dist/runtime/workflow-runner.js +13 -1
- package/dist/runtime/workspace-isolation.d.ts +103 -0
- package/dist/runtime/workspace-isolation.js +373 -0
- package/dist/schema/types.d.ts +1 -0
- package/dist/schema/validate.js +64 -6
- package/dist/schema/yaml-parser.js +7 -2
- package/package.json +1 -1
|
@@ -0,0 +1,725 @@
|
|
|
1
|
+
import { createHash, randomUUID } from "node:crypto";
|
|
2
|
+
import { mkdir, open, readFile, readdir, rename, rm, stat, utimes, } from "node:fs/promises";
|
|
3
|
+
import { dirname, join } from "node:path";
|
|
4
|
+
export const RUN_STORE_SCHEMA_VERSION = 1;
|
|
5
|
+
export const RUN_STATUS_VALUES = [
|
|
6
|
+
"active",
|
|
7
|
+
"cancelling",
|
|
8
|
+
"succeeded",
|
|
9
|
+
"failed",
|
|
10
|
+
"cancelled",
|
|
11
|
+
];
|
|
12
|
+
export class RunStoreError extends Error {
|
|
13
|
+
code;
|
|
14
|
+
constructor(code, message) {
|
|
15
|
+
super(message);
|
|
16
|
+
this.name = "RunStoreError";
|
|
17
|
+
this.code = code;
|
|
18
|
+
}
|
|
19
|
+
}
|
|
20
|
+
const DEFAULT_LOCK_WAIT_MS = 5_000;
|
|
21
|
+
const DEFAULT_LOCK_STALE_MS = 5 * 60_000;
|
|
22
|
+
const DEFAULT_LOCK_RETRY_MS = 25;
|
|
23
|
+
const LEDGER_FILE_NAME = "ledger.json";
|
|
24
|
+
const LOCK_OWNER_FILE_NAME = "owner";
|
|
25
|
+
const LOCK_LEASE_FILE_NAME = "lease";
|
|
26
|
+
const DIRECTORY_MODE = 0o700;
|
|
27
|
+
const FILE_MODE = 0o600;
|
|
28
|
+
function assertPositiveInteger(name, value) {
|
|
29
|
+
if (!Number.isSafeInteger(value) || value <= 0) {
|
|
30
|
+
throw new RunStoreError("invalid_options", `${name} must be a positive safe integer`);
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
function resolveOptions(options) {
|
|
34
|
+
if (!options.root_directory.trim()) {
|
|
35
|
+
throw new RunStoreError("invalid_options", "root_directory must not be empty");
|
|
36
|
+
}
|
|
37
|
+
const lockStaleMs = options.lock_stale_ms ?? DEFAULT_LOCK_STALE_MS;
|
|
38
|
+
const lockHeartbeatMs = options.lock_heartbeat_ms ?? Math.max(1, Math.floor(lockStaleMs / 3));
|
|
39
|
+
const resolved = {
|
|
40
|
+
root_directory: options.root_directory,
|
|
41
|
+
lock_wait_ms: options.lock_wait_ms ?? DEFAULT_LOCK_WAIT_MS,
|
|
42
|
+
lock_stale_ms: lockStaleMs,
|
|
43
|
+
lock_retry_ms: options.lock_retry_ms ?? DEFAULT_LOCK_RETRY_MS,
|
|
44
|
+
lock_heartbeat_ms: lockHeartbeatMs,
|
|
45
|
+
};
|
|
46
|
+
assertPositiveInteger("lock_wait_ms", resolved.lock_wait_ms);
|
|
47
|
+
assertPositiveInteger("lock_stale_ms", resolved.lock_stale_ms);
|
|
48
|
+
assertPositiveInteger("lock_retry_ms", resolved.lock_retry_ms);
|
|
49
|
+
assertPositiveInteger("lock_heartbeat_ms", resolved.lock_heartbeat_ms);
|
|
50
|
+
if (resolved.lock_heartbeat_ms >= resolved.lock_stale_ms) {
|
|
51
|
+
throw new RunStoreError("invalid_options", "lock_heartbeat_ms must be less than lock_stale_ms");
|
|
52
|
+
}
|
|
53
|
+
return resolved;
|
|
54
|
+
}
|
|
55
|
+
function runStorageKey(runId) {
|
|
56
|
+
return createHash("sha256").update(runId, "utf8").digest("hex");
|
|
57
|
+
}
|
|
58
|
+
function isErrno(error, code) {
|
|
59
|
+
return (error instanceof Error &&
|
|
60
|
+
"code" in error &&
|
|
61
|
+
error.code === code);
|
|
62
|
+
}
|
|
63
|
+
function sleep(ms) {
|
|
64
|
+
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
65
|
+
}
|
|
66
|
+
function stableJson(value) {
|
|
67
|
+
if (Array.isArray(value)) {
|
|
68
|
+
return `[${value.map(stableJson).join(",")}]`;
|
|
69
|
+
}
|
|
70
|
+
if (value !== null && typeof value === "object") {
|
|
71
|
+
const entries = Object.entries(value)
|
|
72
|
+
.filter(([, entry]) => entry !== undefined)
|
|
73
|
+
.sort(([left], [right]) => left.localeCompare(right));
|
|
74
|
+
return `{${entries.map(([key, entry]) => `${JSON.stringify(key)}:${stableJson(entry)}`).join(",")}}`;
|
|
75
|
+
}
|
|
76
|
+
return JSON.stringify(value) ?? "null";
|
|
77
|
+
}
|
|
78
|
+
function recordsEqual(left, right) {
|
|
79
|
+
return stableJson(left) === stableJson(right);
|
|
80
|
+
}
|
|
81
|
+
function assertString(value, label) {
|
|
82
|
+
if (typeof value !== "string" || value.length === 0) {
|
|
83
|
+
throw new RunStoreError("invalid_record", `${label} must be a non-empty string`);
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
function assertRunRecord(record, expectedRunId) {
|
|
87
|
+
assertString(record.run_id, "run.run_id");
|
|
88
|
+
assertString(record.workflow_id, "run.workflow_id");
|
|
89
|
+
assertString(record.plan_digest, "run.plan_digest");
|
|
90
|
+
assertString(record.created_at, "run.created_at");
|
|
91
|
+
assertString(record.updated_at, "run.updated_at");
|
|
92
|
+
if (expectedRunId !== undefined && record.run_id !== expectedRunId) {
|
|
93
|
+
throw new RunStoreError("identity_conflict", `run record ${record.run_id} does not match ${expectedRunId}`);
|
|
94
|
+
}
|
|
95
|
+
if (!RUN_STATUS_VALUES.includes(record.status)) {
|
|
96
|
+
throw new RunStoreError("invalid_record", `unsupported run status: ${String(record.status)}`);
|
|
97
|
+
}
|
|
98
|
+
if (record.status === "active" && record.terminal !== null) {
|
|
99
|
+
throw new RunStoreError("invalid_record", "an active run cannot contain a terminal record");
|
|
100
|
+
}
|
|
101
|
+
if (["succeeded", "failed", "cancelled"].includes(record.status) &&
|
|
102
|
+
record.terminal === null) {
|
|
103
|
+
throw new RunStoreError("invalid_record", `terminal run status ${record.status} requires a terminal record`);
|
|
104
|
+
}
|
|
105
|
+
if (record.terminal !== null &&
|
|
106
|
+
(record.status !== record.terminal.status ||
|
|
107
|
+
!["succeeded", "failed", "cancelled"].includes(record.status))) {
|
|
108
|
+
throw new RunStoreError("invalid_record", "run status and terminal status must agree");
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
function assertCorrelatedRunId(actual, expected, label) {
|
|
112
|
+
if (actual !== expected) {
|
|
113
|
+
throw new RunStoreError("identity_conflict", `${label} belongs to run ${actual}, expected ${expected}`);
|
|
114
|
+
}
|
|
115
|
+
}
|
|
116
|
+
function assertSnapshot(value, expectedRunId) {
|
|
117
|
+
if (value === null || typeof value !== "object" || Array.isArray(value)) {
|
|
118
|
+
throw new RunStoreError("invalid_record", "run ledger must be a JSON object");
|
|
119
|
+
}
|
|
120
|
+
const candidate = value;
|
|
121
|
+
if (candidate.schema_version !== RUN_STORE_SCHEMA_VERSION) {
|
|
122
|
+
throw new RunStoreError("unsupported_schema", `unsupported run-store schema version: ${String(candidate.schema_version)}`);
|
|
123
|
+
}
|
|
124
|
+
if (!Number.isSafeInteger(candidate.record_version) ||
|
|
125
|
+
(candidate.record_version ?? 0) < 1) {
|
|
126
|
+
throw new RunStoreError("invalid_record", "record_version must be a positive safe integer");
|
|
127
|
+
}
|
|
128
|
+
if (!candidate.run ||
|
|
129
|
+
!Array.isArray(candidate.tasks) ||
|
|
130
|
+
!Array.isArray(candidate.attempts) ||
|
|
131
|
+
!Array.isArray(candidate.claims) ||
|
|
132
|
+
!Array.isArray(candidate.events) ||
|
|
133
|
+
!Array.isArray(candidate.results)) {
|
|
134
|
+
throw new RunStoreError("invalid_record", "run ledger is missing a required record collection");
|
|
135
|
+
}
|
|
136
|
+
assertString(candidate.created_at, "ledger.created_at");
|
|
137
|
+
assertString(candidate.updated_at, "ledger.updated_at");
|
|
138
|
+
assertRunRecord(candidate.run, expectedRunId);
|
|
139
|
+
validateCollections(candidate);
|
|
140
|
+
return candidate;
|
|
141
|
+
}
|
|
142
|
+
function validateCollections(snapshot) {
|
|
143
|
+
const runId = snapshot.run.run_id;
|
|
144
|
+
const taskByStep = new Map();
|
|
145
|
+
for (const task of snapshot.tasks) {
|
|
146
|
+
assertCorrelatedRunId(task.run_id, runId, "task");
|
|
147
|
+
if (taskByStep.has(task.step_id)) {
|
|
148
|
+
throw new RunStoreError("identity_conflict", `duplicate task for step ${task.step_id}`);
|
|
149
|
+
}
|
|
150
|
+
taskByStep.set(task.step_id, task);
|
|
151
|
+
}
|
|
152
|
+
const attemptById = new Map();
|
|
153
|
+
const attemptByWorker = new Map();
|
|
154
|
+
for (const attempt of snapshot.attempts) {
|
|
155
|
+
assertCorrelatedRunId(attempt.run_id, runId, "attempt");
|
|
156
|
+
if (attempt.process_id !== undefined
|
|
157
|
+
&& attempt.process_id !== null
|
|
158
|
+
&& (!Number.isSafeInteger(attempt.process_id)
|
|
159
|
+
|| attempt.process_id <= 0)) {
|
|
160
|
+
throw new RunStoreError("invalid_record", `attempt ${attempt.attempt_id} has an invalid process_id`);
|
|
161
|
+
}
|
|
162
|
+
if (attemptById.has(attempt.attempt_id)) {
|
|
163
|
+
throw new RunStoreError("identity_conflict", `duplicate attempt ${attempt.attempt_id}`);
|
|
164
|
+
}
|
|
165
|
+
if (attemptByWorker.has(attempt.worker_session_id)) {
|
|
166
|
+
throw new RunStoreError("identity_conflict", `worker session ${attempt.worker_session_id} is bound to multiple attempts`);
|
|
167
|
+
}
|
|
168
|
+
if (!taskByStep.has(attempt.step_id)) {
|
|
169
|
+
throw new RunStoreError("invalid_record", `attempt ${attempt.attempt_id} has no task`);
|
|
170
|
+
}
|
|
171
|
+
attemptById.set(attempt.attempt_id, attempt);
|
|
172
|
+
attemptByWorker.set(attempt.worker_session_id, attempt);
|
|
173
|
+
}
|
|
174
|
+
const claims = new Set();
|
|
175
|
+
for (const claim of snapshot.claims) {
|
|
176
|
+
assertCorrelatedRunId(claim.run_id, runId, "claim");
|
|
177
|
+
if (claims.has(claim.claim_token)) {
|
|
178
|
+
throw new RunStoreError("identity_conflict", `duplicate claim token ${claim.claim_token}`);
|
|
179
|
+
}
|
|
180
|
+
const attempt = attemptById.get(claim.attempt_id);
|
|
181
|
+
if (!attempt ||
|
|
182
|
+
attempt.step_id !== claim.step_id ||
|
|
183
|
+
attempt.worker_session_id !== claim.worker_session_id) {
|
|
184
|
+
throw new RunStoreError("invalid_record", `claim ${claim.claim_token} does not match its attempt`);
|
|
185
|
+
}
|
|
186
|
+
claims.add(claim.claim_token);
|
|
187
|
+
}
|
|
188
|
+
const eventIds = new Set();
|
|
189
|
+
const eventKeys = new Set();
|
|
190
|
+
let previousSequence = 0;
|
|
191
|
+
for (const event of snapshot.events) {
|
|
192
|
+
assertCorrelatedRunId(event.run_id, runId, "event");
|
|
193
|
+
if (eventIds.has(event.event_id) || eventKeys.has(event.idempotency_key)) {
|
|
194
|
+
throw new RunStoreError("event_conflict", `duplicate stored event ${event.event_id}`);
|
|
195
|
+
}
|
|
196
|
+
if (!Number.isSafeInteger(event.sequence) ||
|
|
197
|
+
event.sequence !== previousSequence + 1) {
|
|
198
|
+
throw new RunStoreError("invalid_record", `event ${event.event_id} has non-contiguous sequence ${event.sequence}`);
|
|
199
|
+
}
|
|
200
|
+
eventIds.add(event.event_id);
|
|
201
|
+
eventKeys.add(event.idempotency_key);
|
|
202
|
+
previousSequence = event.sequence;
|
|
203
|
+
}
|
|
204
|
+
const results = new Set();
|
|
205
|
+
const resultById = new Map();
|
|
206
|
+
const resultByAttempt = new Map();
|
|
207
|
+
for (const result of snapshot.results) {
|
|
208
|
+
assertCorrelatedRunId(result.run_id, runId, "result");
|
|
209
|
+
if (results.has(result.result_id)) {
|
|
210
|
+
throw new RunStoreError("result_conflict", `duplicate committed result ${result.result_id}`);
|
|
211
|
+
}
|
|
212
|
+
const attempt = attemptById.get(result.attempt_id);
|
|
213
|
+
if (!attempt ||
|
|
214
|
+
attempt.phase !== "terminal" ||
|
|
215
|
+
attempt.step_id !== result.step_id ||
|
|
216
|
+
attempt.worker_session_id !== result.worker_session_id) {
|
|
217
|
+
throw new RunStoreError("invalid_record", `result ${result.result_id} does not match a terminal attempt`);
|
|
218
|
+
}
|
|
219
|
+
if (resultByAttempt.has(result.attempt_id)) {
|
|
220
|
+
throw new RunStoreError("result_conflict", `attempt ${result.attempt_id} has multiple committed results`);
|
|
221
|
+
}
|
|
222
|
+
results.add(result.result_id);
|
|
223
|
+
resultById.set(result.result_id, result);
|
|
224
|
+
resultByAttempt.set(result.attempt_id, result);
|
|
225
|
+
}
|
|
226
|
+
for (const task of snapshot.tasks) {
|
|
227
|
+
if (task.current_attempt_id !== null &&
|
|
228
|
+
attemptById.get(task.current_attempt_id)?.step_id !== task.step_id) {
|
|
229
|
+
throw new RunStoreError("invalid_record", `task ${task.step_id} references an unknown current attempt`);
|
|
230
|
+
}
|
|
231
|
+
if (task.committed_result_id !== null &&
|
|
232
|
+
resultById.get(task.committed_result_id)?.step_id !== task.step_id) {
|
|
233
|
+
throw new RunStoreError("invalid_record", `task ${task.step_id} references an unknown committed result`);
|
|
234
|
+
}
|
|
235
|
+
}
|
|
236
|
+
}
|
|
237
|
+
function eventReplayEquivalent(existing, incoming) {
|
|
238
|
+
return recordsEqual({
|
|
239
|
+
run_id: existing.run_id,
|
|
240
|
+
step_id: existing.step_id,
|
|
241
|
+
attempt_id: existing.attempt_id,
|
|
242
|
+
worker_session_id: existing.worker_session_id,
|
|
243
|
+
result_id: existing.result_id,
|
|
244
|
+
idempotency_key: existing.idempotency_key,
|
|
245
|
+
type: existing.type,
|
|
246
|
+
payload: existing.payload,
|
|
247
|
+
}, {
|
|
248
|
+
run_id: incoming.run_id,
|
|
249
|
+
step_id: incoming.step_id,
|
|
250
|
+
attempt_id: incoming.attempt_id,
|
|
251
|
+
worker_session_id: incoming.worker_session_id,
|
|
252
|
+
result_id: incoming.result_id,
|
|
253
|
+
idempotency_key: incoming.idempotency_key,
|
|
254
|
+
type: incoming.type,
|
|
255
|
+
payload: incoming.payload,
|
|
256
|
+
});
|
|
257
|
+
}
|
|
258
|
+
function upsertRecords(existing, incoming, keyOf, assertMutable) {
|
|
259
|
+
const records = [...existing];
|
|
260
|
+
const indexes = new Map(records.map((record, index) => [keyOf(record), index]));
|
|
261
|
+
let changed = false;
|
|
262
|
+
for (const record of incoming) {
|
|
263
|
+
const key = keyOf(record);
|
|
264
|
+
const index = indexes.get(key);
|
|
265
|
+
if (index === undefined) {
|
|
266
|
+
indexes.set(key, records.length);
|
|
267
|
+
records.push(record);
|
|
268
|
+
changed = true;
|
|
269
|
+
continue;
|
|
270
|
+
}
|
|
271
|
+
const current = records[index];
|
|
272
|
+
if (recordsEqual(current, record))
|
|
273
|
+
continue;
|
|
274
|
+
assertMutable(current, record);
|
|
275
|
+
records[index] = record;
|
|
276
|
+
changed = true;
|
|
277
|
+
}
|
|
278
|
+
return { records, changed };
|
|
279
|
+
}
|
|
280
|
+
function applyUpdate(current, update) {
|
|
281
|
+
const runId = current.run.run_id;
|
|
282
|
+
assertCorrelatedRunId(update.run_id, runId, "update");
|
|
283
|
+
let changed = false;
|
|
284
|
+
let run = current.run;
|
|
285
|
+
if (update.run) {
|
|
286
|
+
assertRunRecord(update.run, runId);
|
|
287
|
+
if (!recordsEqual(run, update.run)) {
|
|
288
|
+
if (run.terminal !== null) {
|
|
289
|
+
throw new RunStoreError("immutable_record", `terminal run ${runId} cannot be changed`);
|
|
290
|
+
}
|
|
291
|
+
if (run.workflow_id !== update.run.workflow_id ||
|
|
292
|
+
run.workflow_version !== update.run.workflow_version ||
|
|
293
|
+
run.plan_digest !== update.run.plan_digest ||
|
|
294
|
+
run.created_at !== update.run.created_at) {
|
|
295
|
+
throw new RunStoreError("identity_conflict", `run ${runId} cannot change its plan binding or creation identity`);
|
|
296
|
+
}
|
|
297
|
+
run = update.run;
|
|
298
|
+
changed = true;
|
|
299
|
+
}
|
|
300
|
+
}
|
|
301
|
+
const tasks = upsertRecords(current.tasks, update.tasks ?? [], (task) => task.step_id, (existing, incoming) => {
|
|
302
|
+
if (existing.run_id !== incoming.run_id ||
|
|
303
|
+
existing.step_id !== incoming.step_id ||
|
|
304
|
+
existing.created_at !== incoming.created_at) {
|
|
305
|
+
throw new RunStoreError("identity_conflict", `task ${existing.step_id} cannot change its identity`);
|
|
306
|
+
}
|
|
307
|
+
if (["succeeded", "failed", "skipped", "cancelled"].includes(existing.state)) {
|
|
308
|
+
throw new RunStoreError("immutable_record", `terminal task ${existing.step_id} cannot be changed`);
|
|
309
|
+
}
|
|
310
|
+
});
|
|
311
|
+
changed ||= tasks.changed;
|
|
312
|
+
const attempts = upsertRecords(current.attempts, update.attempts ?? [], (attempt) => attempt.attempt_id, (existing, incoming) => {
|
|
313
|
+
if (existing.run_id !== incoming.run_id ||
|
|
314
|
+
existing.step_id !== incoming.step_id ||
|
|
315
|
+
existing.worker_session_id !== incoming.worker_session_id ||
|
|
316
|
+
existing.attempt_number !== incoming.attempt_number ||
|
|
317
|
+
existing.claim_token !== incoming.claim_token) {
|
|
318
|
+
throw new RunStoreError("identity_conflict", `attempt ${existing.attempt_id} cannot change its identity`);
|
|
319
|
+
}
|
|
320
|
+
if (existing.phase === "terminal") {
|
|
321
|
+
throw new RunStoreError("immutable_record", `terminal attempt ${existing.attempt_id} cannot be changed`);
|
|
322
|
+
}
|
|
323
|
+
});
|
|
324
|
+
changed ||= attempts.changed;
|
|
325
|
+
const claims = upsertRecords(current.claims, update.claims ?? [], (claim) => claim.claim_token, (existing, incoming) => {
|
|
326
|
+
if (existing.run_id !== incoming.run_id ||
|
|
327
|
+
existing.step_id !== incoming.step_id ||
|
|
328
|
+
existing.attempt_id !== incoming.attempt_id ||
|
|
329
|
+
existing.worker_session_id !== incoming.worker_session_id ||
|
|
330
|
+
existing.owner_id !== incoming.owner_id ||
|
|
331
|
+
existing.claimed_record_version !== incoming.claimed_record_version ||
|
|
332
|
+
existing.lease.acquired_at !== incoming.lease.acquired_at) {
|
|
333
|
+
throw new RunStoreError("identity_conflict", `claim ${existing.claim_token} cannot change its ownership identity`);
|
|
334
|
+
}
|
|
335
|
+
});
|
|
336
|
+
changed ||= claims.changed;
|
|
337
|
+
const events = [...current.events];
|
|
338
|
+
const eventById = new Map(events.map((event) => [event.event_id, event]));
|
|
339
|
+
const eventByKey = new Map(events.map((event) => [event.idempotency_key, event]));
|
|
340
|
+
const duplicateEventIds = [];
|
|
341
|
+
for (const event of update.events ?? []) {
|
|
342
|
+
assertCorrelatedRunId(event.run_id, runId, "event");
|
|
343
|
+
const byId = eventById.get(event.event_id);
|
|
344
|
+
const byKey = eventByKey.get(event.idempotency_key);
|
|
345
|
+
const existing = byId ?? byKey;
|
|
346
|
+
if (existing) {
|
|
347
|
+
if ((byId && byKey && byId.event_id !== byKey.event_id) ||
|
|
348
|
+
!eventReplayEquivalent(existing, event)) {
|
|
349
|
+
throw new RunStoreError("event_conflict", `event ${event.event_id} conflicts with a stored event identity`);
|
|
350
|
+
}
|
|
351
|
+
duplicateEventIds.push(existing.event_id);
|
|
352
|
+
continue;
|
|
353
|
+
}
|
|
354
|
+
const expectedSequence = events.length + 1;
|
|
355
|
+
if (event.sequence !== expectedSequence) {
|
|
356
|
+
throw new RunStoreError("event_conflict", `event ${event.event_id} sequence ${event.sequence} must be ${expectedSequence}`);
|
|
357
|
+
}
|
|
358
|
+
events.push(event);
|
|
359
|
+
eventById.set(event.event_id, event);
|
|
360
|
+
eventByKey.set(event.idempotency_key, event);
|
|
361
|
+
changed = true;
|
|
362
|
+
}
|
|
363
|
+
const results = [...current.results];
|
|
364
|
+
const resultById = new Map(results.map((result) => [result.result_id, result]));
|
|
365
|
+
const resultByAttempt = new Map(results.map((result) => [result.attempt_id, result]));
|
|
366
|
+
const duplicateResultIds = [];
|
|
367
|
+
for (const result of update.results ?? []) {
|
|
368
|
+
assertCorrelatedRunId(result.run_id, runId, "result");
|
|
369
|
+
const existing = resultById.get(result.result_id);
|
|
370
|
+
if (existing) {
|
|
371
|
+
if (!recordsEqual(existing, result)) {
|
|
372
|
+
throw new RunStoreError("result_conflict", `committed result ${result.result_id} is immutable`);
|
|
373
|
+
}
|
|
374
|
+
duplicateResultIds.push(existing.result_id);
|
|
375
|
+
continue;
|
|
376
|
+
}
|
|
377
|
+
const attemptResult = resultByAttempt.get(result.attempt_id);
|
|
378
|
+
if (attemptResult) {
|
|
379
|
+
throw new RunStoreError("result_conflict", `attempt ${result.attempt_id} already committed result ${attemptResult.result_id}`);
|
|
380
|
+
}
|
|
381
|
+
results.push(result);
|
|
382
|
+
resultById.set(result.result_id, result);
|
|
383
|
+
resultByAttempt.set(result.attempt_id, result);
|
|
384
|
+
changed = true;
|
|
385
|
+
}
|
|
386
|
+
const now = new Date().toISOString();
|
|
387
|
+
const candidate = {
|
|
388
|
+
schema_version: RUN_STORE_SCHEMA_VERSION,
|
|
389
|
+
record_version: changed
|
|
390
|
+
? current.record_version + 1
|
|
391
|
+
: current.record_version,
|
|
392
|
+
run,
|
|
393
|
+
tasks: tasks.records,
|
|
394
|
+
attempts: attempts.records,
|
|
395
|
+
claims: claims.records,
|
|
396
|
+
events,
|
|
397
|
+
results,
|
|
398
|
+
created_at: current.created_at,
|
|
399
|
+
updated_at: changed ? now : current.updated_at,
|
|
400
|
+
};
|
|
401
|
+
validateCollections(candidate);
|
|
402
|
+
return {
|
|
403
|
+
snapshot: candidate,
|
|
404
|
+
changed,
|
|
405
|
+
duplicate_event_ids: duplicateEventIds,
|
|
406
|
+
duplicate_result_ids: duplicateResultIds,
|
|
407
|
+
};
|
|
408
|
+
}
|
|
409
|
+
function isUnsupportedDirectorySync(error) {
|
|
410
|
+
if (!(error instanceof Error) || !("code" in error))
|
|
411
|
+
return false;
|
|
412
|
+
const code = error.code;
|
|
413
|
+
return (code === "EINVAL" ||
|
|
414
|
+
code === "ENOTSUP" ||
|
|
415
|
+
code === "EISDIR" ||
|
|
416
|
+
(code === "EPERM" && process.platform === "win32"));
|
|
417
|
+
}
|
|
418
|
+
async function syncParentDirectory(filePath) {
|
|
419
|
+
let handle;
|
|
420
|
+
try {
|
|
421
|
+
handle = await open(dirname(filePath), "r");
|
|
422
|
+
}
|
|
423
|
+
catch (error) {
|
|
424
|
+
if (isUnsupportedDirectorySync(error))
|
|
425
|
+
return;
|
|
426
|
+
throw error;
|
|
427
|
+
}
|
|
428
|
+
try {
|
|
429
|
+
await handle.sync();
|
|
430
|
+
}
|
|
431
|
+
catch (error) {
|
|
432
|
+
if (!isUnsupportedDirectorySync(error))
|
|
433
|
+
throw error;
|
|
434
|
+
}
|
|
435
|
+
finally {
|
|
436
|
+
await handle.close();
|
|
437
|
+
}
|
|
438
|
+
}
|
|
439
|
+
async function writeAtomic(filePath, data) {
|
|
440
|
+
await mkdir(dirname(filePath), { recursive: true, mode: DIRECTORY_MODE });
|
|
441
|
+
const temporaryPath = `${filePath}.tmp.${process.pid}.${randomUUID()}`;
|
|
442
|
+
let handle;
|
|
443
|
+
try {
|
|
444
|
+
handle = await open(temporaryPath, "wx", FILE_MODE);
|
|
445
|
+
await handle.writeFile(data, "utf8");
|
|
446
|
+
await handle.sync();
|
|
447
|
+
await handle.close();
|
|
448
|
+
handle = undefined;
|
|
449
|
+
await rename(temporaryPath, filePath);
|
|
450
|
+
await syncParentDirectory(filePath);
|
|
451
|
+
}
|
|
452
|
+
catch (error) {
|
|
453
|
+
await handle?.close().catch(() => undefined);
|
|
454
|
+
await rm(temporaryPath, { force: true }).catch(() => undefined);
|
|
455
|
+
throw error;
|
|
456
|
+
}
|
|
457
|
+
}
|
|
458
|
+
async function readLockOwner(ownerPath) {
|
|
459
|
+
try {
|
|
460
|
+
return (await readFile(ownerPath, "utf8")).trim();
|
|
461
|
+
}
|
|
462
|
+
catch (error) {
|
|
463
|
+
if (isErrno(error, "ENOENT"))
|
|
464
|
+
return null;
|
|
465
|
+
throw error;
|
|
466
|
+
}
|
|
467
|
+
}
|
|
468
|
+
async function lockIsStale(lockDirectory, staleMs) {
|
|
469
|
+
try {
|
|
470
|
+
const lease = await stat(join(lockDirectory, LOCK_LEASE_FILE_NAME));
|
|
471
|
+
return Date.now() - lease.mtimeMs > staleMs;
|
|
472
|
+
}
|
|
473
|
+
catch (error) {
|
|
474
|
+
if (!isErrno(error, "ENOENT"))
|
|
475
|
+
return false;
|
|
476
|
+
}
|
|
477
|
+
try {
|
|
478
|
+
const lock = await stat(lockDirectory);
|
|
479
|
+
return Date.now() - lock.mtimeMs > staleMs;
|
|
480
|
+
}
|
|
481
|
+
catch {
|
|
482
|
+
return false;
|
|
483
|
+
}
|
|
484
|
+
}
|
|
485
|
+
async function recoverStaleLock(lockDirectory, staleMs) {
|
|
486
|
+
if (!(await lockIsStale(lockDirectory, staleMs)))
|
|
487
|
+
return false;
|
|
488
|
+
const quarantine = `${lockDirectory}.reclaimed.${randomUUID()}`;
|
|
489
|
+
try {
|
|
490
|
+
await rename(lockDirectory, quarantine);
|
|
491
|
+
}
|
|
492
|
+
catch (error) {
|
|
493
|
+
if (isErrno(error, "ENOENT"))
|
|
494
|
+
return true;
|
|
495
|
+
throw error;
|
|
496
|
+
}
|
|
497
|
+
await rm(quarantine, { recursive: true, force: true });
|
|
498
|
+
return true;
|
|
499
|
+
}
|
|
500
|
+
async function withRunLock(lockDirectory, options, operation) {
|
|
501
|
+
const ownerToken = `${process.pid}.${Date.now()}.${randomUUID()}`;
|
|
502
|
+
const ownerPath = join(lockDirectory, LOCK_OWNER_FILE_NAME);
|
|
503
|
+
const leasePath = join(lockDirectory, LOCK_LEASE_FILE_NAME);
|
|
504
|
+
const deadline = Date.now() + options.lock_wait_ms;
|
|
505
|
+
await mkdir(dirname(lockDirectory), {
|
|
506
|
+
recursive: true,
|
|
507
|
+
mode: DIRECTORY_MODE,
|
|
508
|
+
});
|
|
509
|
+
while (true) {
|
|
510
|
+
try {
|
|
511
|
+
await mkdir(lockDirectory, { mode: DIRECTORY_MODE });
|
|
512
|
+
try {
|
|
513
|
+
const owner = await open(ownerPath, "wx", FILE_MODE);
|
|
514
|
+
await owner.writeFile(ownerToken, "utf8");
|
|
515
|
+
await owner.sync();
|
|
516
|
+
await owner.close();
|
|
517
|
+
const lease = await open(leasePath, "wx", FILE_MODE);
|
|
518
|
+
await lease.writeFile(ownerToken, "utf8");
|
|
519
|
+
await lease.sync();
|
|
520
|
+
await lease.close();
|
|
521
|
+
}
|
|
522
|
+
catch (error) {
|
|
523
|
+
await rm(lockDirectory, { recursive: true, force: true });
|
|
524
|
+
throw error;
|
|
525
|
+
}
|
|
526
|
+
break;
|
|
527
|
+
}
|
|
528
|
+
catch (error) {
|
|
529
|
+
if (!isErrno(error, "EEXIST"))
|
|
530
|
+
throw error;
|
|
531
|
+
if (await recoverStaleLock(lockDirectory, options.lock_stale_ms))
|
|
532
|
+
continue;
|
|
533
|
+
if (Date.now() >= deadline) {
|
|
534
|
+
throw new RunStoreError("lock_timeout", `timed out acquiring run-store lock ${lockDirectory}`);
|
|
535
|
+
}
|
|
536
|
+
await sleep(options.lock_retry_ms);
|
|
537
|
+
}
|
|
538
|
+
}
|
|
539
|
+
let heartbeatRunning = false;
|
|
540
|
+
let lostOwnership = false;
|
|
541
|
+
const refresh = async () => {
|
|
542
|
+
if (heartbeatRunning || lostOwnership)
|
|
543
|
+
return;
|
|
544
|
+
heartbeatRunning = true;
|
|
545
|
+
try {
|
|
546
|
+
if ((await readLockOwner(ownerPath)) !== ownerToken) {
|
|
547
|
+
lostOwnership = true;
|
|
548
|
+
return;
|
|
549
|
+
}
|
|
550
|
+
const now = new Date();
|
|
551
|
+
await utimes(leasePath, now, now);
|
|
552
|
+
}
|
|
553
|
+
catch {
|
|
554
|
+
lostOwnership = true;
|
|
555
|
+
}
|
|
556
|
+
finally {
|
|
557
|
+
heartbeatRunning = false;
|
|
558
|
+
}
|
|
559
|
+
};
|
|
560
|
+
const timer = setInterval(() => void refresh(), options.lock_heartbeat_ms);
|
|
561
|
+
timer.unref();
|
|
562
|
+
const lock = {
|
|
563
|
+
async assertOwned() {
|
|
564
|
+
if (lostOwnership ||
|
|
565
|
+
(await readLockOwner(ownerPath)) !== ownerToken) {
|
|
566
|
+
lostOwnership = true;
|
|
567
|
+
throw new RunStoreError("lock_lost", `run-store lock ownership was lost for ${lockDirectory}`);
|
|
568
|
+
}
|
|
569
|
+
},
|
|
570
|
+
};
|
|
571
|
+
try {
|
|
572
|
+
return await operation(lock);
|
|
573
|
+
}
|
|
574
|
+
finally {
|
|
575
|
+
clearInterval(timer);
|
|
576
|
+
while (heartbeatRunning)
|
|
577
|
+
await sleep(1);
|
|
578
|
+
if ((await readLockOwner(ownerPath).catch(() => null)) === ownerToken) {
|
|
579
|
+
await rm(lockDirectory, { recursive: true, force: true });
|
|
580
|
+
}
|
|
581
|
+
}
|
|
582
|
+
}
|
|
583
|
+
export class DurableRunStore {
|
|
584
|
+
options;
|
|
585
|
+
constructor(options) {
|
|
586
|
+
this.options = resolveOptions(options);
|
|
587
|
+
}
|
|
588
|
+
runDirectory(runId) {
|
|
589
|
+
return join(this.options.root_directory, "runs", runStorageKey(runId));
|
|
590
|
+
}
|
|
591
|
+
ledgerPath(runId) {
|
|
592
|
+
return join(this.runDirectory(runId), LEDGER_FILE_NAME);
|
|
593
|
+
}
|
|
594
|
+
lockDirectory(runId) {
|
|
595
|
+
return join(this.options.root_directory, ".locks", `${runStorageKey(runId)}.lock`);
|
|
596
|
+
}
|
|
597
|
+
async readSnapshot(runId) {
|
|
598
|
+
try {
|
|
599
|
+
const raw = await readFile(this.ledgerPath(runId), "utf8");
|
|
600
|
+
return assertSnapshot(JSON.parse(raw), runId);
|
|
601
|
+
}
|
|
602
|
+
catch (error) {
|
|
603
|
+
if (isErrno(error, "ENOENT"))
|
|
604
|
+
return null;
|
|
605
|
+
if (error instanceof SyntaxError) {
|
|
606
|
+
throw new RunStoreError("invalid_record", `run ${runId} contains invalid JSON`);
|
|
607
|
+
}
|
|
608
|
+
throw error;
|
|
609
|
+
}
|
|
610
|
+
}
|
|
611
|
+
async createRun(input) {
|
|
612
|
+
assertRunRecord(input.run);
|
|
613
|
+
const runId = input.run.run_id;
|
|
614
|
+
return withRunLock(this.lockDirectory(runId), this.options, async (lock) => {
|
|
615
|
+
if (await this.readSnapshot(runId)) {
|
|
616
|
+
throw new RunStoreError("run_exists", `run ${runId} already exists`);
|
|
617
|
+
}
|
|
618
|
+
const now = new Date().toISOString();
|
|
619
|
+
const snapshot = {
|
|
620
|
+
schema_version: RUN_STORE_SCHEMA_VERSION,
|
|
621
|
+
record_version: 1,
|
|
622
|
+
run: input.run,
|
|
623
|
+
tasks: [...(input.tasks ?? [])],
|
|
624
|
+
attempts: [],
|
|
625
|
+
claims: [],
|
|
626
|
+
events: [],
|
|
627
|
+
results: [],
|
|
628
|
+
created_at: now,
|
|
629
|
+
updated_at: now,
|
|
630
|
+
};
|
|
631
|
+
validateCollections(snapshot);
|
|
632
|
+
await lock.assertOwned();
|
|
633
|
+
await writeAtomic(this.ledgerPath(runId), `${JSON.stringify(snapshot, null, 2)}\n`);
|
|
634
|
+
return snapshot;
|
|
635
|
+
});
|
|
636
|
+
}
|
|
637
|
+
async loadRun(runId) {
|
|
638
|
+
assertString(runId, "run_id");
|
|
639
|
+
return this.readSnapshot(runId);
|
|
640
|
+
}
|
|
641
|
+
async requireRun(runId) {
|
|
642
|
+
const snapshot = await this.loadRun(runId);
|
|
643
|
+
if (!snapshot) {
|
|
644
|
+
throw new RunStoreError("run_not_found", `run ${runId} was not found`);
|
|
645
|
+
}
|
|
646
|
+
return snapshot;
|
|
647
|
+
}
|
|
648
|
+
async updateRun(update) {
|
|
649
|
+
assertString(update.run_id, "run_id");
|
|
650
|
+
if (!Number.isSafeInteger(update.expected_record_version) ||
|
|
651
|
+
update.expected_record_version < 1) {
|
|
652
|
+
throw new RunStoreError("invalid_record", "expected_record_version must be a positive safe integer");
|
|
653
|
+
}
|
|
654
|
+
return withRunLock(this.lockDirectory(update.run_id), this.options, async (lock) => {
|
|
655
|
+
const current = await this.readSnapshot(update.run_id);
|
|
656
|
+
if (!current) {
|
|
657
|
+
throw new RunStoreError("run_not_found", `run ${update.run_id} was not found`);
|
|
658
|
+
}
|
|
659
|
+
let applied;
|
|
660
|
+
try {
|
|
661
|
+
applied = applyUpdate(current, update);
|
|
662
|
+
}
|
|
663
|
+
catch (error) {
|
|
664
|
+
if (current.record_version !== update.expected_record_version) {
|
|
665
|
+
throw new RunStoreError("version_conflict", `run ${update.run_id} is at version ${current.record_version}, expected ${update.expected_record_version}`);
|
|
666
|
+
}
|
|
667
|
+
throw error;
|
|
668
|
+
}
|
|
669
|
+
if (!applied.changed) {
|
|
670
|
+
return {
|
|
671
|
+
snapshot: current,
|
|
672
|
+
applied: false,
|
|
673
|
+
duplicate_event_ids: applied.duplicate_event_ids,
|
|
674
|
+
duplicate_result_ids: applied.duplicate_result_ids,
|
|
675
|
+
};
|
|
676
|
+
}
|
|
677
|
+
if (current.record_version !== update.expected_record_version) {
|
|
678
|
+
throw new RunStoreError("version_conflict", `run ${update.run_id} is at version ${current.record_version}, expected ${update.expected_record_version}`);
|
|
679
|
+
}
|
|
680
|
+
await lock.assertOwned();
|
|
681
|
+
await writeAtomic(this.ledgerPath(update.run_id), `${JSON.stringify(applied.snapshot, null, 2)}\n`);
|
|
682
|
+
return {
|
|
683
|
+
snapshot: applied.snapshot,
|
|
684
|
+
applied: true,
|
|
685
|
+
duplicate_event_ids: applied.duplicate_event_ids,
|
|
686
|
+
duplicate_result_ids: applied.duplicate_result_ids,
|
|
687
|
+
};
|
|
688
|
+
});
|
|
689
|
+
}
|
|
690
|
+
/**
|
|
691
|
+
* Reads every valid run ledger after restart. No worker-session directory is
|
|
692
|
+
* consulted, and this store intentionally exposes no worker cleanup operation.
|
|
693
|
+
*/
|
|
694
|
+
async loadAllRuns() {
|
|
695
|
+
const runsDirectory = join(this.options.root_directory, "runs");
|
|
696
|
+
let entries;
|
|
697
|
+
try {
|
|
698
|
+
entries = await readdir(runsDirectory, { withFileTypes: true });
|
|
699
|
+
}
|
|
700
|
+
catch (error) {
|
|
701
|
+
if (isErrno(error, "ENOENT"))
|
|
702
|
+
return [];
|
|
703
|
+
throw error;
|
|
704
|
+
}
|
|
705
|
+
const snapshots = [];
|
|
706
|
+
for (const entry of entries.sort((left, right) => left.name.localeCompare(right.name))) {
|
|
707
|
+
if (!entry.isDirectory())
|
|
708
|
+
continue;
|
|
709
|
+
const ledgerPath = join(runsDirectory, entry.name, LEDGER_FILE_NAME);
|
|
710
|
+
try {
|
|
711
|
+
const raw = await readFile(ledgerPath, "utf8");
|
|
712
|
+
snapshots.push(assertSnapshot(JSON.parse(raw)));
|
|
713
|
+
}
|
|
714
|
+
catch (error) {
|
|
715
|
+
if (isErrno(error, "ENOENT"))
|
|
716
|
+
continue;
|
|
717
|
+
if (error instanceof SyntaxError) {
|
|
718
|
+
throw new RunStoreError("invalid_record", `run ledger ${ledgerPath} contains invalid JSON`);
|
|
719
|
+
}
|
|
720
|
+
throw error;
|
|
721
|
+
}
|
|
722
|
+
}
|
|
723
|
+
return snapshots;
|
|
724
|
+
}
|
|
725
|
+
}
|