@signalridge/pi-workflows 1.2.1 → 1.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +36 -0
- package/README.md +142 -17
- package/package.json +14 -8
- package/skills/workflow-authoring/SKILL.md +30 -0
- package/skills/workflow-authoring/examples/adversarial-verification.js +63 -0
- package/skills/workflow-authoring/examples/bounded-semantic-retry.js +56 -0
- package/skills/workflow-authoring/examples/classify-and-act.js +67 -0
- package/skills/workflow-authoring/examples/defensive-json-parsing.js +63 -0
- package/skills/workflow-authoring/examples/fan-out-and-synthesize.js +47 -0
- package/skills/workflow-authoring/examples/generate-and-filter.js +68 -0
- package/skills/workflow-authoring/examples/loop-until-done.js +68 -0
- package/skills/workflow-authoring/examples/phased-budgets.js +75 -0
- package/skills/workflow-authoring/examples/saved-nested-workflows.js +44 -0
- package/skills/workflow-authoring/examples/structured-output.js +37 -0
- package/skills/workflow-authoring/examples/tournament.js +81 -0
- package/skills/workflow-authoring/examples/validated-gate.js +63 -0
- package/skills/workflow-authoring/references/capabilities.md +44 -0
- package/skills/workflow-authoring/references/capability-details.md +294 -0
- package/skills/workflow-authoring/references/common-helpers.md +4 -0
- package/skills/workflow-authoring/references/debugging.md +29 -0
- package/skills/workflow-authoring/references/focused-recipes.md +12 -0
- package/skills/workflow-authoring/references/helpers.md +9 -0
- package/skills/workflow-authoring/references/lifecycle.md +37 -0
- package/skills/workflow-authoring/references/pattern-selection.md +15 -0
- package/skills/workflow-authoring/references/quality-helpers.md +8 -0
- package/skills/workflow-authoring/references/registry-ownership.md +17 -0
- package/skills/workflow-authoring/references/retry-helper.md +5 -0
- package/skills/workflow-authoring/references/review.md +43 -0
- package/skills/workflow-authoring/references/runtime.md +29 -0
- package/skills/workflow-authoring/references/specialized-helpers.md +19 -0
- package/skills/workflow-authoring/references/versions.md +13 -0
- package/skills/workflow-patterns/SKILL.md +51 -0
- package/src/arming.ts +69 -0
- package/src/builtins.ts +331 -0
- package/src/capabilities.ts +318 -0
- package/src/code-review-scope.ts +144 -0
- package/src/engine.ts +1158 -2561
- package/src/errors.ts +136 -0
- package/src/index.ts +750 -103
- package/src/journal.ts +594 -1001
- package/src/rpc-client.ts +95 -24
- package/src/runtime.ts +1708 -0
- package/src/saved-workflows.ts +121 -0
- package/src/shared-store.ts +132 -0
- package/src/ui.ts +77 -40
- package/src/workflow-settings.ts +100 -0
- package/src/schema.ts +0 -215
- package/src/state-machine.ts +0 -79
- package/src/synthesis.ts +0 -140
package/src/journal.ts
CHANGED
|
@@ -1,27 +1,47 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* journal.ts — durable journal for script-based workflow runs (schema v3).
|
|
3
|
+
*
|
|
4
|
+
* A script run is journaled as an ordered event stream in the session entry
|
|
5
|
+
* store. `run_created` carries the raw script, its hash, and its parsed meta
|
|
6
|
+
* (the runtime re-derives call indexes from the script itself, so no task
|
|
7
|
+
* array is persisted). Each completed `agent()`/`checkpoint()` call journals a
|
|
8
|
+
* `call_result` keyed by its lexical call index; a resume rebuilds the
|
|
9
|
+
* runtime's `resumeJournal` map from these entries and replays the longest
|
|
10
|
+
* unchanged prefix.
|
|
11
|
+
*
|
|
12
|
+
* The atomic recovery machinery (run/terminal/attempt recovery with generation
|
|
13
|
+
* rotations, quarantine of malformed runs, stale-generation rejection) is
|
|
14
|
+
* preserved from the declarative journal and re-keyed to call indexes, because
|
|
15
|
+
* interrupted managed spawns still need generation rotation on restore.
|
|
16
|
+
*
|
|
17
|
+
* Schema v2 runs (declarative DAG journals) are quarantined rather than
|
|
18
|
+
* replayed — the event shapes are incompatible and a script run must not
|
|
19
|
+
* inherit DAG bookkeeping.
|
|
20
|
+
*/
|
|
21
|
+
|
|
1
22
|
import { createHash } from "node:crypto";
|
|
2
23
|
import type { ManagedOwner } from "@signalridge/pi-subagents-protocol";
|
|
3
|
-
import {
|
|
4
|
-
import {
|
|
5
|
-
canTransitionTask,
|
|
6
|
-
canTransitionWorkflow,
|
|
7
|
-
isTerminalTask,
|
|
8
|
-
type TaskStatus,
|
|
9
|
-
type WorkflowStatus,
|
|
10
|
-
} from "./state-machine.js";
|
|
24
|
+
import type { WorkflowMeta } from "./runtime.js";
|
|
11
25
|
|
|
12
26
|
export const JOURNAL_ENTRY_TYPE = "pi-workflows:journal";
|
|
13
|
-
export const JOURNAL_SCHEMA_VERSION =
|
|
14
|
-
|
|
15
|
-
export
|
|
27
|
+
export const JOURNAL_SCHEMA_VERSION = 3;
|
|
28
|
+
/** Schema v2 (declarative DAG) and v1 are quarantined on replay, not migrated. */
|
|
29
|
+
export const JOURNAL_LEGACY_SCHEMA_VERSION = 2;
|
|
30
|
+
export type JournalSchemaVersion = 2 | 3;
|
|
16
31
|
export const JOURNAL_TEXT_LIMIT = 8_000;
|
|
32
|
+
export const JOURNAL_SCRIPT_LIMIT = 200_000;
|
|
17
33
|
export const JOURNAL_ERROR_LIMIT = 2_000;
|
|
18
34
|
export const JOURNAL_ID_LIMIT = 256;
|
|
19
35
|
export const JOURNAL_TIMESTAMP_LIMIT = 8_640_000_000_000_000;
|
|
20
|
-
export const JOURNAL_RECOVERY_ID_PREFIX = "
|
|
36
|
+
export const JOURNAL_RECOVERY_ID_PREFIX = "r3-";
|
|
21
37
|
export const JOURNAL_RECOVERY_ID_LIMIT = JOURNAL_RECOVERY_ID_PREFIX.length + 64;
|
|
22
38
|
/** Maximum number of distinct atomic recoveries retained while replaying one run. */
|
|
23
39
|
export const JOURNAL_RECOVERY_SEEN_LIMIT = 4_096;
|
|
24
|
-
|
|
40
|
+
/** A script run may issue up to 1000 agent()/checkpoint() calls. */
|
|
41
|
+
export const JOURNAL_CALL_LIMIT = 1_000;
|
|
42
|
+
/** One atomic recovery event may rotate every active call in a run. */
|
|
43
|
+
export const JOURNAL_RECOVERY_NODE_LIMIT = JOURNAL_CALL_LIMIT;
|
|
44
|
+
const RECOVERY_ID_PATTERN = /^r3-[0-9a-f]{64}$/;
|
|
25
45
|
|
|
26
46
|
export class StaleJournalGenerationError extends Error {
|
|
27
47
|
constructor(message: string) {
|
|
@@ -30,80 +50,90 @@ export class StaleJournalGenerationError extends Error {
|
|
|
30
50
|
}
|
|
31
51
|
}
|
|
32
52
|
|
|
53
|
+
export type WorkflowStatus =
|
|
54
|
+
| "pending"
|
|
55
|
+
| "running"
|
|
56
|
+
| "pausing"
|
|
57
|
+
| "paused"
|
|
58
|
+
| "stopping"
|
|
59
|
+
| "completed"
|
|
60
|
+
| "failed"
|
|
61
|
+
| "stopped"
|
|
62
|
+
| "interrupted";
|
|
63
|
+
|
|
64
|
+
export type CallStatus = "running" | "completed" | "failed" | "stopped";
|
|
65
|
+
|
|
33
66
|
const WORKFLOW_STATUS_VALUES = new Set<WorkflowStatus>([
|
|
34
67
|
"pending",
|
|
35
68
|
"running",
|
|
36
69
|
"pausing",
|
|
37
70
|
"paused",
|
|
38
|
-
"synthesizing",
|
|
39
71
|
"stopping",
|
|
40
72
|
"completed",
|
|
41
73
|
"failed",
|
|
42
74
|
"stopped",
|
|
43
75
|
"interrupted",
|
|
44
76
|
]);
|
|
45
|
-
const
|
|
46
|
-
"pending",
|
|
47
|
-
"ready",
|
|
48
|
-
"dispatching",
|
|
49
|
-
"queued",
|
|
50
|
-
"running",
|
|
51
|
-
"completed",
|
|
52
|
-
"failed",
|
|
53
|
-
"stopped",
|
|
54
|
-
"blocked",
|
|
55
|
-
]);
|
|
77
|
+
const CALL_STATUS_VALUES = new Set<CallStatus>(["running", "completed", "failed", "stopped"]);
|
|
56
78
|
const JOURNAL_KINDS = new Set([
|
|
57
79
|
"run_created",
|
|
80
|
+
"run_revision",
|
|
81
|
+
"run_removed",
|
|
58
82
|
"workflow_transition",
|
|
59
83
|
"run_recovery",
|
|
60
84
|
"terminal_recovery",
|
|
61
85
|
"attempt_recovery",
|
|
62
|
-
"
|
|
63
|
-
"
|
|
64
|
-
"
|
|
65
|
-
"task_compacted",
|
|
66
|
-
"synthesis_result",
|
|
86
|
+
"call_transition",
|
|
87
|
+
"call_attempt",
|
|
88
|
+
"call_result",
|
|
67
89
|
]);
|
|
68
90
|
|
|
69
|
-
/** One atomic recovery event may rotate every active node in a run. */
|
|
70
|
-
export const JOURNAL_RECOVERY_NODE_LIMIT = 129; // 128 tasks plus synthesis
|
|
71
|
-
const JOURNAL_GENERATION_LIMIT = 1_000_000;
|
|
72
91
|
export type WorkflowOwner = ManagedOwner;
|
|
73
92
|
|
|
74
|
-
|
|
93
|
+
/** A completed agent()/checkpoint() call, journaled for resume replay. */
|
|
94
|
+
export interface CallResult {
|
|
75
95
|
status: "completed" | "failed" | "stopped";
|
|
76
96
|
agentId?: string;
|
|
77
97
|
attemptId?: string;
|
|
78
|
-
text
|
|
98
|
+
/** The JSON-serializable call result (text for plain agents, parsed value for schema agents). */
|
|
99
|
+
result?: unknown;
|
|
79
100
|
error?: string;
|
|
80
|
-
/** Persisted output transcript pointer, when pi-subagents provided one. */
|
|
81
101
|
outputFile?: string;
|
|
82
|
-
/** Total input + output token count, when lifecycle accounting provided it. */
|
|
83
102
|
tokenCount?: number;
|
|
84
103
|
compactionCount: number;
|
|
85
|
-
truncated?: boolean;
|
|
86
104
|
updatedAt: number;
|
|
87
105
|
}
|
|
88
106
|
|
|
89
|
-
export interface
|
|
107
|
+
export interface ScriptRun {
|
|
90
108
|
runId: string;
|
|
91
109
|
schemaVersion: number;
|
|
92
|
-
|
|
110
|
+
script: string;
|
|
111
|
+
scriptHash: string;
|
|
112
|
+
meta: WorkflowMeta;
|
|
113
|
+
/** Number of the latest edited-script revision. */
|
|
114
|
+
revision?: number;
|
|
115
|
+
/** Toolset marker persisted across resume (e.g. "web-research" for deep-research). */
|
|
116
|
+
toolset?: string;
|
|
117
|
+
/** Frozen run params — persisted so resume after restore keeps original budget/scale. */
|
|
118
|
+
frozenMaxAgents?: number;
|
|
119
|
+
frozenConcurrency?: number;
|
|
120
|
+
frozenAgentRetries?: number;
|
|
121
|
+
frozenTokenBudget?: number | null;
|
|
122
|
+
frozenAgentTimeoutMs?: number | null;
|
|
123
|
+
frozenExcludeTools?: string[];
|
|
93
124
|
status: WorkflowStatus;
|
|
94
|
-
|
|
125
|
+
/** Call index → status. Absent entries were never dispatched (replayed or skipped). */
|
|
126
|
+
callStatus: Record<string, CallStatus>;
|
|
95
127
|
agentIds: Record<string, string>;
|
|
96
|
-
/** Monotonic per-
|
|
128
|
+
/** Monotonic per-call attempt generation; completed facts are never reset. */
|
|
97
129
|
attempts: Record<string, number>;
|
|
98
130
|
attemptIds: Record<string, string>;
|
|
99
|
-
/** True when the journal carries generation-specific owner IDs. */
|
|
100
131
|
attemptTracking?: boolean;
|
|
101
|
-
|
|
132
|
+
/** Call index → journaled result (the resume journal source). */
|
|
133
|
+
callResults: Record<string, CallResult>;
|
|
102
134
|
compactions: Record<string, number>;
|
|
103
135
|
startedAt: number;
|
|
104
136
|
updatedAt: number;
|
|
105
|
-
synthesisAgentId?: string;
|
|
106
|
-
synthesisResult?: WorkflowTaskResult;
|
|
107
137
|
error?: string;
|
|
108
138
|
nonResumable?: boolean;
|
|
109
139
|
/** Terminal cleanup intent while active agents settle. */
|
|
@@ -116,14 +146,9 @@ interface JournalBase {
|
|
|
116
146
|
timestamp: number;
|
|
117
147
|
}
|
|
118
148
|
|
|
119
|
-
/** State
|
|
120
|
-
export type RecoverySourceStatus =
|
|
149
|
+
/** State observed immediately before an atomic recovery rotation. */
|
|
150
|
+
export type RecoverySourceStatus = CallStatus | "running";
|
|
121
151
|
|
|
122
|
-
/**
|
|
123
|
-
* A generation rotation is self-contained: replay can clear the superseded
|
|
124
|
-
* attempt and make the new attempt ready without relying on a following
|
|
125
|
-
* task_transition entry.
|
|
126
|
-
*/
|
|
127
152
|
export interface RecoveryRotation {
|
|
128
153
|
nodeId: string;
|
|
129
154
|
sourceAttemptId: string;
|
|
@@ -138,7 +163,7 @@ export interface RecoveryRotation {
|
|
|
138
163
|
export interface RecoveryTerminalResult {
|
|
139
164
|
nodeId: string;
|
|
140
165
|
attemptId: string;
|
|
141
|
-
result:
|
|
166
|
+
result: CallResult;
|
|
142
167
|
owner: WorkflowOwner;
|
|
143
168
|
}
|
|
144
169
|
|
|
@@ -153,11 +178,7 @@ export interface RunRecoveryEvent extends JournalBase {
|
|
|
153
178
|
legacyMigration?: true;
|
|
154
179
|
timestamp: number;
|
|
155
180
|
}
|
|
156
|
-
|
|
157
|
-
* A terminal-intent recovery is one durable state transition. It settles every
|
|
158
|
-
* unresolved node before publishing the final workflow outcome, so replay never
|
|
159
|
-
* has to infer a terminal state from a prefix of result/transition events.
|
|
160
|
-
*/
|
|
181
|
+
|
|
161
182
|
export interface TerminalRecoveryEvent extends JournalBase {
|
|
162
183
|
kind: "terminal_recovery";
|
|
163
184
|
status: "failed" | "stopped";
|
|
@@ -182,10 +203,29 @@ export interface AttemptRecoveryEvent extends JournalBase {
|
|
|
182
203
|
export type JournalEvent =
|
|
183
204
|
| (JournalBase & {
|
|
184
205
|
kind: "run_created";
|
|
185
|
-
|
|
206
|
+
script: string;
|
|
207
|
+
scriptHash: string;
|
|
208
|
+
meta: WorkflowMeta;
|
|
209
|
+
toolset?: string;
|
|
210
|
+
frozenMaxAgents?: number;
|
|
211
|
+
frozenConcurrency?: number;
|
|
212
|
+
frozenAgentRetries?: number;
|
|
213
|
+
frozenTokenBudget?: number | null;
|
|
214
|
+
frozenAgentTimeoutMs?: number | null;
|
|
215
|
+
frozenExcludeTools?: string[];
|
|
186
216
|
attempts?: Record<string, number>;
|
|
187
217
|
attemptIds?: Record<string, string>;
|
|
188
218
|
})
|
|
219
|
+
| (JournalBase & {
|
|
220
|
+
kind: "run_revision";
|
|
221
|
+
revision: number;
|
|
222
|
+
script: string;
|
|
223
|
+
scriptHash: string;
|
|
224
|
+
meta: WorkflowMeta;
|
|
225
|
+
})
|
|
226
|
+
| (JournalBase & {
|
|
227
|
+
kind: "run_removed";
|
|
228
|
+
})
|
|
189
229
|
| (JournalBase & {
|
|
190
230
|
kind: "workflow_transition";
|
|
191
231
|
status: WorkflowStatus;
|
|
@@ -196,37 +236,28 @@ export type JournalEvent =
|
|
|
196
236
|
| TerminalRecoveryEvent
|
|
197
237
|
| AttemptRecoveryEvent
|
|
198
238
|
| (JournalBase & {
|
|
199
|
-
kind: "
|
|
239
|
+
kind: "call_transition";
|
|
200
240
|
nodeId: string;
|
|
201
|
-
status:
|
|
241
|
+
status: CallStatus;
|
|
202
242
|
agentId?: string;
|
|
203
243
|
attemptId?: string;
|
|
204
244
|
owner?: WorkflowOwner;
|
|
205
245
|
})
|
|
206
246
|
| (JournalBase & {
|
|
207
|
-
kind: "
|
|
247
|
+
kind: "call_attempt";
|
|
208
248
|
nodeId: string;
|
|
209
249
|
attemptId: string;
|
|
210
250
|
generation: number;
|
|
211
251
|
owner?: WorkflowOwner;
|
|
212
252
|
})
|
|
213
253
|
| (JournalBase & {
|
|
214
|
-
kind: "
|
|
254
|
+
kind: "call_result";
|
|
215
255
|
nodeId: string;
|
|
216
|
-
result:
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
kind: "task_compacted";
|
|
222
|
-
nodeId: string;
|
|
223
|
-
compactionCount: number;
|
|
224
|
-
attemptId?: string;
|
|
225
|
-
owner?: WorkflowOwner;
|
|
226
|
-
})
|
|
227
|
-
| (JournalBase & {
|
|
228
|
-
kind: "synthesis_result";
|
|
229
|
-
result: WorkflowTaskResult;
|
|
256
|
+
result: CallResult;
|
|
257
|
+
/** Stable hash of the call inputs (prompt/model/tier/phase/schema). */
|
|
258
|
+
callHash?: string;
|
|
259
|
+
/** Store write delta to replay additively on resume. */
|
|
260
|
+
storeDelta?: Record<string, unknown>;
|
|
230
261
|
attemptId?: string;
|
|
231
262
|
owner?: WorkflowOwner;
|
|
232
263
|
});
|
|
@@ -242,7 +273,6 @@ export interface SessionEntryLike {
|
|
|
242
273
|
}
|
|
243
274
|
|
|
244
275
|
export interface ReplayJournalOptions {
|
|
245
|
-
/** Called at most once for malformed/forged/future journal data. */
|
|
246
276
|
onInvalid?: (diagnostic: string) => void;
|
|
247
277
|
}
|
|
248
278
|
|
|
@@ -259,817 +289,308 @@ function boundedString(value: unknown, label: string, limit: number): string {
|
|
|
259
289
|
|
|
260
290
|
function validTimestamp(value: unknown, label: string): number {
|
|
261
291
|
if (typeof value !== "number" || !Number.isInteger(value) || value < 0 || value > JOURNAL_TIMESTAMP_LIMIT) {
|
|
262
|
-
throw new Error(`${label} is invalid`);
|
|
292
|
+
throw new Error(`${label} timestamp is invalid`);
|
|
263
293
|
}
|
|
264
294
|
return value;
|
|
265
295
|
}
|
|
266
296
|
|
|
267
|
-
function
|
|
268
|
-
const
|
|
269
|
-
|
|
270
|
-
|
|
297
|
+
function parseCallIndex(value: unknown, label: string): string {
|
|
298
|
+
const index = boundedString(value, label, JOURNAL_ID_LIMIT);
|
|
299
|
+
if (!/^(?:0|[1-9]\d*)$/u.test(index) || Number(index) >= JOURNAL_CALL_LIMIT) {
|
|
300
|
+
throw new Error(`${label} is not a valid call index`);
|
|
271
301
|
}
|
|
302
|
+
return index;
|
|
272
303
|
}
|
|
273
304
|
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
if (value === null) return "null";
|
|
292
|
-
if (typeof value === "string" || typeof value === "number" || typeof value === "boolean") {
|
|
293
|
-
return JSON.stringify(value);
|
|
294
|
-
}
|
|
295
|
-
if (Array.isArray(value)) return `[${value.map((item) => canonicalJson(item)).join(",")}]`;
|
|
296
|
-
if (isRecord(value)) {
|
|
297
|
-
return `{${Object.keys(value)
|
|
298
|
-
.sort()
|
|
299
|
-
.map((key) => `${JSON.stringify(key)}:${canonicalJson(value[key])}`)
|
|
300
|
-
.join(",")}}`;
|
|
305
|
+
function parseMeta(raw: unknown): WorkflowMeta {
|
|
306
|
+
if (!isRecord(raw)) throw new Error("meta is invalid");
|
|
307
|
+
const name = boundedString(raw.name, "meta.name", 512);
|
|
308
|
+
const description = boundedString(raw.description, "meta.description", 100_000);
|
|
309
|
+
const meta: WorkflowMeta = { name, description };
|
|
310
|
+
if (raw.model !== undefined) meta.model = boundedString(raw.model, "meta.model", 512);
|
|
311
|
+
if (raw.phases !== undefined) {
|
|
312
|
+
if (!Array.isArray(raw.phases)) throw new Error("meta.phases must be an array");
|
|
313
|
+
meta.phases = raw.phases.map((phase, i) => {
|
|
314
|
+
if (!isRecord(phase) || typeof phase.title !== "string" || phase.title.length === 0) {
|
|
315
|
+
throw new Error(`meta.phases[${i}].title is invalid`);
|
|
316
|
+
}
|
|
317
|
+
const out: NonNullable<WorkflowMeta["phases"]>[number] = { title: phase.title.slice(0, 512) };
|
|
318
|
+
if (phase.detail !== undefined) out.detail = String(phase.detail).slice(0, 2_000);
|
|
319
|
+
if (phase.model !== undefined) out.model = String(phase.model).slice(0, 512);
|
|
320
|
+
return out;
|
|
321
|
+
});
|
|
301
322
|
}
|
|
302
|
-
|
|
323
|
+
return meta;
|
|
303
324
|
}
|
|
304
325
|
|
|
305
|
-
function
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
return left < right ? -1 : left > right ? 1 : 0;
|
|
311
|
-
}
|
|
312
|
-
|
|
313
|
-
function sortedRecoveryRotations(rotations: readonly RecoveryRotation[]): RecoveryRotation[] {
|
|
314
|
-
return [...rotations].sort((left, right) => {
|
|
315
|
-
const nodeOrder = compareStrings(left.nodeId, right.nodeId);
|
|
316
|
-
if (nodeOrder !== 0) return nodeOrder;
|
|
317
|
-
const attemptOrder = compareStrings(left.sourceAttemptId, right.sourceAttemptId);
|
|
318
|
-
if (attemptOrder !== 0) return attemptOrder;
|
|
319
|
-
return left.sourceGeneration - right.sourceGeneration;
|
|
320
|
-
});
|
|
321
|
-
}
|
|
322
|
-
|
|
323
|
-
function sortedRecoveryTerminals(terminals: readonly RecoveryTerminalResult[]): RecoveryTerminalResult[] {
|
|
324
|
-
return [...terminals].sort((left, right) => {
|
|
325
|
-
const nodeOrder = compareStrings(left.nodeId, right.nodeId);
|
|
326
|
-
if (nodeOrder !== 0) return nodeOrder;
|
|
327
|
-
return compareStrings(left.attemptId, right.attemptId);
|
|
328
|
-
});
|
|
329
|
-
}
|
|
330
|
-
|
|
331
|
-
function recoverySemanticPayload(event: RecoveryIdInput): Record<string, unknown> {
|
|
332
|
-
if ("kind" in event && event.kind === "terminal_recovery") {
|
|
333
|
-
return {
|
|
334
|
-
blockedNodeIds: [...event.blockedNodeIds].sort(compareStrings),
|
|
335
|
-
branchGeneration: event.branchGeneration,
|
|
336
|
-
...(event.error === undefined ? {} : { error: event.error }),
|
|
337
|
-
kind: event.kind,
|
|
338
|
-
runId: event.runId,
|
|
339
|
-
schemaVersion: JOURNAL_SCHEMA_VERSION,
|
|
340
|
-
status: event.status,
|
|
341
|
-
terminalIntent: event.terminalIntent,
|
|
342
|
-
terminalResults: sortedRecoveryTerminals(event.terminalResults),
|
|
343
|
-
};
|
|
326
|
+
function parseResult(raw: unknown, label: string): CallResult {
|
|
327
|
+
if (!isRecord(raw)) throw new Error(`${label} is invalid`);
|
|
328
|
+
const status = raw.status;
|
|
329
|
+
if (status !== "completed" && status !== "failed" && status !== "stopped") {
|
|
330
|
+
throw new Error(`${label} status is invalid`);
|
|
344
331
|
}
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
rotations: sortedRecoveryRotations(event.rotations),
|
|
349
|
-
runId: event.runId,
|
|
350
|
-
schemaVersion: JOURNAL_SCHEMA_VERSION,
|
|
351
|
-
status: event.status,
|
|
352
|
-
terminalResults: sortedRecoveryTerminals(event.terminalResults ?? []),
|
|
353
|
-
};
|
|
354
|
-
}
|
|
355
|
-
|
|
356
|
-
/** Bounded digest of all normalized semantic fields, excluding timestamp and migration metadata. */
|
|
357
|
-
export function recoverySemanticFingerprint(event: RecoveryIdInput): string {
|
|
358
|
-
return `s1-${digestCanonical(recoverySemanticPayload(event))}`;
|
|
359
|
-
}
|
|
360
|
-
|
|
361
|
-
/**
|
|
362
|
-
* Derive the durable recovery identity from bounded source/target metadata.
|
|
363
|
-
* Terminal recovery includes the complete terminal fact set because it is the
|
|
364
|
-
* state transition, not merely a generation rotation.
|
|
365
|
-
*/
|
|
366
|
-
export function deriveRecoveryId(event: RecoveryIdInput): string {
|
|
367
|
-
if ("kind" in event && event.kind === "terminal_recovery") {
|
|
368
|
-
return `${JOURNAL_RECOVERY_ID_PREFIX}${digestCanonical({
|
|
369
|
-
blockedNodeIds: [...event.blockedNodeIds].sort(compareStrings),
|
|
370
|
-
branchNamespace: event.branchGeneration,
|
|
371
|
-
...(event.error === undefined ? {} : { error: event.error }),
|
|
372
|
-
runId: event.runId,
|
|
373
|
-
status: event.status,
|
|
374
|
-
terminalIntent: event.terminalIntent,
|
|
375
|
-
terminalResults: sortedRecoveryTerminals(event.terminalResults),
|
|
376
|
-
version: 1,
|
|
377
|
-
})}`;
|
|
332
|
+
const compactionCount = raw.compactionCount;
|
|
333
|
+
if (typeof compactionCount !== "number" || !Number.isInteger(compactionCount) || compactionCount < 0) {
|
|
334
|
+
throw new Error(`${label} compactionCount is invalid`);
|
|
378
335
|
}
|
|
379
|
-
const
|
|
380
|
-
const
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
...(terminal.result.agentId === undefined ? {} : { agentId: terminal.result.agentId }),
|
|
404
|
-
...(terminal.result.outputFile === undefined ? {} : { outputFile: terminal.result.outputFile }),
|
|
405
|
-
...(terminal.result.tokenCount === undefined ? {} : { tokenCount: terminal.result.tokenCount }),
|
|
406
|
-
compactionCount: terminal.result.compactionCount,
|
|
407
|
-
...(terminal.result.truncated === undefined ? {} : { truncated: terminal.result.truncated }),
|
|
408
|
-
})),
|
|
409
|
-
version: 1,
|
|
336
|
+
const updatedAt = validTimestamp(raw.updatedAt, label);
|
|
337
|
+
const result: CallResult = {
|
|
338
|
+
status,
|
|
339
|
+
compactionCount,
|
|
340
|
+
updatedAt,
|
|
341
|
+
...(raw.agentId === undefined ? {} : { agentId: boundedString(raw.agentId, `${label}.agentId`, JOURNAL_ID_LIMIT) }),
|
|
342
|
+
...(raw.attemptId === undefined
|
|
343
|
+
? {}
|
|
344
|
+
: { attemptId: boundedString(raw.attemptId, `${label}.attemptId`, JOURNAL_ID_LIMIT) }),
|
|
345
|
+
...(raw.result === undefined ? {} : { result: raw.result }),
|
|
346
|
+
...(raw.error === undefined ? {} : { error: boundedString(raw.error, `${label}.error`, JOURNAL_ERROR_LIMIT) }),
|
|
347
|
+
...(raw.outputFile === undefined
|
|
348
|
+
? {}
|
|
349
|
+
: { outputFile: boundedString(raw.outputFile, `${label}.outputFile`, JOURNAL_ID_LIMIT) }),
|
|
350
|
+
...(raw.tokenCount === undefined
|
|
351
|
+
? {}
|
|
352
|
+
: {
|
|
353
|
+
tokenCount: (() => {
|
|
354
|
+
if (typeof raw.tokenCount !== "number" || !Number.isInteger(raw.tokenCount) || raw.tokenCount < 0) {
|
|
355
|
+
throw new Error(`${label}.tokenCount is invalid`);
|
|
356
|
+
}
|
|
357
|
+
return raw.tokenCount;
|
|
358
|
+
})(),
|
|
359
|
+
}),
|
|
410
360
|
};
|
|
411
|
-
return
|
|
361
|
+
return result;
|
|
412
362
|
}
|
|
413
363
|
|
|
414
|
-
function
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
if (!isRecord(value)) throw new Error("owner must be an object");
|
|
421
|
-
assertKeys(value, ["extension", "runId", "nodeId", "attemptId"], "owner");
|
|
422
|
-
const extension = boundedString(value.extension, "owner.extension", 64);
|
|
423
|
-
const ownerRunId = boundedString(value.runId, "owner.runId", JOURNAL_ID_LIMIT);
|
|
424
|
-
const ownerNodeId = boundedString(value.nodeId, "owner.nodeId", JOURNAL_ID_LIMIT);
|
|
425
|
-
const attemptId =
|
|
426
|
-
value.attemptId === undefined ? undefined : boundedString(value.attemptId, "owner.attemptId", JOURNAL_ID_LIMIT);
|
|
427
|
-
if (extension !== "pi-workflows" || ownerRunId !== runId || ownerNodeId !== nodeId) {
|
|
428
|
-
throw new Error("journal owner does not match its workflow node");
|
|
364
|
+
function parseOwner(raw: unknown, runId: string, nodeId: string): WorkflowOwner {
|
|
365
|
+
if (!isRecord(raw) || raw.extension !== "pi-workflows") throw new Error("journal owner is invalid");
|
|
366
|
+
const ownerRunId = boundedString(raw.runId, "owner.runId", JOURNAL_ID_LIMIT);
|
|
367
|
+
const ownerNodeId = boundedString(raw.nodeId, "owner.nodeId", JOURNAL_ID_LIMIT);
|
|
368
|
+
if (ownerRunId !== runId || ownerNodeId !== nodeId) {
|
|
369
|
+
throw new Error("journal owner does not match its event");
|
|
429
370
|
}
|
|
430
371
|
return {
|
|
431
372
|
extension: "pi-workflows",
|
|
432
373
|
runId: ownerRunId,
|
|
433
374
|
nodeId: ownerNodeId,
|
|
434
|
-
...(attemptId
|
|
435
|
-
};
|
|
436
|
-
}
|
|
437
|
-
|
|
438
|
-
function parseResult(value: unknown, label: string): WorkflowTaskResult {
|
|
439
|
-
if (!isRecord(value)) throw new Error(`${label} must be an object`);
|
|
440
|
-
assertKeys(
|
|
441
|
-
value,
|
|
442
|
-
[
|
|
443
|
-
"status",
|
|
444
|
-
"agentId",
|
|
445
|
-
"attemptId",
|
|
446
|
-
"text",
|
|
447
|
-
"error",
|
|
448
|
-
"outputFile",
|
|
449
|
-
"tokenCount",
|
|
450
|
-
"compactionCount",
|
|
451
|
-
"truncated",
|
|
452
|
-
"updatedAt",
|
|
453
|
-
],
|
|
454
|
-
label,
|
|
455
|
-
);
|
|
456
|
-
const status = value.status;
|
|
457
|
-
if (status !== "completed" && status !== "failed" && status !== "stopped") {
|
|
458
|
-
throw new Error(`${label}.status is invalid`);
|
|
459
|
-
}
|
|
460
|
-
const compactionCount = value.compactionCount;
|
|
461
|
-
if (typeof compactionCount !== "number" || !Number.isInteger(compactionCount) || compactionCount < 0) {
|
|
462
|
-
throw new Error(`${label}.compactionCount is invalid`);
|
|
463
|
-
}
|
|
464
|
-
const tokenCount = value.tokenCount;
|
|
465
|
-
if (tokenCount !== undefined && (typeof tokenCount !== "number" || !Number.isInteger(tokenCount) || tokenCount < 0)) {
|
|
466
|
-
throw new Error(`${label}.tokenCount is invalid`);
|
|
467
|
-
}
|
|
468
|
-
if (value.truncated !== undefined && typeof value.truncated !== "boolean") {
|
|
469
|
-
throw new Error(`${label}.truncated is invalid`);
|
|
470
|
-
}
|
|
471
|
-
return {
|
|
472
|
-
status,
|
|
473
|
-
...(value.agentId === undefined
|
|
474
|
-
? {}
|
|
475
|
-
: { agentId: boundedString(value.agentId, `${label}.agentId`, JOURNAL_ID_LIMIT) }),
|
|
476
|
-
...(value.attemptId === undefined
|
|
477
|
-
? {}
|
|
478
|
-
: { attemptId: boundedString(value.attemptId, `${label}.attemptId`, JOURNAL_ID_LIMIT) }),
|
|
479
|
-
...(value.text === undefined ? {} : { text: boundedString(value.text, `${label}.text`, JOURNAL_TEXT_LIMIT + 32) }),
|
|
480
|
-
...(value.error === undefined
|
|
481
|
-
? {}
|
|
482
|
-
: { error: boundedString(value.error, `${label}.error`, JOURNAL_ERROR_LIMIT + 32) }),
|
|
483
|
-
...(value.outputFile === undefined
|
|
375
|
+
...(raw.attemptId === undefined
|
|
484
376
|
? {}
|
|
485
|
-
: {
|
|
486
|
-
...(tokenCount === undefined ? {} : { tokenCount }),
|
|
487
|
-
compactionCount,
|
|
488
|
-
...(value.truncated === undefined ? {} : { truncated: value.truncated }),
|
|
489
|
-
updatedAt: validTimestamp(value.updatedAt, `${label}.updatedAt`),
|
|
377
|
+
: { attemptId: boundedString(raw.attemptId, "owner.attemptId", JOURNAL_ID_LIMIT) }),
|
|
490
378
|
};
|
|
491
379
|
}
|
|
492
380
|
|
|
493
|
-
function
|
|
494
|
-
|
|
495
|
-
}
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
const result: Record<string, number> = {};
|
|
501
|
-
for (const [nodeId, generation] of Object.entries(value)) {
|
|
502
|
-
if (!allowed.has(nodeId)) throw new Error(`${label} contains an unknown node`);
|
|
503
|
-
if (typeof generation !== "number" || !Number.isInteger(generation) || generation < 1 || generation > 1_000_000) {
|
|
504
|
-
throw new Error(`${label}.${nodeId} is invalid`);
|
|
381
|
+
function parseAttemptMap(raw: unknown, label: string): Record<string, number> {
|
|
382
|
+
if (!isRecord(raw)) throw new Error(`${label} is invalid`);
|
|
383
|
+
const out: Record<string, number> = {};
|
|
384
|
+
for (const [nodeId, generation] of Object.entries(raw)) {
|
|
385
|
+
parseCallIndex(nodeId, `${label} key`);
|
|
386
|
+
if (typeof generation !== "number" || !Number.isInteger(generation) || generation < 1) {
|
|
387
|
+
throw new Error(`${label}.${nodeId} generation is invalid`);
|
|
505
388
|
}
|
|
506
|
-
|
|
389
|
+
out[nodeId] = generation;
|
|
507
390
|
}
|
|
508
|
-
return
|
|
391
|
+
return out;
|
|
509
392
|
}
|
|
510
393
|
|
|
511
|
-
function parseAttemptIdMap(
|
|
512
|
-
if (!isRecord(
|
|
513
|
-
const
|
|
514
|
-
const
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
result[nodeId] = boundedString(attemptId, `${label}.${nodeId}`, JOURNAL_ID_LIMIT);
|
|
394
|
+
function parseAttemptIdMap(raw: unknown, label: string): Record<string, string> {
|
|
395
|
+
if (!isRecord(raw)) throw new Error(`${label} is invalid`);
|
|
396
|
+
const out: Record<string, string> = {};
|
|
397
|
+
for (const [nodeId, attemptId] of Object.entries(raw)) {
|
|
398
|
+
parseCallIndex(nodeId, `${label} key`);
|
|
399
|
+
out[nodeId] = boundedString(attemptId, `${label}.${nodeId}`, JOURNAL_ID_LIMIT);
|
|
518
400
|
}
|
|
519
|
-
return
|
|
401
|
+
return out;
|
|
520
402
|
}
|
|
521
403
|
|
|
522
|
-
function
|
|
523
|
-
|
|
524
|
-
throw new Error(`${label} is invalid`);
|
|
525
|
-
}
|
|
526
|
-
return value;
|
|
404
|
+
function defaultAttemptId(runId: string, nodeId: string, generation = 1): string {
|
|
405
|
+
return `${runId}/${nodeId}/attempt-${generation}`;
|
|
527
406
|
}
|
|
528
407
|
|
|
529
|
-
function
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
|
|
535
|
-
|
|
536
|
-
);
|
|
537
|
-
}
|
|
538
|
-
|
|
539
|
-
function activeTaskStatus(status: TaskStatus): boolean {
|
|
540
|
-
return status === "dispatching" || status === "queued" || status === "running";
|
|
541
|
-
}
|
|
542
|
-
|
|
543
|
-
function parseRecoveryRotationShape(
|
|
544
|
-
value: unknown,
|
|
545
|
-
runId: string,
|
|
546
|
-
nodes: ReadonlySet<string>,
|
|
547
|
-
label: string,
|
|
548
|
-
): RecoveryRotation {
|
|
549
|
-
if (!isRecord(value)) throw new Error(`${label} must be an object`);
|
|
550
|
-
assertKeys(
|
|
551
|
-
value,
|
|
552
|
-
[
|
|
553
|
-
"nodeId",
|
|
554
|
-
"sourceAttemptId",
|
|
555
|
-
"sourceGeneration",
|
|
556
|
-
"sourceStatus",
|
|
557
|
-
"attemptId",
|
|
558
|
-
"generation",
|
|
559
|
-
"owner",
|
|
560
|
-
"supersededAgentId",
|
|
561
|
-
],
|
|
562
|
-
label,
|
|
563
|
-
);
|
|
564
|
-
const nodeId = boundedString(value.nodeId, `${label}.nodeId`, JOURNAL_ID_LIMIT);
|
|
565
|
-
if (!nodes.has(nodeId)) throw new Error(`${label}.nodeId references an unknown node`);
|
|
566
|
-
const sourceAttemptId = boundedString(value.sourceAttemptId, `${label}.sourceAttemptId`, JOURNAL_ID_LIMIT);
|
|
567
|
-
const sourceGeneration = value.sourceGeneration;
|
|
568
|
-
if (
|
|
569
|
-
typeof sourceGeneration !== "number" ||
|
|
570
|
-
!Number.isInteger(sourceGeneration) ||
|
|
571
|
-
sourceGeneration < 1 ||
|
|
572
|
-
sourceGeneration > JOURNAL_GENERATION_LIMIT
|
|
573
|
-
) {
|
|
574
|
-
throw new Error(`${label}.sourceGeneration is invalid`);
|
|
575
|
-
}
|
|
576
|
-
const sourceStatus = value.sourceStatus;
|
|
577
|
-
if (nodeId === "__synthesis__") {
|
|
578
|
-
if (sourceStatus !== "synthesizing") throw new Error(`${label}.sourceStatus is invalid for synthesis`);
|
|
579
|
-
} else if (typeof sourceStatus !== "string" || !TASK_STATUS_VALUES.has(sourceStatus as TaskStatus)) {
|
|
580
|
-
throw new Error(`${label}.sourceStatus is invalid`);
|
|
408
|
+
function parseRotation(raw: unknown, runId: string): RecoveryRotation {
|
|
409
|
+
if (!isRecord(raw)) throw new Error("recovery rotation is invalid");
|
|
410
|
+
const nodeId = parseCallIndex(raw.nodeId, "rotation.nodeId");
|
|
411
|
+
const sourceAttemptId = boundedString(raw.sourceAttemptId, "rotation.sourceAttemptId", JOURNAL_ID_LIMIT);
|
|
412
|
+
const sourceGeneration = raw.sourceGeneration;
|
|
413
|
+
if (typeof sourceGeneration !== "number" || !Number.isInteger(sourceGeneration) || sourceGeneration < 1) {
|
|
414
|
+
throw new Error("rotation.sourceGeneration is invalid");
|
|
581
415
|
}
|
|
582
|
-
const
|
|
416
|
+
const sourceStatus = raw.sourceStatus;
|
|
583
417
|
if (
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
|
|
587
|
-
|
|
418
|
+
sourceStatus !== "running" &&
|
|
419
|
+
sourceStatus !== "completed" &&
|
|
420
|
+
sourceStatus !== "failed" &&
|
|
421
|
+
sourceStatus !== "stopped"
|
|
588
422
|
) {
|
|
589
|
-
throw new Error(
|
|
423
|
+
throw new Error("rotation.sourceStatus is invalid");
|
|
590
424
|
}
|
|
591
|
-
const attemptId = boundedString(
|
|
592
|
-
const
|
|
593
|
-
if (
|
|
594
|
-
|
|
595
|
-
|
|
596
|
-
|
|
597
|
-
|
|
598
|
-
|
|
599
|
-
: boundedString(value.supersededAgentId, `${label}.supersededAgentId`, JOURNAL_ID_LIMIT);
|
|
425
|
+
const attemptId = boundedString(raw.attemptId, "rotation.attemptId", JOURNAL_ID_LIMIT);
|
|
426
|
+
const generation = raw.generation;
|
|
427
|
+
if (typeof generation !== "number" || !Number.isInteger(generation) || generation < 1) {
|
|
428
|
+
throw new Error("rotation.generation is invalid");
|
|
429
|
+
}
|
|
430
|
+
if (generation <= sourceGeneration) throw new Error("rotation generation must exceed its source");
|
|
431
|
+
const owner = parseOwner(raw.owner, runId, nodeId);
|
|
432
|
+
if (owner.attemptId !== attemptId) throw new Error("rotation owner does not match its attempt");
|
|
600
433
|
return {
|
|
601
434
|
nodeId,
|
|
602
435
|
sourceAttemptId,
|
|
603
436
|
sourceGeneration,
|
|
604
|
-
sourceStatus
|
|
437
|
+
sourceStatus,
|
|
605
438
|
attemptId,
|
|
606
439
|
generation,
|
|
607
440
|
owner,
|
|
608
|
-
...(supersededAgentId === undefined
|
|
441
|
+
...(raw.supersededAgentId === undefined
|
|
442
|
+
? {}
|
|
443
|
+
: { supersededAgentId: boundedString(raw.supersededAgentId, "rotation.supersededAgentId", JOURNAL_ID_LIMIT) }),
|
|
609
444
|
};
|
|
610
445
|
}
|
|
611
446
|
|
|
612
|
-
function
|
|
613
|
-
|
|
614
|
-
|
|
615
|
-
|
|
616
|
-
|
|
617
|
-
|
|
618
|
-
|
|
619
|
-
|
|
620
|
-
if (run.attemptIds[nodeId] !== sourceAttemptId) {
|
|
621
|
-
throw new Error(`${label}.sourceAttemptId does not match the current attempt`);
|
|
622
|
-
}
|
|
623
|
-
if (run.attempts[nodeId] !== sourceGeneration) {
|
|
624
|
-
throw new Error(`${label}.sourceGeneration does not match the current generation`);
|
|
625
|
-
}
|
|
626
|
-
if (nodeId === "__synthesis__") {
|
|
627
|
-
const synthesisSourceAllowed =
|
|
628
|
-
run.status === "synthesizing" || (allowInterruptedSynthesis && run.status === "interrupted");
|
|
629
|
-
if (sourceStatus !== "synthesizing" || !synthesisSourceAllowed) {
|
|
630
|
-
throw new Error(`${label}.sourceStatus is invalid for synthesis`);
|
|
631
|
-
}
|
|
632
|
-
} else {
|
|
633
|
-
const taskStatus = run.taskStatus[nodeId];
|
|
634
|
-
if (taskStatus !== sourceStatus || !recoverableTaskStatus(taskStatus)) {
|
|
635
|
-
throw new Error(`${label}.sourceStatus does not match the current task`);
|
|
636
|
-
}
|
|
637
|
-
}
|
|
638
|
-
return rotation;
|
|
639
|
-
}
|
|
640
|
-
|
|
641
|
-
function parseRecoveryRotationsShape(
|
|
642
|
-
raw: unknown,
|
|
643
|
-
runId: string,
|
|
644
|
-
nodes: ReadonlySet<string>,
|
|
645
|
-
terminalNodes: ReadonlySet<string> = new Set(),
|
|
646
|
-
): RecoveryRotation[] {
|
|
647
|
-
if (!Array.isArray(raw)) throw new Error("journal recovery rotations must be an array");
|
|
648
|
-
if (raw.length > JOURNAL_RECOVERY_NODE_LIMIT) throw new Error("journal recovery has too many rotations");
|
|
649
|
-
const seen = new Set<string>();
|
|
650
|
-
return raw.map((value, index) => {
|
|
651
|
-
const rotation = parseRecoveryRotationShape(value, runId, nodes, `rotations[${index}]`);
|
|
652
|
-
if (seen.has(rotation.nodeId)) throw new Error("journal recovery contains duplicate node rotations");
|
|
653
|
-
if (terminalNodes.has(rotation.nodeId)) throw new Error("journal recovery both settles and rotates a node");
|
|
654
|
-
seen.add(rotation.nodeId);
|
|
655
|
-
return rotation;
|
|
656
|
-
});
|
|
447
|
+
function parseTerminalResult(raw: unknown, runId: string): RecoveryTerminalResult {
|
|
448
|
+
if (!isRecord(raw)) throw new Error("recovery terminal result is invalid");
|
|
449
|
+
const nodeId = parseCallIndex(raw.nodeId, "terminal.nodeId");
|
|
450
|
+
const attemptId = boundedString(raw.attemptId, "terminal.attemptId", JOURNAL_ID_LIMIT);
|
|
451
|
+
const result = parseResult(raw.result, "terminal.result");
|
|
452
|
+
const owner = parseOwner(raw.owner, runId, nodeId);
|
|
453
|
+
if (owner.attemptId !== attemptId) throw new Error("terminal owner does not match its attempt");
|
|
454
|
+
return { nodeId, attemptId, result, owner };
|
|
657
455
|
}
|
|
658
456
|
|
|
659
|
-
function
|
|
660
|
-
raw
|
|
661
|
-
|
|
662
|
-
|
|
663
|
-
): RecoveryRotation[] {
|
|
664
|
-
if (!Array.isArray(raw)) throw new Error("journal recovery rotations must be an array");
|
|
665
|
-
if (raw.length > JOURNAL_RECOVERY_NODE_LIMIT) throw new Error("journal recovery has too many rotations");
|
|
666
|
-
const seen = new Set<string>();
|
|
667
|
-
const rotations = raw.map((value, index) => {
|
|
668
|
-
const rotation = parseRecoveryRotation(value, run, `rotations[${index}]`);
|
|
669
|
-
if (seen.has(rotation.nodeId)) throw new Error("journal recovery contains duplicate node rotations");
|
|
670
|
-
if (terminalNodes.has(rotation.nodeId)) throw new Error("journal recovery both settles and rotates a node");
|
|
671
|
-
seen.add(rotation.nodeId);
|
|
672
|
-
return rotation;
|
|
673
|
-
});
|
|
674
|
-
for (const task of run.definition.tasks) {
|
|
675
|
-
if (!activeTaskStatus(run.taskStatus[task.id]) || terminalNodes.has(task.id)) continue;
|
|
676
|
-
if (!seen.has(task.id)) throw new Error(`journal recovery omits active task ${task.id}`);
|
|
457
|
+
function parseRunRecovery(raw: Record<string, unknown>, run: ScriptRun, timestamp: number): RunRecoveryEvent {
|
|
458
|
+
if (raw.status !== "interrupted") throw new Error("run recovery status is invalid");
|
|
459
|
+
if (typeof raw.branchGeneration !== "number" || !Number.isInteger(raw.branchGeneration) || raw.branchGeneration < 0) {
|
|
460
|
+
throw new Error("run recovery branchGeneration is invalid");
|
|
677
461
|
}
|
|
678
|
-
if (
|
|
679
|
-
run
|
|
680
|
-
!run.synthesisResult &&
|
|
681
|
-
!terminalNodes.has("__synthesis__") &&
|
|
682
|
-
!seen.has("__synthesis__")
|
|
683
|
-
) {
|
|
684
|
-
throw new Error("journal recovery omits active synthesis");
|
|
685
|
-
}
|
|
686
|
-
return rotations;
|
|
687
|
-
}
|
|
688
|
-
|
|
689
|
-
function parseRecoveryTerminalResultShape(
|
|
690
|
-
value: unknown,
|
|
691
|
-
runId: string,
|
|
692
|
-
nodes: ReadonlySet<string>,
|
|
693
|
-
label: string,
|
|
694
|
-
): RecoveryTerminalResult {
|
|
695
|
-
if (!isRecord(value)) throw new Error(`${label} must be an object`);
|
|
696
|
-
assertKeys(value, ["nodeId", "attemptId", "result", "owner"], label);
|
|
697
|
-
const nodeId = boundedString(value.nodeId, `${label}.nodeId`, JOURNAL_ID_LIMIT);
|
|
698
|
-
if (!nodes.has(nodeId)) throw new Error("journal recovery terminal result references an unknown node");
|
|
699
|
-
const attemptId = boundedString(value.attemptId, `${label}.attemptId`, JOURNAL_ID_LIMIT);
|
|
700
|
-
const owner = parseOwner(value.owner, runId, nodeId);
|
|
701
|
-
if (owner.attemptId !== attemptId) throw new Error("journal recovery terminal result owner is invalid");
|
|
702
|
-
const parsed = parseResult(value.result, `${label}.result`);
|
|
703
|
-
if (parsed.attemptId !== undefined && parsed.attemptId !== attemptId) {
|
|
704
|
-
throw new Error("journal recovery terminal result payload references a superseded attempt");
|
|
462
|
+
if (!Array.isArray(raw.rotations) || raw.rotations.length > JOURNAL_RECOVERY_NODE_LIMIT) {
|
|
463
|
+
throw new Error("run recovery rotations are invalid");
|
|
705
464
|
}
|
|
465
|
+
const rotations = raw.rotations.map((rotation) => parseRotation(rotation, run.runId));
|
|
466
|
+
const terminalResults =
|
|
467
|
+
raw.terminalResults === undefined
|
|
468
|
+
? undefined
|
|
469
|
+
: (() => {
|
|
470
|
+
if (!Array.isArray(raw.terminalResults)) throw new Error("run recovery terminalResults is invalid");
|
|
471
|
+
return raw.terminalResults.map((terminal) => parseTerminalResult(terminal, run.runId));
|
|
472
|
+
})();
|
|
473
|
+
const recoveryId = boundedString(raw.recoveryId, "recoveryId", JOURNAL_RECOVERY_ID_LIMIT);
|
|
474
|
+
if (!RECOVERY_ID_PATTERN.test(recoveryId)) throw new Error("recoveryId is malformed");
|
|
706
475
|
return {
|
|
707
|
-
|
|
708
|
-
|
|
709
|
-
|
|
710
|
-
|
|
476
|
+
kind: "run_recovery",
|
|
477
|
+
schemaVersion: JOURNAL_SCHEMA_VERSION,
|
|
478
|
+
runId: run.runId,
|
|
479
|
+
status: "interrupted",
|
|
480
|
+
branchGeneration: raw.branchGeneration,
|
|
481
|
+
rotations,
|
|
482
|
+
...(terminalResults === undefined || terminalResults.length === 0 ? {} : { terminalResults }),
|
|
483
|
+
recoveryId,
|
|
484
|
+
timestamp,
|
|
711
485
|
};
|
|
712
486
|
}
|
|
713
487
|
|
|
714
|
-
function
|
|
715
|
-
raw
|
|
716
|
-
|
|
717
|
-
|
|
718
|
-
): RecoveryTerminalResult[] {
|
|
719
|
-
if (raw === undefined) return [];
|
|
720
|
-
if (!Array.isArray(raw)) throw new Error("journal recovery terminal results must be an array");
|
|
721
|
-
if (raw.length > JOURNAL_RECOVERY_NODE_LIMIT) throw new Error("journal recovery has too many terminal results");
|
|
722
|
-
const seen = new Set<string>();
|
|
723
|
-
return raw.map((value, index) => {
|
|
724
|
-
const terminal = parseRecoveryTerminalResultShape(value, runId, nodes, `terminalResults[${index}]`);
|
|
725
|
-
if (seen.has(terminal.nodeId)) throw new Error("journal recovery contains duplicate terminal results");
|
|
726
|
-
seen.add(terminal.nodeId);
|
|
727
|
-
return terminal;
|
|
728
|
-
});
|
|
729
|
-
}
|
|
730
|
-
|
|
731
|
-
function parseRecoveryTerminalResults(raw: unknown, run: WorkflowRun): RecoveryTerminalResult[] {
|
|
732
|
-
const terminals = parseRecoveryTerminalResultShapes(raw, run.runId, taskNodeIds(run));
|
|
733
|
-
for (const terminal of terminals) {
|
|
734
|
-
const currentStatus = terminal.nodeId === "__synthesis__" ? undefined : run.taskStatus[terminal.nodeId];
|
|
735
|
-
if (terminal.nodeId === "__synthesis__" && run.synthesisResult) {
|
|
736
|
-
throw new Error("journal recovery terminal result references settled synthesis");
|
|
737
|
-
}
|
|
738
|
-
if (currentStatus !== undefined && isTerminalTask(currentStatus)) {
|
|
739
|
-
throw new Error("journal recovery terminal result references a settled task");
|
|
740
|
-
}
|
|
741
|
-
if (terminal.attemptId !== run.attemptIds[terminal.nodeId]) {
|
|
742
|
-
throw new Error("journal recovery terminal result references a superseded attempt");
|
|
743
|
-
}
|
|
744
|
-
}
|
|
745
|
-
return terminals;
|
|
746
|
-
}
|
|
747
|
-
|
|
748
|
-
function parseRunRecovery(raw: Record<string, unknown>, run: WorkflowRun, timestamp: number): RunRecoveryEvent {
|
|
749
|
-
if (raw.schemaVersion !== JOURNAL_SCHEMA_VERSION) throw new Error("run recovery schema version is unsupported");
|
|
750
|
-
assertKeys(
|
|
751
|
-
raw,
|
|
752
|
-
[
|
|
753
|
-
"kind",
|
|
754
|
-
"schemaVersion",
|
|
755
|
-
"runId",
|
|
756
|
-
"status",
|
|
757
|
-
"branchGeneration",
|
|
758
|
-
"rotations",
|
|
759
|
-
"terminalResults",
|
|
760
|
-
"recoveryId",
|
|
761
|
-
"legacyMigration",
|
|
762
|
-
"timestamp",
|
|
763
|
-
],
|
|
764
|
-
"run recovery",
|
|
765
|
-
);
|
|
766
|
-
if (raw.legacyMigration !== undefined && raw.legacyMigration !== true) {
|
|
767
|
-
throw new Error("run recovery legacy migration marker is invalid");
|
|
488
|
+
function parseTerminalRecovery(raw: Record<string, unknown>, run: ScriptRun, timestamp: number): TerminalRecoveryEvent {
|
|
489
|
+
if (raw.status !== "failed" && raw.status !== "stopped") throw new Error("terminal recovery status is invalid");
|
|
490
|
+
if (raw.terminalIntent !== "failure" && raw.terminalIntent !== "stop") {
|
|
491
|
+
throw new Error("terminal recovery intent is invalid");
|
|
768
492
|
}
|
|
769
|
-
if (
|
|
770
|
-
throw new Error(
|
|
493
|
+
if (typeof raw.branchGeneration !== "number" || !Number.isInteger(raw.branchGeneration) || raw.branchGeneration < 0) {
|
|
494
|
+
throw new Error("terminal recovery branchGeneration is invalid");
|
|
771
495
|
}
|
|
772
|
-
if (
|
|
773
|
-
|
|
774
|
-
const branchGeneration = parseBranchGeneration(raw.branchGeneration, "branchGeneration");
|
|
775
|
-
const terminalResults = parseRecoveryTerminalResults(raw.terminalResults, run);
|
|
776
|
-
if (Array.isArray(raw.rotations) && raw.rotations.length + terminalResults.length > JOURNAL_RECOVERY_NODE_LIMIT) {
|
|
777
|
-
throw new Error("journal recovery affects too many nodes");
|
|
496
|
+
if (!Array.isArray(raw.terminalResults) || raw.terminalResults.length > JOURNAL_RECOVERY_NODE_LIMIT) {
|
|
497
|
+
throw new Error("terminal recovery terminalResults are invalid");
|
|
778
498
|
}
|
|
779
|
-
const
|
|
780
|
-
|
|
781
|
-
|
|
782
|
-
runId: run.runId,
|
|
783
|
-
status: "interrupted" as const,
|
|
784
|
-
branchGeneration,
|
|
785
|
-
rotations: parseRecoveryRotations(raw.rotations, run, new Set(terminalResults.map((item) => item.nodeId))),
|
|
786
|
-
...(terminalResults.length > 0 ? { terminalResults } : {}),
|
|
787
|
-
};
|
|
788
|
-
const expectedRecoveryId = deriveRecoveryId(base);
|
|
789
|
-
const legacyMigration = raw.recoveryId === undefined;
|
|
790
|
-
const recoveryId = legacyMigration ? expectedRecoveryId : parseRecoveryId(raw.recoveryId, "recoveryId");
|
|
791
|
-
if (recoveryId !== expectedRecoveryId) {
|
|
792
|
-
throw new Error("run recovery recoveryId does not match its semantic source and target fields");
|
|
499
|
+
const terminalResults = raw.terminalResults.map((terminal) => parseTerminalResult(terminal, run.runId));
|
|
500
|
+
if (!Array.isArray(raw.blockedNodeIds) || raw.blockedNodeIds.length > JOURNAL_RECOVERY_NODE_LIMIT) {
|
|
501
|
+
throw new Error("terminal recovery blockedNodeIds are invalid");
|
|
793
502
|
}
|
|
503
|
+
const blockedNodeIds = raw.blockedNodeIds.map((nodeId) => parseCallIndex(nodeId, "blockedNodeId"));
|
|
504
|
+
const recoveryId = boundedString(raw.recoveryId, "recoveryId", JOURNAL_RECOVERY_ID_LIMIT);
|
|
505
|
+
if (!RECOVERY_ID_PATTERN.test(recoveryId)) throw new Error("recoveryId is malformed");
|
|
794
506
|
return {
|
|
795
|
-
|
|
507
|
+
kind: "terminal_recovery",
|
|
508
|
+
schemaVersion: JOURNAL_SCHEMA_VERSION,
|
|
509
|
+
runId: run.runId,
|
|
510
|
+
status: raw.status,
|
|
511
|
+
terminalIntent: raw.terminalIntent,
|
|
512
|
+
branchGeneration: raw.branchGeneration,
|
|
513
|
+
terminalResults,
|
|
514
|
+
blockedNodeIds,
|
|
515
|
+
...(raw.error === undefined
|
|
516
|
+
? {}
|
|
517
|
+
: { error: boundedString(raw.error, "terminal recovery error", JOURNAL_ERROR_LIMIT) }),
|
|
796
518
|
recoveryId,
|
|
797
|
-
...(legacyMigration ? { legacyMigration: true } : {}),
|
|
798
519
|
timestamp,
|
|
799
520
|
};
|
|
800
521
|
}
|
|
801
522
|
|
|
802
|
-
function
|
|
803
|
-
if (raw.
|
|
804
|
-
|
|
805
|
-
raw,
|
|
806
|
-
[
|
|
807
|
-
"kind",
|
|
808
|
-
"schemaVersion",
|
|
809
|
-
"runId",
|
|
810
|
-
"status",
|
|
811
|
-
"branchGeneration",
|
|
812
|
-
"rotations",
|
|
813
|
-
"terminalResults",
|
|
814
|
-
"recoveryId",
|
|
815
|
-
"legacyMigration",
|
|
816
|
-
"timestamp",
|
|
817
|
-
],
|
|
818
|
-
"run recovery duplicate",
|
|
819
|
-
);
|
|
820
|
-
if (raw.legacyMigration !== undefined && raw.legacyMigration !== true) {
|
|
821
|
-
throw new Error("run recovery legacy migration marker is invalid");
|
|
822
|
-
}
|
|
823
|
-
const runId = boundedString(raw.runId, "runId", JOURNAL_ID_LIMIT);
|
|
824
|
-
if (runId !== run.runId) throw new Error("run recovery duplicate references a different run");
|
|
825
|
-
if (raw.status !== "interrupted") throw new Error("run recovery status must be interrupted");
|
|
826
|
-
const branchGeneration = parseBranchGeneration(raw.branchGeneration, "branchGeneration");
|
|
827
|
-
const nodes = taskNodeIds(run);
|
|
828
|
-
const terminalResults = parseRecoveryTerminalResultShapes(raw.terminalResults, runId, nodes);
|
|
829
|
-
if (Array.isArray(raw.rotations) && raw.rotations.length + terminalResults.length > JOURNAL_RECOVERY_NODE_LIMIT) {
|
|
830
|
-
throw new Error("journal recovery affects too many nodes");
|
|
831
|
-
}
|
|
832
|
-
const base: RecoveryIdInput = {
|
|
833
|
-
kind: "run_recovery" as const,
|
|
834
|
-
runId,
|
|
835
|
-
status: "interrupted",
|
|
836
|
-
branchGeneration,
|
|
837
|
-
rotations: parseRecoveryRotationsShape(
|
|
838
|
-
raw.rotations,
|
|
839
|
-
runId,
|
|
840
|
-
nodes,
|
|
841
|
-
new Set(terminalResults.map((item) => item.nodeId)),
|
|
842
|
-
),
|
|
843
|
-
...(terminalResults.length > 0 ? { terminalResults } : {}),
|
|
844
|
-
};
|
|
845
|
-
const expectedRecoveryId = deriveRecoveryId(base);
|
|
846
|
-
const legacyMigration = raw.recoveryId === undefined;
|
|
847
|
-
const recoveryId = legacyMigration ? expectedRecoveryId : parseRecoveryId(raw.recoveryId, "recoveryId");
|
|
848
|
-
if (recoveryId !== expectedRecoveryId) {
|
|
849
|
-
throw new Error("run recovery recoveryId does not match its semantic source and target fields");
|
|
523
|
+
function parseAttemptRecovery(raw: Record<string, unknown>, run: ScriptRun, timestamp: number): AttemptRecoveryEvent {
|
|
524
|
+
if (typeof raw.branchGeneration !== "number" || !Number.isInteger(raw.branchGeneration) || raw.branchGeneration < 0) {
|
|
525
|
+
throw new Error("attempt recovery branchGeneration is invalid");
|
|
850
526
|
}
|
|
527
|
+
const rotation = parseRotation(raw.rotation, run.runId);
|
|
851
528
|
return {
|
|
852
|
-
|
|
853
|
-
|
|
854
|
-
|
|
855
|
-
|
|
856
|
-
|
|
857
|
-
|
|
529
|
+
kind: "attempt_recovery",
|
|
530
|
+
schemaVersion: JOURNAL_SCHEMA_VERSION,
|
|
531
|
+
runId: run.runId,
|
|
532
|
+
nodeId: rotation.nodeId,
|
|
533
|
+
branchGeneration: raw.branchGeneration,
|
|
534
|
+
rotation,
|
|
535
|
+
...(raw.legacyMigration === true ? { legacyMigration: true } : {}),
|
|
536
|
+
timestamp,
|
|
858
537
|
};
|
|
859
538
|
}
|
|
860
539
|
|
|
861
|
-
|
|
862
|
-
|
|
863
|
-
|
|
864
|
-
|
|
865
|
-
return raw.map((value, index) => {
|
|
866
|
-
const nodeId = boundedString(value, `blockedNodeIds[${index}]`, JOURNAL_ID_LIMIT);
|
|
867
|
-
if (nodeId === "__synthesis__" || !nodes.has(nodeId)) {
|
|
868
|
-
throw new Error("terminal recovery blockedNodeIds references an unknown task node");
|
|
869
|
-
}
|
|
870
|
-
if (seen.has(nodeId)) throw new Error("terminal recovery contains duplicate blocked nodes");
|
|
871
|
-
seen.add(nodeId);
|
|
872
|
-
return nodeId;
|
|
873
|
-
});
|
|
540
|
+
/** Digest of an atomic recovery's semantic content, for duplicate detection. */
|
|
541
|
+
export function recoverySemanticFingerprint(event: RunRecoveryEvent | TerminalRecoveryEvent): string {
|
|
542
|
+
const { recoveryId: _recoveryId, ...rest } = event;
|
|
543
|
+
return createHash("sha256").update(JSON.stringify(rest)).digest("hex");
|
|
874
544
|
}
|
|
875
545
|
|
|
876
|
-
|
|
877
|
-
|
|
878
|
-
|
|
879
|
-
|
|
880
|
-
runId: string,
|
|
881
|
-
nodes: ReadonlySet<string>,
|
|
882
|
-
label: string,
|
|
883
|
-
): TerminalRecoveryBase {
|
|
884
|
-
if (raw.schemaVersion !== JOURNAL_SCHEMA_VERSION) throw new Error("terminal recovery schema version is unsupported");
|
|
885
|
-
assertKeys(
|
|
886
|
-
raw,
|
|
887
|
-
[
|
|
888
|
-
"kind",
|
|
889
|
-
"schemaVersion",
|
|
890
|
-
"runId",
|
|
891
|
-
"status",
|
|
892
|
-
"terminalIntent",
|
|
893
|
-
"branchGeneration",
|
|
894
|
-
"terminalResults",
|
|
895
|
-
"blockedNodeIds",
|
|
896
|
-
"error",
|
|
897
|
-
"recoveryId",
|
|
898
|
-
"timestamp",
|
|
899
|
-
],
|
|
900
|
-
label,
|
|
901
|
-
);
|
|
902
|
-
if (raw.kind !== "terminal_recovery") throw new Error(`${label} kind is invalid`);
|
|
903
|
-
const parsedRunId = boundedString(raw.runId, `${label}.runId`, JOURNAL_ID_LIMIT);
|
|
904
|
-
if (parsedRunId !== runId) throw new Error(`${label} references a different workflow run`);
|
|
905
|
-
const statusValue = raw.status;
|
|
906
|
-
if (statusValue !== "failed" && statusValue !== "stopped") throw new Error(`${label}.status is invalid`);
|
|
907
|
-
const status: "failed" | "stopped" = statusValue;
|
|
908
|
-
const terminalIntentValue = raw.terminalIntent;
|
|
909
|
-
if (terminalIntentValue !== "failure" && terminalIntentValue !== "stop") {
|
|
910
|
-
throw new Error(`${label}.terminalIntent is invalid`);
|
|
911
|
-
}
|
|
912
|
-
const terminalIntent: "failure" | "stop" = terminalIntentValue;
|
|
913
|
-
if ((terminalIntent === "failure" && status !== "failed") || (terminalIntent === "stop" && status !== "stopped")) {
|
|
914
|
-
throw new Error(`${label}.status does not match terminalIntent`);
|
|
915
|
-
}
|
|
916
|
-
const branchGeneration = parseBranchGeneration(raw.branchGeneration, `${label}.branchGeneration`);
|
|
917
|
-
if (!Array.isArray(raw.terminalResults)) throw new Error(`${label}.terminalResults must be an array`);
|
|
918
|
-
const terminalResults = parseRecoveryTerminalResultShapes(raw.terminalResults, parsedRunId, nodes);
|
|
919
|
-
const blockedNodeIds = parseBlockedNodeIds(raw.blockedNodeIds, nodes);
|
|
920
|
-
const affected = new Set<string>();
|
|
921
|
-
for (const terminal of terminalResults) {
|
|
922
|
-
if (affected.has(terminal.nodeId)) throw new Error(`${label} settles and blocks the same node`);
|
|
923
|
-
affected.add(terminal.nodeId);
|
|
546
|
+
function parseDuplicateRunRecovery(raw: Record<string, unknown>, run: ScriptRun): RunRecoveryEvent {
|
|
547
|
+
if (raw.status !== "interrupted") throw new Error("run recovery status is invalid");
|
|
548
|
+
if (!Array.isArray(raw.rotations) || raw.rotations.length > JOURNAL_RECOVERY_NODE_LIMIT) {
|
|
549
|
+
throw new Error("run recovery rotations are invalid");
|
|
924
550
|
}
|
|
925
|
-
|
|
926
|
-
|
|
927
|
-
|
|
928
|
-
|
|
929
|
-
|
|
930
|
-
|
|
931
|
-
|
|
932
|
-
|
|
933
|
-
|
|
934
|
-
|
|
935
|
-
|
|
936
|
-
|
|
937
|
-
|
|
938
|
-
|
|
939
|
-
|
|
551
|
+
const rotations = raw.rotations.map((rotation) => parseRotation(rotation, run.runId));
|
|
552
|
+
const terminalResults =
|
|
553
|
+
raw.terminalResults === undefined
|
|
554
|
+
? undefined
|
|
555
|
+
: (() => {
|
|
556
|
+
if (!Array.isArray(raw.terminalResults)) throw new Error("run recovery terminalResults is invalid");
|
|
557
|
+
return raw.terminalResults.map((terminal) => parseTerminalResult(terminal, run.runId));
|
|
558
|
+
})();
|
|
559
|
+
return {
|
|
560
|
+
kind: "run_recovery",
|
|
561
|
+
schemaVersion: JOURNAL_SCHEMA_VERSION,
|
|
562
|
+
runId: run.runId,
|
|
563
|
+
status: "interrupted",
|
|
564
|
+
branchGeneration: run.updatedAt, // not meaningful for duplicates; fingerprint covers content
|
|
565
|
+
rotations,
|
|
566
|
+
...(terminalResults === undefined || terminalResults.length === 0 ? {} : { terminalResults }),
|
|
567
|
+
recoveryId: boundedString(raw.recoveryId, "recoveryId", JOURNAL_RECOVERY_ID_LIMIT),
|
|
568
|
+
timestamp: run.updatedAt,
|
|
940
569
|
};
|
|
941
|
-
const expectedRecoveryId = deriveRecoveryId(base);
|
|
942
|
-
const recoveryId = parseRecoveryId(raw.recoveryId, `${label}.recoveryId`);
|
|
943
|
-
if (recoveryId !== expectedRecoveryId) {
|
|
944
|
-
throw new Error(`${label}.recoveryId does not match its semantic terminal fields`);
|
|
945
|
-
}
|
|
946
|
-
return { ...base, recoveryId };
|
|
947
|
-
}
|
|
948
|
-
|
|
949
|
-
function parseTerminalRecovery(
|
|
950
|
-
raw: Record<string, unknown>,
|
|
951
|
-
run: WorkflowRun,
|
|
952
|
-
timestamp: number,
|
|
953
|
-
): TerminalRecoveryEvent {
|
|
954
|
-
const base = parseTerminalRecoveryBase(raw, run.runId, taskNodeIds(run), "terminal recovery");
|
|
955
|
-
if (run.status !== "stopping" || run.terminalIntent !== base.terminalIntent) {
|
|
956
|
-
throw new Error(`invalid terminal recovery source state: ${run.status}`);
|
|
957
|
-
}
|
|
958
|
-
const terminalNodes = new Set(base.terminalResults.map((terminal) => terminal.nodeId));
|
|
959
|
-
for (const terminal of base.terminalResults) {
|
|
960
|
-
if (terminal.nodeId === "__synthesis__") {
|
|
961
|
-
if (!run.definition.synthesis) throw new Error("terminal recovery references undefined synthesis");
|
|
962
|
-
if (run.synthesisResult && JSON.stringify(run.synthesisResult) !== JSON.stringify(terminal.result)) {
|
|
963
|
-
throw new Error("terminal recovery synthesis fact contradicts an existing result");
|
|
964
|
-
}
|
|
965
|
-
if (!run.synthesisResult && terminal.attemptId !== run.attemptIds.__synthesis__) {
|
|
966
|
-
throw new Error("terminal recovery synthesis fact references a superseded attempt");
|
|
967
|
-
}
|
|
968
|
-
} else {
|
|
969
|
-
const existingResult = run.taskResults[terminal.nodeId];
|
|
970
|
-
if (existingResult && JSON.stringify(existingResult) !== JSON.stringify(terminal.result)) {
|
|
971
|
-
throw new Error("terminal recovery task fact contradicts an existing result");
|
|
972
|
-
}
|
|
973
|
-
if (!existingResult && isTerminalTask(run.taskStatus[terminal.nodeId])) {
|
|
974
|
-
throw new Error("terminal recovery task fact references a settled task");
|
|
975
|
-
}
|
|
976
|
-
if (!existingResult && terminal.attemptId !== run.attemptIds[terminal.nodeId]) {
|
|
977
|
-
throw new Error("terminal recovery task fact references a superseded attempt");
|
|
978
|
-
}
|
|
979
|
-
}
|
|
980
|
-
}
|
|
981
|
-
for (const task of run.definition.tasks) {
|
|
982
|
-
if (isTerminalTask(run.taskStatus[task.id]) || run.taskResults[task.id]) continue;
|
|
983
|
-
if (!terminalNodes.has(task.id) && !base.blockedNodeIds.includes(task.id)) {
|
|
984
|
-
throw new Error(`terminal recovery omits unresolved task ${task.id}`);
|
|
985
|
-
}
|
|
986
|
-
}
|
|
987
|
-
const synthesisUnresolved = run.definition.synthesis !== undefined && !run.synthesisResult;
|
|
988
|
-
if (synthesisUnresolved && !terminalNodes.has("__synthesis__")) {
|
|
989
|
-
throw new Error("terminal recovery omits unresolved synthesis");
|
|
990
|
-
}
|
|
991
|
-
return { ...base, timestamp };
|
|
992
|
-
}
|
|
993
|
-
|
|
994
|
-
function parseDuplicateTerminalRecovery(raw: Record<string, unknown>, run: WorkflowRun): TerminalRecoveryEvent {
|
|
995
|
-
const base = parseTerminalRecoveryBase(raw, run.runId, taskNodeIds(run), "terminal recovery duplicate");
|
|
996
|
-
return { ...base, timestamp: validTimestamp(raw.timestamp, "timestamp") };
|
|
997
570
|
}
|
|
998
571
|
|
|
999
|
-
function
|
|
1000
|
-
if (raw.
|
|
1001
|
-
|
|
1002
|
-
|
|
1003
|
-
["kind", "schemaVersion", "runId", "nodeId", "branchGeneration", "rotation", "legacyMigration", "timestamp"],
|
|
1004
|
-
"attempt recovery",
|
|
1005
|
-
);
|
|
1006
|
-
const legacyMigration = raw.legacyMigration === true;
|
|
1007
|
-
if (
|
|
1008
|
-
run.terminalIntent !== undefined ||
|
|
1009
|
-
["completed", "failed", "stopped"].includes(run.status) ||
|
|
1010
|
-
(run.status === "interrupted" && !legacyMigration)
|
|
1011
|
-
) {
|
|
1012
|
-
throw new Error(`invalid attempt recovery source state: ${run.status}`);
|
|
1013
|
-
}
|
|
1014
|
-
if (raw.legacyMigration !== undefined && raw.legacyMigration !== true) {
|
|
1015
|
-
throw new Error("attempt recovery legacy migration marker is invalid");
|
|
572
|
+
function parseDuplicateTerminalRecovery(raw: Record<string, unknown>, run: ScriptRun): TerminalRecoveryEvent {
|
|
573
|
+
if (raw.status !== "failed" && raw.status !== "stopped") throw new Error("terminal recovery status is invalid");
|
|
574
|
+
if (!Array.isArray(raw.terminalResults) || raw.terminalResults.length > JOURNAL_RECOVERY_NODE_LIMIT) {
|
|
575
|
+
throw new Error("terminal recovery terminalResults are invalid");
|
|
1016
576
|
}
|
|
1017
|
-
const
|
|
1018
|
-
const nodeId = boundedString(raw.nodeId, "nodeId", JOURNAL_ID_LIMIT);
|
|
1019
|
-
const rotation = parseRecoveryRotation(raw.rotation, run, "rotation", legacyMigration);
|
|
1020
|
-
if (rotation.nodeId !== nodeId) throw new Error("attempt recovery node does not match its rotation");
|
|
577
|
+
const terminalResults = raw.terminalResults.map((terminal) => parseTerminalResult(terminal, run.runId));
|
|
1021
578
|
return {
|
|
1022
|
-
kind: "
|
|
579
|
+
kind: "terminal_recovery",
|
|
1023
580
|
schemaVersion: JOURNAL_SCHEMA_VERSION,
|
|
1024
581
|
runId: run.runId,
|
|
1025
|
-
|
|
1026
|
-
|
|
1027
|
-
|
|
1028
|
-
|
|
1029
|
-
|
|
582
|
+
status: raw.status,
|
|
583
|
+
terminalIntent: raw.terminalIntent === "failure" ? "failure" : "stop",
|
|
584
|
+
branchGeneration: run.updatedAt,
|
|
585
|
+
terminalResults,
|
|
586
|
+
blockedNodeIds: [],
|
|
587
|
+
recoveryId: boundedString(raw.recoveryId, "recoveryId", JOURNAL_RECOVERY_ID_LIMIT),
|
|
588
|
+
timestamp: run.updatedAt,
|
|
1030
589
|
};
|
|
1031
590
|
}
|
|
1032
591
|
|
|
1033
|
-
function
|
|
1034
|
-
|
|
1035
|
-
}
|
|
1036
|
-
|
|
1037
|
-
function parseEvent(raw: unknown, runs: Map<string, WorkflowRun>): JournalEvent {
|
|
1038
|
-
if (!isRecord(raw)) throw new Error("journal event must be an object");
|
|
1039
|
-
assertKeys(
|
|
1040
|
-
raw,
|
|
1041
|
-
[
|
|
1042
|
-
"kind",
|
|
1043
|
-
"schemaVersion",
|
|
1044
|
-
"runId",
|
|
1045
|
-
"definition",
|
|
1046
|
-
"attempts",
|
|
1047
|
-
"attemptIds",
|
|
1048
|
-
"status",
|
|
1049
|
-
"error",
|
|
1050
|
-
"terminalIntent",
|
|
1051
|
-
"nodeId",
|
|
1052
|
-
"agentId",
|
|
1053
|
-
"attemptId",
|
|
1054
|
-
"generation",
|
|
1055
|
-
"owner",
|
|
1056
|
-
"result",
|
|
1057
|
-
"compactionCount",
|
|
1058
|
-
"branchGeneration",
|
|
1059
|
-
"rotations",
|
|
1060
|
-
"rotation",
|
|
1061
|
-
"sourceAttemptId",
|
|
1062
|
-
"sourceGeneration",
|
|
1063
|
-
"sourceStatus",
|
|
1064
|
-
"supersededAgentId",
|
|
1065
|
-
"legacyMigration",
|
|
1066
|
-
"terminalResults",
|
|
1067
|
-
"blockedNodeIds",
|
|
1068
|
-
"recoveryId",
|
|
1069
|
-
"timestamp",
|
|
1070
|
-
],
|
|
1071
|
-
"journal event",
|
|
1072
|
-
);
|
|
592
|
+
function parseEvent(raw: unknown, runs: Map<string, ScriptRun>): JournalEvent {
|
|
593
|
+
if (!isRecord(raw)) throw new Error("journal event is not an object");
|
|
1073
594
|
if (typeof raw.kind !== "string" || !JOURNAL_KINDS.has(raw.kind)) throw new Error("journal event kind is unknown");
|
|
1074
595
|
if (raw.schemaVersion !== JOURNAL_SCHEMA_VERSION && raw.schemaVersion !== JOURNAL_LEGACY_SCHEMA_VERSION) {
|
|
1075
596
|
throw new Error("journal schema version is unsupported");
|
|
@@ -1079,23 +600,90 @@ function parseEvent(raw: unknown, runs: Map<string, WorkflowRun>): JournalEvent
|
|
|
1079
600
|
|
|
1080
601
|
if (raw.kind === "run_created") {
|
|
1081
602
|
if (runs.has(runId)) throw new Error("duplicate workflow run_created event");
|
|
1082
|
-
const
|
|
603
|
+
const script = boundedString(raw.script, "script", JOURNAL_SCRIPT_LIMIT);
|
|
604
|
+
const scriptHash = boundedString(raw.scriptHash, "scriptHash", 128);
|
|
605
|
+
const meta = parseMeta(raw.meta);
|
|
606
|
+
const toolset =
|
|
607
|
+
raw.toolset === undefined
|
|
608
|
+
? undefined
|
|
609
|
+
: typeof raw.toolset === "string" && raw.toolset.length > 0 && raw.toolset.length <= 64
|
|
610
|
+
? raw.toolset
|
|
611
|
+
: undefined;
|
|
612
|
+
const frozenMaxAgents =
|
|
613
|
+
typeof raw.frozenMaxAgents === "number" && Number.isInteger(raw.frozenMaxAgents) && raw.frozenMaxAgents >= 1
|
|
614
|
+
? raw.frozenMaxAgents
|
|
615
|
+
: undefined;
|
|
616
|
+
const frozenConcurrency =
|
|
617
|
+
typeof raw.frozenConcurrency === "number" && Number.isInteger(raw.frozenConcurrency) && raw.frozenConcurrency >= 1
|
|
618
|
+
? raw.frozenConcurrency
|
|
619
|
+
: undefined;
|
|
620
|
+
const frozenAgentRetries =
|
|
621
|
+
typeof raw.frozenAgentRetries === "number" &&
|
|
622
|
+
Number.isInteger(raw.frozenAgentRetries) &&
|
|
623
|
+
raw.frozenAgentRetries >= 0
|
|
624
|
+
? raw.frozenAgentRetries
|
|
625
|
+
: undefined;
|
|
626
|
+
const frozenTokenBudget =
|
|
627
|
+
raw.frozenTokenBudget === null
|
|
628
|
+
? null
|
|
629
|
+
: typeof raw.frozenTokenBudget === "number" &&
|
|
630
|
+
Number.isFinite(raw.frozenTokenBudget) &&
|
|
631
|
+
raw.frozenTokenBudget >= 1
|
|
632
|
+
? raw.frozenTokenBudget
|
|
633
|
+
: undefined;
|
|
634
|
+
const frozenAgentTimeoutMs =
|
|
635
|
+
raw.frozenAgentTimeoutMs === null
|
|
636
|
+
? null
|
|
637
|
+
: typeof raw.frozenAgentTimeoutMs === "number" &&
|
|
638
|
+
Number.isFinite(raw.frozenAgentTimeoutMs) &&
|
|
639
|
+
raw.frozenAgentTimeoutMs >= 1
|
|
640
|
+
? raw.frozenAgentTimeoutMs
|
|
641
|
+
: undefined;
|
|
642
|
+
const frozenExcludeTools =
|
|
643
|
+
Array.isArray(raw.frozenExcludeTools) && raw.frozenExcludeTools.every((v) => typeof v === "string")
|
|
644
|
+
? (raw.frozenExcludeTools as string[])
|
|
645
|
+
: undefined;
|
|
1083
646
|
return {
|
|
1084
647
|
kind: "run_created",
|
|
1085
648
|
schemaVersion: raw.schemaVersion as JournalSchemaVersion,
|
|
1086
649
|
runId,
|
|
1087
|
-
|
|
650
|
+
script,
|
|
651
|
+
scriptHash,
|
|
652
|
+
meta,
|
|
653
|
+
...(toolset ? { toolset } : {}),
|
|
654
|
+
...(frozenMaxAgents !== undefined ? { frozenMaxAgents } : {}),
|
|
655
|
+
...(frozenConcurrency !== undefined ? { frozenConcurrency } : {}),
|
|
656
|
+
...(frozenAgentRetries !== undefined ? { frozenAgentRetries } : {}),
|
|
657
|
+
...(frozenTokenBudget !== undefined ? { frozenTokenBudget } : {}),
|
|
658
|
+
...(frozenAgentTimeoutMs !== undefined ? { frozenAgentTimeoutMs } : {}),
|
|
659
|
+
...(frozenExcludeTools ? { frozenExcludeTools } : {}),
|
|
1088
660
|
timestamp,
|
|
1089
|
-
...(raw.attempts === undefined ? {} : { attempts: parseAttemptMap(raw.attempts,
|
|
1090
|
-
...(raw.attemptIds === undefined
|
|
1091
|
-
? {}
|
|
1092
|
-
: { attemptIds: parseAttemptIdMap(raw.attemptIds, definition, "attemptIds") }),
|
|
661
|
+
...(raw.attempts === undefined ? {} : { attempts: parseAttemptMap(raw.attempts, "attempts") }),
|
|
662
|
+
...(raw.attemptIds === undefined ? {} : { attemptIds: parseAttemptIdMap(raw.attemptIds, "attemptIds") }),
|
|
1093
663
|
};
|
|
1094
664
|
}
|
|
1095
665
|
|
|
1096
666
|
const run = runs.get(runId);
|
|
1097
667
|
if (!run) throw new Error("journal event references an unknown run");
|
|
1098
|
-
|
|
668
|
+
if (raw.kind === "run_revision") {
|
|
669
|
+
const revision = raw.revision;
|
|
670
|
+
if (typeof revision !== "number" || !Number.isInteger(revision) || revision <= (run.revision ?? 0))
|
|
671
|
+
throw new Error("workflow revision is invalid or stale");
|
|
672
|
+
const script = boundedString(raw.script, "revision.script", JOURNAL_SCRIPT_LIMIT);
|
|
673
|
+
const scriptHash = boundedString(raw.scriptHash, "revision.scriptHash", 128);
|
|
674
|
+
return {
|
|
675
|
+
kind: "run_revision",
|
|
676
|
+
schemaVersion: JOURNAL_SCHEMA_VERSION,
|
|
677
|
+
runId,
|
|
678
|
+
revision,
|
|
679
|
+
script,
|
|
680
|
+
scriptHash,
|
|
681
|
+
meta: parseMeta(raw.meta),
|
|
682
|
+
timestamp,
|
|
683
|
+
};
|
|
684
|
+
}
|
|
685
|
+
if (raw.kind === "run_removed")
|
|
686
|
+
return { kind: "run_removed", schemaVersion: JOURNAL_SCHEMA_VERSION, runId, timestamp };
|
|
1099
687
|
if (raw.kind === "run_recovery") return parseRunRecovery(raw, run, timestamp);
|
|
1100
688
|
if (raw.kind === "terminal_recovery") return parseTerminalRecovery(raw, run, timestamp);
|
|
1101
689
|
if (raw.kind === "attempt_recovery") return parseAttemptRecovery(raw, run, timestamp);
|
|
@@ -1118,63 +706,14 @@ function parseEvent(raw: unknown, runs: Map<string, WorkflowRun>): JournalEvent
|
|
|
1118
706
|
};
|
|
1119
707
|
}
|
|
1120
708
|
|
|
1121
|
-
// Terminal workflow outcomes are immutable. A late
|
|
1122
|
-
//
|
|
1123
|
-
// valid run
|
|
709
|
+
// Terminal workflow outcomes are immutable. A late call fact from the
|
|
710
|
+
// superseded execution is stale, not a reason to quarantine an otherwise
|
|
711
|
+
// valid run.
|
|
1124
712
|
if (isTerminalWorkflowStatus(run.status) && raw.kind !== "workflow_transition") {
|
|
1125
713
|
throw new StaleJournalGenerationError("journal event follows a terminal workflow outcome");
|
|
1126
714
|
}
|
|
1127
715
|
|
|
1128
|
-
|
|
1129
|
-
const result = parseResult(raw.result, "synthesis result");
|
|
1130
|
-
const rawAttemptId =
|
|
1131
|
-
raw.attemptId === undefined ? undefined : boundedString(raw.attemptId, "attemptId", JOURNAL_ID_LIMIT);
|
|
1132
|
-
if (rawAttemptId !== undefined && result.attemptId !== undefined && rawAttemptId !== result.attemptId) {
|
|
1133
|
-
throw new StaleJournalGenerationError("synthesis result carries conflicting attempt generations");
|
|
1134
|
-
}
|
|
1135
|
-
const synthesisAttemptId = rawAttemptId ?? result.attemptId ?? run.attemptIds.__synthesis__;
|
|
1136
|
-
const owner = raw.owner === undefined ? undefined : parseOwner(raw.owner, runId, "__synthesis__");
|
|
1137
|
-
const currentAttemptId = run.attemptIds.__synthesis__;
|
|
1138
|
-
if (
|
|
1139
|
-
run.attemptTracking &&
|
|
1140
|
-
(rawAttemptId === undefined ||
|
|
1141
|
-
result.attemptId === undefined ||
|
|
1142
|
-
owner === undefined ||
|
|
1143
|
-
owner.attemptId === undefined)
|
|
1144
|
-
) {
|
|
1145
|
-
throw new Error("synthesis result is missing a required raw, payload, or owner attemptId");
|
|
1146
|
-
}
|
|
1147
|
-
if (
|
|
1148
|
-
run.attemptTracking &&
|
|
1149
|
-
(rawAttemptId !== result.attemptId ||
|
|
1150
|
-
synthesisAttemptId !== currentAttemptId ||
|
|
1151
|
-
owner?.attemptId !== currentAttemptId)
|
|
1152
|
-
) {
|
|
1153
|
-
throw new StaleJournalGenerationError("synthesis result references a stale attempt");
|
|
1154
|
-
}
|
|
1155
|
-
if (synthesisAttemptId !== currentAttemptId) {
|
|
1156
|
-
throw new StaleJournalGenerationError("synthesis result references a superseded attempt");
|
|
1157
|
-
}
|
|
1158
|
-
if (owner?.attemptId && currentAttemptId && owner.attemptId !== currentAttemptId) {
|
|
1159
|
-
throw new StaleJournalGenerationError("journal synthesis owner references a superseded attempt");
|
|
1160
|
-
}
|
|
1161
|
-
return {
|
|
1162
|
-
kind: "synthesis_result",
|
|
1163
|
-
schemaVersion: JOURNAL_SCHEMA_VERSION,
|
|
1164
|
-
runId,
|
|
1165
|
-
result: result.attemptId === undefined ? { ...result, attemptId: synthesisAttemptId } : result,
|
|
1166
|
-
attemptId: synthesisAttemptId,
|
|
1167
|
-
timestamp,
|
|
1168
|
-
...(owner ? { owner } : {}),
|
|
1169
|
-
};
|
|
1170
|
-
}
|
|
1171
|
-
|
|
1172
|
-
const nodeId = boundedString(raw.nodeId, "nodeId", JOURNAL_ID_LIMIT);
|
|
1173
|
-
if (
|
|
1174
|
-
!nodes.has(nodeId) ||
|
|
1175
|
-
(nodeId === "__synthesis__" && raw.kind !== "task_compacted" && raw.kind !== "task_attempt")
|
|
1176
|
-
)
|
|
1177
|
-
throw new Error("journal event references an unknown task node");
|
|
716
|
+
const nodeId = parseCallIndex(raw.nodeId, "nodeId");
|
|
1178
717
|
const owner = raw.owner === undefined ? undefined : parseOwner(raw.owner, runId, nodeId);
|
|
1179
718
|
const rawAttemptId =
|
|
1180
719
|
raw.attemptId === undefined ? undefined : boundedString(raw.attemptId, "attemptId", JOURNAL_ID_LIMIT);
|
|
@@ -1182,27 +721,27 @@ function parseEvent(raw: unknown, runs: Map<string, WorkflowRun>): JournalEvent
|
|
|
1182
721
|
if (run.attemptTracking && (rawAttemptId === undefined || owner === undefined || owner.attemptId === undefined)) {
|
|
1183
722
|
throw new Error("journal attempt-aware event is missing a required raw or owner attemptId");
|
|
1184
723
|
}
|
|
1185
|
-
if (raw.kind !== "
|
|
724
|
+
if (raw.kind !== "call_attempt" && owner?.attemptId && attemptId && owner.attemptId !== attemptId) {
|
|
1186
725
|
throw new StaleJournalGenerationError("journal owner references a different attempt");
|
|
1187
726
|
}
|
|
1188
|
-
if (raw.kind !== "
|
|
727
|
+
if (raw.kind !== "call_attempt" && attemptId !== run.attemptIds[nodeId]) {
|
|
1189
728
|
throw new StaleJournalGenerationError("journal event references a superseded attempt");
|
|
1190
729
|
}
|
|
1191
730
|
|
|
1192
|
-
if (raw.kind === "
|
|
731
|
+
if (raw.kind === "call_attempt") {
|
|
1193
732
|
const generation = raw.generation;
|
|
1194
733
|
if (typeof generation !== "number" || !Number.isInteger(generation) || generation < 1 || generation > 1_000_000) {
|
|
1195
|
-
throw new Error("
|
|
734
|
+
throw new Error("call attempt generation is invalid");
|
|
1196
735
|
}
|
|
1197
736
|
if (generation <= (run.attempts[nodeId] ?? 0)) {
|
|
1198
|
-
throw new StaleJournalGenerationError("
|
|
737
|
+
throw new StaleJournalGenerationError("call attempt generation is stale");
|
|
1199
738
|
}
|
|
1200
739
|
const expectedAttemptId = `${runId}/${nodeId}/attempt-${generation}`;
|
|
1201
740
|
if (attemptId !== expectedAttemptId || owner?.attemptId !== attemptId) {
|
|
1202
|
-
throw new Error("
|
|
741
|
+
throw new Error("call attempt does not identify its next generation");
|
|
1203
742
|
}
|
|
1204
743
|
return {
|
|
1205
|
-
kind: "
|
|
744
|
+
kind: "call_attempt",
|
|
1206
745
|
schemaVersion: JOURNAL_SCHEMA_VERSION,
|
|
1207
746
|
runId,
|
|
1208
747
|
nodeId,
|
|
@@ -1213,20 +752,15 @@ function parseEvent(raw: unknown, runs: Map<string, WorkflowRun>): JournalEvent
|
|
|
1213
752
|
};
|
|
1214
753
|
}
|
|
1215
754
|
|
|
1216
|
-
if (raw.kind === "
|
|
1217
|
-
if (!
|
|
1218
|
-
const previous = run.taskStatus[nodeId];
|
|
1219
|
-
const legacyDirectStart = previous === "pending" && raw.status === "running";
|
|
1220
|
-
if (!previous || (!canTransitionTask(previous, raw.status as TaskStatus) && !legacyDirectStart)) {
|
|
1221
|
-
throw new Error(`invalid task transition: ${String(previous)} -> ${String(raw.status)}`);
|
|
1222
|
-
}
|
|
755
|
+
if (raw.kind === "call_transition") {
|
|
756
|
+
if (!CALL_STATUS_VALUES.has(raw.status as CallStatus)) throw new Error("call status is unknown");
|
|
1223
757
|
const agentId = raw.agentId === undefined ? undefined : boundedString(raw.agentId, "agentId", JOURNAL_ID_LIMIT);
|
|
1224
758
|
return {
|
|
1225
|
-
kind: "
|
|
759
|
+
kind: "call_transition",
|
|
1226
760
|
schemaVersion: JOURNAL_SCHEMA_VERSION,
|
|
1227
761
|
runId,
|
|
1228
762
|
nodeId,
|
|
1229
|
-
status: raw.status as
|
|
763
|
+
status: raw.status as CallStatus,
|
|
1230
764
|
timestamp,
|
|
1231
765
|
...(agentId ? { agentId } : {}),
|
|
1232
766
|
...(attemptId ? { attemptId } : {}),
|
|
@@ -1234,40 +768,28 @@ function parseEvent(raw: unknown, runs: Map<string, WorkflowRun>): JournalEvent
|
|
|
1234
768
|
};
|
|
1235
769
|
}
|
|
1236
770
|
|
|
1237
|
-
|
|
1238
|
-
if (
|
|
1239
|
-
typeof raw.compactionCount !== "number" ||
|
|
1240
|
-
!Number.isInteger(raw.compactionCount) ||
|
|
1241
|
-
raw.compactionCount < 0 ||
|
|
1242
|
-
raw.compactionCount > 1_000_000
|
|
1243
|
-
) {
|
|
1244
|
-
throw new Error("compaction count is invalid");
|
|
1245
|
-
}
|
|
1246
|
-
return {
|
|
1247
|
-
kind: "task_compacted",
|
|
1248
|
-
schemaVersion: JOURNAL_SCHEMA_VERSION,
|
|
1249
|
-
runId,
|
|
1250
|
-
nodeId,
|
|
1251
|
-
compactionCount: raw.compactionCount,
|
|
1252
|
-
attemptId,
|
|
1253
|
-
timestamp,
|
|
1254
|
-
...(owner ? { owner } : {}),
|
|
1255
|
-
};
|
|
1256
|
-
}
|
|
1257
|
-
|
|
1258
|
-
const result = parseResult(raw.result, "task result");
|
|
771
|
+
const result = parseResult(raw.result, "call result");
|
|
1259
772
|
if (run.attemptTracking && result.attemptId === undefined) {
|
|
1260
|
-
throw new Error("
|
|
773
|
+
throw new Error("call result is missing a required payload attemptId");
|
|
1261
774
|
}
|
|
1262
775
|
if (result.attemptId && attemptId && result.attemptId !== attemptId) {
|
|
1263
|
-
throw new StaleJournalGenerationError("
|
|
776
|
+
throw new StaleJournalGenerationError("call result payload references a superseded attempt");
|
|
1264
777
|
}
|
|
1265
778
|
return {
|
|
1266
|
-
kind: "
|
|
779
|
+
kind: "call_result",
|
|
1267
780
|
schemaVersion: JOURNAL_SCHEMA_VERSION,
|
|
1268
781
|
runId,
|
|
1269
782
|
nodeId,
|
|
1270
783
|
result: result.attemptId === undefined && attemptId ? { ...result, attemptId } : result,
|
|
784
|
+
...(raw.callHash === undefined ? {} : { callHash: boundedString(raw.callHash, "callHash", 128) }),
|
|
785
|
+
...(raw.storeDelta === undefined
|
|
786
|
+
? {}
|
|
787
|
+
: {
|
|
788
|
+
storeDelta: (() => {
|
|
789
|
+
if (!isRecord(raw.storeDelta)) throw new Error("call result storeDelta is invalid");
|
|
790
|
+
return raw.storeDelta as Record<string, unknown>;
|
|
791
|
+
})(),
|
|
792
|
+
}),
|
|
1271
793
|
attemptId,
|
|
1272
794
|
timestamp,
|
|
1273
795
|
...(owner ? { owner } : {}),
|
|
@@ -1288,9 +810,10 @@ export function boundedError(value: unknown): string | undefined {
|
|
|
1288
810
|
export function replayJournal(
|
|
1289
811
|
entries: readonly SessionEntryLike[],
|
|
1290
812
|
options: ReplayJournalOptions = {},
|
|
1291
|
-
): Map<string,
|
|
1292
|
-
const runs = new Map<string,
|
|
813
|
+
): Map<string, ScriptRun> {
|
|
814
|
+
const runs = new Map<string, ScriptRun>();
|
|
1293
815
|
const quarantined = new Set<string>();
|
|
816
|
+
const removed = new Set<string>();
|
|
1294
817
|
/** Recovery identities are scoped to one replayed run and bounded explicitly. */
|
|
1295
818
|
const seenRecoveries = new Map<string, Map<string, string>>();
|
|
1296
819
|
let reported = false;
|
|
@@ -1304,7 +827,19 @@ export function replayJournal(
|
|
|
1304
827
|
const raw = entry.data;
|
|
1305
828
|
const rawRunId = isRecord(raw) && typeof raw.runId === "string" ? raw.runId : undefined;
|
|
1306
829
|
if (rawRunId && quarantined.has(rawRunId)) continue;
|
|
830
|
+
if (rawRunId && removed.has(rawRunId)) continue;
|
|
1307
831
|
try {
|
|
832
|
+
if (isRecord(raw) && raw.kind === "run_created" && rawRunId && runs.has(rawRunId)) {
|
|
833
|
+
const existing = runs.get(rawRunId);
|
|
834
|
+
if (
|
|
835
|
+
existing &&
|
|
836
|
+
existing.script === raw.script &&
|
|
837
|
+
existing.scriptHash === raw.scriptHash &&
|
|
838
|
+
JSON.stringify(existing.meta) === JSON.stringify(raw.meta)
|
|
839
|
+
)
|
|
840
|
+
continue;
|
|
841
|
+
throw new Error("conflicting duplicate workflow run_created event");
|
|
842
|
+
}
|
|
1308
843
|
if (
|
|
1309
844
|
isRecord(raw) &&
|
|
1310
845
|
(raw.kind === "run_recovery" || raw.kind === "terminal_recovery") &&
|
|
@@ -1312,7 +847,6 @@ export function replayJournal(
|
|
|
1312
847
|
) {
|
|
1313
848
|
for (const [seenRunId, seen] of seenRecoveries) {
|
|
1314
849
|
if (seenRunId === rawRunId || !seen.has(raw.recoveryId)) continue;
|
|
1315
|
-
// A colliding recovery ID belongs to the malformed current entry; keep the previously validated owner intact.
|
|
1316
850
|
throw new Error(`recoveryId ${raw.recoveryId} is already owned by workflow run ${seenRunId}`);
|
|
1317
851
|
}
|
|
1318
852
|
}
|
|
@@ -1320,9 +854,6 @@ export function replayJournal(
|
|
|
1320
854
|
const run = runs.get(rawRunId);
|
|
1321
855
|
const seen = seenRecoveries.get(rawRunId);
|
|
1322
856
|
if (run && seen) {
|
|
1323
|
-
// Parse duplicates without consulting current generation/status: the
|
|
1324
|
-
// first copy already advanced those fields. The normalized semantic
|
|
1325
|
-
// digest still validates every bounded field before accepting a no-op.
|
|
1326
857
|
const candidate =
|
|
1327
858
|
raw.kind === "run_recovery"
|
|
1328
859
|
? parseDuplicateRunRecovery(raw, run)
|
|
@@ -1364,6 +895,9 @@ export function replayJournal(
|
|
|
1364
895
|
continue;
|
|
1365
896
|
}
|
|
1366
897
|
applyJournalEvent(runs, event);
|
|
898
|
+
if (event.kind === "run_removed") {
|
|
899
|
+
removed.add(event.runId);
|
|
900
|
+
}
|
|
1367
901
|
} catch (error: unknown) {
|
|
1368
902
|
if (error instanceof StaleJournalGenerationError) continue;
|
|
1369
903
|
if (rawRunId) {
|
|
@@ -1387,42 +921,29 @@ export function replayJournal(
|
|
|
1387
921
|
return runs;
|
|
1388
922
|
}
|
|
1389
923
|
|
|
1390
|
-
function applyRecoveryRotation(run:
|
|
924
|
+
function applyRecoveryRotation(run: ScriptRun, rotation: RecoveryRotation): void {
|
|
1391
925
|
run.attemptTracking = true;
|
|
1392
926
|
run.attempts[rotation.nodeId] = rotation.generation;
|
|
1393
927
|
run.attemptIds[rotation.nodeId] = rotation.attemptId;
|
|
1394
928
|
delete run.agentIds[rotation.nodeId];
|
|
1395
|
-
delete run.
|
|
929
|
+
delete run.callResults[rotation.nodeId];
|
|
1396
930
|
run.compactions[rotation.nodeId] = 0;
|
|
1397
|
-
|
|
1398
|
-
run.synthesisAgentId = undefined;
|
|
1399
|
-
run.synthesisResult = undefined;
|
|
1400
|
-
} else {
|
|
1401
|
-
run.taskStatus[rotation.nodeId] = "ready";
|
|
1402
|
-
}
|
|
931
|
+
run.callStatus[rotation.nodeId] = "running";
|
|
1403
932
|
}
|
|
1404
933
|
|
|
1405
|
-
function applyRecoveryTerminalResult(run:
|
|
934
|
+
function applyRecoveryTerminalResult(run: ScriptRun, terminal: RecoveryTerminalResult): void {
|
|
1406
935
|
run.compactions[terminal.nodeId] = Math.max(run.compactions[terminal.nodeId] ?? 0, terminal.result.compactionCount);
|
|
1407
|
-
|
|
1408
|
-
|
|
1409
|
-
|
|
1410
|
-
} else {
|
|
1411
|
-
run.taskResults[terminal.nodeId] = terminal.result;
|
|
1412
|
-
run.taskStatus[terminal.nodeId] = terminal.result.status;
|
|
1413
|
-
if (terminal.result.agentId) run.agentIds[terminal.nodeId] = terminal.result.agentId;
|
|
1414
|
-
}
|
|
936
|
+
run.callResults[terminal.nodeId] = terminal.result;
|
|
937
|
+
run.callStatus[terminal.nodeId] = terminal.result.status;
|
|
938
|
+
if (terminal.result.agentId) run.agentIds[terminal.nodeId] = terminal.result.agentId;
|
|
1415
939
|
}
|
|
1416
940
|
|
|
1417
|
-
export function applyRecoveryTerminalResults(
|
|
1418
|
-
run: WorkflowRun,
|
|
1419
|
-
terminalResults: readonly RecoveryTerminalResult[],
|
|
1420
|
-
): void {
|
|
941
|
+
export function applyRecoveryTerminalResults(run: ScriptRun, terminalResults: readonly RecoveryTerminalResult[]): void {
|
|
1421
942
|
for (const terminal of terminalResults) applyRecoveryTerminalResult(run, terminal);
|
|
1422
943
|
}
|
|
1423
944
|
|
|
1424
945
|
/** Apply an already validated atomic recovery event after its append succeeds. */
|
|
1425
|
-
export function applyRecoveryEvent(run:
|
|
946
|
+
export function applyRecoveryEvent(run: ScriptRun, event: RunRecoveryEvent | AttemptRecoveryEvent): void {
|
|
1426
947
|
if (event.kind === "run_recovery") {
|
|
1427
948
|
run.status = "interrupted";
|
|
1428
949
|
run.terminalIntent = undefined;
|
|
@@ -1434,10 +955,10 @@ export function applyRecoveryEvent(run: WorkflowRun, event: RunRecoveryEvent | A
|
|
|
1434
955
|
run.updatedAt = event.timestamp;
|
|
1435
956
|
}
|
|
1436
957
|
/** Apply a validated terminal recovery after its durable append succeeds. */
|
|
1437
|
-
export function applyTerminalRecoveryEvent(run:
|
|
958
|
+
export function applyTerminalRecoveryEvent(run: ScriptRun, event: TerminalRecoveryEvent): void {
|
|
1438
959
|
for (const terminal of event.terminalResults) applyRecoveryTerminalResult(run, terminal);
|
|
1439
960
|
for (const nodeId of event.blockedNodeIds) {
|
|
1440
|
-
if (
|
|
961
|
+
if (run.callStatus[nodeId] !== "completed") run.callStatus[nodeId] = "stopped";
|
|
1441
962
|
}
|
|
1442
963
|
run.status = event.status;
|
|
1443
964
|
run.terminalIntent = event.terminalIntent;
|
|
@@ -1446,29 +967,37 @@ export function applyTerminalRecoveryEvent(run: WorkflowRun, event: TerminalReco
|
|
|
1446
967
|
run.updatedAt = event.timestamp;
|
|
1447
968
|
}
|
|
1448
969
|
|
|
1449
|
-
function applyJournalEvent(runs: Map<string,
|
|
970
|
+
function applyJournalEvent(runs: Map<string, ScriptRun>, event: JournalEvent): void {
|
|
1450
971
|
if (event.kind === "run_created") {
|
|
1451
|
-
const
|
|
972
|
+
const callStatus: Record<string, CallStatus> = {};
|
|
1452
973
|
const attempts: Record<string, number> = {};
|
|
1453
974
|
const attemptIds: Record<string, string> = {};
|
|
1454
|
-
for (const
|
|
1455
|
-
|
|
1456
|
-
attempts[
|
|
1457
|
-
attemptIds[
|
|
975
|
+
for (const nodeId of Object.keys(event.attemptIds ?? {})) {
|
|
976
|
+
callStatus[nodeId] = "running";
|
|
977
|
+
attempts[nodeId] = event.attempts?.[nodeId] ?? 1;
|
|
978
|
+
attemptIds[nodeId] = event.attemptIds?.[nodeId] ?? defaultAttemptId(event.runId, nodeId);
|
|
1458
979
|
}
|
|
1459
|
-
attempts.__synthesis__ = event.attempts?.__synthesis__ ?? 1;
|
|
1460
|
-
attemptIds.__synthesis__ = event.attemptIds?.__synthesis__ ?? defaultAttemptId(event.runId, "__synthesis__");
|
|
1461
980
|
runs.set(event.runId, {
|
|
1462
981
|
runId: event.runId,
|
|
1463
982
|
schemaVersion: event.schemaVersion,
|
|
1464
|
-
|
|
983
|
+
script: event.script,
|
|
984
|
+
scriptHash: event.scriptHash,
|
|
985
|
+
meta: event.meta,
|
|
986
|
+
revision: 0,
|
|
987
|
+
...(event.toolset ? { toolset: event.toolset } : {}),
|
|
988
|
+
...(event.frozenMaxAgents !== undefined ? { frozenMaxAgents: event.frozenMaxAgents } : {}),
|
|
989
|
+
...(event.frozenConcurrency !== undefined ? { frozenConcurrency: event.frozenConcurrency } : {}),
|
|
990
|
+
...(event.frozenAgentRetries !== undefined ? { frozenAgentRetries: event.frozenAgentRetries } : {}),
|
|
991
|
+
...(event.frozenTokenBudget !== undefined ? { frozenTokenBudget: event.frozenTokenBudget } : {}),
|
|
992
|
+
...(event.frozenAgentTimeoutMs !== undefined ? { frozenAgentTimeoutMs: event.frozenAgentTimeoutMs } : {}),
|
|
993
|
+
...(event.frozenExcludeTools ? { frozenExcludeTools: event.frozenExcludeTools } : {}),
|
|
1465
994
|
status: "pending",
|
|
1466
|
-
|
|
995
|
+
callStatus,
|
|
1467
996
|
agentIds: {},
|
|
1468
997
|
attempts,
|
|
1469
998
|
attemptIds,
|
|
1470
999
|
attemptTracking: event.attemptIds !== undefined,
|
|
1471
|
-
|
|
1000
|
+
callResults: {},
|
|
1472
1001
|
compactions: {},
|
|
1473
1002
|
startedAt: event.timestamp,
|
|
1474
1003
|
updatedAt: event.timestamp,
|
|
@@ -1478,8 +1007,18 @@ function applyJournalEvent(runs: Map<string, WorkflowRun>, event: JournalEvent):
|
|
|
1478
1007
|
|
|
1479
1008
|
const run = runs.get(event.runId);
|
|
1480
1009
|
if (!run) return;
|
|
1010
|
+
if (event.kind === "run_removed") {
|
|
1011
|
+
runs.delete(event.runId);
|
|
1012
|
+
return;
|
|
1013
|
+
}
|
|
1481
1014
|
run.updatedAt = event.timestamp;
|
|
1482
1015
|
switch (event.kind) {
|
|
1016
|
+
case "run_revision":
|
|
1017
|
+
run.revision = event.revision;
|
|
1018
|
+
run.script = event.script;
|
|
1019
|
+
run.scriptHash = event.scriptHash;
|
|
1020
|
+
run.meta = event.meta;
|
|
1021
|
+
break;
|
|
1483
1022
|
case "workflow_transition":
|
|
1484
1023
|
run.status = event.status;
|
|
1485
1024
|
if (event.error !== undefined) run.error = event.error;
|
|
@@ -1492,93 +1031,147 @@ function applyJournalEvent(runs: Map<string, WorkflowRun>, event: JournalEvent):
|
|
|
1492
1031
|
case "terminal_recovery":
|
|
1493
1032
|
applyTerminalRecoveryEvent(run, event);
|
|
1494
1033
|
break;
|
|
1495
|
-
case "
|
|
1034
|
+
case "call_attempt":
|
|
1496
1035
|
run.attemptTracking = true;
|
|
1497
1036
|
run.attempts[event.nodeId] = event.generation;
|
|
1498
1037
|
run.attemptIds[event.nodeId] = event.attemptId;
|
|
1499
1038
|
delete run.agentIds[event.nodeId];
|
|
1500
|
-
delete run.
|
|
1501
|
-
|
|
1502
|
-
run.synthesisAgentId = undefined;
|
|
1503
|
-
run.synthesisResult = undefined;
|
|
1504
|
-
} else {
|
|
1505
|
-
run.taskStatus[event.nodeId] = "ready";
|
|
1506
|
-
}
|
|
1039
|
+
delete run.callResults[event.nodeId];
|
|
1040
|
+
run.callStatus[event.nodeId] = "running";
|
|
1507
1041
|
run.compactions[event.nodeId] = 0;
|
|
1508
1042
|
break;
|
|
1509
|
-
case "
|
|
1043
|
+
case "call_transition":
|
|
1510
1044
|
if (event.attemptId && event.attemptId !== run.attemptIds[event.nodeId]) {
|
|
1511
|
-
throw new Error("
|
|
1045
|
+
throw new Error("call transition references a superseded attempt");
|
|
1512
1046
|
}
|
|
1513
|
-
run.
|
|
1047
|
+
run.callStatus[event.nodeId] = event.status;
|
|
1514
1048
|
if (event.agentId) run.agentIds[event.nodeId] = event.agentId;
|
|
1515
1049
|
break;
|
|
1516
|
-
case "
|
|
1050
|
+
case "call_result":
|
|
1517
1051
|
if (event.attemptId && event.attemptId !== run.attemptIds[event.nodeId]) {
|
|
1518
|
-
throw new Error("
|
|
1052
|
+
throw new Error("call result references a superseded attempt");
|
|
1519
1053
|
}
|
|
1520
1054
|
if (
|
|
1521
|
-
run.
|
|
1522
|
-
JSON.stringify(run.
|
|
1055
|
+
run.callResults[event.nodeId] &&
|
|
1056
|
+
JSON.stringify(run.callResults[event.nodeId]) !== JSON.stringify(event.result)
|
|
1523
1057
|
) {
|
|
1524
|
-
throw new Error("contradictory terminal
|
|
1058
|
+
throw new Error("contradictory terminal call result");
|
|
1525
1059
|
}
|
|
1526
|
-
run.
|
|
1527
|
-
run.
|
|
1060
|
+
run.callResults[event.nodeId] = event.result;
|
|
1061
|
+
run.callStatus[event.nodeId] = event.result.status;
|
|
1528
1062
|
run.compactions[event.nodeId] = Math.max(run.compactions[event.nodeId] ?? 0, event.result.compactionCount);
|
|
1529
1063
|
if (event.result.agentId) run.agentIds[event.nodeId] = event.result.agentId;
|
|
1530
1064
|
break;
|
|
1531
|
-
case "task_compacted":
|
|
1532
|
-
if (event.attemptId && event.attemptId !== run.attemptIds[event.nodeId]) {
|
|
1533
|
-
throw new Error("compaction references a superseded attempt");
|
|
1534
|
-
}
|
|
1535
|
-
run.compactions[event.nodeId] = event.compactionCount;
|
|
1536
|
-
break;
|
|
1537
|
-
case "synthesis_result":
|
|
1538
|
-
if (run.synthesisResult && JSON.stringify(run.synthesisResult) !== JSON.stringify(event.result)) {
|
|
1539
|
-
throw new Error("contradictory synthesis result");
|
|
1540
|
-
}
|
|
1541
|
-
run.synthesisResult = event.result;
|
|
1542
|
-
run.compactions.__synthesis__ = Math.max(run.compactions.__synthesis__ ?? 0, event.result.compactionCount);
|
|
1543
|
-
if (event.result.agentId) run.synthesisAgentId = event.result.agentId;
|
|
1544
|
-
break;
|
|
1545
1065
|
}
|
|
1546
1066
|
}
|
|
1547
1067
|
|
|
1548
|
-
function validateAggregate(run:
|
|
1549
|
-
const
|
|
1550
|
-
for (const nodeId of nodes) {
|
|
1068
|
+
function validateAggregate(run: ScriptRun): void {
|
|
1069
|
+
for (const nodeId of Object.keys(run.attemptIds)) {
|
|
1551
1070
|
if (!Number.isInteger(run.attempts[nodeId]) || run.attempts[nodeId] < 1) {
|
|
1552
1071
|
throw new Error(`missing attempt generation for ${nodeId}`);
|
|
1553
1072
|
}
|
|
1554
1073
|
boundedString(run.attemptIds[nodeId], `attemptIds.${nodeId}`, JOURNAL_ID_LIMIT);
|
|
1555
1074
|
}
|
|
1556
1075
|
if (isTerminalWorkflowStatus(run.status)) {
|
|
1557
|
-
const statuses = Object.values(run.
|
|
1076
|
+
const statuses = Object.values(run.callStatus);
|
|
1558
1077
|
if (run.status === "completed" && statuses.some((status) => status !== "completed")) {
|
|
1559
|
-
throw new Error("completed workflow has non-completed
|
|
1078
|
+
throw new Error("completed workflow has non-completed calls");
|
|
1560
1079
|
}
|
|
1561
|
-
if (
|
|
1562
|
-
|
|
1563
|
-
|
|
1564
|
-
|
|
1565
|
-
throw new Error("
|
|
1566
|
-
}
|
|
1567
|
-
if (run.status === "failed" || run.status === "stopped") {
|
|
1568
|
-
if (run.synthesisAgentId && !run.synthesisResult) {
|
|
1569
|
-
throw new Error(`${run.status} workflow has an active synthesis agent`);
|
|
1570
|
-
}
|
|
1080
|
+
if (
|
|
1081
|
+
run.status === "stopped" &&
|
|
1082
|
+
statuses.some((status) => status !== "completed" && status !== "stopped" && status !== "failed")
|
|
1083
|
+
) {
|
|
1084
|
+
throw new Error("stopped workflow has active calls");
|
|
1571
1085
|
}
|
|
1572
1086
|
}
|
|
1573
|
-
if (run.synthesisResult && Object.values(run.taskStatus).some((status) => !isTerminalTask(status))) {
|
|
1574
|
-
throw new Error("synthesis settled before workflow tasks");
|
|
1575
|
-
}
|
|
1576
1087
|
}
|
|
1577
1088
|
|
|
1578
1089
|
function isTerminalWorkflowStatus(status: WorkflowStatus): boolean {
|
|
1579
1090
|
return status === "completed" || status === "failed" || status === "stopped";
|
|
1580
1091
|
}
|
|
1581
1092
|
|
|
1582
|
-
|
|
1583
|
-
|
|
1093
|
+
function canTransitionWorkflow(previous: WorkflowStatus, next: WorkflowStatus): boolean {
|
|
1094
|
+
if (previous === next) return true;
|
|
1095
|
+
switch (previous) {
|
|
1096
|
+
case "pending":
|
|
1097
|
+
return next === "running" || next === "stopping";
|
|
1098
|
+
case "running":
|
|
1099
|
+
return (
|
|
1100
|
+
next === "pausing" ||
|
|
1101
|
+
next === "paused" ||
|
|
1102
|
+
next === "stopping" ||
|
|
1103
|
+
next === "interrupted" ||
|
|
1104
|
+
next === "completed" ||
|
|
1105
|
+
next === "failed"
|
|
1106
|
+
);
|
|
1107
|
+
case "pausing":
|
|
1108
|
+
return next === "paused" || next === "stopping";
|
|
1109
|
+
case "paused":
|
|
1110
|
+
return next === "running" || next === "stopping";
|
|
1111
|
+
case "interrupted":
|
|
1112
|
+
return next === "running" || next === "stopping";
|
|
1113
|
+
case "stopping":
|
|
1114
|
+
return next === "stopped" || next === "failed" || next === "interrupted";
|
|
1115
|
+
default:
|
|
1116
|
+
return false;
|
|
1117
|
+
}
|
|
1118
|
+
}
|
|
1119
|
+
|
|
1120
|
+
export function snapshotRun(run: ScriptRun): ScriptRun {
|
|
1121
|
+
return JSON.parse(JSON.stringify(run)) as ScriptRun;
|
|
1122
|
+
}
|
|
1123
|
+
|
|
1124
|
+
/**
|
|
1125
|
+
* Build the runtime's resume journal from raw session entries.
|
|
1126
|
+
*
|
|
1127
|
+
* The runtime keys its cache by `${runId}:${callIndex}` (the same namespacing
|
|
1128
|
+
* the store deltas use), so a nested workflow() call with its own runId can
|
|
1129
|
+
* never replay the parent's entry. Every `call_result` event that carries a
|
|
1130
|
+
* `callHash` is a replayable agent()/checkpoint() fact; entries without a hash
|
|
1131
|
+
* (legacy or malformed) are skipped rather than quarantined — a resume must
|
|
1132
|
+
* tolerate a foreign or partial journal and simply re-run the misses live.
|
|
1133
|
+
*/
|
|
1134
|
+
export function buildResumeJournal(
|
|
1135
|
+
entries: readonly SessionEntryLike[],
|
|
1136
|
+
): Map<string, { index: number; runId?: string; hash: string; result: unknown; storeDelta?: Record<string, unknown> }> {
|
|
1137
|
+
const journal = new Map<
|
|
1138
|
+
string,
|
|
1139
|
+
{ index: number; runId?: string; hash: string; result: unknown; storeDelta?: Record<string, unknown> }
|
|
1140
|
+
>();
|
|
1141
|
+
for (const entry of entries) {
|
|
1142
|
+
if (entry.type !== "custom" || entry.customType !== JOURNAL_ENTRY_TYPE) continue;
|
|
1143
|
+
const raw = entry.data;
|
|
1144
|
+
if (!isRecord(raw) || raw.kind !== "call_result" || typeof raw.callHash !== "string") continue;
|
|
1145
|
+
if (raw.schemaVersion !== JOURNAL_SCHEMA_VERSION) continue;
|
|
1146
|
+
const runId = boundedString(raw.runId, "runId", JOURNAL_ID_LIMIT);
|
|
1147
|
+
let nodeId: string;
|
|
1148
|
+
try {
|
|
1149
|
+
nodeId = parseCallIndex(raw.nodeId, "nodeId");
|
|
1150
|
+
} catch {
|
|
1151
|
+
continue;
|
|
1152
|
+
}
|
|
1153
|
+
const result = isRecord(raw.result) ? raw.result : undefined;
|
|
1154
|
+
// Failed/stopped lifecycle facts are diagnostic history, not successful
|
|
1155
|
+
// replay values. Replaying them as `undefined` would silently skip a call
|
|
1156
|
+
// and bypass the runtime's retry/error policy.
|
|
1157
|
+
if (result?.status !== "completed" || !Object.hasOwn(result, "result")) continue;
|
|
1158
|
+
journal.set(`${runId}:${nodeId}`, {
|
|
1159
|
+
index: Number(nodeId),
|
|
1160
|
+
runId,
|
|
1161
|
+
hash: raw.callHash,
|
|
1162
|
+
result: result.result,
|
|
1163
|
+
...(isRecord(raw.storeDelta) && Object.keys(raw.storeDelta).length > 0
|
|
1164
|
+
? { storeDelta: raw.storeDelta as Record<string, unknown> }
|
|
1165
|
+
: {}),
|
|
1166
|
+
});
|
|
1167
|
+
}
|
|
1168
|
+
return journal;
|
|
1169
|
+
}
|
|
1170
|
+
|
|
1171
|
+
/** Derive a stable recovery id from a recovery event's content. */
|
|
1172
|
+
export function deriveRecoveryId(event: { runId: string; rotations: unknown[]; timestamp: number }): string {
|
|
1173
|
+
const digest = createHash("sha256")
|
|
1174
|
+
.update(JSON.stringify({ runId: event.runId, rotations: event.rotations, timestamp: event.timestamp }))
|
|
1175
|
+
.digest("hex");
|
|
1176
|
+
return `${JOURNAL_RECOVERY_ID_PREFIX}${digest}`;
|
|
1584
1177
|
}
|