@signalridge/pi-workflows 1.2.0 → 1.3.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (49) hide show
  1. package/CHANGELOG.md +45 -0
  2. package/README.md +142 -17
  3. package/package.json +14 -8
  4. package/skills/workflow-authoring/SKILL.md +30 -0
  5. package/skills/workflow-authoring/examples/adversarial-verification.js +63 -0
  6. package/skills/workflow-authoring/examples/bounded-semantic-retry.js +56 -0
  7. package/skills/workflow-authoring/examples/classify-and-act.js +67 -0
  8. package/skills/workflow-authoring/examples/defensive-json-parsing.js +63 -0
  9. package/skills/workflow-authoring/examples/fan-out-and-synthesize.js +47 -0
  10. package/skills/workflow-authoring/examples/generate-and-filter.js +68 -0
  11. package/skills/workflow-authoring/examples/loop-until-done.js +68 -0
  12. package/skills/workflow-authoring/examples/phased-budgets.js +75 -0
  13. package/skills/workflow-authoring/examples/saved-nested-workflows.js +44 -0
  14. package/skills/workflow-authoring/examples/structured-output.js +37 -0
  15. package/skills/workflow-authoring/examples/tournament.js +81 -0
  16. package/skills/workflow-authoring/examples/validated-gate.js +63 -0
  17. package/skills/workflow-authoring/references/capabilities.md +44 -0
  18. package/skills/workflow-authoring/references/capability-details.md +294 -0
  19. package/skills/workflow-authoring/references/common-helpers.md +4 -0
  20. package/skills/workflow-authoring/references/debugging.md +29 -0
  21. package/skills/workflow-authoring/references/focused-recipes.md +12 -0
  22. package/skills/workflow-authoring/references/helpers.md +9 -0
  23. package/skills/workflow-authoring/references/lifecycle.md +37 -0
  24. package/skills/workflow-authoring/references/pattern-selection.md +15 -0
  25. package/skills/workflow-authoring/references/quality-helpers.md +8 -0
  26. package/skills/workflow-authoring/references/registry-ownership.md +17 -0
  27. package/skills/workflow-authoring/references/retry-helper.md +5 -0
  28. package/skills/workflow-authoring/references/review.md +43 -0
  29. package/skills/workflow-authoring/references/runtime.md +29 -0
  30. package/skills/workflow-authoring/references/specialized-helpers.md +19 -0
  31. package/skills/workflow-authoring/references/versions.md +13 -0
  32. package/skills/workflow-patterns/SKILL.md +51 -0
  33. package/src/arming.ts +69 -0
  34. package/src/builtins.ts +331 -0
  35. package/src/capabilities.ts +318 -0
  36. package/src/code-review-scope.ts +144 -0
  37. package/src/engine.ts +1158 -2561
  38. package/src/errors.ts +136 -0
  39. package/src/index.ts +750 -103
  40. package/src/journal.ts +594 -1001
  41. package/src/rpc-client.ts +95 -24
  42. package/src/runtime.ts +1708 -0
  43. package/src/saved-workflows.ts +121 -0
  44. package/src/shared-store.ts +132 -0
  45. package/src/ui.ts +77 -40
  46. package/src/workflow-settings.ts +100 -0
  47. package/src/schema.ts +0 -215
  48. package/src/state-machine.ts +0 -79
  49. package/src/synthesis.ts +0 -140
package/src/engine.ts CHANGED
@@ -1,936 +1,596 @@
1
- import { randomUUID } from "node:crypto";
1
+ /**
2
+ * engine.ts — script-driven workflow engine.
3
+ *
4
+ * A workflow is a raw JavaScript module (see runtime.ts). This engine owns the
5
+ * durable side: it journals the script (`run_created` with script text, hash,
6
+ * and meta), dispatches each live `agent()` call through the spawn-managed
7
+ * protocol with A4 generation-rotated spawn keys, persists every completed
8
+ * call as a `call_result` fact (the resume journal source), and exposes
9
+ * list/get/pause/resume/stop control plus branch-change quiescence and
10
+ * session-tree restore.
11
+ *
12
+ * Resume semantics: the engine rebuilds the runtime's resume journal from
13
+ * journaled call_result events, re-runs the script with the SAME runId, and
14
+ * the runtime replays the longest unchanged prefix from cache while calls at
15
+ * or after the first hash miss run live. Live dispatches of a previously
16
+ * journaled call index rotate the spawnKey generation (A4), so an edited
17
+ * script never collides with pi-subagents' fingerprint tombstone.
18
+ */
19
+
20
+ import { createHash, randomUUID } from "node:crypto";
21
+ import type { ManagedSpawnResponse } from "@signalridge/pi-subagents-protocol";
22
+ import { WorkflowError, WorkflowErrorCode, wrapError } from "./errors.js";
2
23
  import {
3
- type AttemptRecoveryEvent,
4
- applyRecoveryEvent,
5
- applyTerminalRecoveryEvent,
6
- boundedError,
24
+ buildResumeJournal,
25
+ type CallResult,
7
26
  deriveRecoveryId,
27
+ JOURNAL_ENTRY_TYPE,
8
28
  JOURNAL_SCHEMA_VERSION,
9
29
  type JournalEvent,
10
30
  type JournalWriter,
11
- type RecoveryRotation,
12
- type RecoverySourceStatus,
13
31
  type RecoveryTerminalResult,
14
- type RunRecoveryEvent,
15
32
  replayJournal,
33
+ type ScriptRun,
16
34
  type SessionEntryLike,
17
35
  snapshotRun,
18
36
  type TerminalRecoveryEvent,
19
37
  type WorkflowOwner,
20
- type WorkflowRun,
21
38
  } from "./journal.js";
22
- import type {
23
- ManagedSpawnClient,
24
- ManagedSpawnResponse,
25
- ManagedTerminalSnapshot,
26
- WorkflowEventBus,
27
- } from "./rpc-client.js";
28
- import { registerLifecycleListener } from "./rpc-client.js";
29
- import { readyTaskIds, type WorkflowDefinition, type WorkflowTask } from "./schema.js";
30
39
  import {
31
- canTransitionTask,
32
- isTerminalTask,
33
- isTerminalWorkflow,
34
- type TaskStatus,
35
- transitionTask,
36
- transitionWorkflow,
37
- type WorkflowStatus,
38
- } from "./state-machine.js";
40
+ createManagedSpawnClient,
41
+ type DispatchTask,
42
+ type ManagedSpawnClient,
43
+ type WorkflowEventBus,
44
+ } from "./rpc-client.js";
39
45
  import {
40
- buildBoundedResultSection,
41
- buildSynthesisPrompt,
42
- cap,
43
- MAX_DISPATCH_PROMPT_CHARS,
44
- MAX_TASK_INPUT_CHARS,
45
- MAX_TASK_RESULT_CHARS,
46
- MIN_TASK_INPUT_CHARS,
47
- type ResultEntry,
48
- resultFromLifecycle,
49
- TASK_INPUT_HEADER,
50
- } from "./synthesis.js";
51
-
52
- const SYNTHESIS_NODE_ID = "__synthesis__";
46
+ type AgentRunOptions,
47
+ type AgentUsage,
48
+ type JournalEntry,
49
+ parseWorkflowScript,
50
+ runWorkflow,
51
+ type WorkflowAgentRunner,
52
+ type WorkflowRunResult,
53
+ } from "./runtime.js";
54
+
55
+ export const MAX_ATTEMPTS_PER_NODE = 3;
56
+
57
+ const BRANCH_QUIESCE_TIMEOUT_MS = 8_000;
53
58
  const MAX_WAITERS = 256;
54
- const MAX_ATTEMPTS_PER_NODE = 3;
55
- const BRANCH_QUIESCE_TIMEOUT_MS = 5_000;
56
- const MAX_QUARANTINED_AGENT_IDS = 4_096;
57
- const MAX_RECOVERY_MEMO_ENTRIES = 4_096;
58
-
59
- export interface WorkflowQuiesceResult {
60
- settled: boolean;
61
- pending: string[];
62
- diagnostic?: string;
63
- }
64
-
65
- export interface WorkflowSummary {
66
- runId: string;
67
- name: string;
68
- description?: string;
69
- status: WorkflowStatus;
70
- startedAt: number;
71
- updatedAt: number;
72
- elapsedMs: number;
73
- taskCount: number;
74
- phaseCount: number;
75
- completedPhases: number;
76
- completedTasks: number;
77
- failedTasks: number;
78
- activeTasks: number;
79
- tokenCount?: number;
80
- synthesisPreview?: string;
81
- error?: string;
82
- nonResumable?: boolean;
83
- tasks?: WorkflowTaskDetail[];
84
- synthesis?: WorkflowSynthesisDetail;
85
- }
86
-
87
- export interface WorkflowTaskDetail {
88
- id: string;
89
- status: TaskStatus;
90
- agentId?: string;
91
- compactions: number;
92
- resultPreview?: string;
93
- errorPreview?: string;
94
- }
59
+ const JOURNAL_RETRY_INITIAL_DELAY_MS = 25;
60
+ const JOURNAL_RETRY_MAX_DELAY_MS = 2_000;
61
+ const JOURNAL_RETRY_MAX_ATTEMPTS = 8;
62
+ const PROVIDER_RETRY_DEFAULT_MS = 60_000;
63
+ const PROVIDER_RETRY_MAX_MS = 24 * 60 * 60 * 1_000;
95
64
 
96
- export interface WorkflowSynthesisDetail {
97
- status: "completed" | "failed" | "stopped";
98
- agentId?: string;
99
- compactions: number;
100
- resultPreview?: string;
101
- errorPreview?: string;
65
+ class JournalAppendError extends Error {
66
+ constructor(kind: string, cause: unknown) {
67
+ super(`journal append failed for ${kind}: ${cause instanceof Error ? cause.message : String(cause)}`);
68
+ this.name = "JournalAppendError";
69
+ }
102
70
  }
103
71
 
104
- export interface WorkflowControlResult {
105
- action: "list" | "get" | "pause" | "resume" | "stop";
106
- run?: WorkflowSummary;
107
- runs?: WorkflowSummary[];
72
+ export type WorkflowQuiesceResult = { settled: boolean; pending: string[]; diagnostic?: string };
73
+
74
+ export interface ScriptStartOptions {
75
+ args?: unknown;
76
+ background?: boolean;
77
+ maxAgents?: number;
78
+ concurrency?: number;
79
+ agentRetries?: number;
80
+ tokenBudget?: number | null;
81
+ agentTimeoutMs?: number | null;
82
+ toolset?: string;
83
+ excludeTools?: string[];
84
+ signal?: AbortSignal;
85
+ /** Foreground human confirmation for checkpoint() — absent in headless runs. */
86
+ confirm?: (
87
+ promptText: string,
88
+ options: {
89
+ default?: unknown;
90
+ headless?: "default" | "abort";
91
+ kind?: "confirm" | "input" | "select";
92
+ choices?: string[];
93
+ timeoutMs?: number;
94
+ },
95
+ ) => Promise<unknown>;
96
+ loadSavedWorkflow?: (name: string) => string | undefined;
97
+ mainModel?: string;
108
98
  }
109
99
 
110
- export interface WorkflowStartResult {
100
+ export interface ScriptStartResult {
111
101
  runId: string;
112
- status: WorkflowStatus;
102
+ status: string;
113
103
  background: boolean;
114
104
  result?: string;
115
105
  error?: string;
116
106
  waitAborted?: boolean;
117
107
  }
118
108
 
119
- interface Waiter {
120
- resolve: (run: WorkflowRun) => void;
121
- reject: (error: Error) => void;
122
- cleanup: () => void;
123
- settled: boolean;
124
- }
125
-
126
- type RecoveryEvent = RunRecoveryEvent | AttemptRecoveryEvent;
127
-
128
- interface RecoveryMemo {
129
- branchGeneration: number;
130
- runId: string;
131
- key: string;
132
- event: RecoveryEvent;
133
- }
134
-
135
- interface TerminalRecoveryMemo {
136
- branchGeneration: number;
137
- runId: string;
138
- key: string;
139
- event: TerminalRecoveryEvent;
140
- }
141
-
142
- interface RecoveryPlanItem {
143
- nodeId: string;
144
- sourceAttemptId: string;
145
- sourceGeneration: number;
146
- sourceStatus: RecoverySourceStatus;
147
- oldAgentId: string;
148
- }
149
-
150
- interface DeferredInterruptedRecord {
151
- kind: "interrupted";
152
- runId: string;
153
- nodeId: string;
154
- oldAgentId: string;
155
- }
156
-
157
- interface DeferredTerminalRecord extends RecoveryTerminalResult {
158
- kind: "terminal";
159
- runId: string;
160
- }
161
-
162
- interface TerminalCleanupTarget {
163
- agentId: string;
164
- owner: WorkflowOwner;
165
- fallbackStopRequested: boolean;
166
- }
167
-
168
- interface EngineRestoreSnapshot {
169
- runs: Map<string, WorkflowRun>;
170
- recoveryMemo: Map<string, RecoveryMemo>;
171
- terminalRecoveryMemo: Map<string, TerminalRecoveryMemo>;
172
- recoveryBranchGeneration: number;
173
- branchEpoch: number;
174
- branchChanging: boolean;
175
- branchEventsClosed: boolean;
109
+ export interface ScriptRunState {
110
+ run: ScriptRun;
111
+ /** Frozen run parameters — resume reuses the original values, ignoring new budget/toolset overrides. */
112
+ frozenOptions: Pick<
113
+ ScriptStartOptions,
114
+ "tokenBudget" | "maxAgents" | "concurrency" | "agentRetries" | "agentTimeoutMs" | "toolset" | "excludeTools"
115
+ >;
116
+ /** Abort controller for this run's script execution. */
117
+ controller: AbortController;
118
+ /** The in-flight runWorkflow promise, or undefined before start/after settle. */
119
+ execution?: Promise<WorkflowRunResult<unknown>>;
120
+ /** Monotonic execution generation; stale pause/resume continuations cannot mutate a newer run. */
121
+ executionGeneration: number;
122
+ /** The script's final return value, captured when the run settles. */
123
+ result?: unknown;
124
+ /** Per-call-index generation base for A4 spawnKey rotation (seeded from journal). */
125
+ generations: Map<string, number>;
126
+ /** Live dispatch waiters keyed by agent id. */
127
+ agentWaiters: Map<
128
+ string,
129
+ {
130
+ callIndex: string;
131
+ generation: number;
132
+ onUsage?: (usage: AgentUsage) => void;
133
+ onHistory?: (history: unknown[]) => void;
134
+ executionGeneration: number;
135
+ resolve: (r: ManagedSpawnResponse) => void;
136
+ reject: (e: unknown) => void;
137
+ settled: boolean;
138
+ }
139
+ >;
140
+ /** Allocations whose managed-spawn reply has not arrived yet. */
141
+ pendingSpawns: Map<string, { spawnKey: string; owner: WorkflowOwner }>;
142
+ /** Journal retry bookkeeping. */
143
+ pendingJournal: Set<string>;
144
+ /** Whether a lifecycle suspension is in effect (branch change). */
176
145
  lifecycleSuspended: boolean;
177
- quarantinedAgentIds: Set<string>;
178
- }
179
- export class WorkflowWaitAbortedError extends Error {
180
- constructor() {
181
- super("foreground workflow wait aborted; the workflow run continues");
182
- this.name = "WorkflowWaitAbortedError";
183
- }
184
- }
185
-
186
- function isRecord(value: unknown): value is Record<string, unknown> {
187
- return typeof value === "object" && value !== null && !Array.isArray(value);
188
- }
189
-
190
- const WORKFLOW_TIER_VALUES = new Set(["small", "medium", "large"]);
191
- const WORKFLOW_THINKING_LEVELS = new Set(["minimal", "low", "medium", "high", "xhigh", "max"]);
192
- const WORKFLOW_CONFIGURED_THINKING = new Set([...WORKFLOW_THINKING_LEVELS, "inherit"]);
193
- const TIER_SNAPSHOT_KEYS = new Set([
194
- "tier",
195
- "model",
196
- "thinking",
197
- "configuredModel",
198
- "configuredThinking",
199
- "requestedThinking",
200
- "modelSource",
201
- "thinkingSource",
202
- "clamped",
203
- "diagnostic",
204
- ]);
205
-
206
- function isValidTierSnapshot(value: unknown, tier: string): boolean {
207
- if (!isRecord(value) || value.tier !== tier || !WORKFLOW_TIER_VALUES.has(tier)) return false;
208
- if (Object.keys(value).some((key) => !TIER_SNAPSHOT_KEYS.has(key))) return false;
209
- for (const key of ["model", "configuredModel"] as const) {
210
- const field = value[key];
211
- if (field !== undefined && (typeof field !== "string" || !field.trim() || field.length > 2_000)) return false;
212
- if (key === "model" && typeof field === "string" && !field.includes("/")) return false;
213
- }
214
- for (const key of ["thinking", "requestedThinking"] as const) {
215
- const field = value[key];
216
- if (field !== undefined && (typeof field !== "string" || !WORKFLOW_THINKING_LEVELS.has(field))) return false;
217
- }
218
- const configuredThinking = value.configuredThinking;
219
- if (
220
- configuredThinking !== undefined &&
221
- (typeof configuredThinking !== "string" || !WORKFLOW_CONFIGURED_THINKING.has(configuredThinking))
222
- )
223
- return false;
224
- if (value.modelSource !== "frontmatter" && value.modelSource !== "tier" && value.modelSource !== "parent")
225
- return false;
226
- if (value.thinkingSource !== "frontmatter" && value.thinkingSource !== "tier" && value.thinkingSource !== "parent")
227
- return false;
228
- if (value.clamped !== undefined && typeof value.clamped !== "boolean") return false;
229
- if (value.diagnostic !== undefined && (typeof value.diagnostic !== "string" || value.diagnostic.length > 2_000))
230
- return false;
231
- return true;
232
- }
233
-
234
- function isAbortSignal(value: unknown): value is AbortSignal {
235
- return (
236
- isRecord(value) &&
237
- typeof value.aborted === "boolean" &&
238
- typeof value.addEventListener === "function" &&
239
- typeof value.removeEventListener === "function"
240
- );
241
- }
242
-
243
- function ownerFor(data: Record<string, unknown>): { runId: string; nodeId: string; attemptId?: string } | undefined {
244
- const owner = data.owner;
245
- if (!isRecord(owner) || owner.extension !== "pi-workflows") return undefined;
246
- const validId = (value: unknown): value is string =>
247
- typeof value === "string" && value.length > 0 && value.length <= 256;
248
- if (!validId(owner.runId) || !validId(owner.nodeId)) return undefined;
249
- if (owner.attemptId !== undefined && !validId(owner.attemptId)) return undefined;
250
- return {
251
- runId: owner.runId,
252
- nodeId: owner.nodeId,
253
- ...(typeof owner.attemptId === "string" ? { attemptId: owner.attemptId } : {}),
254
- };
255
146
  }
256
147
 
257
- function statusFromLifecycle(eventName: string, data: Record<string, unknown>): "completed" | "failed" | "stopped" {
258
- if (eventName === "subagents:completed") return "completed";
259
- const status = data.status;
260
- return status === "stopped" ? "stopped" : "failed";
148
+ function isTerminalWorkflow(status: string): boolean {
149
+ return status === "completed" || status === "failed" || status === "stopped";
261
150
  }
262
151
 
263
- function activeStatus(status: TaskStatus): boolean {
264
- return status === "dispatching" || status === "queued" || status === "running";
152
+ function managedTerminalStatus(snapshot: { status: string }): "completed" | "failed" | "stopped" {
153
+ return snapshot.status === "completed" ? "completed" : snapshot.status === "failed" ? "failed" : "stopped";
265
154
  }
266
155
 
267
- function resultPreview(result: { text?: string; error?: string } | undefined): {
268
- resultPreview?: string;
269
- errorPreview?: string;
270
- } {
156
+ function resultFromLifecycle(
157
+ status: "completed" | "failed" | "stopped",
158
+ agentId: string | undefined,
159
+ result: string | undefined,
160
+ error: string | undefined,
161
+ compactionCount: number,
162
+ tokenCount?: number,
163
+ ): CallResult {
271
164
  return {
272
- ...(result?.text ? { resultPreview: result.text.slice(0, 2_000) } : {}),
273
- ...(result?.error ? { errorPreview: result.error.slice(0, 2_000) } : {}),
165
+ status,
166
+ ...(agentId ? { agentId } : {}),
167
+ ...(result !== undefined && result.length > 0 ? { result } : {}),
168
+ ...(error !== undefined && error.length > 0 ? { error } : {}),
169
+ ...(tokenCount !== undefined ? { tokenCount } : {}),
170
+ compactionCount,
171
+ updatedAt: Date.now(),
274
172
  };
275
173
  }
276
174
 
277
- function tokenCountFrom(value: unknown): number | undefined {
278
- if (!isRecord(value) || typeof value.total !== "number" || !Number.isFinite(value.total) || value.total <= 0) {
279
- return undefined;
280
- }
281
- return Math.floor(value.total);
282
- }
283
-
284
- function normalizeManagedSpawnResponse(response: ManagedSpawnResponse | string): ManagedSpawnResponse {
285
- return typeof response === "string"
286
- ? { id: response, state: "running", created: true }
287
- : { ...response, created: response.created ?? true };
288
- }
289
-
290
- function managedTerminalStatus(snapshot: ManagedTerminalSnapshot): "completed" | "failed" | "stopped" {
291
- if (snapshot.status === "completed") return "completed";
292
- if (snapshot.status === "stopped") return "stopped";
293
- return "failed";
294
- }
295
-
296
- class JournalAppendError extends Error {
297
- constructor(eventKind: JournalEvent["kind"], cause: unknown) {
298
- super(`${eventKind} append failed: ${cause instanceof Error ? cause.message : String(cause)}`);
299
- this.name = "JournalAppendError";
300
- }
301
- }
302
-
303
- const JOURNAL_RETRY_INITIAL_DELAY_MS = 25;
304
- const JOURNAL_RETRY_MAX_DELAY_MS = 2_000;
305
-
306
- function spawnManaged(
307
- client: ManagedSpawnClient,
308
- task: WorkflowTask,
309
- runId: string,
310
- nodeId: string,
311
- attemptId: string | undefined,
312
- ): Promise<ManagedSpawnResponse | string> {
313
- // Protocol-v3 clients receive the attempt as the fourth argument. Keep the
314
- // old in-process fixture shape working without passing an ExtensionContext
315
- // back into the engine.
316
- if (client.spawn.length >= 5) {
317
- return client.spawn(task, runId, nodeId, undefined, attemptId);
318
- }
319
- return client.spawn(task, runId, nodeId, attemptId);
175
+ /** Parse a journaled attemptId of the form `<runId>/<nodeId>/attempt-<generation>`. */
176
+ function generationFromAttemptId(attemptId: string | undefined): number {
177
+ if (!attemptId) return 0;
178
+ const match = /\/attempt-(\d+)$/u.exec(attemptId);
179
+ return match ? Number(match[1]) : 0;
320
180
  }
321
181
 
322
182
  export class WorkflowEngine {
323
- private readonly runs = new Map<string, WorkflowRun>();
324
- /** Managed RPC calls that have not settled yet, including dispatching tasks. */
325
- private readonly dispatches = new Map<string, Set<Promise<void>>>();
326
- /** Late managed spawn responses discovered while terminal cleanup waits. */
327
- private readonly lateDispatchTargets = new Map<string, TerminalCleanupTarget[]>();
328
- /** Late branch-replacement spawns whose owner-scoped quiescence did not settle. */
329
- private readonly lateDispatchQuiescence = new Map<string, string[]>();
330
- private readonly waiters = new Map<string, Set<Waiter>>();
331
- private readonly pumping = new Set<string>();
332
- private readonly settling = new Map<string, Promise<void>>();
183
+ private readonly runs = new Map<string, ScriptRunState>();
184
+ /** Fired when a run settles terminal; wired by the UI surface for widget refresh. */
185
+ onRunSettled: (() => void) | undefined;
186
+ private readonly waiters = new Map<
187
+ string,
188
+ Set<{ resolve: (run: ScriptRun) => void; reject: (e: unknown) => void; settled: boolean; cleanup: () => void }>
189
+ >();
333
190
  private unsubscribeLifecycle: () => void;
334
- private lifecycleSuspended = false;
335
- private branchChanging = false;
336
- private branchEpoch = 0;
337
- private branchEventsClosed = false;
191
+ private lifecyclePaused = false;
338
192
  private branchQuiescePromise: Promise<WorkflowQuiesceResult> | undefined;
339
- /** Agent ids quarantined across a branch replacement; never accept late events. */
340
- private readonly quarantinedAgentIds = new Set<string>();
341
193
  private recoveryBranchGeneration = 0;
342
- /**
343
- * Durable recovery rotations already emitted for this engine/branch. The
344
- * key is semantic (branch, run, node, source attempt), never a journal
345
- * payload hash. A bounded map prevents repeated reloads from retaining
346
- * unbounded history.
347
- */
348
- private readonly recoveryMemo = new Map<string, RecoveryMemo>();
349
- /** Terminal cleanup facts already emitted for a stale branch snapshot. */
350
- private readonly terminalRecoveryMemo = new Map<string, TerminalRecoveryMemo>();
351
- /** Candidate replay state used to roll back a failed recovery append. */
352
- private restoreFailureSnapshot: EngineRestoreSnapshot | undefined;
353
- /** Lifecycle journal writes that failed transiently and are waiting to retry. */
354
194
  private readonly journalRetries = new Map<string, { timer?: ReturnType<typeof setTimeout>; attempt: number }>();
355
- /** Prefixes durably written before a lifecycle batch append failed. */
356
- private readonly journalBatchProgress = new Map<string, { serialized: string; nextIndex: number }>();
357
- /** Runs whose lifecycle journal cannot be made durable after bounded retries. */
358
195
  private readonly journalBlockedRuns = new Set<string>();
196
+ private readonly providerResumeTimers = new Map<string, ReturnType<typeof setTimeout>>();
359
197
  private journalMutationKey: string | undefined;
198
+ private readonly quarantinedRunIds = new Set<string>();
360
199
 
361
200
  constructor(
362
201
  readonly events: WorkflowEventBus,
363
202
  private readonly client: ManagedSpawnClient,
364
203
  private readonly journal: JournalWriter,
204
+ private readonly readEntries: () => readonly SessionEntryLike[] = () => [],
365
205
  ) {
366
- this.unsubscribeLifecycle = registerLifecycleListener(events, (eventName, data) => {
367
- this.onLifecycle(eventName, data);
368
- });
206
+ this.unsubscribeLifecycle = this.attachLifecycle();
207
+ }
208
+
209
+ private attachLifecycle(): () => void {
210
+ const unsubs = [
211
+ "subagents:created",
212
+ "subagents:started",
213
+ "subagents:completed",
214
+ "subagents:failed",
215
+ "subagents:compacted",
216
+ ].map((eventName) =>
217
+ this.events.on(eventName, (raw: unknown) => {
218
+ this.onLifecycle(eventName, raw);
219
+ }),
220
+ );
221
+ return () => {
222
+ for (const unsubscribe of unsubs) unsubscribe();
223
+ };
369
224
  }
370
225
 
371
- dispose(): void {
372
- this.branchEpoch += 1;
373
- this.clearJournalRetries();
374
- this.unsubscribeLifecycle();
375
- this.quarantinedAgentIds.clear();
376
- this.recoveryMemo.clear();
377
- this.terminalRecoveryMemo.clear();
378
- this.lateDispatchTargets.clear();
379
- this.lateDispatchQuiescence.clear();
380
- this.recoveryBranchGeneration = 0;
381
- for (const waiters of this.waiters.values()) {
382
- for (const waiter of waiters) {
383
- if (waiter.settled) continue;
226
+ private onLifecycle(eventName: string, raw: unknown): void {
227
+ if (!raw || typeof raw !== "object") return;
228
+ const data = raw as Record<string, unknown>;
229
+ const agentId = typeof data.id === "string" ? data.id : typeof data.agentId === "string" ? data.agentId : undefined;
230
+ if (!agentId) return;
231
+
232
+ // Resolve a live dispatch waiter on terminal lifecycle events.
233
+ if (eventName === "subagents:completed" || eventName === "subagents:failed") {
234
+ const lifecycleStatus =
235
+ eventName === "subagents:completed"
236
+ ? "completed"
237
+ : data.status === "stopped" || data.status === "interrupted"
238
+ ? "stopped"
239
+ : "failed";
240
+ for (const state of this.runs.values()) {
241
+ const waiter = state.agentWaiters.get(agentId);
242
+ if (!waiter || waiter.settled || waiter.executionGeneration !== state.executionGeneration) continue;
243
+ const owner = data.owner;
244
+ if (!owner || typeof owner !== "object" || Array.isArray(owner)) continue;
245
+ const ownerRecord = owner as Record<string, unknown>;
246
+ if (
247
+ ownerRecord.extension !== "pi-workflows" ||
248
+ ownerRecord.runId !== state.run.runId ||
249
+ ownerRecord.nodeId !== `call-${waiter.callIndex}` ||
250
+ ownerRecord.attemptId !== `attempt-${waiter.generation}`
251
+ )
252
+ continue;
253
+ const tokenCount =
254
+ typeof data.tokenCount === "number"
255
+ ? data.tokenCount
256
+ : data.tokens &&
257
+ typeof data.tokens === "object" &&
258
+ typeof (data.tokens as Record<string, unknown>).total === "number"
259
+ ? ((data.tokens as Record<string, unknown>).total as number)
260
+ : undefined;
384
261
  waiter.settled = true;
385
- waiter.cleanup();
386
- waiter.reject(new Error("workflow engine disposed"));
262
+ const terminal: ManagedSpawnResponse = {
263
+ id: agentId,
264
+ terminal: {
265
+ status: lifecycleStatus,
266
+ ...(typeof data.result === "string" ? { result: data.result } : {}),
267
+ ...(typeof data.error === "string" ? { error: data.error } : {}),
268
+ ...(typeof data.outputFile === "string" ? { outputFile: data.outputFile } : {}),
269
+ ...(tokenCount === undefined ? {} : { tokenCount }),
270
+ compactionCount: typeof data.compactionCount === "number" ? data.compactionCount : 0,
271
+ completedAt: typeof data.completedAt === "number" ? data.completedAt : Date.now(),
272
+ },
273
+ };
274
+ state.agentWaiters.delete(agentId);
275
+ if (tokenCount !== undefined)
276
+ waiter.onUsage?.({ input: 0, output: tokenCount, total: tokenCount, cost: 0, cacheRead: 0, cacheWrite: 0 });
277
+ if (Array.isArray(data.history)) waiter.onHistory?.(data.history);
278
+ waiter.resolve(terminal);
387
279
  }
388
280
  }
389
- this.waiters.clear();
281
+ }
282
+
283
+ dispose(): void {
284
+ this.unsubscribeLifecycle();
285
+ this.quarantinedRunIds.clear();
286
+ for (const state of this.runs.values()) {
287
+ this.cleanupAgents(state.run.runId);
288
+ state.controller.abort(new DOMException("Workflow engine disposed", "AbortError"));
289
+ }
290
+ this.rejectWaiters(new WorkflowWaitAbortedError());
291
+ this.runs.clear();
292
+ for (const timer of this.providerResumeTimers.values()) clearTimeout(timer);
293
+ this.providerResumeTimers.clear();
294
+ this.branchQuiescePromise = undefined;
390
295
  }
391
296
 
392
297
  suspendLifecycle(): void {
393
- if (this.lifecycleSuspended) return;
394
- this.lifecycleSuspended = true;
298
+ if (this.lifecyclePaused) return;
299
+ this.lifecyclePaused = true;
395
300
  this.unsubscribeLifecycle();
396
301
  }
397
302
 
398
303
  resumeLifecycle(): void {
399
- if (!this.lifecycleSuspended) return;
400
- this.lifecycleSuspended = false;
401
- this.unsubscribeLifecycle = registerLifecycleListener(this.events, (eventName, data) => {
402
- this.onLifecycle(eventName, data);
403
- });
304
+ if (!this.lifecyclePaused) return;
305
+ this.lifecyclePaused = false;
306
+ this.unsubscribeLifecycle = this.attachLifecycle();
404
307
  }
405
308
 
406
309
  /** Stop owned agents while Pi replaces the active session-tree branch. */
407
310
  async quiesceForBranchChange(): Promise<WorkflowQuiesceResult> {
408
311
  if (this.branchQuiescePromise) return this.branchQuiescePromise;
409
- this.branchChanging = true;
410
312
  this.clearJournalRetries();
411
- this.lateDispatchTargets.clear();
412
- this.lateDispatchQuiescence.clear();
413
- this.branchEpoch += 1;
414
- this.branchEventsClosed = false;
415
- this.branchQuiescePromise = (async () => {
416
- const operations: Array<Promise<{ runId: string; result: WorkflowQuiesceResult }>> = [];
417
- for (const run of this.runs.values()) {
418
- if (isTerminalWorkflow(run.status)) continue;
419
- const activeNodes = Object.entries(run.taskStatus)
420
- .filter(([, status]) => activeStatus(status))
421
- .map(([nodeId]) => ({ nodeId, agentId: run.agentIds[nodeId] }))
422
- .filter((item): item is { nodeId: string; agentId: string } => Boolean(item.agentId));
423
- const agentIds = activeNodes.map(({ agentId }) => agentId);
424
- const owners = activeNodes.map(({ nodeId }) => this.ownerForNode(run, nodeId));
425
- const hasDispatches = (this.dispatches.get(run.runId)?.size ?? 0) > 0;
426
- if (run.synthesisAgentId && !run.synthesisResult) {
427
- agentIds.push(run.synthesisAgentId);
428
- owners.push(this.ownerForNode(run, SYNTHESIS_NODE_ID));
429
- }
430
- if (agentIds.length === 0 && !hasDispatches) continue;
431
- for (const agentId of agentIds) this.quarantineAgent(agentId);
432
- const includeDispatches = (
433
- result: WorkflowQuiesceResult,
434
- waitForDispatches: boolean,
435
- ): Promise<{ runId: string; result: WorkflowQuiesceResult }> => {
436
- const settled = waitForDispatches
437
- ? this.waitForDispatches(run.runId, BRANCH_QUIESCE_TIMEOUT_MS)
438
- : Promise.resolve(true);
439
- return settled.then((dispatchSettled) => {
440
- const latePending = dispatchSettled ? this.takeLateDispatchQuiescence(run.runId) : [];
441
- const diagnostic = [
442
- result.diagnostic,
443
- latePending.length > 0 ? `late spawn quiescence incomplete: ${latePending.join(", ")}` : undefined,
444
- ]
445
- .filter((value): value is string => Boolean(value))
446
- .join("; ");
447
- return {
448
- runId: run.runId,
449
- result: {
450
- settled: result.settled && dispatchSettled && latePending.length === 0,
451
- pending: [
452
- ...new Set([
453
- ...result.pending,
454
- ...latePending,
455
- ...(dispatchSettled ? [] : [`${run.runId}:dispatch`]),
456
- ]),
457
- ].slice(0, 256),
458
- ...(diagnostic ? { diagnostic } : {}),
459
- },
460
- };
461
- });
462
- };
463
- if (this.client.quiesceOwned && (agentIds.length > 0 || hasDispatches)) {
464
- operations.push(
465
- this.client
466
- .quiesceOwned(run.runId, agentIds, BRANCH_QUIESCE_TIMEOUT_MS, owners)
467
- .then((result) =>
468
- includeDispatches(
469
- {
470
- settled: result.settled && result.pending.length === 0,
471
- pending: [...new Set(result.pending)].slice(0, 256),
472
- },
473
- true,
474
- ),
475
- )
476
- .catch((error: unknown) =>
477
- includeDispatches(
478
- {
479
- settled: false,
480
- pending: agentIds.slice(0, 256),
481
- diagnostic: `owned quiescence RPC failed: ${boundedError(error) ?? "unknown error"}`,
482
- },
483
- true,
484
- ),
485
- ),
486
- );
487
- } else {
488
- const stopOperations = activeNodes.map(({ nodeId, agentId }) => this.stopOwned(run, nodeId, agentId));
489
- if (run.synthesisAgentId && !run.synthesisResult) {
490
- stopOperations.push(this.stopOwned(run, SYNTHESIS_NODE_ID, run.synthesisAgentId));
491
- }
492
- operations.push(
493
- Promise.allSettled(stopOperations).then(() =>
494
- includeDispatches(
495
- {
496
- settled: agentIds.length === 0 && !hasDispatches,
497
- pending: [...agentIds, ...(hasDispatches ? [`${run.runId}:dispatch`] : [])].slice(0, 256),
498
- diagnostic: hasDispatches
499
- ? "dispatch quiescence is unavailable; pending spawn calls were quarantined conservatively"
500
- : agentIds.length > 0
501
- ? "owned quiescence is unavailable; stopped agents were quarantined conservatively"
502
- : undefined,
503
- },
504
- true,
505
- ),
506
- ),
507
- );
508
- }
509
- }
510
313
 
511
- const observations = await Promise.all(operations);
512
- const pending = [...new Set(observations.flatMap(({ result }) => result.pending))].slice(0, 256);
513
- const diagnostics: string[] = [];
514
- for (const { runId, result } of observations) {
515
- if (result.diagnostic) diagnostics.push(result.diagnostic);
516
- if (result.settled) continue;
517
- const run = this.runs.get(runId);
518
- if (!run || isTerminalWorkflow(run.status)) continue;
519
- const diagnostic = `branch quiescence incomplete for workflow ${runId}: ${
520
- result.pending.length > 0 ? `pending agents ${result.pending.join(", ")}` : "terminal state was not confirmed"
521
- }${result.diagnostic ? ` (${result.diagnostic})` : ""}`;
522
- diagnostics.push(diagnostic);
523
- try {
524
- this.setWorkflowStatus(run, run.status, boundedError(diagnostic) ?? diagnostic);
525
- } catch (error: unknown) {
526
- diagnostics.push(
527
- `workflow quiescence diagnostic was not journaled: ${boundedError(error) ?? "unknown error"}`,
528
- );
314
+ // Suspend and abort synchronously, before any await can let a stale runtime
315
+ // dispatch another call or let a provider-limit continuation recreate an old
316
+ // branch agent. `restore()` creates fresh states with suspension cleared.
317
+ const targets = new Map<string, { agentIds: string[]; owners: WorkflowOwner[] }>();
318
+ for (const state of this.runs.values()) {
319
+ if (isTerminalWorkflow(state.run.status)) continue;
320
+ const agentIds = [...state.agentWaiters.keys()];
321
+ const owners = agentIds.map((agentId) => {
322
+ const waiter = state.agentWaiters.get(agentId);
323
+ return {
324
+ extension: "pi-workflows" as const,
325
+ runId: state.run.runId,
326
+ nodeId: waiter ? `call-${waiter.callIndex}` : "",
327
+ attemptId: `attempt-${waiter?.generation ?? 1}`,
328
+ };
329
+ });
330
+ targets.set(state.run.runId, { agentIds, owners });
331
+ state.lifecycleSuspended = true;
332
+ state.controller.abort(new DOMException("Workflow branch change", "AbortError"));
333
+ }
334
+
335
+ const operation = (async () => {
336
+ const operations: Array<Promise<WorkflowQuiesceResult>> = [];
337
+ for (const state of this.runs.values()) {
338
+ const run = state.run;
339
+ const target = targets.get(run.runId);
340
+ if (isTerminalWorkflow(run.status)) {
341
+ this.reconcilePendingSpawns(state);
342
+ continue;
529
343
  }
344
+ if (state.pendingSpawns.size > 0) operations.push(this.quiescePendingSpawns(state));
345
+ const activeAgentIds = target?.agentIds ?? [];
346
+ if (activeAgentIds.length === 0) continue;
347
+ operations.push(
348
+ this.client
349
+ .quiesceOwned?.(run.runId, activeAgentIds, BRANCH_QUIESCE_TIMEOUT_MS, target?.owners)
350
+ .then((result) => ({
351
+ settled: result.settled && result.pending.length === 0,
352
+ pending: [...new Set(result.pending)].slice(0, 256),
353
+ }))
354
+ .catch((error: unknown) => ({
355
+ settled: false,
356
+ pending: activeAgentIds.slice(0, 256),
357
+ diagnostic: error instanceof Error ? error.message : String(error),
358
+ })) ?? Promise.resolve({ settled: false, pending: activeAgentIds.slice(0, 256) }),
359
+ );
530
360
  }
531
- const diagnostic = boundedError(diagnostics.join("; ")) ?? undefined;
532
- const result: WorkflowQuiesceResult = {
533
- settled: pending.length === 0 && observations.every(({ result: item }) => item.settled),
361
+ const results = await Promise.all(operations);
362
+ const pending = [...new Set(results.flatMap((r) => r.pending))].slice(0, 256);
363
+ const diagnostics = results.map((r) => r.diagnostic).filter((d): d is string => Boolean(d));
364
+ return {
365
+ settled: results.every((r) => r.settled) && pending.length === 0,
534
366
  pending,
535
- ...(diagnostic ? { diagnostic } : {}),
367
+ ...(diagnostics.length > 0 ? { diagnostic: diagnostics.join("; ") } : {}),
536
368
  };
537
- // After this point no lifecycle event from the old branch is accepted.
538
- // Events are already ignored while branchChanging; the subagents manager
539
- // detaches timed-out records before returning, and the generation gate
540
- // below also protects fixtures and legacy RPC peers without quiescence.
541
- this.branchEventsClosed = true;
542
- this.suspendLifecycle();
543
- return result;
544
369
  })();
545
- return this.branchQuiescePromise;
370
+ const tracked = operation.then(
371
+ (result) => {
372
+ if (this.branchQuiescePromise === tracked) this.branchQuiescePromise = undefined;
373
+ return result;
374
+ },
375
+ (error: unknown) => {
376
+ if (this.branchQuiescePromise === tracked) this.branchQuiescePromise = undefined;
377
+ throw error;
378
+ },
379
+ );
380
+ this.branchQuiescePromise = tracked;
381
+ return tracked;
546
382
  }
547
383
 
384
+ // ── start / control ────────────────────────────────────────────────────────
385
+
548
386
  /**
549
- * Restore a session-tree branch. Reusing the same branch generation is
550
- * intentionally idempotent: a stale snapshot may be replayed while Pi is
551
- * still delivering tree events. Callers that replace the branch must pass a
552
- * new generation (the no-argument path also advances after quiescence).
387
+ * Start a script run. The script is parsed once here for the journal's meta;
388
+ * the runtime re-parses it inside the sandbox (single source of truth for
389
+ * the body it executes).
553
390
  */
554
- restore(entries: readonly SessionEntryLike[], branchGeneration?: number): void {
555
- const snapshot = this.captureRestoreSnapshot();
556
- try {
557
- this.restoreInternal(entries, branchGeneration);
558
- this.restoreFailureSnapshot = undefined;
559
- } catch (error: unknown) {
560
- // Recovery is a transaction from the engine's point of view. In
561
- // particular, an append failure must not leave a half-rotated run in
562
- // memory or advance the branch namespace without its durable event.
563
- const rollback = this.restoreFailureSnapshot ?? snapshot;
564
- this.restoreFailureSnapshot = undefined;
565
- this.restoreAfterFailure(rollback);
566
- throw error;
567
- }
568
- }
569
-
570
- private captureRestoreSnapshot(): EngineRestoreSnapshot {
571
- return {
572
- runs: new Map([...this.runs].map(([runId, run]) => [runId, snapshotRun(run)])),
573
- recoveryMemo: new Map(
574
- [...this.recoveryMemo].map(([key, memo]) => [
575
- key,
576
- {
577
- ...memo,
578
- event: structuredClone(memo.event),
579
- },
580
- ]),
581
- ),
582
- terminalRecoveryMemo: new Map(
583
- [...this.terminalRecoveryMemo].map(([key, memo]) => [
584
- key,
585
- {
586
- ...memo,
587
- event: structuredClone(memo.event),
588
- },
589
- ]),
590
- ),
591
- recoveryBranchGeneration: this.recoveryBranchGeneration,
592
- branchEpoch: this.branchEpoch,
593
- branchChanging: this.branchChanging,
594
- branchEventsClosed: this.branchEventsClosed,
595
- lifecycleSuspended: this.lifecycleSuspended,
596
- quarantinedAgentIds: new Set(this.quarantinedAgentIds),
597
- };
598
- }
599
-
600
- private restoreAfterFailure(snapshot: EngineRestoreSnapshot): void {
601
- const invalidationEpoch = this.branchEpoch + 1;
602
- const lifecycleWasSuspended = this.lifecycleSuspended;
603
- this.dispatches.clear();
604
- this.lateDispatchTargets.clear();
605
- this.pumping.clear();
606
- this.settling.clear();
607
- this.runs.clear();
608
- for (const [runId, run] of snapshot.runs) this.runs.set(runId, run);
609
- this.recoveryMemo.clear();
610
- for (const [key, memo] of snapshot.recoveryMemo) this.recoveryMemo.set(key, memo);
611
- this.terminalRecoveryMemo.clear();
612
- for (const [key, memo] of snapshot.terminalRecoveryMemo) this.terminalRecoveryMemo.set(key, memo);
613
- this.recoveryBranchGeneration = snapshot.recoveryBranchGeneration;
614
- this.branchEpoch = Math.max(invalidationEpoch, snapshot.branchEpoch + 1);
615
- this.branchChanging = snapshot.branchChanging;
616
- this.branchEventsClosed = snapshot.branchEventsClosed;
617
- if (snapshot.lifecycleSuspended && !lifecycleWasSuspended) this.suspendLifecycle();
618
- else if (!snapshot.lifecycleSuspended && lifecycleWasSuspended) this.resumeLifecycle();
619
- this.quarantinedAgentIds.clear();
620
- for (const agentId of snapshot.quarantinedAgentIds) this.quarantinedAgentIds.add(agentId);
621
- this.branchQuiescePromise = undefined;
622
- }
623
-
624
- private restoreInternal(entries: readonly SessionEntryLike[], branchGeneration?: number): void {
625
- const branchReplacing = this.branchChanging;
626
- this.prepareRecoveryNamespace(branchGeneration, branchReplacing);
627
-
628
- // Invalidate dispatch promises from the replaced branch before clearing the
629
- // run map. A managed RPC reply arriving after this point must be stopped,
630
- // never journaled into the new branch.
631
- this.branchEpoch += 1;
632
- this.branchChanging = false;
633
- this.branchEventsClosed = false;
634
- this.branchQuiescePromise = undefined;
635
- this.dispatches.clear();
636
- this.lateDispatchTargets.clear();
637
- this.pumping.clear();
638
- this.settling.clear();
639
- this.runs.clear();
640
- for (const [runId, run] of replayJournal(entries, {
641
- onInvalid: (diagnostic) => console.warn(`[pi-workflows] ${diagnostic}`),
642
- }))
643
- this.runs.set(runId, run);
644
- this.restoreFailureSnapshot = this.captureRestoreSnapshot();
645
-
646
- // Persisted subagent records are the terminal source of truth when a
647
- // session was interrupted between lifecycle delivery and workflow
648
- // journaling. Interrupted records are deferred: recovery must append one
649
- // complete event before changing the in-memory attempt generation.
650
- const recoveredNodes = new Map<string, DeferredInterruptedRecord>();
651
- const terminalRecords = new Map<string, DeferredTerminalRecord>();
652
- const recoveryKey = (runId: string, nodeId: string): string => `${runId}\u0000${nodeId}`;
653
- for (const entry of entries) {
654
- const candidates = [this.reconcileSubagentRecord(entry), this.reconcileManagedSpawnTombstone(entry)];
655
- for (const reconciled of candidates) {
656
- if (!reconciled) continue;
657
- const key = recoveryKey(reconciled.runId, reconciled.nodeId);
658
- if (reconciled.kind === "interrupted") recoveredNodes.set(key, reconciled);
659
- else if (!terminalRecords.has(key)) terminalRecords.set(key, reconciled);
660
- }
661
- }
662
-
663
- for (const run of this.runs.values()) {
664
- const wasActive =
665
- run.status === "running" ||
666
- run.status === "pausing" ||
667
- run.status === "synthesizing" ||
668
- run.status === "stopping";
669
- const runTerminalRecords = [...terminalRecords.values()].filter((record) => record.runId === run.runId);
670
- if (!wasActive && runTerminalRecords.length > 0) {
671
- // Non-active legacy/paused runs retain the existing reconciliation
672
- // behavior. Active terminal-intent runs are handled below so each
673
- // recovered terminal fact is journaled before the final transition.
674
- this.commitDeferredTerminals(run, runTerminalRecords);
675
- }
676
- if (wasActive) {
677
- if (run.terminalIntent === "stop" || run.terminalIntent === "failure") {
678
- // A user stop/failure is an immutable terminal fact, never an
679
- // interrupted run that can be resumed after reload. Finish every
680
- // active node first; otherwise replay can leave a terminal workflow
681
- // with live task statuses and a later restore can resurrect them.
682
- this.recoverTerminalIntent(run, runTerminalRecords);
683
- } else {
684
- // A run-level event owns the workflow interruption and every active
685
- // task/synthesis rotation. append() happens before any mutation, so
686
- // an injected failure leaves this replayed run untouched and an
687
- // exact retry can safely emit the same generation once.
688
- this.recoverInterruptedRun(run, this.recoveryPlan(run, recoveredNodes, terminalRecords), runTerminalRecords);
689
- }
690
- } else if (run.status === "interrupted") {
691
- // A legacy sequence could have persisted workflow_transition before
692
- // its task_attempt. Migrate each still-active node independently;
693
- // every attempt_recovery event is valid on its own prefix.
694
- for (const item of this.recoveryPlan(run, recoveredNodes, terminalRecords)) {
695
- this.recoverInterruptedNode(run, item.nodeId, item.oldAgentId, false, true);
696
- }
697
- }
698
-
699
- if (isTerminalWorkflow(run.status)) continue;
700
-
701
- // A terminal record can arrive without its lifecycle result having been
702
- // journaled. Preserve the existing reconciliation behavior for that
703
- // record; it is not part of the generation rotation itself.
704
- for (const task of run.definition.tasks) {
705
- const status = run.taskStatus[task.id];
706
- if (run.taskResults[task.id] && !isTerminalTask(status)) {
707
- this.setTaskStatus(run, task.id, run.taskResults[task.id].status, run.agentIds[task.id]);
708
- }
709
- }
710
- }
711
- for (const waiters of this.waiters.values()) {
712
- for (const waiter of waiters) {
713
- if (waiter.settled) continue;
714
- waiter.settled = true;
715
- waiter.cleanup();
716
- waiter.reject(new Error("workflow session branch changed; the old run is no longer active"));
717
- }
718
- }
719
- this.waiters.clear();
720
- }
721
-
722
- private prepareRecoveryNamespace(branchGeneration: number | undefined, branchReplacing: boolean): void {
723
- if (branchGeneration !== undefined && (!Number.isSafeInteger(branchGeneration) || branchGeneration < 0)) {
724
- throw new Error("workflow branch generation must be a non-negative safe integer");
725
- }
726
- const nextGeneration =
727
- branchGeneration === undefined
728
- ? branchReplacing
729
- ? this.recoveryBranchGeneration + 1
730
- : this.recoveryBranchGeneration
731
- : branchGeneration;
732
- if (nextGeneration === this.recoveryBranchGeneration) {
733
- if (branchReplacing) {
734
- this.recoveryMemo.clear();
735
- this.terminalRecoveryMemo.clear();
736
- }
737
- return;
738
- }
739
- this.recoveryMemo.clear();
740
- this.terminalRecoveryMemo.clear();
741
- this.recoveryBranchGeneration = nextGeneration;
742
- }
743
-
744
- private recoveryMemoKey(runId: string, kind: RecoveryEvent["kind"], sourceKey: string): string {
745
- return JSON.stringify([this.recoveryBranchGeneration, runId, kind, sourceKey]);
746
- }
747
-
748
- private sourceAttemptId(run: WorkflowRun, nodeId: string): string {
749
- return run.attemptIds[nodeId] ?? `${run.runId}/${nodeId}/attempt-${run.attempts[nodeId] ?? 1}`;
750
- }
751
-
752
- private recoveryPlan(
753
- run: WorkflowRun,
754
- deferred: ReadonlyMap<string, DeferredInterruptedRecord>,
755
- terminalRecords: ReadonlyMap<string, DeferredTerminalRecord> = new Map(),
756
- ): RecoveryPlanItem[] {
757
- const activeWorkflow =
758
- run.status === "running" ||
759
- run.status === "pausing" ||
760
- run.status === "synthesizing" ||
761
- run.status === "stopping";
762
- const legacyInterrupted = run.status === "interrupted";
763
- if ((!activeWorkflow && !legacyInterrupted) || run.terminalIntent) return [];
764
- const plan: RecoveryPlanItem[] = [];
765
- for (const task of run.definition.tasks) {
766
- const status = run.taskStatus[task.id];
767
- const key = `${run.runId}\u0000${task.id}`;
768
- const deferredRecord = deferred.get(key);
769
- if (run.taskResults[task.id] || isTerminalTask(status) || terminalRecords.has(key)) continue;
770
- if (!activeStatus(status) && deferredRecord === undefined) continue;
771
- plan.push({
772
- nodeId: task.id,
773
- sourceAttemptId: this.sourceAttemptId(run, task.id),
774
- sourceGeneration: run.attempts[task.id],
775
- sourceStatus: status,
776
- oldAgentId: deferredRecord?.oldAgentId ?? run.agentIds[task.id] ?? "",
777
- });
778
- }
779
- if ((run.status === "synthesizing" || (legacyInterrupted && run.synthesisAgentId)) && !run.synthesisResult) {
780
- const synthesisKey = `${run.runId}\u0000${SYNTHESIS_NODE_ID}`;
781
- const deferredRecord = deferred.get(synthesisKey);
782
- if (terminalRecords.has(synthesisKey)) return plan;
783
- plan.push({
784
- nodeId: SYNTHESIS_NODE_ID,
785
- sourceAttemptId: this.sourceAttemptId(run, SYNTHESIS_NODE_ID),
786
- sourceGeneration: run.attempts[SYNTHESIS_NODE_ID],
787
- sourceStatus: "synthesizing",
788
- oldAgentId: deferredRecord?.oldAgentId ?? run.synthesisAgentId ?? "",
789
- });
790
- }
791
- return plan;
792
- }
793
-
794
- private recoverySourceKey(
795
- run: WorkflowRun,
796
- plan: readonly RecoveryPlanItem[],
797
- terminalResults: readonly DeferredTerminalRecord[] = [],
798
- ): string {
799
- return JSON.stringify({
800
- status: run.status,
801
- terminalIntent: run.terminalIntent,
802
- rotations: plan.map((item) => [
803
- item.nodeId,
804
- item.sourceAttemptId,
805
- item.sourceGeneration,
806
- item.sourceStatus,
807
- item.oldAgentId,
808
- ]),
809
- terminalResults: terminalResults.map((item) => [
810
- item.nodeId,
811
- item.attemptId,
812
- item.result.status,
813
- item.result.agentId,
814
- item.result.text,
815
- item.result.error,
816
- item.result.outputFile,
817
- item.result.tokenCount,
818
- item.result.compactionCount,
819
- item.result.truncated,
820
- ]),
821
- });
822
- }
823
-
824
- private rememberRecovery(memo: RecoveryMemo): void {
825
- const key = this.recoveryMemoKey(memo.runId, memo.event.kind, memo.key);
826
- this.recoveryMemo.delete(key);
827
- this.recoveryMemo.set(key, memo);
828
- while (this.recoveryMemo.size > MAX_RECOVERY_MEMO_ENTRIES) {
829
- const oldest = this.recoveryMemo.keys().next().value;
830
- if (typeof oldest !== "string") break;
831
- this.recoveryMemo.delete(oldest);
832
- }
833
- }
834
-
835
- private memoizedRecovery(runId: string, kind: RecoveryEvent["kind"], sourceKey: string): RecoveryEvent | undefined {
836
- return this.recoveryMemo.get(this.recoveryMemoKey(runId, kind, sourceKey))?.event;
837
- }
838
-
839
- list(): WorkflowSummary[] {
840
- return [...this.runs.values()].sort((a, b) => b.startedAt - a.startedAt).map((run) => this.summary(run));
841
- }
842
-
843
- get(runId: string): WorkflowSummary | undefined {
844
- const run = this.runs.get(runId);
845
- return run ? this.summary(run) : undefined;
846
- }
847
-
848
- getRun(runId: string): WorkflowRun | undefined {
849
- return this.runs.get(runId);
850
- }
851
-
852
- async start(definition: WorkflowDefinition, signal?: AbortSignal): Promise<WorkflowStartResult>;
853
- /** @deprecated The context argument is ignored; pass the foreground signal as the second argument. */
854
- async start(
855
- definition: WorkflowDefinition,
856
- _legacyContext: unknown,
857
- signal?: AbortSignal,
858
- ): Promise<WorkflowStartResult>;
859
- async start(
860
- definition: WorkflowDefinition,
861
- signalOrLegacyContext?: AbortSignal | unknown,
862
- legacySignal?: AbortSignal,
863
- ): Promise<WorkflowStartResult> {
864
- const signal = legacySignal ?? (isAbortSignal(signalOrLegacyContext) ? signalOrLegacyContext : undefined);
391
+ async start(script: string, options: ScriptStartOptions = {}): Promise<ScriptStartResult> {
392
+ const { meta } = parseWorkflowScript(script);
865
393
  const now = Date.now();
866
394
  const runId = randomUUID();
867
- const run: WorkflowRun = {
395
+ const scriptHash = hashScript(script);
396
+ const toolset = options.toolset;
397
+ const run: ScriptRun = {
868
398
  runId,
869
399
  schemaVersion: JOURNAL_SCHEMA_VERSION,
870
- definition,
400
+ script,
401
+ scriptHash,
402
+ meta,
403
+ ...(toolset ? { toolset } : {}),
404
+ ...(options.maxAgents !== undefined ? { frozenMaxAgents: options.maxAgents } : {}),
405
+ ...(options.concurrency !== undefined ? { frozenConcurrency: options.concurrency } : {}),
406
+ ...(options.agentRetries !== undefined ? { frozenAgentRetries: options.agentRetries } : {}),
407
+ ...(options.tokenBudget !== undefined ? { frozenTokenBudget: options.tokenBudget } : {}),
408
+ ...(options.agentTimeoutMs !== undefined ? { frozenAgentTimeoutMs: options.agentTimeoutMs } : {}),
409
+ ...(options.excludeTools ? { frozenExcludeTools: [...options.excludeTools] } : {}),
871
410
  status: "pending",
872
- taskStatus: Object.fromEntries(definition.tasks.map((task) => [task.id, "pending" as const])),
411
+ callStatus: {},
873
412
  agentIds: {},
874
- attempts: Object.fromEntries([
875
- ...definition.tasks.map((task) => [task.id, 1] as const),
876
- [SYNTHESIS_NODE_ID, 1] as const,
877
- ]),
878
- attemptIds: Object.fromEntries([
879
- ...definition.tasks.map((task) => [task.id, `${runId}/${task.id}/attempt-1`] as const),
880
- [SYNTHESIS_NODE_ID, `${runId}/${SYNTHESIS_NODE_ID}/attempt-1`] as const,
881
- ]),
882
- attemptTracking: true,
883
- taskResults: {},
413
+ attempts: {},
414
+ attemptIds: {},
415
+ callResults: {},
884
416
  compactions: {},
885
417
  startedAt: now,
886
418
  updatedAt: now,
887
419
  };
888
- this.write({
420
+ this.appendRequired({
889
421
  kind: "run_created",
890
422
  schemaVersion: JOURNAL_SCHEMA_VERSION,
891
- runId: run.runId,
892
- definition,
893
- attempts: run.attempts,
894
- attemptIds: run.attemptIds,
423
+ runId,
424
+ script,
425
+ scriptHash,
426
+ meta,
427
+ ...(toolset ? { toolset } : {}),
428
+ ...(options.maxAgents !== undefined ? { frozenMaxAgents: options.maxAgents } : {}),
429
+ ...(options.concurrency !== undefined ? { frozenConcurrency: options.concurrency } : {}),
430
+ ...(options.agentRetries !== undefined ? { frozenAgentRetries: options.agentRetries } : {}),
431
+ ...(options.tokenBudget !== undefined ? { frozenTokenBudget: options.tokenBudget } : {}),
432
+ ...(options.agentTimeoutMs !== undefined ? { frozenAgentTimeoutMs: options.agentTimeoutMs } : {}),
433
+ ...(options.excludeTools ? { frozenExcludeTools: [...options.excludeTools] } : {}),
895
434
  timestamp: now,
896
435
  });
436
+ const state: ScriptRunState = {
437
+ run,
438
+ frozenOptions: {
439
+ ...(options.tokenBudget !== undefined ? { tokenBudget: options.tokenBudget } : {}),
440
+ ...(options.maxAgents !== undefined ? { maxAgents: options.maxAgents } : {}),
441
+ ...(options.concurrency !== undefined ? { concurrency: options.concurrency } : {}),
442
+ ...(options.agentRetries !== undefined ? { agentRetries: options.agentRetries } : {}),
443
+ ...(options.agentTimeoutMs !== undefined ? { agentTimeoutMs: options.agentTimeoutMs } : {}),
444
+ ...(options.toolset ? { toolset: options.toolset } : {}),
445
+ ...(options.excludeTools ? { excludeTools: [...options.excludeTools] } : {}),
446
+ },
447
+ controller: new AbortController(),
448
+ generations: new Map(),
449
+ agentWaiters: new Map(),
450
+ pendingSpawns: new Map(),
451
+ pendingJournal: new Set(),
452
+ lifecycleSuspended: false,
453
+ executionGeneration: 0,
454
+ };
455
+ this.runs.set(runId, state);
897
456
  this.setWorkflowStatus(run, "running");
898
- this.runs.set(run.runId, run);
899
- void this.pump(run);
900
- if (definition.background) return { runId: run.runId, status: run.status, background: true };
457
+ state.executionGeneration += 1;
458
+ state.execution = this.execute(runId, script, options, undefined, state.executionGeneration);
459
+ // A run-fatal abort/stop already journals its terminal state; never let a
460
+ // leftover rejection become an unhandled promise rejection.
461
+ void state.execution.catch(() => {});
462
+ if (options.background) return { runId, status: run.status, background: true };
901
463
  try {
902
- const settled = await this.waitFor(run.runId, signal);
464
+ const settled = await this.waitFor(runId, options.signal);
465
+ const state = this.runs.get(runId);
466
+ const scriptResult = state?.result;
903
467
  return {
904
- runId: run.runId,
468
+ runId,
905
469
  status: settled.status,
906
470
  background: false,
907
- ...(settled.synthesisResult?.text ? { result: settled.synthesisResult.text } : {}),
471
+ ...(scriptResult !== undefined ? { result: String(scriptResult) } : {}),
908
472
  ...(settled.error ? { error: settled.error } : {}),
909
473
  };
910
474
  } catch (error: unknown) {
911
475
  if (!(error instanceof WorkflowWaitAbortedError)) throw error;
912
- const current = this.runs.get(run.runId) ?? run;
913
- return {
914
- runId: current.runId,
915
- status: current.status,
916
- background: false,
917
- waitAborted: true,
918
- error: error.message,
919
- };
476
+ const current = this.runs.get(runId)?.run ?? run;
477
+ return { runId: current.runId, status: current.status, background: false, waitAborted: true };
478
+ }
479
+ }
480
+
481
+ /**
482
+ * Resume an interrupted/paused run: rebuild the resume journal from journaled
483
+ * call_result events and re-run the same script under the same runId. The
484
+ * runtime replays the longest unchanged prefix; every live dispatch of a
485
+ * previously-journaled call index rotates the spawnKey generation.
486
+ */
487
+ async resume(
488
+ runId: string,
489
+ entries: readonly SessionEntryLike[],
490
+ options: ScriptStartOptions = {},
491
+ replacementScript?: string,
492
+ ): Promise<ScriptStartResult | undefined> {
493
+ const state = this.runs.get(runId);
494
+ if (!state) return undefined;
495
+ if (state.lifecycleSuspended) return undefined;
496
+ const run = state.run;
497
+ const providerTimer = this.providerResumeTimers.get(runId);
498
+ if (providerTimer) {
499
+ clearTimeout(providerTimer);
500
+ this.providerResumeTimers.delete(runId);
501
+ }
502
+ if (run.nonResumable || (isTerminalWorkflow(run.status) && run.status !== "failed" && run.status !== "stopped"))
503
+ return undefined;
504
+ if (replacementScript !== undefined) {
505
+ const parsed = parseWorkflowScript(replacementScript);
506
+ const revision = (run.revision ?? 0) + 1;
507
+ run.script = replacementScript;
508
+ run.scriptHash = hashScript(replacementScript);
509
+ run.meta = parsed.meta;
510
+ run.revision = revision;
511
+ this.persist(runId, `revision:${revision}`, () =>
512
+ this.write({
513
+ kind: "run_revision",
514
+ schemaVersion: JOURNAL_SCHEMA_VERSION,
515
+ runId,
516
+ revision,
517
+ script: replacementScript,
518
+ scriptHash: run.scriptHash,
519
+ meta: parsed.meta,
520
+ timestamp: Date.now(),
521
+ }),
522
+ );
523
+ }
524
+ const resumeJournal = buildResumeJournal(entries);
525
+ // Seed generation bases from journaled attemptIds so a live re-dispatch of a
526
+ // previously-journaled call never reuses a spawnKey (A4).
527
+ for (const [callIndex, attemptId] of Object.entries(run.attemptIds)) {
528
+ state.generations.set(callIndex, generationFromAttemptId(attemptId));
529
+ }
530
+ this.setWorkflowStatus(run, "running");
531
+ state.controller = new AbortController();
532
+ // Freeze semantics: original run parameters win over new options.
533
+ // Only signal/confirm/args/background/loadSavedWorkflow/mainModel are per-resume context;
534
+ // budget/scale/toolset are frozen from the original start.
535
+ const frozen = state.frozenOptions;
536
+ const resumeOptions: ScriptStartOptions = {
537
+ ...options,
538
+ tokenBudget: frozen.tokenBudget,
539
+ maxAgents: frozen.maxAgents,
540
+ concurrency: frozen.concurrency,
541
+ agentRetries: frozen.agentRetries,
542
+ agentTimeoutMs: frozen.agentTimeoutMs,
543
+ toolset: frozen.toolset,
544
+ excludeTools: frozen.excludeTools ? [...frozen.excludeTools] : undefined,
545
+ };
546
+ state.executionGeneration += 1;
547
+ state.execution = this.execute(runId, run.script, resumeOptions, resumeJournal, state.executionGeneration);
548
+ void state.execution.catch(() => {});
549
+ if (options.background) return { runId, status: run.status, background: true };
550
+ try {
551
+ const settled = await this.waitFor(runId, options.signal);
552
+ return { runId, status: settled.status, background: false };
553
+ } catch (error: unknown) {
554
+ if (!(error instanceof WorkflowWaitAbortedError)) throw error;
555
+ return { runId, status: run.status, background: false, waitAborted: true };
920
556
  }
921
557
  }
922
558
 
923
- async control(action: "list" | "get" | "pause" | "resume" | "stop", runId?: string): Promise<WorkflowControlResult> {
559
+ async control(
560
+ action: "list" | "get" | "pause" | "resume" | "stop" | "rm",
561
+ runId?: string,
562
+ entries?: readonly SessionEntryLike[],
563
+ ): Promise<{ action: string; runs?: unknown[]; run?: unknown }> {
924
564
  if (action === "list") return { action, runs: this.list() };
925
565
  if (!runId) throw new Error(`${action} requires runId`);
926
- const run = this.runs.get(runId);
927
- if (!run) throw new Error(`workflow run not found: ${runId}`);
566
+ const state = this.runs.get(runId);
567
+ if (!state) throw new Error(`workflow run not found: ${runId}`);
568
+ const run = state.run;
928
569
  if (action === "get") return { action, run: this.summary(run, true) };
929
570
  if (action === "pause") {
930
571
  if (run.status === "running") {
931
572
  this.setWorkflowStatus(run, "pausing");
932
- this.maybePause(run);
933
- } else if (run.status !== "paused") {
573
+ this.reconcilePendingSpawns(state);
574
+ // Abort the script execution; the run settles as interrupted and can be
575
+ // resumed from the journal. The script's in-flight agents are stopped by
576
+ // the runtime's abort signal via the dispatch waiters.
577
+ state.controller.abort();
578
+ this.persist(runId, "pause-recovery", () =>
579
+ this.write({
580
+ kind: "run_recovery",
581
+ schemaVersion: JOURNAL_SCHEMA_VERSION,
582
+ runId,
583
+ status: "interrupted",
584
+ branchGeneration: this.recoveryBranchGeneration,
585
+ rotations: [],
586
+ recoveryId: deriveRecoveryId({ runId, rotations: [], timestamp: Date.now() }),
587
+ timestamp: Date.now(),
588
+ }),
589
+ );
590
+ run.status = "interrupted";
591
+ run.updatedAt = Date.now();
592
+ this.settleWaiters(runId, run);
593
+ } else if (run.status !== "paused" && run.status !== "interrupted") {
934
594
  throw new Error(`cannot pause workflow in state ${run.status}`);
935
595
  }
936
596
  } else if (action === "resume") {
@@ -938,24 +598,45 @@ export class WorkflowEngine {
938
598
  if (run.status !== "paused" && run.status !== "interrupted") {
939
599
  throw new Error(`cannot resume workflow in state ${run.status}`);
940
600
  }
941
- this.setWorkflowStatus(run, "running");
942
- void this.pump(run);
601
+ const sessionEntries = entries ?? this.readEntries();
602
+ const resumed = await this.resume(runId, sessionEntries, {
603
+ background: true,
604
+ mainModel: undefined,
605
+ });
606
+ if (!resumed) throw new Error(`cannot resume workflow run: ${runId}`);
607
+ return { action, run: this.summary(this.runs.get(runId)?.run ?? run) };
943
608
  } else if (action === "stop") {
944
- await this.stop(run);
609
+ await this.stop(state);
610
+ } else if (action === "rm") {
611
+ if (!isTerminalWorkflow(run.status)) throw new Error("cannot remove a running workflow");
612
+ this.persist(runId, "run-removed", () =>
613
+ this.write({ kind: "run_removed", schemaVersion: JOURNAL_SCHEMA_VERSION, runId, timestamp: Date.now() }),
614
+ );
615
+ this.runs.delete(runId);
616
+ this.waiters.delete(runId);
617
+ return { action, run: this.summary(run) };
945
618
  }
946
619
  return { action, run: this.summary(run) };
947
620
  }
948
621
 
949
- async waitFor(runId: string, signal?: AbortSignal): Promise<WorkflowRun> {
950
- const run = this.runs.get(runId);
951
- if (!run) throw new Error(`workflow run not found: ${runId}`);
952
- if (isTerminalWorkflow(run.status)) return run;
953
- const all = this.waiters.get(runId) ?? new Set<Waiter>();
622
+ async waitFor(runId: string, signal?: AbortSignal): Promise<ScriptRun> {
623
+ const state = this.runs.get(runId);
624
+ if (!state) throw new Error(`workflow run not found: ${runId}`);
625
+ if (isTerminalWorkflow(state.run.status) || state.run.status === "paused" || state.run.status === "interrupted")
626
+ return state.run;
627
+ const all =
628
+ this.waiters.get(runId) ??
629
+ new Set<{
630
+ resolve: (run: ScriptRun) => void;
631
+ reject: (e: unknown) => void;
632
+ settled: boolean;
633
+ cleanup: () => void;
634
+ }>();
954
635
  if (all.size >= MAX_WAITERS) throw new Error("too many workflow waiters");
955
636
  this.waiters.set(runId, all);
956
- return new Promise<WorkflowRun>((resolve, reject) => {
637
+ return new Promise<ScriptRun>((resolve, reject) => {
957
638
  let abort = () => {};
958
- const waiter: Waiter = {
639
+ const waiter = {
959
640
  resolve,
960
641
  reject,
961
642
  settled: false,
@@ -970,223 +651,518 @@ export class WorkflowEngine {
970
651
  reject(new WorkflowWaitAbortedError());
971
652
  };
972
653
  all.add(waiter);
973
- if (signal) {
974
- if (signal.aborted) {
975
- abort();
976
- return;
977
- }
978
- signal.addEventListener("abort", abort, { once: true });
979
- }
980
- if (isTerminalWorkflow(run.status)) {
981
- all.delete(waiter);
982
- if (all.size === 0) this.waiters.delete(runId);
983
- waiter.settled = true;
984
- waiter.cleanup();
985
- resolve(run);
986
- }
987
- });
988
- }
989
-
990
- private trackDispatch(runId: string, dispatch: Promise<void>): void {
991
- const pending = this.dispatches.get(runId) ?? new Set<Promise<void>>();
992
- pending.add(dispatch);
993
- this.dispatches.set(runId, pending);
994
- // Use both settlement branches so a rejected dispatch never creates an
995
- // unhandled rejection while its bookkeeping is removed.
996
- void dispatch.then(
997
- () => this.untrackDispatch(runId, dispatch),
998
- () => this.untrackDispatch(runId, dispatch),
999
- );
1000
- }
1001
-
1002
- private untrackDispatch(runId: string, dispatch: Promise<void>): void {
1003
- const pending = this.dispatches.get(runId);
1004
- if (!pending) return;
1005
- pending.delete(dispatch);
1006
- if (pending.size === 0) this.dispatches.delete(runId);
1007
- }
1008
-
1009
- private async waitForDispatches(runId: string, timeoutMs: number): Promise<boolean> {
1010
- const pending = this.dispatches.get(runId);
1011
- if (!pending || pending.size === 0) return true;
1012
- let timer: ReturnType<typeof setTimeout> | undefined;
1013
- const allSettled = Promise.allSettled([...pending]).then(() => true);
1014
- const timedOut = new Promise<boolean>((resolve) => {
1015
- timer = setTimeout(() => resolve(false), timeoutMs);
654
+ signal?.addEventListener("abort", abort, { once: true });
1016
655
  });
1017
- const settled = await Promise.race([allSettled, timedOut]);
1018
- if (timer) clearTimeout(timer);
1019
- return settled;
1020
- }
1021
-
1022
- private quarantineAgent(agentId: string): void {
1023
- this.quarantinedAgentIds.add(agentId);
1024
- if (this.quarantinedAgentIds.size > MAX_QUARANTINED_AGENT_IDS) {
1025
- const oldest = this.quarantinedAgentIds.values().next().value;
1026
- if (typeof oldest === "string") this.quarantinedAgentIds.delete(oldest);
1027
- }
1028
656
  }
1029
657
 
1030
- private stopOwned(run: WorkflowRun, nodeId: string, agentId: string, attemptId?: string): Promise<void> {
1031
- const owner = {
1032
- ...this.ownerForNode(run, nodeId),
1033
- ...(attemptId === undefined ? {} : { attemptId }),
1034
- };
1035
- if (!this.client.stopOwned) {
1036
- return Promise.reject(new Error("workflow owner-scoped stop is unavailable; refusing unowned stop fallback"));
658
+ private rejectWaiters(error: Error): void {
659
+ for (const all of this.waiters.values()) {
660
+ for (const waiter of [...all]) {
661
+ if (waiter.settled) continue;
662
+ waiter.settled = true;
663
+ all.delete(waiter);
664
+ waiter.cleanup();
665
+ waiter.reject(error);
666
+ }
1037
667
  }
1038
- return this.client.stopOwned(agentId, owner);
668
+ this.waiters.clear();
1039
669
  }
1040
670
 
1041
- private rememberLateDispatchTarget(
1042
- run: WorkflowRun,
1043
- nodeId: string,
1044
- agentId: string,
1045
- attemptId: string | undefined,
1046
- ): void {
1047
- const owner: WorkflowOwner = {
1048
- ...this.ownerForNode(run, nodeId),
1049
- ...(attemptId === undefined ? {} : { attemptId }),
1050
- };
1051
- const targets = this.lateDispatchTargets.get(run.runId) ?? [];
1052
- if (
1053
- !targets.some(
1054
- (target) =>
1055
- target.agentId === agentId &&
1056
- target.owner.nodeId === owner.nodeId &&
1057
- target.owner.attemptId === owner.attemptId,
1058
- )
1059
- ) {
1060
- targets.push({ agentId, owner, fallbackStopRequested: false });
671
+ private settleWaiters(runId: string, run: ScriptRun): void {
672
+ const all = this.waiters.get(runId);
673
+ if (!all) return;
674
+ for (const waiter of [...all]) {
675
+ if (waiter.settled) continue;
676
+ waiter.settled = true;
677
+ all.delete(waiter);
678
+ waiter.cleanup();
679
+ waiter.resolve(run);
1061
680
  }
1062
- this.lateDispatchTargets.set(run.runId, targets);
681
+ if (all.size === 0) this.waiters.delete(runId);
1063
682
  }
1064
683
 
1065
- private takeLateDispatchTargets(run: WorkflowRun): TerminalCleanupTarget[] {
1066
- const targets = this.lateDispatchTargets.get(run.runId);
1067
- if (!targets) return [];
1068
- this.lateDispatchTargets.delete(run.runId);
1069
- return targets;
1070
- }
684
+ // ── execution ──────────────────────────────────────────────────────────────
1071
685
 
1072
- private rememberLateDispatchQuiescence(runId: string, agentId: string): void {
1073
- const pending = this.lateDispatchQuiescence.get(runId) ?? [];
1074
- if (!pending.includes(agentId)) pending.push(agentId);
1075
- this.lateDispatchQuiescence.set(runId, pending.slice(0, 256));
1076
- }
686
+ private async execute(
687
+ runId: string,
688
+ script: string,
689
+ options: ScriptStartOptions,
690
+ resumeJournal?: Map<string, JournalEntry>,
691
+ executionGeneration?: number,
692
+ ): Promise<WorkflowRunResult<unknown>> {
693
+ const state = this.runs.get(runId);
694
+ if (!state) throw new Error(`workflow run not found: ${runId}`);
695
+ const generation = executionGeneration ?? state.executionGeneration;
1077
696
 
1078
- private takeLateDispatchQuiescence(runId: string): string[] {
1079
- const pending = this.lateDispatchQuiescence.get(runId) ?? [];
1080
- this.lateDispatchQuiescence.delete(runId);
1081
- return pending;
1082
- }
697
+ const runner: WorkflowAgentRunner = {
698
+ run: async (prompt: string, runOptions: AgentRunOptions = {}) => {
699
+ return this.dispatchAgent(runId, prompt, runOptions);
700
+ },
701
+ };
1083
702
 
1084
- private initialTerminalCleanupTargets(run: WorkflowRun): TerminalCleanupTarget[] {
1085
- const targets: TerminalCleanupTarget[] = [];
1086
- for (const [nodeId, status] of Object.entries(run.taskStatus)) {
1087
- if (!activeStatus(status)) continue;
1088
- const agentId = run.agentIds[nodeId];
1089
- if (!agentId) continue;
1090
- targets.push({ agentId, owner: this.ownerForNode(run, nodeId), fallbackStopRequested: false });
1091
- }
1092
- if (run.synthesisAgentId && !run.synthesisResult) {
1093
- targets.push({
1094
- agentId: run.synthesisAgentId,
1095
- owner: this.ownerForNode(run, SYNTHESIS_NODE_ID),
1096
- fallbackStopRequested: false,
703
+ try {
704
+ const result = await runWorkflow(script, {
705
+ args: options.args,
706
+ agent: runner,
707
+ mainModel: options.mainModel,
708
+ concurrency: options.concurrency,
709
+ agentRetries: options.agentRetries,
710
+ tokenBudget: options.tokenBudget,
711
+ agentTimeoutMs: options.agentTimeoutMs,
712
+ signal: state.controller.signal,
713
+ maxAgents: options.maxAgents,
714
+ runId,
715
+ resumeJournal,
716
+ toolset: options.toolset,
717
+ excludeTools: options.excludeTools,
718
+ confirm: options.confirm,
719
+ loadSavedWorkflow: options.loadSavedWorkflow,
720
+ onAgentJournal: (entry) => {
721
+ // Nested workflow frames use their own run ids. They are intentionally
722
+ // not persisted as top-level call facts until a namespaced journal exists;
723
+ // mixing them into the parent would quarantine the run on replay.
724
+ if (entry.runId === undefined || entry.runId === runId) this.journalCallResult(runId, entry);
725
+ },
726
+ onRuntimeEvent: () => {},
1097
727
  });
728
+ const live = this.runs.get(runId);
729
+ if (live && state.executionGeneration === generation) live.result = result.result;
730
+ const run = live?.run;
731
+ if (run && state.executionGeneration === generation && !isTerminalWorkflow(run.status)) {
732
+ this.setWorkflowStatus(run, "completed");
733
+ }
734
+ return result;
735
+ } catch (error: unknown) {
736
+ const run = this.runs.get(runId)?.run;
737
+ if (run && state.executionGeneration === generation && !isTerminalWorkflow(run.status)) {
738
+ const message = error instanceof Error ? error.message : String(error);
739
+ const workflowError = error instanceof WorkflowError ? error : wrapError(error);
740
+ if (workflowError.code === WorkflowErrorCode.PROVIDER_USAGE_LIMIT) {
741
+ this.setWorkflowStatus(run, "paused", message);
742
+ this.cleanupAgents(runId);
743
+ this.scheduleProviderResume(runId, workflowError.resetHint);
744
+ } else if (workflowError.code === WorkflowErrorCode.WORKFLOW_ABORTED && state.controller.signal.aborted) {
745
+ this.setWorkflowStatus(run, "interrupted");
746
+ } else {
747
+ this.setWorkflowStatus(run, "failed", message);
748
+ this.cleanupAgents(runId);
749
+ }
750
+ }
751
+ throw error;
752
+ } finally {
753
+ // Reject any waiters that never settled (agents left in flight).
754
+ const run = this.runs.get(runId)?.run;
755
+ if (run && state.executionGeneration === generation) this.settleWaiters(runId, run);
1098
756
  }
1099
- return targets;
1100
757
  }
1101
758
 
1102
- private async stopAndQuiesceOwned(
1103
- run: WorkflowRun,
1104
- nodeId: string,
1105
- agentId: string,
1106
- attemptId: string | undefined,
1107
- branchEpoch: number,
1108
- quiesce = true,
1109
- allowStaleBranch = false,
1110
- ): Promise<boolean> {
1111
- const isCurrent = (): boolean => {
1112
- if (this.runs.get(run.runId) !== run) return false;
1113
- return allowStaleBranch || (!this.branchChanging && branchEpoch === this.branchEpoch);
759
+ /**
760
+ * Dispatch one live agent() call through spawn-managed with an A4-rotated
761
+ * spawnKey: `${runId}/call-${callIndex}/attempt-${generation}`. The generation
762
+ * combines the journaled base (so a resume of an edited call lands on a fresh
763
+ * key) with the runtime's per-attempt counter.
764
+ */
765
+ private async dispatchAgent(runId: string, prompt: string, runOptions: AgentRunOptions): Promise<unknown> {
766
+ const state = this.runs.get(runId);
767
+ if (!state) throw new Error(`workflow run not found: ${runId}`);
768
+ if (state.lifecycleSuspended || state.controller.signal.aborted) {
769
+ throw new WorkflowError("workflow branch is quiescing", WorkflowErrorCode.WORKFLOW_ABORTED, {
770
+ recoverable: true,
771
+ });
772
+ }
773
+ const run = state.run;
774
+ const executionGeneration = state.executionGeneration;
775
+ const callIndex = runOptions.callIndex ?? state.generations.size;
776
+ // The spawnKey identity carries the `call-` prefix (A4):
777
+ // spawnKey = `${runId}/call-${callIndex}/attempt-${generation}`
778
+ // The journal node identity is the bare call index, and the journal
779
+ // attemptId is the full `${runId}/${nodeId}/attempt-${generation}` path the
780
+ // journal replay validates.
781
+ const spawnNodeId = `call-${callIndex}`;
782
+ const nodeId = String(callIndex);
783
+ const base = state.generations.get(nodeId) ?? 0;
784
+ const generation = base + (runOptions.attempt ?? 1);
785
+ state.generations.set(nodeId, generation);
786
+ const spawnAttemptId = `attempt-${generation}`;
787
+ const attemptId = `${runId}/${nodeId}/attempt-${generation}`;
788
+
789
+ const type = runOptions.agentType ?? "general-purpose";
790
+ const tier =
791
+ runOptions.tier === "small" || runOptions.tier === "medium" || runOptions.tier === "large"
792
+ ? runOptions.tier
793
+ : undefined;
794
+ const task: DispatchTask = {
795
+ subagent_type: type,
796
+ prompt,
797
+ description: runOptions.label ?? `workflow call ${callIndex}`,
798
+ ...(tier === undefined ? {} : { tier }),
799
+ ...(runOptions.model === undefined ? {} : { model: runOptions.model }),
800
+ ...(runOptions.thread === undefined ? {} : { thread: runOptions.thread }),
801
+ ...(runOptions.toolset === undefined ? {} : { toolset: runOptions.toolset }),
802
+ ...(runOptions.excludeTools === undefined ? {} : { excludeTools: runOptions.excludeTools }),
803
+ ...(runOptions.isolation === undefined ? {} : { isolation: runOptions.isolation }),
804
+ };
805
+
806
+ // Journal the attempt rotation so a later resume knows the generation base.
807
+ this.persist(runId, `call-attempt:${nodeId}`, () =>
808
+ this.write({
809
+ kind: "call_attempt",
810
+ schemaVersion: JOURNAL_SCHEMA_VERSION,
811
+ runId,
812
+ nodeId,
813
+ attemptId,
814
+ generation,
815
+ owner: { extension: "pi-workflows", runId, nodeId, attemptId },
816
+ timestamp: Date.now(),
817
+ }),
818
+ );
819
+ run.callStatus[nodeId] = "running";
820
+ run.attempts[nodeId] = generation;
821
+ run.attemptIds[nodeId] = attemptId;
822
+ run.updatedAt = Date.now();
823
+
824
+ // Abort linkage: an external signal (pause/stop/Esc) cancels the wait and
825
+ // stops the owned agent. Track the spawn key before the RPC reply arrives so
826
+ // a lost reply can still be reconciled during shutdown or branch replacement.
827
+ const externalSignal = runOptions.signal ?? state.controller.signal;
828
+ const spawnKey = `${runId}/${spawnNodeId}/${spawnAttemptId}`;
829
+ const owner: WorkflowOwner = {
830
+ extension: "pi-workflows",
831
+ runId,
832
+ nodeId: spawnNodeId,
833
+ attemptId: spawnAttemptId,
1114
834
  };
1115
- await this.stopOwned(run, nodeId, agentId, attemptId).catch(() => {});
1116
- if (!quiesce) return true;
1117
- if (!this.client.quiesceOwned) return false;
1118
- const owner = {
1119
- ...this.ownerForNode(run, nodeId),
1120
- ...(attemptId === undefined ? {} : { attemptId }),
835
+ const pendingKey = `${nodeId}\u0000${generation}`;
836
+ state.pendingSpawns.set(pendingKey, { spawnKey, owner });
837
+ const reconcilePending = () => {
838
+ const reconciliation = this.client.reconcileManaged?.(spawnKey, owner);
839
+ void reconciliation?.catch(() => {});
840
+ };
841
+ const onPendingAbort = () => reconcilePending();
842
+ externalSignal.addEventListener("abort", onPendingAbort, { once: true });
843
+ if (externalSignal.aborted) onPendingAbort();
844
+
845
+ let pendingId = "";
846
+ let response: ManagedSpawnResponse | string;
847
+ try {
848
+ response = await this.client.spawn(task, runId, spawnNodeId, spawnAttemptId, undefined, externalSignal);
849
+ } finally {
850
+ externalSignal.removeEventListener("abort", onPendingAbort);
851
+ state.pendingSpawns.delete(pendingKey);
852
+ }
853
+ const responseId = typeof response === "string" ? response : response.id;
854
+ if (
855
+ state.executionGeneration !== executionGeneration ||
856
+ state.controller.signal.aborted ||
857
+ externalSignal.aborted
858
+ ) {
859
+ await this.client
860
+ .stopOwned?.(responseId, { extension: "pi-workflows", runId, nodeId: spawnNodeId, attemptId: spawnAttemptId })
861
+ .catch(() => {});
862
+ throw new WorkflowError(
863
+ "workflow dispatch became stale during pause/resume",
864
+ WorkflowErrorCode.WORKFLOW_ABORTED,
865
+ { recoverable: true },
866
+ );
867
+ }
868
+ if (typeof response === "string") {
869
+ pendingId = response;
870
+ } else if (response.terminal) {
871
+ // The spawn response already carries a terminal snapshot (fast completion
872
+ // or interrupted replay). The runtime journals successful calls itself
873
+ // (onAgentJournal); a terminal snapshot that is NOT a success is
874
+ // journaled here so the resume journal reflects the failure.
875
+ const terminal = response.terminal;
876
+ if (typeof terminal.tokenCount === "number")
877
+ runOptions.onUsage?.({
878
+ input: 0,
879
+ output: terminal.tokenCount,
880
+ total: terminal.tokenCount,
881
+ cost: 0,
882
+ cacheRead: 0,
883
+ cacheWrite: 0,
884
+ });
885
+ const status = managedTerminalStatus(terminal);
886
+ if (status !== "completed") {
887
+ const result = resultFromLifecycle(
888
+ status,
889
+ response.id,
890
+ terminal.result,
891
+ terminal.error,
892
+ terminal.compactionCount,
893
+ terminal.tokenCount,
894
+ );
895
+ result.attemptId = attemptId;
896
+ this.commitCallTerminal(runId, nodeId, result);
897
+ if (status === "failed" && terminal.error) {
898
+ throw wrapError(new Error(terminal.error), { agentLabel: runOptions.label });
899
+ }
900
+ }
901
+ return terminal.result ?? "";
902
+ } else {
903
+ pendingId = response.id;
904
+ }
905
+
906
+ const waitAbort = () => {
907
+ const waiter = state.agentWaiters.get(pendingId);
908
+ if (waiter && !waiter.settled) {
909
+ waiter.settled = true;
910
+ state.agentWaiters.delete(pendingId);
911
+ void this.client
912
+ .stopOwned?.(pendingId, {
913
+ extension: "pi-workflows",
914
+ runId,
915
+ nodeId: spawnNodeId,
916
+ attemptId: spawnAttemptId,
917
+ })
918
+ .catch(() => {});
919
+ waiter.reject(new WorkflowError("workflow aborted", WorkflowErrorCode.WORKFLOW_ABORTED, { recoverable: true }));
920
+ }
1121
921
  };
1122
- while (isCurrent()) {
1123
- const result = await this.client
1124
- .quiesceOwned(run.runId, [agentId], BRANCH_QUIESCE_TIMEOUT_MS, [owner])
1125
- .catch((): WorkflowQuiesceResult => ({ settled: false, pending: [agentId] }));
1126
- if (result.settled && result.pending.length === 0) return true;
1127
- if (allowStaleBranch) return false;
1128
- if (!isCurrent()) return false;
1129
- await new Promise<void>((resolve) => setTimeout(resolve, 100));
922
+
923
+ // Wait for the terminal lifecycle event.
924
+ const terminal = await new Promise<ManagedSpawnResponse>((resolve, reject) => {
925
+ const waiter = {
926
+ callIndex: String(callIndex),
927
+ generation,
928
+ executionGeneration,
929
+ onUsage: runOptions.onUsage,
930
+ onHistory: runOptions.onHistory,
931
+ settled: false,
932
+ resolve,
933
+ reject,
934
+ };
935
+ state.agentWaiters.set(pendingId, waiter);
936
+ externalSignal.addEventListener("abort", waitAbort, { once: true });
937
+ if (externalSignal.aborted) waitAbort();
938
+ }).finally(() => {
939
+ externalSignal.removeEventListener("abort", waitAbort);
940
+ });
941
+
942
+ const snapshot = terminal.terminal;
943
+ if (!snapshot)
944
+ throw new WorkflowError("subagent settled without a terminal snapshot", WorkflowErrorCode.AGENT_EXECUTION_ERROR, {
945
+ recoverable: true,
946
+ });
947
+ const status = managedTerminalStatus(snapshot);
948
+ // Successful calls are journaled by the runtime's onAgentJournal (the
949
+ // single writer for completed results); only failures settle here so the
950
+ // resume journal reflects them.
951
+ if (status !== "completed") {
952
+ const result = resultFromLifecycle(
953
+ status,
954
+ terminal.id,
955
+ snapshot.result,
956
+ snapshot.error,
957
+ snapshot.compactionCount,
958
+ snapshot.tokenCount,
959
+ );
960
+ result.attemptId = attemptId;
961
+ this.commitCallTerminal(runId, nodeId, result);
962
+ if (status === "failed" && snapshot.error) {
963
+ throw wrapError(new Error(snapshot.error), { agentLabel: runOptions.label });
964
+ }
1130
965
  }
1131
- return false;
966
+ return snapshot.result ?? "";
1132
967
  }
1133
968
 
1134
- private clearJournalRetries(): void {
1135
- for (const retry of this.journalRetries.values()) {
1136
- if (retry.timer) clearTimeout(retry.timer);
969
+ private journalCallResult(runId: string, entry: JournalEntry): void {
970
+ const state = this.runs.get(runId);
971
+ if (!state) return;
972
+ const run = state.run;
973
+ const nodeId = String(entry.index);
974
+ const result = entry.result;
975
+ const callResult: CallResult = {
976
+ status: "completed",
977
+ ...(typeof result === "string" && result.length === 0 ? {} : { result }),
978
+ compactionCount: 0,
979
+ updatedAt: Date.now(),
980
+ };
981
+ const attemptId = run.attemptIds[nodeId];
982
+ if (attemptId) callResult.attemptId = attemptId;
983
+ const event: JournalEvent = {
984
+ kind: "call_result",
985
+ schemaVersion: JOURNAL_SCHEMA_VERSION,
986
+ runId,
987
+ nodeId,
988
+ result: callResult,
989
+ callHash: entry.hash,
990
+ ...(entry.storeDelta !== undefined && Object.keys(entry.storeDelta).length > 0
991
+ ? { storeDelta: entry.storeDelta }
992
+ : {}),
993
+ ...(attemptId ? { attemptId } : {}),
994
+ ...(attemptId ? { owner: { extension: "pi-workflows", runId, nodeId, attemptId } } : {}),
995
+ timestamp: Date.now(),
996
+ };
997
+ this.persist(runId, `call-result:${nodeId}`, () => this.write(event));
998
+ run.callResults[nodeId] = callResult;
999
+ run.callStatus[nodeId] = "completed";
1000
+ run.updatedAt = Date.now();
1001
+ }
1002
+
1003
+ private commitCallTerminal(runId: string, nodeId: string, result: CallResult): void {
1004
+ const state = this.runs.get(runId);
1005
+ if (!state) return;
1006
+ const run = state.run;
1007
+ const attemptId = result.attemptId ?? run.attemptIds[nodeId];
1008
+ const event: JournalEvent = {
1009
+ kind: "call_result",
1010
+ schemaVersion: JOURNAL_SCHEMA_VERSION,
1011
+ runId,
1012
+ nodeId,
1013
+ result,
1014
+ ...(attemptId ? { attemptId } : {}),
1015
+ ...(attemptId ? { owner: { extension: "pi-workflows", runId, nodeId, attemptId } } : {}),
1016
+ timestamp: Date.now(),
1017
+ };
1018
+ this.persist(runId, `call-terminal:${nodeId}`, () => this.write(event));
1019
+ run.callResults[nodeId] = result;
1020
+ run.callStatus[nodeId] = result.status;
1021
+ if (result.agentId) run.agentIds[nodeId] = result.agentId;
1022
+ run.compactions[nodeId] = Math.max(run.compactions[nodeId] ?? 0, result.compactionCount);
1023
+ run.updatedAt = Date.now();
1024
+ // A recoverable child failure is a value for runtime retry/handling. The
1025
+ // script, not this lifecycle callback, decides whether the whole run fails.
1026
+ }
1027
+
1028
+ private reconcilePendingSpawns(state: ScriptRunState): void {
1029
+ for (const pending of state.pendingSpawns.values()) {
1030
+ const reconciliation = this.client.reconcileManaged?.(pending.spawnKey, pending.owner);
1031
+ void reconciliation?.catch(() => {});
1032
+ }
1033
+ state.pendingSpawns.clear();
1034
+ }
1035
+
1036
+ private async quiescePendingSpawns(state: ScriptRunState): Promise<WorkflowQuiesceResult> {
1037
+ const pending = [...state.pendingSpawns.values()];
1038
+ if (pending.length === 0) return { settled: true, pending: [] };
1039
+ if (!this.client.reconcileManaged) {
1040
+ return { settled: false, pending: pending.map(({ spawnKey }) => spawnKey).slice(0, 256) };
1041
+ }
1042
+ const failed: string[] = [];
1043
+ await Promise.all(
1044
+ pending.map(async ({ spawnKey, owner }) => {
1045
+ try {
1046
+ const result = await this.client.reconcileManaged?.(spawnKey, owner);
1047
+ if (result === undefined) failed.push(spawnKey);
1048
+ } catch {
1049
+ failed.push(spawnKey);
1050
+ }
1051
+ }),
1052
+ );
1053
+ return { settled: failed.length === 0, pending: failed.slice(0, 256) };
1054
+ }
1055
+
1056
+ private cleanupAgents(runId: string): void {
1057
+ const state = this.runs.get(runId);
1058
+ if (!state) return;
1059
+ this.reconcilePendingSpawns(state);
1060
+ for (const [agentId, waiter] of [...state.agentWaiters]) {
1061
+ if (waiter.settled) continue;
1062
+ waiter.settled = true;
1063
+ state.agentWaiters.delete(agentId);
1064
+ waiter.reject(
1065
+ new WorkflowError("workflow failed while agent in flight", WorkflowErrorCode.WORKFLOW_ABORTED, {
1066
+ recoverable: true,
1067
+ }),
1068
+ );
1069
+ void this.client
1070
+ .stopOwned?.(agentId, {
1071
+ extension: "pi-workflows",
1072
+ runId,
1073
+ nodeId: `call-${waiter.callIndex}`,
1074
+ attemptId: `attempt-${waiter.generation}`,
1075
+ })
1076
+ .catch(() => {});
1137
1077
  }
1138
- this.journalRetries.clear();
1139
- this.journalBatchProgress.clear();
1140
- this.journalBlockedRuns.clear();
1141
1078
  }
1142
1079
 
1143
- private hasPendingJournalWork(runId: string): boolean {
1144
- if (this.journalBlockedRuns.has(runId)) return true;
1145
- const retryMarker = `\\u0000${runId}\\u0000`;
1146
- if ([...this.journalRetries.keys()].some((key) => key.includes(retryMarker))) return true;
1147
- const batchMarker = `${runId}:`;
1148
- return [...this.journalBatchProgress.keys()].some((key) => key.startsWith(batchMarker));
1080
+ private async stop(state: ScriptRunState): Promise<void> {
1081
+ const run = state.run;
1082
+ if (isTerminalWorkflow(run.status)) return;
1083
+ this.setWorkflowStatus(run, "stopping", "workflow stopped", "stop");
1084
+ run.nonResumable = true;
1085
+ state.controller.abort();
1086
+ this.cleanupAgents(run.runId);
1087
+ const terminalResults: RecoveryTerminalResult[] = [];
1088
+ for (const [nodeId, status] of Object.entries(run.callStatus)) {
1089
+ if (status !== "completed") {
1090
+ const result = resultFromLifecycle(
1091
+ "stopped",
1092
+ run.agentIds[nodeId],
1093
+ undefined,
1094
+ "workflow stopped",
1095
+ run.compactions[nodeId] ?? 0,
1096
+ );
1097
+ result.attemptId = run.attemptIds[nodeId];
1098
+ terminalResults.push({
1099
+ nodeId,
1100
+ attemptId: run.attemptIds[nodeId] ?? "",
1101
+ result,
1102
+ owner: { extension: "pi-workflows", runId: run.runId, nodeId, attemptId: run.attemptIds[nodeId] ?? "" },
1103
+ });
1104
+ run.callResults[nodeId] = result;
1105
+ run.callStatus[nodeId] = "stopped";
1106
+ }
1107
+ }
1108
+ const event: TerminalRecoveryEvent = {
1109
+ kind: "terminal_recovery",
1110
+ schemaVersion: JOURNAL_SCHEMA_VERSION,
1111
+ runId: run.runId,
1112
+ status: "stopped",
1113
+ terminalIntent: "stop",
1114
+ branchGeneration: this.recoveryBranchGeneration,
1115
+ terminalResults,
1116
+ blockedNodeIds: [],
1117
+ error: "workflow stopped",
1118
+ recoveryId: deriveRecoveryId({ runId: run.runId, rotations: [], timestamp: Date.now() }),
1119
+ timestamp: Date.now(),
1120
+ };
1121
+ this.persist(run.runId, "stop-terminal", () => this.write(event));
1122
+ run.status = "stopped";
1123
+ run.updatedAt = Date.now();
1124
+ this.settleWaiters(run.runId, run);
1149
1125
  }
1150
1126
 
1127
+ // ── journal writes ─────────────────────────────────────────────────────────
1128
+
1151
1129
  /**
1152
1130
  * A Pi lifecycle event is delivered once. If its durable append is transiently
1153
- * unavailable, retain the immutable mutation and retry it without requiring
1154
- * the subagents peer to emit a second terminal callback.
1131
+ * unavailable, retain the mutation in memory and retry the append with
1132
+ * backoff without requiring the peer to re-emit. The in-memory run state is
1133
+ * authoritative; the journal is best-effort with bounded retries, and a run
1134
+ * whose appends keep failing is marked blocked (reported once, not spammed).
1155
1135
  */
1156
- private scheduleJournalRetry(run: WorkflowRun, key: string, mutation: () => void): void {
1157
- const epoch = this.branchEpoch;
1158
- const retryKey = `${epoch}\\u0000${run.runId}\\u0000${key}`;
1136
+ private scheduleJournalRetry(runId: string, key: string, mutation: () => void): void {
1137
+ const retryKey = `${runId}\u0000${key}`;
1159
1138
  if (this.journalRetries.has(retryKey)) return;
1160
1139
  const retry = { attempt: 0, timer: undefined as ReturnType<typeof setTimeout> | undefined };
1161
1140
  const attempt = (): void => {
1162
1141
  const current = this.journalRetries.get(retryKey);
1163
1142
  if (!current) return;
1164
1143
  current.timer = undefined;
1165
- if (
1166
- this.branchEpoch !== epoch ||
1167
- this.branchChanging ||
1168
- this.lifecycleSuspended ||
1169
- this.runs.get(run.runId) !== run
1170
- ) {
1144
+ if (this.lifecyclePaused || !this.runs.has(runId)) {
1171
1145
  this.journalRetries.delete(retryKey);
1172
1146
  return;
1173
1147
  }
1174
1148
  const previousKey = this.journalMutationKey;
1175
- this.journalMutationKey = `${run.runId}:${key}`;
1149
+ this.journalMutationKey = `${runId}:${key}`;
1176
1150
  try {
1177
1151
  mutation();
1178
1152
  this.journalRetries.delete(retryKey);
1179
1153
  } catch (error: unknown) {
1180
1154
  if (!(error instanceof JournalAppendError)) {
1181
1155
  this.journalRetries.delete(retryKey);
1182
- console.warn(`[pi-workflows] lifecycle retry abandoned: ${boundedError(error) ?? "unknown error"}`);
1156
+ console.warn(
1157
+ `[pi-workflows] journal retry abandoned: ${error instanceof Error ? error.message : String(error)}`,
1158
+ );
1183
1159
  return;
1184
1160
  }
1185
1161
  current.attempt += 1;
1186
- if (current.attempt >= 8) {
1162
+ if (current.attempt >= JOURNAL_RETRY_MAX_ATTEMPTS) {
1187
1163
  this.journalRetries.delete(retryKey);
1188
- this.journalBlockedRuns.add(run.runId);
1189
- console.warn(`[pi-workflows] lifecycle journal retry exhausted: ${error.message}`);
1164
+ this.journalBlockedRuns.add(runId);
1165
+ console.warn(`[pi-workflows] journal retry exhausted for ${runId}: ${error.message}`);
1190
1166
  return;
1191
1167
  }
1192
1168
  const delay = Math.min(
@@ -1202,1611 +1178,232 @@ export class WorkflowEngine {
1202
1178
  queueMicrotask(attempt);
1203
1179
  }
1204
1180
 
1205
- private tryLifecycleMutation(run: WorkflowRun, key: string, mutation: () => void): boolean {
1206
- const previousKey = this.journalMutationKey;
1207
- this.journalMutationKey = `${run.runId}:${key}`;
1181
+ private appendRequired(event: JournalEvent): void {
1208
1182
  try {
1209
- mutation();
1210
- return true;
1183
+ this.journal.append(event);
1211
1184
  } catch (error: unknown) {
1212
- if (!(error instanceof JournalAppendError)) throw error;
1213
- this.scheduleJournalRetry(run, key, mutation);
1214
- return false;
1215
- } finally {
1216
- this.journalMutationKey = previousKey;
1185
+ throw new WorkflowError(
1186
+ `Could not persist workflow start: ${error instanceof Error ? error.message : String(error)}`,
1187
+ WorkflowErrorCode.PERSISTENCE_ERROR,
1188
+ { recoverable: true, details: error },
1189
+ );
1217
1190
  }
1218
1191
  }
1219
1192
 
1220
1193
  private write(event: JournalEvent): void {
1194
+ // The in-memory mutation is applied by the caller BEFORE calling write;
1195
+ // here we only persist. A transient failure schedules a retry of this exact
1196
+ // append. Terminal bookkeeping already happened, so a retry that never
1197
+ // succeeds degrades to a warning rather than corrupting run state.
1221
1198
  try {
1222
1199
  this.journal.append(event);
1223
1200
  } catch (error: unknown) {
1224
- throw new JournalAppendError(event.kind, error);
1201
+ if (this.journalMutationKey) {
1202
+ throw new JournalAppendError(event.kind, error);
1203
+ }
1204
+ console.warn(`[pi-workflows] journal append failed: ${error instanceof Error ? error.message : String(error)}`);
1225
1205
  }
1226
1206
  }
1227
1207
 
1228
- private appendEvents(events: readonly JournalEvent[]): void {
1229
- const key = this.journalMutationKey;
1230
- const serialized = key ? JSON.stringify(events) : undefined;
1231
- let start = 0;
1232
- if (key && serialized !== undefined) {
1233
- const prior = this.journalBatchProgress.get(key);
1234
- if (prior && prior.serialized === serialized) {
1235
- start = prior.nextIndex;
1236
- } else {
1237
- this.journalBatchProgress.set(key, { serialized, nextIndex: 0 });
1238
- }
1239
- }
1240
- for (let index = start; index < events.length; index += 1) {
1241
- const event = events[index];
1242
- if (!event) continue;
1243
- this.write(event);
1244
- if (key) {
1245
- const progress = this.journalBatchProgress.get(key);
1246
- if (progress) progress.nextIndex = index + 1;
1208
+ /**
1209
+ * Persist one journal event with bounded retry. The caller applies the
1210
+ * in-memory mutation itself; `persist` only makes the append durable and, on
1211
+ * a transient failure, schedules a retry of the exact same append.
1212
+ */
1213
+ private persist(runId: string, key: string, mutation: () => void): void {
1214
+ const previousKey = this.journalMutationKey;
1215
+ this.journalMutationKey = `${runId}:${key}`;
1216
+ try {
1217
+ mutation();
1218
+ } catch (error: unknown) {
1219
+ if (error instanceof JournalAppendError) {
1220
+ this.scheduleJournalRetry(runId, key, mutation);
1221
+ return;
1247
1222
  }
1223
+ console.warn(`[pi-workflows] journal append failed: ${error instanceof Error ? error.message : String(error)}`);
1224
+ } finally {
1225
+ this.journalMutationKey = previousKey;
1248
1226
  }
1249
- if (key) this.journalBatchProgress.delete(key);
1250
1227
  }
1251
1228
 
1252
- private ownerForNode(run: WorkflowRun, nodeId: string): WorkflowOwner {
1253
- return {
1254
- extension: "pi-workflows",
1255
- runId: run.runId,
1256
- nodeId,
1257
- ...(run.attemptIds[nodeId] ? { attemptId: run.attemptIds[nodeId] } : {}),
1258
- };
1229
+ private scheduleProviderResume(runId: string, hint?: string): void {
1230
+ if (this.providerResumeTimers.has(runId)) return;
1231
+ const match = hint ? /(\d+(?:\.\d+)?)\s*(seconds?|minutes?|hours?|days?)/i.exec(hint) : undefined;
1232
+ const unit = match?.[2]?.toLowerCase();
1233
+ const multiplier = unit?.startsWith("second")
1234
+ ? 1_000
1235
+ : unit?.startsWith("minute")
1236
+ ? 60_000
1237
+ : unit?.startsWith("hour")
1238
+ ? 3_600_000
1239
+ : unit?.startsWith("day")
1240
+ ? 86_400_000
1241
+ : undefined;
1242
+ const delay = Math.min(
1243
+ PROVIDER_RETRY_MAX_MS,
1244
+ match && multiplier ? Math.max(1_000, Number(match[1]) * multiplier) : PROVIDER_RETRY_DEFAULT_MS,
1245
+ );
1246
+ const timer = setTimeout(() => {
1247
+ this.providerResumeTimers.delete(runId);
1248
+ const state = this.runs.get(runId);
1249
+ if (state?.run.status !== "paused") return;
1250
+ void this.resume(runId, this.readEntries(), { background: true }).catch((error: unknown) => {
1251
+ console.warn(
1252
+ `[pi-workflows] automatic provider-limit resume failed: ${error instanceof Error ? error.message : String(error)}`,
1253
+ );
1254
+ });
1255
+ }, delay);
1256
+ timer.unref?.();
1257
+ this.providerResumeTimers.set(runId, timer);
1258
+ }
1259
+ private clearJournalRetries(): void {
1260
+ for (const { timer } of this.journalRetries.values()) {
1261
+ if (timer) clearTimeout(timer);
1262
+ }
1263
+ this.journalRetries.clear();
1264
+ this.journalBlockedRuns.clear();
1265
+ for (const timer of this.providerResumeTimers.values()) clearTimeout(timer);
1266
+ this.providerResumeTimers.clear();
1259
1267
  }
1260
1268
 
1261
1269
  private setWorkflowStatus(
1262
- run: WorkflowRun,
1263
- status: WorkflowStatus,
1270
+ run: ScriptRun,
1271
+ status: ScriptRun["status"],
1264
1272
  error?: string,
1265
- terminalIntent: WorkflowRun["terminalIntent"] = run.terminalIntent,
1273
+ terminalIntent?: "stop" | "failure",
1266
1274
  ): void {
1267
- const nextStatus = transitionWorkflow(run.status, status);
1275
+ if (isTerminalWorkflow(run.status) && !isTerminalWorkflow(status)) return;
1268
1276
  const timestamp = Date.now();
1269
- const event: JournalEvent = {
1270
- kind: "workflow_transition",
1271
- schemaVersion: JOURNAL_SCHEMA_VERSION,
1272
- runId: run.runId,
1273
- status: nextStatus,
1274
- timestamp,
1275
- ...(error === undefined ? {} : { error }),
1276
- ...(terminalIntent === undefined ? {} : { terminalIntent }),
1277
- };
1278
- // Do not advance the aggregate before the durable prefix accepts this
1279
- // event. A caller can retry the same transition after append() throws.
1280
- this.write(event);
1281
- run.status = nextStatus;
1282
- run.updatedAt = timestamp;
1277
+ this.persist(run.runId, `transition:${status}`, () =>
1278
+ this.write({
1279
+ kind: "workflow_transition",
1280
+ schemaVersion: JOURNAL_SCHEMA_VERSION,
1281
+ runId: run.runId,
1282
+ status,
1283
+ ...(error !== undefined ? { error } : {}),
1284
+ ...(terminalIntent !== undefined ? { terminalIntent } : {}),
1285
+ timestamp,
1286
+ }),
1287
+ );
1288
+ run.status = status;
1283
1289
  if (error !== undefined) run.error = error;
1284
1290
  if (terminalIntent !== undefined) run.terminalIntent = terminalIntent;
1285
- if (isTerminalWorkflow(nextStatus)) this.resolveWaiters(run);
1286
- }
1287
-
1288
- private setTaskStatus(run: WorkflowRun, nodeId: string, status: TaskStatus, agentId?: string): void {
1289
- const previous = run.taskStatus[nodeId];
1290
- if (!previous) return;
1291
- if (!canTransitionTask(previous, status)) {
1292
- if (isTerminalTask(previous)) return;
1293
- throw new Error(`invalid task transition: ${previous} -> ${status}`);
1294
- }
1295
- const nextStatus = transitionTask(previous, status);
1296
- const timestamp = Date.now();
1297
- const event: JournalEvent = {
1298
- kind: "task_transition",
1299
- schemaVersion: JOURNAL_SCHEMA_VERSION,
1300
- runId: run.runId,
1301
- nodeId,
1302
- status: nextStatus,
1303
- ...(agentId ? { agentId } : {}),
1304
- ...(run.attemptIds[nodeId] ? { attemptId: run.attemptIds[nodeId] } : {}),
1305
- owner: this.ownerForNode(run, nodeId),
1306
- timestamp,
1307
- };
1308
- this.write(event);
1309
- run.taskStatus[nodeId] = nextStatus;
1310
1291
  run.updatedAt = timestamp;
1311
- if (agentId) run.agentIds[nodeId] = agentId;
1312
- }
1313
-
1314
- private blockedDependentIds(run: WorkflowRun, failedNodeId: string): string[] {
1315
- const queue = [failedNodeId];
1316
- const blocked = new Set<string>();
1317
- while (queue.length > 0) {
1318
- const dependency = queue.shift();
1319
- if (!dependency) continue;
1320
- for (const task of run.definition.tasks) {
1321
- if (!task.depends_on.includes(dependency) || blocked.has(task.id)) continue;
1322
- const status = run.taskStatus[task.id];
1323
- if (status !== "pending" && status !== "ready") continue;
1324
- blocked.add(task.id);
1325
- queue.push(task.id);
1326
- }
1292
+ if (isTerminalWorkflow(status)) {
1293
+ this.settleWaiters(run.runId, run);
1294
+ this.onRunSettled?.();
1327
1295
  }
1328
- return [...blocked];
1329
1296
  }
1330
1297
 
1331
- private commitTaskTerminal(
1332
- run: WorkflowRun,
1333
- nodeId: string,
1334
- result: WorkflowRun["taskResults"][string],
1335
- status: "completed" | "failed" | "stopped",
1336
- failureMessage?: string,
1337
- blockDependents = true,
1338
- ): void {
1339
- const previous = run.taskStatus[nodeId];
1340
- if (!previous || run.taskResults[nodeId] || isTerminalTask(previous)) return;
1341
- if (!canTransitionTask(previous, status)) {
1342
- throw new Error(`invalid task transition: ${previous} -> ${status}`);
1343
- }
1344
- const timestamp = result.updatedAt;
1345
- const resultEvent: JournalEvent = {
1346
- kind: "task_result",
1347
- schemaVersion: JOURNAL_SCHEMA_VERSION,
1348
- runId: run.runId,
1349
- nodeId,
1350
- result,
1351
- attemptId: run.attemptIds[nodeId],
1352
- owner: this.ownerForNode(run, nodeId),
1353
- timestamp,
1354
- };
1355
- const transitionEvent: JournalEvent = {
1356
- kind: "task_transition",
1357
- schemaVersion: JOURNAL_SCHEMA_VERSION,
1358
- runId: run.runId,
1359
- nodeId,
1360
- status,
1361
- ...(result.agentId ? { agentId: result.agentId } : {}),
1362
- ...(run.attemptIds[nodeId] ? { attemptId: run.attemptIds[nodeId] } : {}),
1363
- owner: this.ownerForNode(run, nodeId),
1364
- timestamp,
1365
- };
1366
- const blockedIds = blockDependents && status !== "completed" ? this.blockedDependentIds(run, nodeId) : [];
1367
- const blockedEvents: JournalEvent[] = blockedIds.map((blockedNodeId) => ({
1368
- kind: "task_transition",
1369
- schemaVersion: JOURNAL_SCHEMA_VERSION,
1370
- runId: run.runId,
1371
- nodeId: blockedNodeId,
1372
- status: "blocked",
1373
- ...(run.attemptIds[blockedNodeId] ? { attemptId: run.attemptIds[blockedNodeId] } : {}),
1374
- owner: this.ownerForNode(run, blockedNodeId),
1375
- timestamp,
1376
- }));
1377
- const message = failureMessage ?? result.error ?? `workflow task ${nodeId} ${status}`;
1378
- const needsFailureCleanup =
1379
- status !== "completed" &&
1380
- run.terminalIntent !== "stop" &&
1381
- (run.status !== "stopping" || run.terminalIntent !== "failure" || run.error !== message);
1382
- const workflowEvent: JournalEvent | undefined = needsFailureCleanup
1383
- ? {
1384
- kind: "workflow_transition",
1385
- schemaVersion: JOURNAL_SCHEMA_VERSION,
1386
- runId: run.runId,
1387
- status: "stopping",
1388
- terminalIntent: "failure",
1389
- error: message,
1390
- timestamp,
1391
- }
1392
- : undefined;
1393
- this.appendEvents([resultEvent, transitionEvent, ...blockedEvents, ...(workflowEvent ? [workflowEvent] : [])]);
1298
+ // ── restore / summary ──────────────────────────────────────────────────────
1394
1299
 
1395
- run.taskResults[nodeId] = result;
1396
- run.taskStatus[nodeId] = status;
1397
- if (result.agentId) run.agentIds[nodeId] = result.agentId;
1398
- run.compactions[nodeId] = Math.max(run.compactions[nodeId] ?? 0, result.compactionCount);
1399
- run.updatedAt = timestamp;
1400
- for (const blockedNodeId of blockedIds) {
1401
- run.taskStatus[blockedNodeId] = "blocked";
1402
- run.updatedAt = timestamp;
1300
+ restore(entries: readonly SessionEntryLike[], branchGeneration = 0): void {
1301
+ // A branch replay is a replacement, not a merge. Runs absent from the new
1302
+ // branch must not remain controllable or append facts into it.
1303
+ this.clearJournalRetries();
1304
+ for (const state of this.runs.values()) {
1305
+ this.cleanupAgents(state.run.runId);
1306
+ state.controller.abort(new DOMException("Workflow branch replaced", "AbortError"));
1403
1307
  }
1404
- if (workflowEvent) {
1405
- run.status = "stopping";
1406
- run.terminalIntent = "failure";
1407
- run.error = message;
1408
- run.updatedAt = timestamp;
1409
- this.startFailureCleanup(run, message);
1308
+ this.rejectWaiters(new WorkflowWaitAbortedError());
1309
+ this.runs.clear();
1310
+ this.recoveryBranchGeneration = branchGeneration;
1311
+ const runs = replayJournal(entries, {
1312
+ onInvalid: (diagnostic) => console.warn(`[pi-workflows] ${diagnostic}`),
1313
+ });
1314
+ for (const run of runs.values()) {
1315
+ const state: ScriptRunState = {
1316
+ run,
1317
+ frozenOptions: {
1318
+ ...(run.frozenTokenBudget !== undefined ? { tokenBudget: run.frozenTokenBudget } : {}),
1319
+ ...(run.frozenMaxAgents !== undefined ? { maxAgents: run.frozenMaxAgents } : {}),
1320
+ ...(run.frozenConcurrency !== undefined ? { concurrency: run.frozenConcurrency } : {}),
1321
+ ...(run.frozenAgentRetries !== undefined ? { agentRetries: run.frozenAgentRetries } : {}),
1322
+ ...(run.frozenAgentTimeoutMs !== undefined ? { agentTimeoutMs: run.frozenAgentTimeoutMs } : {}),
1323
+ ...(run.toolset ? { toolset: run.toolset } : {}),
1324
+ ...(run.frozenExcludeTools ? { excludeTools: [...run.frozenExcludeTools] } : {}),
1325
+ },
1326
+ controller: new AbortController(),
1327
+ generations: new Map(),
1328
+ agentWaiters: new Map(),
1329
+ pendingSpawns: new Map(),
1330
+ pendingJournal: new Set(),
1331
+ lifecycleSuspended: false,
1332
+ executionGeneration: 0,
1333
+ };
1334
+ for (const [callIndex, attemptId] of Object.entries(run.attemptIds)) {
1335
+ state.generations.set(callIndex, generationFromAttemptId(attemptId));
1336
+ }
1337
+ // Only non-terminal runs can be resumed; terminal runs are inert history.
1338
+ if (isTerminalWorkflow(run.status)) {
1339
+ this.runs.set(run.runId, state);
1340
+ continue;
1341
+ }
1342
+ // A restored run that never settled was cut off mid-execution (branch
1343
+ // change, reload, or crash). Mark it interrupted so it is visibly
1344
+ // resumable rather than looking live-but-frozen.
1345
+ run.status = "interrupted";
1346
+ run.updatedAt = Date.now();
1347
+ this.runs.set(run.runId, state);
1410
1348
  }
1411
1349
  }
1412
1350
 
1413
- private commitSynthesisTerminal(
1414
- run: WorkflowRun,
1415
- result: WorkflowRun["synthesisResult"],
1416
- finalStatus: "completed" | "failed" | "stopped",
1417
- ): void {
1418
- if (run.synthesisResult || !result) return;
1419
- const timestamp = result.updatedAt;
1420
- const resultEvent: JournalEvent = {
1421
- kind: "synthesis_result",
1422
- schemaVersion: JOURNAL_SCHEMA_VERSION,
1423
- runId: run.runId,
1424
- result,
1425
- attemptId: run.attemptIds[SYNTHESIS_NODE_ID],
1426
- owner: this.ownerForNode(run, SYNTHESIS_NODE_ID),
1427
- timestamp,
1428
- };
1429
- const settling = run.status === "stopping" || this.settling.has(run.runId);
1430
- const workflowEvent: JournalEvent | undefined = settling
1431
- ? undefined
1432
- : {
1433
- kind: "workflow_transition",
1434
- schemaVersion: JOURNAL_SCHEMA_VERSION,
1435
- runId: run.runId,
1436
- status: finalStatus,
1437
- ...(result.error ? { error: result.error } : {}),
1438
- ...(run.terminalIntent ? { terminalIntent: run.terminalIntent } : {}),
1439
- timestamp,
1440
- };
1441
- this.appendEvents([resultEvent, ...(workflowEvent ? [workflowEvent] : [])]);
1442
-
1443
- run.synthesisResult = result;
1444
- if (result.agentId) run.synthesisAgentId = result.agentId;
1445
- run.compactions[SYNTHESIS_NODE_ID] = Math.max(run.compactions[SYNTHESIS_NODE_ID] ?? 0, result.compactionCount);
1446
- run.updatedAt = timestamp;
1447
- if (workflowEvent) {
1448
- run.status = finalStatus;
1449
- run.updatedAt = timestamp;
1450
- if (result.error) run.error = result.error;
1451
- if (isTerminalWorkflow(finalStatus)) this.resolveWaiters(run);
1452
- }
1351
+ list(): Array<Record<string, unknown>> {
1352
+ return [...this.runs.values()]
1353
+ .sort((a, b) => b.run.startedAt - a.run.startedAt)
1354
+ .map((state) => this.summary(state.run));
1453
1355
  }
1454
1356
 
1455
- private commitDeferredTerminals(run: WorkflowRun, records: readonly DeferredTerminalRecord[]): void {
1456
- const pending = records.filter((record) =>
1457
- record.nodeId === SYNTHESIS_NODE_ID
1458
- ? run.synthesisResult === undefined
1459
- : run.taskResults[record.nodeId] === undefined && !isTerminalTask(run.taskStatus[record.nodeId]),
1460
- );
1461
- if (pending.length === 0) return;
1462
- const events: JournalEvent[] = pending.map((record) =>
1463
- record.nodeId === SYNTHESIS_NODE_ID
1464
- ? {
1465
- kind: "synthesis_result",
1466
- schemaVersion: JOURNAL_SCHEMA_VERSION,
1467
- runId: run.runId,
1468
- result: record.result,
1469
- attemptId: run.attemptIds[SYNTHESIS_NODE_ID],
1470
- owner: this.ownerForNode(run, SYNTHESIS_NODE_ID),
1471
- timestamp: Math.max(run.updatedAt, record.result.updatedAt),
1472
- }
1473
- : {
1474
- kind: "task_result",
1475
- schemaVersion: JOURNAL_SCHEMA_VERSION,
1476
- runId: run.runId,
1477
- nodeId: record.nodeId,
1478
- result: record.result,
1479
- attemptId: run.attemptIds[record.nodeId],
1480
- owner: this.ownerForNode(run, record.nodeId),
1481
- timestamp: Math.max(run.updatedAt, record.result.updatedAt),
1482
- },
1483
- );
1484
- const transitions: JournalEvent[] = pending
1485
- .filter((record) => record.nodeId !== SYNTHESIS_NODE_ID)
1486
- .map((record) => ({
1487
- kind: "task_transition",
1488
- schemaVersion: JOURNAL_SCHEMA_VERSION,
1489
- runId: run.runId,
1490
- nodeId: record.nodeId,
1491
- status: record.result.status,
1492
- ...(record.result.agentId ? { agentId: record.result.agentId } : {}),
1493
- attemptId: run.attemptIds[record.nodeId],
1494
- owner: this.ownerForNode(run, record.nodeId),
1495
- timestamp: Math.max(run.updatedAt, record.result.updatedAt),
1496
- }));
1497
- this.appendEvents([...events, ...transitions]);
1498
- for (const record of pending) {
1499
- if (record.nodeId === SYNTHESIS_NODE_ID) {
1500
- run.synthesisResult = record.result;
1501
- if (record.result.agentId) run.synthesisAgentId = record.result.agentId;
1502
- } else {
1503
- run.taskResults[record.nodeId] = record.result;
1504
- run.taskStatus[record.nodeId] = record.result.status;
1505
- if (record.result.agentId) run.agentIds[record.nodeId] = record.result.agentId;
1506
- }
1507
- run.compactions[record.nodeId] = Math.max(run.compactions[record.nodeId] ?? 0, record.result.compactionCount);
1508
- run.updatedAt = Math.max(run.updatedAt, record.result.updatedAt);
1509
- }
1510
- }
1511
- private async pump(run: WorkflowRun): Promise<void> {
1512
- if (this.branchChanging || this.pumping.has(run.runId) || run.status !== "running") return;
1513
- this.pumping.add(run.runId);
1514
- const branchEpoch = this.branchEpoch;
1515
- const isCurrent = (): boolean =>
1516
- !this.branchChanging && branchEpoch === this.branchEpoch && this.runs.get(run.runId) === run;
1517
- try {
1518
- if (!isCurrent()) return;
1519
- const completed = new Set(
1520
- Object.entries(run.taskStatus)
1521
- .filter(([, status]) => status === "completed")
1522
- .map(([nodeId]) => nodeId),
1523
- );
1524
- const ready = readyTaskIds(run.definition, completed);
1525
- for (const nodeId of ready) {
1526
- if (!isCurrent() || run.status !== "running") break;
1527
- if (run.taskStatus[nodeId] !== "pending" && run.taskStatus[nodeId] !== "ready") continue;
1528
- if (run.taskStatus[nodeId] === "pending") this.setTaskStatus(run, nodeId, "ready");
1529
- this.setTaskStatus(run, nodeId, "dispatching");
1530
- const dispatch = this.dispatchTask(run, nodeId, branchEpoch);
1531
- this.trackDispatch(run.runId, dispatch);
1532
- }
1533
- if (!isCurrent() || run.status !== "running") return;
1534
- const statuses = Object.values(run.taskStatus);
1535
- if (statuses.some((status) => status === "failed" || status === "blocked" || status === "stopped")) {
1536
- if (statuses.some((status) => status === "stopped") && run.terminalIntent === "stop") {
1537
- if (!isCurrent()) return;
1538
- await this.stop(run);
1539
- } else {
1540
- if (!isCurrent()) return;
1541
- await this.failRun(
1542
- run,
1543
- statuses.some((status) => status === "stopped")
1544
- ? "one or more workflow tasks stopped unexpectedly"
1545
- : "one or more workflow tasks failed",
1546
- );
1547
- }
1548
- return;
1549
- }
1550
- if (!isCurrent() || statuses.some((status) => !isTerminalTask(status))) return;
1551
- if (run.definition.synthesis && !run.synthesisResult && (!run.synthesisAgentId || run.status === "running")) {
1552
- if (!isCurrent()) return;
1553
- this.setWorkflowStatus(run, "synthesizing");
1554
- const dispatch = this.dispatchSynthesis(run);
1555
- this.trackDispatch(run.runId, dispatch);
1556
- return;
1557
- }
1558
- if (!isCurrent()) return;
1559
- if (!run.definition.synthesis) {
1560
- this.setWorkflowStatus(run, "completed");
1561
- } else if (run.synthesisResult) {
1562
- const synthesisStatus =
1563
- run.synthesisResult.status === "completed"
1564
- ? "completed"
1565
- : run.synthesisResult.status === "stopped"
1566
- ? "stopped"
1567
- : "failed";
1568
- this.setWorkflowStatus(run, synthesisStatus, run.synthesisResult.error);
1569
- }
1570
- } finally {
1571
- if (this.runs.get(run.runId) === run && this.branchEpoch === branchEpoch) {
1572
- this.pumping.delete(run.runId);
1573
- }
1574
- }
1357
+ /** Raw run list for the navigator (keeps live ScriptRun state). */
1358
+ getState(runId: string): ScriptRunState | undefined {
1359
+ return this.runs.get(runId);
1575
1360
  }
1576
1361
 
1577
- private recoverInterruptedRun(
1578
- run: WorkflowRun,
1579
- plan: readonly RecoveryPlanItem[],
1580
- terminalResults: readonly DeferredTerminalRecord[],
1581
- ): void {
1582
- const sourceKey = this.recoverySourceKey(run, plan, terminalResults);
1583
- const memo = this.memoizedRecovery(run.runId, "run_recovery", sourceKey);
1584
- if (memo?.kind === "run_recovery") {
1585
- // The durable event belongs to this branch and source snapshot. Reapply
1586
- // it to the freshly replayed run without another append.
1587
- applyRecoveryEvent(run, memo);
1588
- return;
1589
- }
1590
-
1591
- const exhausted = plan.find((item) => item.sourceGeneration + 1 > MAX_ATTEMPTS_PER_NODE);
1592
- if (exhausted) {
1593
- const message = `workflow node ${exhausted.nodeId} exceeded interrupted-attempt limit`;
1594
- if (exhausted.nodeId === SYNTHESIS_NODE_ID) {
1595
- const result = resultFromLifecycle(
1596
- "failed",
1597
- run.synthesisAgentId,
1598
- undefined,
1599
- message,
1600
- run.compactions[SYNTHESIS_NODE_ID] ?? 0,
1601
- );
1602
- result.attemptId = run.attemptIds[SYNTHESIS_NODE_ID];
1603
- this.commitSynthesisTerminal(run, result, "failed");
1604
- } else {
1605
- this.failTask(run, exhausted.nodeId, message);
1606
- }
1607
- return;
1608
- }
1362
+ getRun(runId: string): ScriptRun | undefined {
1363
+ return this.runs.get(runId)?.run;
1364
+ }
1609
1365
 
1610
- const rotations: RecoveryRotation[] = plan.map((item) => {
1611
- const generation = item.sourceGeneration + 1;
1612
- const attemptId = `${run.runId}/${item.nodeId}/attempt-${generation}`;
1613
- return {
1614
- nodeId: item.nodeId,
1615
- sourceAttemptId: item.sourceAttemptId,
1616
- sourceGeneration: item.sourceGeneration,
1617
- sourceStatus: item.sourceStatus,
1618
- attemptId,
1619
- generation,
1620
- owner: { extension: "pi-workflows", runId: run.runId, nodeId: item.nodeId, attemptId },
1621
- ...(item.oldAgentId ? { supersededAgentId: item.oldAgentId } : {}),
1622
- };
1623
- });
1624
- const recoveryBase = {
1625
- kind: "run_recovery" as const,
1626
- schemaVersion: JOURNAL_SCHEMA_VERSION as 2,
1366
+ summary(run: ScriptRun, detailed = false): Record<string, unknown> {
1367
+ const base: Record<string, unknown> = {
1627
1368
  runId: run.runId,
1628
- status: "interrupted" as const,
1629
- branchGeneration: this.recoveryBranchGeneration,
1630
- rotations,
1631
- ...(terminalResults.length > 0
1632
- ? {
1633
- terminalResults: terminalResults.map(
1634
- ({ kind: _kind, runId: _runId, nodeId, attemptId, result, owner }) => ({
1635
- nodeId,
1636
- attemptId,
1637
- result: { ...result, updatedAt: run.updatedAt },
1638
- owner,
1639
- }),
1640
- ),
1641
- }
1642
- : {}),
1643
- };
1644
- const event: RunRecoveryEvent = {
1645
- ...recoveryBase,
1646
- recoveryId: deriveRecoveryId(recoveryBase),
1647
- timestamp: Date.now(),
1369
+ status: run.status,
1370
+ name: run.meta.name,
1371
+ description: run.meta.description.slice(0, 500),
1372
+ agentCount: Object.keys(run.callResults).length,
1373
+ phases: run.meta.phases?.map((p) => p.title) ?? [],
1374
+ startedAt: run.startedAt,
1375
+ updatedAt: run.updatedAt,
1376
+ ...(run.error ? { error: run.error.slice(0, 2_000) } : {}),
1648
1377
  };
1649
- // This is the only persistence boundary for the complete recovery. Do not
1650
- // mutate the run or memoize until append returns successfully.
1651
- this.write(event);
1652
- applyRecoveryEvent(run, event);
1653
- this.rememberRecovery({
1654
- branchGeneration: this.recoveryBranchGeneration,
1655
- runId: run.runId,
1656
- key: sourceKey,
1657
- event,
1658
- });
1378
+ if (!detailed) return base;
1379
+ const calls = Object.entries(run.callStatus)
1380
+ .sort(([a], [b]) => Number(a) - Number(b))
1381
+ .map(([nodeId, status]) => {
1382
+ const result = run.callResults[nodeId];
1383
+ return {
1384
+ index: Number(nodeId),
1385
+ status,
1386
+ ...(result?.result !== undefined ? { resultPreview: String(result.result).slice(0, 300) } : {}),
1387
+ ...(result?.error ? { error: result.error.slice(0, 300) } : {}),
1388
+ ...(result?.tokenCount !== undefined ? { tokenCount: result.tokenCount } : {}),
1389
+ ...(run.agentIds[nodeId] ? { agentId: run.agentIds[nodeId] } : {}),
1390
+ attemptId: run.attemptIds[nodeId],
1391
+ };
1392
+ });
1393
+ return { ...base, calls };
1659
1394
  }
1395
+ }
1660
1396
 
1661
- private recoverInterruptedNode(
1662
- run: WorkflowRun,
1663
- nodeId: string,
1664
- oldAgentId: string,
1665
- resumeImmediately = true,
1666
- legacyMigration = false,
1667
- ): void {
1668
- const sourceGeneration = run.attempts[nodeId] ?? 1;
1669
- const sourceAttemptId = this.sourceAttemptId(run, nodeId);
1670
- const sourceStatus: RecoverySourceStatus = nodeId === SYNTHESIS_NODE_ID ? "synthesizing" : run.taskStatus[nodeId];
1671
- if (nodeId !== SYNTHESIS_NODE_ID && isTerminalTask(run.taskStatus[nodeId])) return;
1672
- const plan: RecoveryPlanItem = {
1673
- nodeId,
1674
- sourceAttemptId,
1675
- sourceGeneration,
1676
- sourceStatus,
1677
- oldAgentId,
1678
- };
1679
- const sourceKey = this.recoverySourceKey(run, [plan]);
1680
- const memo = this.memoizedRecovery(run.runId, "attempt_recovery", sourceKey);
1681
- if (memo?.kind === "attempt_recovery") {
1682
- applyRecoveryEvent(run, memo);
1683
- this.resumeRecoveredNode(run, nodeId, resumeImmediately);
1684
- return;
1685
- }
1397
+ function hashScript(script: string): string {
1398
+ return createHash("sha256").update(script).digest("hex");
1399
+ }
1686
1400
 
1687
- const generation = sourceGeneration + 1;
1688
- if (generation > MAX_ATTEMPTS_PER_NODE) {
1689
- const message = `workflow node ${nodeId} exceeded interrupted-attempt limit`;
1690
- if (nodeId === SYNTHESIS_NODE_ID) {
1691
- const result = resultFromLifecycle(
1692
- "failed",
1693
- run.synthesisAgentId,
1694
- undefined,
1695
- message,
1696
- run.compactions[nodeId] ?? 0,
1697
- );
1698
- result.attemptId = run.attemptIds[nodeId];
1699
- this.commitSynthesisTerminal(run, result, "failed");
1700
- } else {
1701
- this.failTask(run, nodeId, message);
1702
- }
1703
- return;
1704
- }
1705
-
1706
- const attemptId = `${run.runId}/${nodeId}/attempt-${generation}`;
1707
- const rotation: RecoveryRotation = {
1708
- nodeId,
1709
- sourceAttemptId,
1710
- sourceGeneration,
1711
- sourceStatus,
1712
- attemptId,
1713
- generation,
1714
- owner: { extension: "pi-workflows", runId: run.runId, nodeId, attemptId },
1715
- ...(oldAgentId ? { supersededAgentId: oldAgentId } : {}),
1716
- };
1717
- const event: AttemptRecoveryEvent = {
1718
- kind: "attempt_recovery",
1719
- schemaVersion: JOURNAL_SCHEMA_VERSION,
1720
- runId: run.runId,
1721
- nodeId,
1722
- branchGeneration: this.recoveryBranchGeneration,
1723
- rotation,
1724
- ...(legacyMigration ? { legacyMigration: true } : {}),
1725
- timestamp: Date.now(),
1726
- };
1727
- this.write(event);
1728
- applyRecoveryEvent(run, event);
1729
- this.rememberRecovery({
1730
- branchGeneration: this.recoveryBranchGeneration,
1731
- runId: run.runId,
1732
- key: sourceKey,
1733
- event,
1734
- });
1735
- this.resumeRecoveredNode(run, nodeId, resumeImmediately);
1736
- }
1737
-
1738
- private resumeRecoveredNode(run: WorkflowRun, nodeId: string, resumeImmediately: boolean): void {
1739
- if (!resumeImmediately) return;
1740
- if (nodeId === SYNTHESIS_NODE_ID && run.status === "synthesizing") {
1741
- const dispatch = this.dispatchSynthesis(run);
1742
- this.trackDispatch(run.runId, dispatch);
1743
- } else if (nodeId !== SYNTHESIS_NODE_ID && run.status === "running") {
1744
- void this.pump(run);
1745
- }
1746
- }
1747
-
1748
- private applyManagedTerminal(run: WorkflowRun, nodeId: string, response: ManagedSpawnResponse): void {
1749
- const snapshot = response.terminal;
1750
- if (!snapshot) return;
1751
- if (snapshot.status === "interrupted") {
1752
- this.recoverInterruptedNode(run, nodeId, response.id);
1753
- return;
1754
- }
1755
- const status = managedTerminalStatus(snapshot);
1756
- const compactionCount = Math.max(run.compactions[nodeId] ?? 0, snapshot.compactionCount);
1757
- const result = resultFromLifecycle(
1758
- status,
1759
- response.id,
1760
- snapshot.result,
1761
- snapshot.error,
1762
- compactionCount,
1763
- snapshot.completedAt,
1764
- snapshot.outputFile,
1765
- snapshot.tokenCount,
1766
- );
1767
- result.attemptId = run.attemptIds[nodeId];
1768
-
1769
- if (nodeId === SYNTHESIS_NODE_ID) {
1770
- this.commitSynthesisTerminal(run, result, status);
1771
- return;
1772
- }
1773
- if (!run.taskStatus[nodeId] || run.taskResults[nodeId] || isTerminalTask(run.taskStatus[nodeId])) return;
1774
- this.commitTaskTerminal(run, nodeId, result, status);
1775
- if (status === "completed" && run.status !== "stopping" && !this.settling.has(run.runId)) {
1776
- this.maybePause(run);
1777
- if (run.status === "running") void this.pump(run);
1778
- }
1779
- }
1780
-
1781
- /**
1782
- * Append the journaled results of `task.inputs` to its prompt.
1783
- *
1784
- * Must be a pure function of the journaled prefix: the composed prompt feeds pi-subagents'
1785
- * managed fingerprint, and a replay that recomposes a different prompt is reported as a spawn
1786
- * key conflict and fails the run non-retryably. So: iterate the declared `inputs` array in
1787
- * order (never `Object.keys(run.taskResults)`, whose order differs between a live run and a
1788
- * replay), read only journaled result fields (never `run.taskStatus`, live agent state, or the
1789
- * clock), and never throw — `dispatchTask` has no error path around composition.
1790
- */
1791
- private composeTaskPrompt(run: WorkflowRun, task: WorkflowTask): string {
1792
- const inputs = task.inputs ?? [];
1793
- if (inputs.length === 0) return task.prompt;
1794
- const separator = "\n\n";
1795
- // pi-subagents-protocol rejects an oversized prompt by throwing, so budget against the
1796
- // remaining headroom rather than adding a fixed section and hoping it fits.
1797
- const headroom = MAX_DISPATCH_PROMPT_CHARS - task.prompt.length - separator.length;
1798
- const budget = Math.min(MAX_TASK_INPUT_CHARS, headroom);
1799
- if (budget < MIN_TASK_INPUT_CHARS) return task.prompt;
1800
- const entries: ResultEntry[] = inputs.map((id) => {
1801
- const dependency = run.definition.tasks.find((candidate) => candidate.id === id);
1802
- return {
1803
- task: dependency ?? { id, description: id },
1804
- result: run.taskResults[id],
1805
- fallbackStatus: "missing",
1806
- };
1807
- });
1808
- const section = buildBoundedResultSection([TASK_INPUT_HEADER], entries, {
1809
- maxPerResult: MAX_TASK_RESULT_CHARS,
1810
- maxTotal: budget,
1811
- overflowMarker: "\n\n[dependency input truncated: additional results omitted]",
1812
- });
1813
- return cap(`${task.prompt}${separator}${section}`, MAX_DISPATCH_PROMPT_CHARS).text;
1814
- }
1815
-
1816
- private async dispatchTask(run: WorkflowRun, nodeId: string, branchEpoch: number): Promise<void> {
1817
- const task = run.definition.tasks.find((candidate) => candidate.id === nodeId);
1818
- if (
1819
- !task ||
1820
- this.branchChanging ||
1821
- branchEpoch !== this.branchEpoch ||
1822
- this.runs.get(run.runId) !== run ||
1823
- run.status !== "running"
1824
- )
1825
- return;
1826
- const attemptId = run.attemptIds[nodeId];
1827
- const prompt = this.composeTaskPrompt(run, task);
1828
- const tierOverride = task.tier === undefined ? run.definition.tier : undefined;
1829
- // Keeping the `task` identity when nothing changes guarantees a byte-identical dispatch
1830
- // for every workflow that does not declare `inputs`.
1831
- const dispatchTaskDefinition: WorkflowTask =
1832
- tierOverride === undefined && prompt === task.prompt
1833
- ? task
1834
- : { ...task, prompt, ...(tierOverride === undefined ? {} : { tier: tierOverride }) };
1835
- let response: ManagedSpawnResponse;
1836
- try {
1837
- response = normalizeManagedSpawnResponse(
1838
- await spawnManaged(this.client, dispatchTaskDefinition, run.runId, nodeId, attemptId),
1839
- );
1840
- } catch (error: unknown) {
1841
- if (branchEpoch !== this.branchEpoch || this.runs.get(run.runId) !== run) return;
1842
- const message = boundedError(error) ?? "managed spawn failed";
1843
- if ((run.status as WorkflowStatus) === "stopping") {
1844
- if (!isTerminalTask(run.taskStatus[nodeId])) this.setTaskStatus(run, nodeId, "stopped", run.agentIds[nodeId]);
1845
- return;
1846
- }
1847
- this.failTask(run, nodeId, message);
1848
- return;
1849
- }
1850
-
1851
- if (
1852
- this.branchChanging ||
1853
- branchEpoch !== this.branchEpoch ||
1854
- this.runs.get(run.runId) !== run ||
1855
- run.status !== "running"
1856
- ) {
1857
- if (
1858
- !this.branchChanging &&
1859
- branchEpoch === this.branchEpoch &&
1860
- this.runs.get(run.runId) === run &&
1861
- (run.status as WorkflowStatus) === "stopping"
1862
- ) {
1863
- run.agentIds[nodeId] = response.id;
1864
- this.rememberLateDispatchTarget(run, nodeId, response.id, attemptId);
1865
- return;
1866
- }
1867
- this.quarantineAgent(response.id);
1868
- const allowStaleBranch = this.branchChanging || branchEpoch !== this.branchEpoch;
1869
- const quiesced = await this.stopAndQuiesceOwned(
1870
- run,
1871
- nodeId,
1872
- response.id,
1873
- attemptId,
1874
- branchEpoch,
1875
- !response.terminal,
1876
- allowStaleBranch,
1877
- );
1878
- if (allowStaleBranch && !quiesced) this.rememberLateDispatchQuiescence(run.runId, response.id);
1879
- return;
1880
- }
1881
- if (response.terminal) {
1882
- // Keep terminal persistence failures outside the spawn-error handler. A
1883
- // journal append failure must leave the current attempt untouched rather
1884
- // than turning an already durable managed result into a new task failure.
1885
- this.tryLifecycleMutation(run, `managed-terminal:${nodeId}:${attemptId}`, () => {
1886
- this.applyManagedTerminal(run, nodeId, response);
1887
- });
1888
- return;
1889
- }
1890
- if (
1891
- this.branchChanging ||
1892
- branchEpoch !== this.branchEpoch ||
1893
- this.runs.get(run.runId) !== run ||
1894
- run.status !== "running"
1895
- ) {
1896
- this.quarantineAgent(response.id);
1897
- const allowStaleBranch = this.branchChanging || branchEpoch !== this.branchEpoch;
1898
- const quiesced = await this.stopAndQuiesceOwned(
1899
- run,
1900
- nodeId,
1901
- response.id,
1902
- attemptId,
1903
- branchEpoch,
1904
- !response.terminal,
1905
- allowStaleBranch,
1906
- );
1907
- if (allowStaleBranch && !quiesced) this.rememberLateDispatchQuiescence(run.runId, response.id);
1908
- return;
1909
- }
1910
- const taskStatus = run.taskStatus[nodeId];
1911
- if (response.state === "running" && (taskStatus === "dispatching" || taskStatus === "queued")) {
1912
- this.setTaskStatus(run, nodeId, "running", response.id);
1913
- } else if (taskStatus === "dispatching") {
1914
- this.setTaskStatus(run, nodeId, "queued", response.id);
1915
- }
1916
- }
1917
-
1918
- private async dispatchSynthesis(run: WorkflowRun, branchEpoch = this.branchEpoch): Promise<void> {
1919
- const synthesis = run.definition.synthesis;
1920
- if (
1921
- !synthesis ||
1922
- this.branchChanging ||
1923
- branchEpoch !== this.branchEpoch ||
1924
- this.runs.get(run.runId) !== run ||
1925
- run.status !== "synthesizing"
1926
- )
1927
- return;
1928
- const prompt = buildSynthesisPrompt(run.definition, run);
1929
- const task: WorkflowTask = {
1930
- id: SYNTHESIS_NODE_ID,
1931
- subagent_type: synthesis.subagent_type,
1932
- tier: synthesis.tier ?? run.definition.tier,
1933
- description: "Synthesize workflow results",
1934
- prompt,
1935
- depends_on: [],
1936
- };
1937
- const attemptId = run.attemptIds[SYNTHESIS_NODE_ID];
1938
- let response: ManagedSpawnResponse;
1939
- try {
1940
- response = normalizeManagedSpawnResponse(
1941
- await spawnManaged(this.client, task, run.runId, SYNTHESIS_NODE_ID, attemptId),
1942
- );
1943
- } catch (error: unknown) {
1944
- if (branchEpoch !== this.branchEpoch || this.runs.get(run.runId) !== run) return;
1945
- const result = resultFromLifecycle("failed", run.synthesisAgentId, undefined, boundedError(error), 0);
1946
- result.attemptId = run.attemptIds[SYNTHESIS_NODE_ID];
1947
- this.commitSynthesisTerminal(run, result, "failed");
1948
- return;
1949
- }
1950
-
1951
- if (
1952
- this.branchChanging ||
1953
- branchEpoch !== this.branchEpoch ||
1954
- this.runs.get(run.runId) !== run ||
1955
- run.status !== "synthesizing"
1956
- ) {
1957
- if (
1958
- !this.branchChanging &&
1959
- branchEpoch === this.branchEpoch &&
1960
- this.runs.get(run.runId) === run &&
1961
- (run.status as WorkflowStatus) === "stopping"
1962
- ) {
1963
- run.synthesisAgentId = response.id;
1964
- this.rememberLateDispatchTarget(run, SYNTHESIS_NODE_ID, response.id, attemptId);
1965
- return;
1966
- }
1967
- this.quarantineAgent(response.id);
1968
- const allowStaleBranch = this.branchChanging || branchEpoch !== this.branchEpoch;
1969
- const quiesced = await this.stopAndQuiesceOwned(
1970
- run,
1971
- SYNTHESIS_NODE_ID,
1972
- response.id,
1973
- attemptId,
1974
- branchEpoch,
1975
- !response.terminal,
1976
- allowStaleBranch,
1977
- );
1978
- if (allowStaleBranch && !quiesced) this.rememberLateDispatchQuiescence(run.runId, response.id);
1979
- return;
1980
- }
1981
- if (response.terminal) {
1982
- // Do not route a terminal journal failure through the RPC error path; the
1983
- // managed terminal fact remains retryable/recoverable as this attempt.
1984
- this.tryLifecycleMutation(run, `managed-terminal:${SYNTHESIS_NODE_ID}:${attemptId}`, () => {
1985
- this.applyManagedTerminal(run, SYNTHESIS_NODE_ID, response);
1986
- });
1987
- return;
1988
- }
1989
- run.synthesisAgentId ??= response.id;
1990
- if (
1991
- this.branchChanging ||
1992
- branchEpoch !== this.branchEpoch ||
1993
- this.runs.get(run.runId) !== run ||
1994
- run.status !== "synthesizing"
1995
- ) {
1996
- this.quarantineAgent(response.id);
1997
- const allowStaleBranch = this.branchChanging || branchEpoch !== this.branchEpoch;
1998
- const quiesced = await this.stopAndQuiesceOwned(
1999
- run,
2000
- SYNTHESIS_NODE_ID,
2001
- response.id,
2002
- attemptId,
2003
- branchEpoch,
2004
- !response.terminal,
2005
- allowStaleBranch,
2006
- );
2007
- if (allowStaleBranch && !quiesced) this.rememberLateDispatchQuiescence(run.runId, response.id);
2008
- return;
2009
- }
2010
- }
2011
- /**
2012
- * Gate lifecycle observations at the workflow boundary. Attempt-aware runs
2013
- * must carry the exact current generation and agent identity; only a created
2014
- * event may establish an agent id when dispatch has not returned one yet.
2015
- * Legacy journal runs intentionally retain the pre-v2 permissive behavior.
2016
- */
2017
- private acceptsLifecycleEvent(
2018
- run: WorkflowRun,
2019
- nodeId: string,
2020
- eventName: string,
2021
- ownerAttemptId: string | undefined,
2022
- agentId: string | undefined,
2023
- ): boolean {
2024
- const expectedAttemptId = run.attemptIds[nodeId];
2025
- if (run.attemptTracking) {
2026
- if (!expectedAttemptId || ownerAttemptId !== expectedAttemptId || !agentId) return false;
2027
- }
2028
-
2029
- const expectedAgentId = nodeId === SYNTHESIS_NODE_ID ? run.synthesisAgentId : run.agentIds[nodeId];
2030
- if (agentId && expectedAgentId && agentId !== expectedAgentId) return false;
2031
- if (run.attemptTracking && !expectedAgentId && eventName !== "subagents:created") return false;
2032
- return true;
2033
- }
2034
-
2035
- private onLifecycle(eventName: string, data: Record<string, unknown>): void {
2036
- if (this.lifecycleSuspended || this.branchChanging || this.branchEventsClosed) return;
2037
- const owner = ownerFor(data);
2038
- if (!owner) return;
2039
- const run = this.runs.get(owner.runId);
2040
- if (!run) return;
2041
- const agentId = typeof data.id === "string" ? data.id : undefined;
2042
- if (agentId && this.quarantinedAgentIds.has(agentId)) return;
2043
- if (owner.nodeId !== SYNTHESIS_NODE_ID && !(owner.nodeId in run.taskStatus)) return;
2044
-
2045
- const settling = run.status === "stopping" || this.settling.has(run.runId);
2046
- if (isTerminalWorkflow(run.status)) return;
2047
- if (!this.acceptsLifecycleEvent(run, owner.nodeId, eventName, owner.attemptId, agentId)) return;
2048
- const terminalEvent = eventName === "subagents:completed" || eventName === "subagents:failed";
2049
- if (run.status === "stopping" && !terminalEvent) return;
2050
- if (
2051
- owner.nodeId === SYNTHESIS_NODE_ID
2052
- ? run.synthesisResult !== undefined
2053
- : run.taskResults[owner.nodeId] !== undefined || isTerminalTask(run.taskStatus[owner.nodeId])
2054
- )
2055
- return;
2056
-
2057
- if (eventName === "subagents:compacted") {
2058
- const previousCount = run.compactions[owner.nodeId] ?? 0;
2059
- const observedCount = data.compactionCount;
2060
- const count =
2061
- typeof observedCount === "number" && Number.isSafeInteger(observedCount) && observedCount >= 0
2062
- ? Math.max(previousCount, observedCount)
2063
- : previousCount + 1;
2064
- const timestamp = Date.now();
2065
- const event: JournalEvent = {
2066
- kind: "task_compacted",
2067
- schemaVersion: JOURNAL_SCHEMA_VERSION,
2068
- runId: run.runId,
2069
- nodeId: owner.nodeId,
2070
- compactionCount: count,
2071
- attemptId: run.attemptIds[owner.nodeId],
2072
- owner: this.ownerForNode(run, owner.nodeId),
2073
- timestamp,
2074
- };
2075
- this.tryLifecycleMutation(run, `compacted:${owner.nodeId}:${count}`, () => {
2076
- this.write(event);
2077
- run.compactions[owner.nodeId] = Math.max(run.compactions[owner.nodeId] ?? 0, count);
2078
- run.updatedAt = timestamp;
2079
- });
2080
- return;
2081
- }
2082
-
2083
- if (owner.nodeId === SYNTHESIS_NODE_ID) {
2084
- if (eventName === "subagents:created" && agentId) run.synthesisAgentId ??= agentId;
2085
- if (eventName === "subagents:started" || !terminalEvent) return;
2086
- const status = statusFromLifecycle(eventName, data);
2087
- const timestamp = Date.now();
2088
- const result = resultFromLifecycle(
2089
- status,
2090
- agentId ?? run.synthesisAgentId,
2091
- data.result,
2092
- data.error,
2093
- run.compactions[SYNTHESIS_NODE_ID] ?? 0,
2094
- timestamp,
2095
- data.outputFile,
2096
- tokenCountFrom(data.tokens),
2097
- );
2098
- result.attemptId = run.attemptIds[SYNTHESIS_NODE_ID];
2099
- this.tryLifecycleMutation(run, `terminal:${owner.nodeId}:${owner.attemptId ?? "legacy"}`, () => {
2100
- this.commitSynthesisTerminal(run, result, status);
2101
- });
2102
- return;
2103
- }
2104
-
2105
- if (eventName === "subagents:created") {
2106
- if (run.taskStatus[owner.nodeId] === "dispatching") {
2107
- this.tryLifecycleMutation(run, `created:${owner.nodeId}:${owner.attemptId ?? "legacy"}`, () => {
2108
- this.setTaskStatus(run, owner.nodeId, "queued", agentId);
2109
- });
2110
- }
2111
- return;
2112
- }
2113
- if (eventName === "subagents:started") {
2114
- if (run.taskStatus[owner.nodeId] === "dispatching" || run.taskStatus[owner.nodeId] === "queued") {
2115
- this.tryLifecycleMutation(run, `started:${owner.nodeId}:${owner.attemptId ?? "legacy"}`, () => {
2116
- this.setTaskStatus(run, owner.nodeId, "running", agentId);
2117
- });
2118
- }
2119
- return;
2120
- }
2121
- if (!terminalEvent) return;
2122
-
2123
- const status = statusFromLifecycle(eventName, data);
2124
- const timestamp = Date.now();
2125
- const result = resultFromLifecycle(
2126
- status,
2127
- agentId ?? run.agentIds[owner.nodeId],
2128
- data.result,
2129
- data.error,
2130
- run.compactions[owner.nodeId] ?? 0,
2131
- timestamp,
2132
- data.outputFile,
2133
- tokenCountFrom(data.tokens),
2134
- );
2135
- result.attemptId = run.attemptIds[owner.nodeId];
2136
- this.tryLifecycleMutation(run, `terminal:${owner.nodeId}:${owner.attemptId ?? "legacy"}`, () => {
2137
- this.commitTaskTerminal(run, owner.nodeId, result, status);
2138
- if (status === "completed" && !settling && run.status !== "stopping") {
2139
- this.maybePause(run);
2140
- if (run.status === "running") void this.pump(run);
2141
- }
2142
- });
2143
- }
2144
-
2145
- private blockDependents(run: WorkflowRun, failedNodeId: string): void {
2146
- const blockedIds = this.blockedDependentIds(run, failedNodeId);
2147
- if (blockedIds.length === 0) return;
2148
- const timestamp = Date.now();
2149
- const events: JournalEvent[] = blockedIds.map((nodeId) => ({
2150
- kind: "task_transition",
2151
- schemaVersion: JOURNAL_SCHEMA_VERSION,
2152
- runId: run.runId,
2153
- nodeId,
2154
- status: "blocked",
2155
- ...(run.attemptIds[nodeId] ? { attemptId: run.attemptIds[nodeId] } : {}),
2156
- owner: this.ownerForNode(run, nodeId),
2157
- timestamp,
2158
- }));
2159
- this.appendEvents(events);
2160
- for (const nodeId of blockedIds) run.taskStatus[nodeId] = "blocked";
2161
- run.updatedAt = timestamp;
2162
- }
2163
-
2164
- private failTask(run: WorkflowRun, nodeId: string, message: string): void {
2165
- if (isTerminalWorkflow(run.status) || run.terminalIntent === "stop") return;
2166
- const result = resultFromLifecycle(
2167
- "failed",
2168
- run.agentIds[nodeId],
2169
- undefined,
2170
- message,
2171
- run.compactions[nodeId] ?? 0,
2172
- );
2173
- result.attemptId = run.attemptIds[nodeId];
2174
- this.commitTaskTerminal(run, nodeId, result, "failed", message);
2175
- }
2176
-
2177
- /**
2178
- * Failure policy: fail-fast for scheduling, but never abandon sibling agents.
2179
- * Mark the intent, stop every remaining owned agent, consume/journal terminal
2180
- * callbacks, then settle the workflow as failed with dependents blocked.
2181
- */
2182
- private beginFailureCleanup(run: WorkflowRun, message: string): void {
2183
- if (isTerminalWorkflow(run.status) || run.terminalIntent === "stop") return;
2184
- if (run.terminalIntent !== "failure" || run.status !== "stopping" || run.error !== message) {
2185
- this.setWorkflowStatus(run, "stopping", message, "failure");
2186
- }
2187
- this.startFailureCleanup(run, message);
2188
- }
2189
-
2190
- private startFailureCleanup(run: WorkflowRun, message: string): void {
2191
- if (isTerminalWorkflow(run.status) || run.terminalIntent !== "failure") return;
2192
- if (this.settling.has(run.runId)) return;
2193
- const cleanup = this.settleTerminalCleanup(run, message);
2194
- this.settling.set(run.runId, cleanup);
2195
- void cleanup.then(
2196
- () => {
2197
- if (this.settling.get(run.runId) === cleanup) this.settling.delete(run.runId);
2198
- },
2199
- () => {
2200
- if (this.settling.get(run.runId) === cleanup) this.settling.delete(run.runId);
2201
- },
2202
- );
2203
- }
2204
-
2205
- private terminalRecoveryKey(runId: string, sourceKey: string): string {
2206
- return JSON.stringify([this.recoveryBranchGeneration, runId, "terminal_recovery", sourceKey]);
2207
- }
2208
-
2209
- private applyTerminalRecoveryMemo(run: WorkflowRun, memo: TerminalRecoveryMemo): void {
2210
- applyTerminalRecoveryEvent(run, memo.event);
2211
- }
2212
-
2213
- private rememberTerminalRecovery(memo: TerminalRecoveryMemo): void {
2214
- const key = this.terminalRecoveryKey(memo.runId, memo.key);
2215
- this.terminalRecoveryMemo.delete(key);
2216
- this.terminalRecoveryMemo.set(key, memo);
2217
- while (this.terminalRecoveryMemo.size > MAX_RECOVERY_MEMO_ENTRIES) {
2218
- const oldest = this.terminalRecoveryMemo.keys().next().value;
2219
- if (typeof oldest !== "string") break;
2220
- this.terminalRecoveryMemo.delete(oldest);
2221
- }
2222
- }
2223
-
2224
- /**
2225
- * Complete a terminal-intent run recovered from a session branch with one
2226
- * durable state transition. The event is built entirely from an immutable
2227
- * source snapshot; append is the only boundary before applying it.
2228
- */
2229
- private recoverTerminalIntent(run: WorkflowRun, terminalRecords: readonly DeferredTerminalRecord[]): void {
2230
- const source = snapshotRun(run);
2231
- const terminalRecordsForKey = [...terminalRecords]
2232
- .sort((left, right) => left.nodeId.localeCompare(right.nodeId) || left.attemptId.localeCompare(right.attemptId))
2233
- .map((record) => ({
2234
- nodeId: record.nodeId,
2235
- attemptId: record.attemptId,
2236
- owner: record.owner,
2237
- result: { ...record.result, updatedAt: 0 },
2238
- }));
2239
- const sourceKey = JSON.stringify({ source, terminalRecords: terminalRecordsForKey });
2240
- const memo = this.terminalRecoveryMemo.get(this.terminalRecoveryKey(run.runId, sourceKey));
2241
- if (memo) {
2242
- this.applyTerminalRecoveryMemo(run, memo);
2243
- return;
2244
- }
2245
-
2246
- const terminalIntent = source.terminalIntent;
2247
- if (terminalIntent === undefined) return;
2248
- const fallbackError =
2249
- source.error ?? (terminalIntent === "failure" ? "workflow failure cleanup" : "workflow stopped");
2250
- const recoveredByNode = new Map(terminalRecords.map((record) => [record.nodeId, record]));
2251
- const finalStatuses: Record<string, TaskStatus> = { ...source.taskStatus };
2252
- const resultForNode = (nodeId: string): WorkflowRun["taskResults"][string] | undefined => {
2253
- const sourceResult = source.taskResults[nodeId];
2254
- if (sourceResult) {
2255
- return { ...sourceResult, attemptId: source.attemptIds[nodeId] };
2256
- }
2257
- const recovered = recoveredByNode.get(nodeId)?.result;
2258
- if (!recovered) return undefined;
2259
- return { ...recovered, attemptId: source.attemptIds[nodeId], updatedAt: source.updatedAt };
2260
- };
2261
- for (const task of source.definition.tasks) {
2262
- const observed = resultForNode(task.id);
2263
- if (observed && !isTerminalTask(source.taskStatus[task.id])) finalStatuses[task.id] = observed.status;
2264
- }
2265
-
2266
- const blockedNodeIds = new Set(
2267
- source.definition.tasks.filter((task) => source.taskStatus[task.id] === "blocked").map((task) => task.id),
2268
- );
2269
- if (terminalIntent === "failure") {
2270
- const failedOrBlocked = new Set(
2271
- source.definition.tasks
2272
- .filter((task) => finalStatuses[task.id] === "failed" || finalStatuses[task.id] === "blocked")
2273
- .map((task) => task.id),
2274
- );
2275
- let changed = true;
2276
- while (changed) {
2277
- changed = false;
2278
- for (const task of source.definition.tasks) {
2279
- if (isTerminalTask(finalStatuses[task.id])) continue;
2280
- if (!task.depends_on.some((dependency) => failedOrBlocked.has(dependency))) continue;
2281
- finalStatuses[task.id] = "blocked";
2282
- failedOrBlocked.add(task.id);
2283
- blockedNodeIds.add(task.id);
2284
- changed = true;
2285
- }
2286
- }
2287
- }
2288
-
2289
- const terminalResults: RecoveryTerminalResult[] = [];
2290
- const terminalFact = (nodeId: string, result: WorkflowRun["taskResults"][string]): RecoveryTerminalResult => {
2291
- const attemptId = source.attemptIds[nodeId];
2292
- return {
2293
- nodeId,
2294
- attemptId,
2295
- result: { ...result, attemptId },
2296
- owner: { extension: "pi-workflows", runId: source.runId, nodeId, attemptId },
2297
- };
2298
- };
2299
- for (const task of source.definition.tasks) {
2300
- const sourceStatus = source.taskStatus[task.id];
2301
- if (isTerminalTask(sourceStatus)) continue;
2302
- if (blockedNodeIds.has(task.id)) continue;
2303
- const result =
2304
- resultForNode(task.id) ??
2305
- resultFromLifecycle(
2306
- "stopped",
2307
- source.agentIds[task.id],
2308
- undefined,
2309
- fallbackError,
2310
- source.compactions[task.id] ?? 0,
2311
- source.updatedAt,
2312
- );
2313
- terminalResults.push(terminalFact(task.id, result));
2314
- }
2315
-
2316
- if (source.definition.synthesis && !source.synthesisResult) {
2317
- const observed = source.synthesisResult ?? recoveredByNode.get(SYNTHESIS_NODE_ID)?.result;
2318
- const result =
2319
- observed ??
2320
- resultFromLifecycle(
2321
- "stopped",
2322
- source.synthesisAgentId,
2323
- undefined,
2324
- fallbackError,
2325
- source.compactions[SYNTHESIS_NODE_ID] ?? 0,
2326
- source.updatedAt,
2327
- );
2328
- terminalResults.push(terminalFact(SYNTHESIS_NODE_ID, result));
2329
- }
2330
-
2331
- const recoveryBase = {
2332
- kind: "terminal_recovery" as const,
2333
- schemaVersion: JOURNAL_SCHEMA_VERSION as 2,
2334
- runId: source.runId,
2335
- status: terminalIntent === "failure" ? ("failed" as const) : ("stopped" as const),
2336
- terminalIntent,
2337
- branchGeneration: this.recoveryBranchGeneration,
2338
- terminalResults,
2339
- blockedNodeIds: [...blockedNodeIds].sort(),
2340
- error: fallbackError,
2341
- };
2342
- const event: TerminalRecoveryEvent = {
2343
- ...recoveryBase,
2344
- recoveryId: deriveRecoveryId(recoveryBase),
2345
- timestamp: source.updatedAt,
2346
- };
2347
- // Do not mutate the replayed run or memoize until this single append succeeds.
2348
- this.write(event);
2349
- applyTerminalRecoveryEvent(run, event);
2350
- this.rememberTerminalRecovery({
2351
- branchGeneration: this.recoveryBranchGeneration,
2352
- runId: run.runId,
2353
- key: sourceKey,
2354
- event,
2355
- });
2356
- }
2357
-
2358
- private async settleTerminalCleanup(
2359
- run: WorkflowRun,
2360
- message: string,
2361
- cleanupEpoch = this.branchEpoch,
2362
- cleanupTargets?: TerminalCleanupTarget[],
2363
- ): Promise<void> {
2364
- const isCurrent = (): boolean =>
2365
- !this.branchChanging && cleanupEpoch === this.branchEpoch && this.runs.get(run.runId) === run;
2366
- // Capture exact owners before the first yield. A pending lifecycle callback
2367
- // may make a target terminal before the cleanup RPC is allowed to run.
2368
- let pendingTargets = cleanupTargets ? [...cleanupTargets] : this.initialTerminalCleanupTargets(run);
2369
- // Let beginFailureCleanup/stop publish the promise in `settling` before a
2370
- // synchronous stop RPC can emit lifecycle callbacks.
2371
- await Promise.resolve();
2372
- if (!isCurrent()) return;
2373
- const addTarget = (target: TerminalCleanupTarget): void => {
2374
- if (
2375
- pendingTargets.some(
2376
- (existing) =>
2377
- existing.agentId === target.agentId &&
2378
- existing.owner.nodeId === target.owner.nodeId &&
2379
- existing.owner.attemptId === target.owner.attemptId,
2380
- )
2381
- )
2382
- return;
2383
- pendingTargets.push(target);
2384
- };
2385
- for (const target of this.takeLateDispatchTargets(run)) addTarget(target);
2386
-
2387
- let agentsSettled = pendingTargets.length === 0;
2388
- if (pendingTargets.length > 0) {
2389
- if (this.client.quiesceOwned) {
2390
- const agentIds = pendingTargets.map(({ agentId }) => agentId);
2391
- const owners = pendingTargets.map(({ owner }) => owner);
2392
- const result = await this.client.quiesceOwned(run.runId, agentIds, BRANCH_QUIESCE_TIMEOUT_MS, owners).catch(
2393
- (error: unknown): WorkflowQuiesceResult => ({
2394
- settled: false,
2395
- pending: agentIds,
2396
- diagnostic: `owned quiescence failed: ${boundedError(error) ?? "unknown error"}`,
2397
- }),
2398
- );
2399
- const pendingIds = new Set(result.pending);
2400
- if (result.settled && result.pending.length === 0) {
2401
- pendingTargets = [];
2402
- } else {
2403
- const retained = pendingTargets.filter((target) => pendingIds.has(target.agentId));
2404
- // A false result without a matching pending id does not prove that
2405
- // any target settled. Keep the exact targets and retry fail-closed.
2406
- pendingTargets = result.pending.length === 0 || retained.length === 0 ? pendingTargets : retained;
2407
- }
2408
- agentsSettled = pendingTargets.length === 0;
2409
- } else {
2410
- // An owner-scoped stop only requests cancellation; without the v3
2411
- // quiescence proof, the AgentSession may still own provider work.
2412
- const stopPromises = pendingTargets
2413
- .filter((target) => !target.fallbackStopRequested)
2414
- .map((target) => {
2415
- target.fallbackStopRequested = true;
2416
- return this.stopOwned(run, target.owner.nodeId, target.agentId, target.owner.attemptId).catch(() => {});
2417
- });
2418
- await Promise.allSettled(stopPromises);
2419
- agentsSettled = false;
2420
- }
2421
- }
2422
-
2423
- const dispatchesSettled = await this.waitForDispatches(run.runId, BRANCH_QUIESCE_TIMEOUT_MS);
2424
- if (!isCurrent()) return;
2425
- let lateTargetsAdded = false;
2426
- for (const target of this.takeLateDispatchTargets(run)) {
2427
- const previousCount = pendingTargets.length;
2428
- addTarget(target);
2429
- lateTargetsAdded ||= pendingTargets.length > previousCount;
2430
- }
2431
- agentsSettled = pendingTargets.length === 0;
2432
- if (lateTargetsAdded) {
2433
- if (isCurrent()) await this.settleTerminalCleanup(run, message, cleanupEpoch, pendingTargets);
2434
- return;
2435
- }
2436
- if (!agentsSettled || !dispatchesSettled) {
2437
- // A stop acknowledgement only requests cancellation; the AgentSession
2438
- // still owns provider work and its pool slot until its promise settles.
2439
- // Stay in `stopping` and retry instead of publishing a false terminal run.
2440
- await new Promise<void>((resolve) => setTimeout(resolve, 100));
2441
- if (isCurrent()) await this.settleTerminalCleanup(run, message, cleanupEpoch, pendingTargets);
2442
- return;
2443
- }
2444
-
2445
- if (this.hasPendingJournalWork(run.runId)) {
2446
- // Do not synthesize a stopped/failed fact over a lifecycle callback whose
2447
- // durable prefix is still retrying (or has become permanently blocked).
2448
- // A retry may commit the observed terminal result and make the synthetic
2449
- // fact unnecessary; a blocked journal must remain nonterminal fail-closed.
2450
- if (this.journalBlockedRuns.has(run.runId)) return;
2451
- await new Promise<void>((resolve) => setTimeout(resolve, 100));
2452
- if (isCurrent()) await this.settleTerminalCleanup(run, message, cleanupEpoch, pendingTargets);
2453
- return;
2454
- }
2455
-
2456
- for (const [nodeId, status] of Object.entries(run.taskStatus)) {
2457
- if (isTerminalTask(status)) continue;
2458
- const result =
2459
- run.taskResults[nodeId] ??
2460
- resultFromLifecycle(
2461
- "stopped",
2462
- run.agentIds[nodeId],
2463
- undefined,
2464
- run.terminalIntent === "failure" ? "workflow failure cleanup" : message,
2465
- run.compactions[nodeId] ?? 0,
2466
- );
2467
- result.attemptId ??= run.attemptIds[nodeId];
2468
- this.commitTaskTerminal(run, nodeId, result, "stopped", undefined, false);
2469
- }
2470
-
2471
- if (run.synthesisAgentId && !run.synthesisResult) {
2472
- const result = resultFromLifecycle(
2473
- "stopped",
2474
- run.synthesisAgentId,
2475
- undefined,
2476
- run.terminalIntent === "failure" ? "workflow failure cleanup" : message,
2477
- run.compactions[SYNTHESIS_NODE_ID] ?? 0,
2478
- );
2479
- result.attemptId = run.attemptIds[SYNTHESIS_NODE_ID];
2480
- this.commitSynthesisTerminal(run, result, "stopped");
2481
- }
2482
-
2483
- if (run.status === "stopping" && isCurrent()) {
2484
- this.setWorkflowStatus(run, run.terminalIntent === "failure" ? "failed" : "stopped", message);
2485
- this.lateDispatchTargets.delete(run.runId);
2486
- }
2487
- }
2488
-
2489
- private async failRun(run: WorkflowRun, message: string): Promise<void> {
2490
- if (isTerminalWorkflow(run.status)) return;
2491
- for (const task of run.definition.tasks) {
2492
- if (run.taskStatus[task.id] === "failed") this.blockDependents(run, task.id);
2493
- }
2494
- this.beginFailureCleanup(run, message);
2495
- const cleanup = this.settling.get(run.runId);
2496
- if (cleanup) await cleanup;
2497
- }
2498
-
2499
- private maybePause(run: WorkflowRun): void {
2500
- if (run.status !== "pausing") return;
2501
- if (!Object.values(run.taskStatus).some(activeStatus) && !run.synthesisAgentId) {
2502
- this.setWorkflowStatus(run, "paused");
2503
- }
2504
- }
2505
-
2506
- private async stop(run: WorkflowRun): Promise<void> {
2507
- if (isTerminalWorkflow(run.status)) return;
2508
- if (run.status !== "stopping" || run.terminalIntent !== "stop" || run.error !== "workflow stopped") {
2509
- this.setWorkflowStatus(run, "stopping", "workflow stopped", "stop");
2510
- }
2511
- run.nonResumable = true;
2512
- const existing = this.settling.get(run.runId);
2513
- if (existing) {
2514
- await existing;
2515
- return;
2516
- }
2517
- const cleanup = this.settleTerminalCleanup(run, "workflow stopped");
2518
- this.settling.set(run.runId, cleanup);
2519
- try {
2520
- await cleanup;
2521
- } finally {
2522
- if (this.settling.get(run.runId) === cleanup) this.settling.delete(run.runId);
2523
- }
2524
- }
2525
-
2526
- private reconcileManagedSpawnTombstone(entry: SessionEntryLike): DeferredTerminalRecord | undefined {
2527
- if (entry.type !== "custom" || entry.customType !== "subagents:managed-spawn" || !isRecord(entry.data)) return;
2528
- const data = entry.data;
2529
- const allowedKeys = new Set([
2530
- "schemaVersion",
2531
- "spawnKey",
2532
- "fingerprint",
2533
- "id",
2534
- "requestId",
2535
- "type",
2536
- "description",
2537
- "owner",
2538
- "tier",
2539
- "tierSnapshot",
2540
- "state",
2541
- "createdAt",
2542
- "updatedAt",
2543
- "compactionCount",
2544
- "terminal",
2545
- ]);
2546
- if (Object.keys(data).some((key) => !allowedKeys.has(key))) return;
2547
- if (data.schemaVersion !== 1 || (data.state !== "completed" && data.state !== "failed" && data.state !== "stopped"))
2548
- return;
2549
- const tier = data.tier;
2550
- if (tier !== undefined && (typeof tier !== "string" || !WORKFLOW_TIER_VALUES.has(tier))) return;
2551
- if (data.tierSnapshot !== undefined && (typeof tier !== "string" || !isValidTierSnapshot(data.tierSnapshot, tier)))
2552
- return;
2553
- const bounded = (value: unknown, max: number): string | undefined =>
2554
- typeof value === "string" && value.length > 0 && value.length <= max && value.trim() === value
2555
- ? value
2556
- : undefined;
2557
- const spawnKey = bounded(data.spawnKey, 256);
2558
- const id = bounded(data.id, 128);
2559
- const fingerprint = bounded(data.fingerprint, 128);
2560
- const requestId = bounded(data.requestId, 128);
2561
- const type = bounded(data.type, 128);
2562
- const description = bounded(data.description, 512);
2563
- if (!spawnKey || !id || !fingerprint || !requestId || !type || !description || !isRecord(data.owner)) return;
2564
- const ownerKeys = new Set(["extension", "runId", "nodeId", "attemptId"]);
2565
- if (Object.keys(data.owner).some((key) => !ownerKeys.has(key))) return;
2566
- const ownerRunId = bounded(data.owner.runId, 256);
2567
- const ownerNodeId = bounded(data.owner.nodeId, 256);
2568
- const ownerAttemptId = bounded(data.owner.attemptId, 256);
2569
- if (data.owner.extension !== "pi-workflows" || !ownerRunId || !ownerNodeId || !ownerAttemptId) return;
2570
- const run = this.runs.get(ownerRunId);
2571
- if (!run || isTerminalWorkflow(run.status)) return;
2572
- const isSynthesis = ownerNodeId === SYNTHESIS_NODE_ID;
2573
- if (!isSynthesis && !(ownerNodeId in run.taskStatus)) return;
2574
- if (isSynthesis && !run.definition.synthesis) return;
2575
- const currentAttemptId = run.attemptIds[ownerNodeId];
2576
- const generation = run.attempts[ownerNodeId];
2577
- if (!currentAttemptId || !Number.isSafeInteger(generation) || generation < 1 || ownerAttemptId !== currentAttemptId)
2578
- return;
2579
- if (currentAttemptId !== `${run.runId}/${ownerNodeId}/attempt-${generation}`) return;
2580
- const expectedSpawnKeys = new Set([
2581
- currentAttemptId,
2582
- `${run.runId}/${ownerNodeId}/${currentAttemptId}`,
2583
- `${run.runId}/${ownerNodeId}/attempt-${generation}`,
2584
- `${run.runId}:${ownerNodeId}:${currentAttemptId}`,
2585
- ]);
2586
- if (!expectedSpawnKeys.has(spawnKey)) return;
2587
- const currentAgentId = isSynthesis ? run.synthesisAgentId : run.agentIds[ownerNodeId];
2588
- if (currentAgentId && currentAgentId !== id) return;
2589
- if (isSynthesis ? run.synthesisResult : run.taskResults[ownerNodeId]) return;
2590
- if (!isSynthesis && isTerminalTask(run.taskStatus[ownerNodeId])) return;
2591
-
2592
- const terminal = data.terminal;
2593
- if (!isRecord(terminal)) return;
2594
- const terminalKeys = new Set([
2595
- "status",
2596
- "result",
2597
- "error",
2598
- "outputFile",
2599
- "tokenCount",
2600
- "compactionCount",
2601
- "completedAt",
2602
- ]);
2603
- if (Object.keys(terminal).some((key) => !terminalKeys.has(key))) return;
2604
- if (terminal.status !== data.state) return;
2605
- const resultText = terminal.result === undefined ? undefined : bounded(terminal.result, 8_000);
2606
- const errorText = terminal.error === undefined ? undefined : bounded(terminal.error, 2_000);
2607
- const outputFile = terminal.outputFile === undefined ? undefined : bounded(terminal.outputFile, 2_000);
2608
- if (
2609
- (terminal.result !== undefined && !resultText) ||
2610
- (terminal.error !== undefined && !errorText) ||
2611
- (terminal.outputFile !== undefined && !outputFile)
2612
- )
2613
- return;
2614
- const compactionCount = terminal.compactionCount;
2615
- const persistedCompactionCount = data.compactionCount;
2616
- const completedAt = terminal.completedAt;
2617
- const createdAt = data.createdAt;
2618
- const updatedAt = data.updatedAt;
2619
- if (
2620
- typeof compactionCount !== "number" ||
2621
- !Number.isSafeInteger(compactionCount) ||
2622
- compactionCount < 0 ||
2623
- typeof persistedCompactionCount !== "number" ||
2624
- !Number.isSafeInteger(persistedCompactionCount) ||
2625
- persistedCompactionCount !== compactionCount ||
2626
- typeof completedAt !== "number" ||
2627
- !Number.isFinite(completedAt) ||
2628
- completedAt < 0 ||
2629
- typeof createdAt !== "number" ||
2630
- !Number.isFinite(createdAt) ||
2631
- createdAt < 0 ||
2632
- typeof updatedAt !== "number" ||
2633
- !Number.isFinite(updatedAt) ||
2634
- updatedAt < createdAt ||
2635
- completedAt < createdAt ||
2636
- completedAt > updatedAt
2637
- )
2638
- return;
2639
- const tokenCount = terminal.tokenCount;
2640
- if (
2641
- tokenCount !== undefined &&
2642
- (typeof tokenCount !== "number" || !Number.isSafeInteger(tokenCount) || tokenCount < 0)
2643
- )
2644
- return;
2645
- const status = data.state === "completed" ? "completed" : data.state === "stopped" ? "stopped" : "failed";
2646
- const result = resultFromLifecycle(
2647
- status,
2648
- id,
2649
- resultText,
2650
- errorText,
2651
- compactionCount,
2652
- completedAt,
2653
- outputFile,
2654
- tokenCount,
2655
- );
2656
- result.attemptId = currentAttemptId;
2657
- return {
2658
- kind: "terminal",
2659
- runId: run.runId,
2660
- nodeId: ownerNodeId,
2661
- attemptId: currentAttemptId,
2662
- result,
2663
- owner: { extension: "pi-workflows", runId: run.runId, nodeId: ownerNodeId, attemptId: currentAttemptId },
2664
- };
2665
- }
2666
- private reconcileSubagentRecord(
2667
- entry: SessionEntryLike,
2668
- ): DeferredInterruptedRecord | DeferredTerminalRecord | undefined {
2669
- if (entry.type !== "custom" || entry.customType !== "subagents:record" || !isRecord(entry.data)) return;
2670
- const owner = ownerFor(entry.data);
2671
- if (!owner) return;
2672
- const run = this.runs.get(owner.runId);
2673
- if (!run) return;
2674
- if (isTerminalWorkflow(run.status)) return;
2675
- if (owner.nodeId !== SYNTHESIS_NODE_ID && !(owner.nodeId in run.taskStatus)) return;
2676
-
2677
- const currentAttemptId = run.attemptIds[owner.nodeId];
2678
- const recordAgentId = typeof entry.data.id === "string" ? entry.data.id : undefined;
2679
- if (!this.acceptsLifecycleEvent(run, owner.nodeId, "subagents:record", owner.attemptId, recordAgentId)) return;
2680
- const currentAgentId = owner.nodeId === SYNTHESIS_NODE_ID ? run.synthesisAgentId : run.agentIds[owner.nodeId];
2681
- const status = entry.data.status;
2682
- if (
2683
- status !== "completed" &&
2684
- status !== "steered" &&
2685
- status !== "error" &&
2686
- status !== "stopped" &&
2687
- status !== "aborted" &&
2688
- status !== "interrupted"
2689
- )
2690
- return;
2691
-
2692
- // An interrupted record is not a failed task result. It means the previous
2693
- // AgentSession disappeared during reload. Legacy records without an
2694
- // attemptId are accepted only while their agent id is still the current
2695
- // journaled agent; after a recovery event clears that id, the same old
2696
- // record is ignored on every later restore.
2697
- if (status === "interrupted") {
2698
- if (run.terminalIntent) return;
2699
- if (!owner.attemptId && (!recordAgentId || !currentAgentId || recordAgentId !== currentAgentId)) return;
2700
- if (owner.nodeId === SYNTHESIS_NODE_ID ? run.synthesisResult : run.taskResults[owner.nodeId]) return;
2701
- return {
2702
- kind: "interrupted",
2703
- runId: run.runId,
2704
- nodeId: owner.nodeId,
2705
- oldAgentId: recordAgentId ?? currentAgentId ?? "",
2706
- };
2707
- }
2708
-
2709
- const resultStatus =
2710
- status === "completed" || status === "steered" ? "completed" : status === "stopped" ? "stopped" : "failed";
2711
- if (owner.nodeId === SYNTHESIS_NODE_ID ? run.synthesisResult : run.taskResults[owner.nodeId]) return;
2712
- if (!currentAttemptId) return;
2713
- const result = resultFromLifecycle(
2714
- resultStatus,
2715
- recordAgentId ?? currentAgentId,
2716
- entry.data.result,
2717
- entry.data.error,
2718
- run.compactions[owner.nodeId] ?? 0,
2719
- Date.now(),
2720
- entry.data.outputFile,
2721
- typeof entry.data.tokens === "number" ? entry.data.tokens : undefined,
2722
- );
2723
- result.attemptId = currentAttemptId;
2724
- return {
2725
- kind: "terminal",
2726
- runId: run.runId,
2727
- nodeId: owner.nodeId,
2728
- attemptId: currentAttemptId,
2729
- result,
2730
- owner: {
2731
- extension: "pi-workflows",
2732
- runId: run.runId,
2733
- nodeId: owner.nodeId,
2734
- attemptId: currentAttemptId,
2735
- },
2736
- };
2737
- }
2738
-
2739
- private resolveWaiters(run: WorkflowRun): void {
2740
- const waiters = this.waiters.get(run.runId);
2741
- if (!waiters) return;
2742
- this.waiters.delete(run.runId);
2743
- for (const waiter of waiters) {
2744
- if (waiter.settled) continue;
2745
- waiter.settled = true;
2746
- waiter.cleanup();
2747
- waiter.resolve(run);
2748
- }
2749
- }
2750
-
2751
- private summary(run: WorkflowRun, includeDetails = false): WorkflowSummary {
2752
- const statuses = Object.values(run.taskStatus);
2753
- const completedTasks = statuses.filter((status) => status === "completed").length;
2754
- const failedTasks = statuses.filter(
2755
- (status) => status === "failed" || status === "blocked" || status === "stopped",
2756
- ).length;
2757
- const activeTasks = statuses.filter(activeStatus).length;
2758
- const phaseCompletion = run.definition.phases.map((phase) => {
2759
- const phaseTasks = run.definition.tasks.filter((task) => task.phase === phase.id);
2760
- return phaseTasks.length > 0 && phaseTasks.every((task) => run.taskStatus[task.id] === "completed");
2761
- });
2762
- const completedPhases = phaseCompletion.filter(Boolean).length;
2763
- const tokenValues = [...Object.values(run.taskResults), ...(run.synthesisResult ? [run.synthesisResult] : [])]
2764
- .map((result) => result.tokenCount)
2765
- .filter((value): value is number => value !== undefined);
2766
- const tokenCount = tokenValues.length > 0 ? tokenValues.reduce((total, value) => total + value, 0) : undefined;
2767
- const taskDetails = includeDetails
2768
- ? run.definition.tasks.map((task) => ({
2769
- id: task.id,
2770
- status: run.taskStatus[task.id] ?? "pending",
2771
- ...(run.agentIds[task.id] ? { agentId: run.agentIds[task.id] } : {}),
2772
- compactions: run.compactions[task.id] ?? 0,
2773
- ...resultPreview(run.taskResults[task.id]),
2774
- }))
2775
- : undefined;
2776
- const synthesisDetails =
2777
- includeDetails && run.synthesisResult
2778
- ? {
2779
- status: run.synthesisResult.status,
2780
- ...(run.synthesisAgentId ? { agentId: run.synthesisAgentId } : {}),
2781
- compactions: run.compactions[SYNTHESIS_NODE_ID] ?? 0,
2782
- ...resultPreview(run.synthesisResult),
2783
- }
2784
- : undefined;
2785
- return {
2786
- runId: run.runId,
2787
- name: run.definition.name,
2788
- ...(run.definition.description ? { description: run.definition.description } : {}),
2789
- status: run.status,
2790
- startedAt: run.startedAt,
2791
- updatedAt: run.updatedAt,
2792
- elapsedMs: Math.max(
2793
- 0,
2794
- (run.status === "completed" || run.status === "failed" || run.status === "stopped"
2795
- ? run.updatedAt
2796
- : Date.now()) - run.startedAt,
2797
- ),
2798
- taskCount: run.definition.tasks.length,
2799
- phaseCount: run.definition.phases.length,
2800
- completedPhases,
2801
- completedTasks,
2802
- failedTasks,
2803
- activeTasks,
2804
- ...(tokenCount === undefined ? {} : { tokenCount }),
2805
- ...(run.synthesisResult?.text ? { synthesisPreview: run.synthesisResult.text.slice(0, 500) } : {}),
2806
- ...(run.error ? { error: run.error.slice(0, 2_000) } : {}),
2807
- ...(run.nonResumable ? { nonResumable: true } : {}),
2808
- ...(taskDetails ? { tasks: taskDetails } : {}),
2809
- ...(synthesisDetails ? { synthesis: synthesisDetails } : {}),
2810
- };
1401
+ export class WorkflowWaitAbortedError extends Error {
1402
+ constructor() {
1403
+ super("workflow wait aborted");
1404
+ this.name = "WorkflowWaitAbortedError";
2811
1405
  }
2812
1406
  }
1407
+
1408
+ export type { ScriptRun, WorkflowOwner };
1409
+ export { createManagedSpawnClient, JOURNAL_ENTRY_TYPE, snapshotRun };