@signalridge/pi-workflows 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +23 -0
- package/LICENSE +21 -0
- package/README.md +39 -0
- package/package.json +56 -0
- package/src/engine.ts +2760 -0
- package/src/index.ts +326 -0
- package/src/journal.ts +1584 -0
- package/src/rpc-client.ts +364 -0
- package/src/schema.ts +186 -0
- package/src/state-machine.ts +73 -0
- package/src/synthesis.ts +95 -0
- package/src/ui.ts +142 -0
package/src/engine.ts
ADDED
|
@@ -0,0 +1,2760 @@
|
|
|
1
|
+
import { randomUUID } from "node:crypto";
|
|
2
|
+
import {
|
|
3
|
+
type AttemptRecoveryEvent,
|
|
4
|
+
applyRecoveryEvent,
|
|
5
|
+
applyTerminalRecoveryEvent,
|
|
6
|
+
boundedError,
|
|
7
|
+
deriveRecoveryId,
|
|
8
|
+
JOURNAL_SCHEMA_VERSION,
|
|
9
|
+
type JournalEvent,
|
|
10
|
+
type JournalWriter,
|
|
11
|
+
type RecoveryRotation,
|
|
12
|
+
type RecoverySourceStatus,
|
|
13
|
+
type RecoveryTerminalResult,
|
|
14
|
+
type RunRecoveryEvent,
|
|
15
|
+
replayJournal,
|
|
16
|
+
type SessionEntryLike,
|
|
17
|
+
snapshotRun,
|
|
18
|
+
type TerminalRecoveryEvent,
|
|
19
|
+
type WorkflowOwner,
|
|
20
|
+
type WorkflowRun,
|
|
21
|
+
} from "./journal.js";
|
|
22
|
+
import type {
|
|
23
|
+
ManagedSpawnClient,
|
|
24
|
+
ManagedSpawnResponse,
|
|
25
|
+
ManagedTerminalSnapshot,
|
|
26
|
+
WorkflowEventBus,
|
|
27
|
+
} from "./rpc-client.js";
|
|
28
|
+
import { registerLifecycleListener } from "./rpc-client.js";
|
|
29
|
+
import { readyTaskIds, type WorkflowDefinition, type WorkflowTask } from "./schema.js";
|
|
30
|
+
import {
|
|
31
|
+
canTransitionTask,
|
|
32
|
+
isTerminalTask,
|
|
33
|
+
isTerminalWorkflow,
|
|
34
|
+
type TaskStatus,
|
|
35
|
+
transitionTask,
|
|
36
|
+
transitionWorkflow,
|
|
37
|
+
type WorkflowStatus,
|
|
38
|
+
} from "./state-machine.js";
|
|
39
|
+
import { buildSynthesisPrompt, resultFromLifecycle } from "./synthesis.js";
|
|
40
|
+
|
|
41
|
+
const SYNTHESIS_NODE_ID = "__synthesis__";
|
|
42
|
+
const MAX_WAITERS = 256;
|
|
43
|
+
const MAX_ATTEMPTS_PER_NODE = 3;
|
|
44
|
+
const BRANCH_QUIESCE_TIMEOUT_MS = 5_000;
|
|
45
|
+
const MAX_QUARANTINED_AGENT_IDS = 4_096;
|
|
46
|
+
const MAX_RECOVERY_MEMO_ENTRIES = 4_096;
|
|
47
|
+
|
|
48
|
+
export interface WorkflowQuiesceResult {
|
|
49
|
+
settled: boolean;
|
|
50
|
+
pending: string[];
|
|
51
|
+
diagnostic?: string;
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
export interface WorkflowSummary {
|
|
55
|
+
runId: string;
|
|
56
|
+
name: string;
|
|
57
|
+
description?: string;
|
|
58
|
+
status: WorkflowStatus;
|
|
59
|
+
startedAt: number;
|
|
60
|
+
updatedAt: number;
|
|
61
|
+
elapsedMs: number;
|
|
62
|
+
taskCount: number;
|
|
63
|
+
phaseCount: number;
|
|
64
|
+
completedPhases: number;
|
|
65
|
+
completedTasks: number;
|
|
66
|
+
failedTasks: number;
|
|
67
|
+
activeTasks: number;
|
|
68
|
+
tokenCount?: number;
|
|
69
|
+
synthesisPreview?: string;
|
|
70
|
+
error?: string;
|
|
71
|
+
nonResumable?: boolean;
|
|
72
|
+
tasks?: WorkflowTaskDetail[];
|
|
73
|
+
synthesis?: WorkflowSynthesisDetail;
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
export interface WorkflowTaskDetail {
|
|
77
|
+
id: string;
|
|
78
|
+
status: TaskStatus;
|
|
79
|
+
agentId?: string;
|
|
80
|
+
compactions: number;
|
|
81
|
+
resultPreview?: string;
|
|
82
|
+
errorPreview?: string;
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
export interface WorkflowSynthesisDetail {
|
|
86
|
+
status: "completed" | "failed" | "stopped";
|
|
87
|
+
agentId?: string;
|
|
88
|
+
compactions: number;
|
|
89
|
+
resultPreview?: string;
|
|
90
|
+
errorPreview?: string;
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
export interface WorkflowControlResult {
|
|
94
|
+
action: "list" | "get" | "pause" | "resume" | "stop";
|
|
95
|
+
run?: WorkflowSummary;
|
|
96
|
+
runs?: WorkflowSummary[];
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
export interface WorkflowStartResult {
|
|
100
|
+
runId: string;
|
|
101
|
+
status: WorkflowStatus;
|
|
102
|
+
background: boolean;
|
|
103
|
+
result?: string;
|
|
104
|
+
error?: string;
|
|
105
|
+
waitAborted?: boolean;
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
interface Waiter {
|
|
109
|
+
resolve: (run: WorkflowRun) => void;
|
|
110
|
+
reject: (error: Error) => void;
|
|
111
|
+
cleanup: () => void;
|
|
112
|
+
settled: boolean;
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
type RecoveryEvent = RunRecoveryEvent | AttemptRecoveryEvent;
|
|
116
|
+
|
|
117
|
+
interface RecoveryMemo {
|
|
118
|
+
branchGeneration: number;
|
|
119
|
+
runId: string;
|
|
120
|
+
key: string;
|
|
121
|
+
event: RecoveryEvent;
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
interface TerminalRecoveryMemo {
|
|
125
|
+
branchGeneration: number;
|
|
126
|
+
runId: string;
|
|
127
|
+
key: string;
|
|
128
|
+
event: TerminalRecoveryEvent;
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
interface RecoveryPlanItem {
|
|
132
|
+
nodeId: string;
|
|
133
|
+
sourceAttemptId: string;
|
|
134
|
+
sourceGeneration: number;
|
|
135
|
+
sourceStatus: RecoverySourceStatus;
|
|
136
|
+
oldAgentId: string;
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
interface DeferredInterruptedRecord {
|
|
140
|
+
kind: "interrupted";
|
|
141
|
+
runId: string;
|
|
142
|
+
nodeId: string;
|
|
143
|
+
oldAgentId: string;
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
interface DeferredTerminalRecord extends RecoveryTerminalResult {
|
|
147
|
+
kind: "terminal";
|
|
148
|
+
runId: string;
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
interface TerminalCleanupTarget {
|
|
152
|
+
agentId: string;
|
|
153
|
+
owner: WorkflowOwner;
|
|
154
|
+
fallbackStopRequested: boolean;
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
interface EngineRestoreSnapshot {
|
|
158
|
+
runs: Map<string, WorkflowRun>;
|
|
159
|
+
recoveryMemo: Map<string, RecoveryMemo>;
|
|
160
|
+
terminalRecoveryMemo: Map<string, TerminalRecoveryMemo>;
|
|
161
|
+
recoveryBranchGeneration: number;
|
|
162
|
+
branchEpoch: number;
|
|
163
|
+
branchChanging: boolean;
|
|
164
|
+
branchEventsClosed: boolean;
|
|
165
|
+
lifecycleSuspended: boolean;
|
|
166
|
+
quarantinedAgentIds: Set<string>;
|
|
167
|
+
}
|
|
168
|
+
export class WorkflowWaitAbortedError extends Error {
|
|
169
|
+
constructor() {
|
|
170
|
+
super("foreground workflow wait aborted; the workflow run continues");
|
|
171
|
+
this.name = "WorkflowWaitAbortedError";
|
|
172
|
+
}
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
function isRecord(value: unknown): value is Record<string, unknown> {
|
|
176
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
const WORKFLOW_TIER_VALUES = new Set(["small", "medium", "large"]);
|
|
180
|
+
const WORKFLOW_THINKING_LEVELS = new Set(["minimal", "low", "medium", "high", "xhigh", "max"]);
|
|
181
|
+
const WORKFLOW_CONFIGURED_THINKING = new Set([...WORKFLOW_THINKING_LEVELS, "inherit"]);
|
|
182
|
+
const TIER_SNAPSHOT_KEYS = new Set([
|
|
183
|
+
"tier",
|
|
184
|
+
"model",
|
|
185
|
+
"thinking",
|
|
186
|
+
"configuredModel",
|
|
187
|
+
"configuredThinking",
|
|
188
|
+
"requestedThinking",
|
|
189
|
+
"modelSource",
|
|
190
|
+
"thinkingSource",
|
|
191
|
+
"clamped",
|
|
192
|
+
"diagnostic",
|
|
193
|
+
]);
|
|
194
|
+
|
|
195
|
+
function isValidTierSnapshot(value: unknown, tier: string): boolean {
|
|
196
|
+
if (!isRecord(value) || value.tier !== tier || !WORKFLOW_TIER_VALUES.has(tier)) return false;
|
|
197
|
+
if (Object.keys(value).some((key) => !TIER_SNAPSHOT_KEYS.has(key))) return false;
|
|
198
|
+
for (const key of ["model", "configuredModel"] as const) {
|
|
199
|
+
const field = value[key];
|
|
200
|
+
if (field !== undefined && (typeof field !== "string" || !field.trim() || field.length > 2_000)) return false;
|
|
201
|
+
if (key === "model" && typeof field === "string" && !field.includes("/")) return false;
|
|
202
|
+
}
|
|
203
|
+
for (const key of ["thinking", "requestedThinking"] as const) {
|
|
204
|
+
const field = value[key];
|
|
205
|
+
if (field !== undefined && (typeof field !== "string" || !WORKFLOW_THINKING_LEVELS.has(field))) return false;
|
|
206
|
+
}
|
|
207
|
+
const configuredThinking = value.configuredThinking;
|
|
208
|
+
if (
|
|
209
|
+
configuredThinking !== undefined &&
|
|
210
|
+
(typeof configuredThinking !== "string" || !WORKFLOW_CONFIGURED_THINKING.has(configuredThinking))
|
|
211
|
+
)
|
|
212
|
+
return false;
|
|
213
|
+
if (value.modelSource !== "frontmatter" && value.modelSource !== "tier" && value.modelSource !== "parent")
|
|
214
|
+
return false;
|
|
215
|
+
if (value.thinkingSource !== "frontmatter" && value.thinkingSource !== "tier" && value.thinkingSource !== "parent")
|
|
216
|
+
return false;
|
|
217
|
+
if (value.clamped !== undefined && typeof value.clamped !== "boolean") return false;
|
|
218
|
+
if (value.diagnostic !== undefined && (typeof value.diagnostic !== "string" || value.diagnostic.length > 2_000))
|
|
219
|
+
return false;
|
|
220
|
+
return true;
|
|
221
|
+
}
|
|
222
|
+
|
|
223
|
+
function isAbortSignal(value: unknown): value is AbortSignal {
|
|
224
|
+
return (
|
|
225
|
+
isRecord(value) &&
|
|
226
|
+
typeof value.aborted === "boolean" &&
|
|
227
|
+
typeof value.addEventListener === "function" &&
|
|
228
|
+
typeof value.removeEventListener === "function"
|
|
229
|
+
);
|
|
230
|
+
}
|
|
231
|
+
|
|
232
|
+
function ownerFor(data: Record<string, unknown>): { runId: string; nodeId: string; attemptId?: string } | undefined {
|
|
233
|
+
const owner = data.owner;
|
|
234
|
+
if (!isRecord(owner) || owner.extension !== "pi-workflows") return undefined;
|
|
235
|
+
const validId = (value: unknown): value is string =>
|
|
236
|
+
typeof value === "string" && value.length > 0 && value.length <= 256;
|
|
237
|
+
if (!validId(owner.runId) || !validId(owner.nodeId)) return undefined;
|
|
238
|
+
if (owner.attemptId !== undefined && !validId(owner.attemptId)) return undefined;
|
|
239
|
+
return {
|
|
240
|
+
runId: owner.runId,
|
|
241
|
+
nodeId: owner.nodeId,
|
|
242
|
+
...(typeof owner.attemptId === "string" ? { attemptId: owner.attemptId } : {}),
|
|
243
|
+
};
|
|
244
|
+
}
|
|
245
|
+
|
|
246
|
+
function statusFromLifecycle(eventName: string, data: Record<string, unknown>): "completed" | "failed" | "stopped" {
|
|
247
|
+
if (eventName === "subagents:completed") return "completed";
|
|
248
|
+
const status = data.status;
|
|
249
|
+
return status === "stopped" ? "stopped" : "failed";
|
|
250
|
+
}
|
|
251
|
+
|
|
252
|
+
function activeStatus(status: TaskStatus): boolean {
|
|
253
|
+
return status === "dispatching" || status === "queued" || status === "running";
|
|
254
|
+
}
|
|
255
|
+
|
|
256
|
+
function resultPreview(result: { text?: string; error?: string } | undefined): {
|
|
257
|
+
resultPreview?: string;
|
|
258
|
+
errorPreview?: string;
|
|
259
|
+
} {
|
|
260
|
+
return {
|
|
261
|
+
...(result?.text ? { resultPreview: result.text.slice(0, 2_000) } : {}),
|
|
262
|
+
...(result?.error ? { errorPreview: result.error.slice(0, 2_000) } : {}),
|
|
263
|
+
};
|
|
264
|
+
}
|
|
265
|
+
|
|
266
|
+
function tokenCountFrom(value: unknown): number | undefined {
|
|
267
|
+
if (!isRecord(value) || typeof value.total !== "number" || !Number.isFinite(value.total) || value.total <= 0) {
|
|
268
|
+
return undefined;
|
|
269
|
+
}
|
|
270
|
+
return Math.floor(value.total);
|
|
271
|
+
}
|
|
272
|
+
|
|
273
|
+
function normalizeManagedSpawnResponse(response: ManagedSpawnResponse | string): ManagedSpawnResponse {
|
|
274
|
+
return typeof response === "string"
|
|
275
|
+
? { id: response, state: "running", created: true }
|
|
276
|
+
: { ...response, created: response.created ?? true };
|
|
277
|
+
}
|
|
278
|
+
|
|
279
|
+
function managedTerminalStatus(snapshot: ManagedTerminalSnapshot): "completed" | "failed" | "stopped" {
|
|
280
|
+
if (snapshot.status === "completed") return "completed";
|
|
281
|
+
if (snapshot.status === "stopped") return "stopped";
|
|
282
|
+
return "failed";
|
|
283
|
+
}
|
|
284
|
+
|
|
285
|
+
class JournalAppendError extends Error {
|
|
286
|
+
constructor(eventKind: JournalEvent["kind"], cause: unknown) {
|
|
287
|
+
super(`${eventKind} append failed: ${cause instanceof Error ? cause.message : String(cause)}`);
|
|
288
|
+
this.name = "JournalAppendError";
|
|
289
|
+
}
|
|
290
|
+
}
|
|
291
|
+
|
|
292
|
+
const JOURNAL_RETRY_INITIAL_DELAY_MS = 25;
|
|
293
|
+
const JOURNAL_RETRY_MAX_DELAY_MS = 2_000;
|
|
294
|
+
|
|
295
|
+
function spawnManaged(
|
|
296
|
+
client: ManagedSpawnClient,
|
|
297
|
+
task: WorkflowTask,
|
|
298
|
+
runId: string,
|
|
299
|
+
nodeId: string,
|
|
300
|
+
attemptId: string | undefined,
|
|
301
|
+
): Promise<ManagedSpawnResponse | string> {
|
|
302
|
+
// Protocol-v3 clients receive the attempt as the fourth argument. Keep the
|
|
303
|
+
// old in-process fixture shape working without passing an ExtensionContext
|
|
304
|
+
// back into the engine.
|
|
305
|
+
if (client.spawn.length >= 5) {
|
|
306
|
+
return client.spawn(task, runId, nodeId, undefined, attemptId);
|
|
307
|
+
}
|
|
308
|
+
return client.spawn(task, runId, nodeId, attemptId);
|
|
309
|
+
}
|
|
310
|
+
|
|
311
|
+
export class WorkflowEngine {
|
|
312
|
+
private readonly runs = new Map<string, WorkflowRun>();
|
|
313
|
+
/** Managed RPC calls that have not settled yet, including dispatching tasks. */
|
|
314
|
+
private readonly dispatches = new Map<string, Set<Promise<void>>>();
|
|
315
|
+
/** Late managed spawn responses discovered while terminal cleanup waits. */
|
|
316
|
+
private readonly lateDispatchTargets = new Map<string, TerminalCleanupTarget[]>();
|
|
317
|
+
/** Late branch-replacement spawns whose owner-scoped quiescence did not settle. */
|
|
318
|
+
private readonly lateDispatchQuiescence = new Map<string, string[]>();
|
|
319
|
+
private readonly waiters = new Map<string, Set<Waiter>>();
|
|
320
|
+
private readonly pumping = new Set<string>();
|
|
321
|
+
private readonly settling = new Map<string, Promise<void>>();
|
|
322
|
+
private unsubscribeLifecycle: () => void;
|
|
323
|
+
private lifecycleSuspended = false;
|
|
324
|
+
private branchChanging = false;
|
|
325
|
+
private branchEpoch = 0;
|
|
326
|
+
private branchEventsClosed = false;
|
|
327
|
+
private branchQuiescePromise: Promise<WorkflowQuiesceResult> | undefined;
|
|
328
|
+
/** Agent ids quarantined across a branch replacement; never accept late events. */
|
|
329
|
+
private readonly quarantinedAgentIds = new Set<string>();
|
|
330
|
+
private recoveryBranchGeneration = 0;
|
|
331
|
+
/**
|
|
332
|
+
* Durable recovery rotations already emitted for this engine/branch. The
|
|
333
|
+
* key is semantic (branch, run, node, source attempt), never a journal
|
|
334
|
+
* payload hash. A bounded map prevents repeated reloads from retaining
|
|
335
|
+
* unbounded history.
|
|
336
|
+
*/
|
|
337
|
+
private readonly recoveryMemo = new Map<string, RecoveryMemo>();
|
|
338
|
+
/** Terminal cleanup facts already emitted for a stale branch snapshot. */
|
|
339
|
+
private readonly terminalRecoveryMemo = new Map<string, TerminalRecoveryMemo>();
|
|
340
|
+
/** Candidate replay state used to roll back a failed recovery append. */
|
|
341
|
+
private restoreFailureSnapshot: EngineRestoreSnapshot | undefined;
|
|
342
|
+
/** Lifecycle journal writes that failed transiently and are waiting to retry. */
|
|
343
|
+
private readonly journalRetries = new Map<string, { timer?: ReturnType<typeof setTimeout>; attempt: number }>();
|
|
344
|
+
/** Prefixes durably written before a lifecycle batch append failed. */
|
|
345
|
+
private readonly journalBatchProgress = new Map<string, { serialized: string; nextIndex: number }>();
|
|
346
|
+
/** Runs whose lifecycle journal cannot be made durable after bounded retries. */
|
|
347
|
+
private readonly journalBlockedRuns = new Set<string>();
|
|
348
|
+
private journalMutationKey: string | undefined;
|
|
349
|
+
|
|
350
|
+
constructor(
|
|
351
|
+
readonly events: WorkflowEventBus,
|
|
352
|
+
private readonly client: ManagedSpawnClient,
|
|
353
|
+
private readonly journal: JournalWriter,
|
|
354
|
+
) {
|
|
355
|
+
this.unsubscribeLifecycle = registerLifecycleListener(events, (eventName, data) => {
|
|
356
|
+
this.onLifecycle(eventName, data);
|
|
357
|
+
});
|
|
358
|
+
}
|
|
359
|
+
|
|
360
|
+
dispose(): void {
|
|
361
|
+
this.branchEpoch += 1;
|
|
362
|
+
this.clearJournalRetries();
|
|
363
|
+
this.unsubscribeLifecycle();
|
|
364
|
+
this.quarantinedAgentIds.clear();
|
|
365
|
+
this.recoveryMemo.clear();
|
|
366
|
+
this.terminalRecoveryMemo.clear();
|
|
367
|
+
this.lateDispatchTargets.clear();
|
|
368
|
+
this.lateDispatchQuiescence.clear();
|
|
369
|
+
this.recoveryBranchGeneration = 0;
|
|
370
|
+
for (const waiters of this.waiters.values()) {
|
|
371
|
+
for (const waiter of waiters) {
|
|
372
|
+
if (waiter.settled) continue;
|
|
373
|
+
waiter.settled = true;
|
|
374
|
+
waiter.cleanup();
|
|
375
|
+
waiter.reject(new Error("workflow engine disposed"));
|
|
376
|
+
}
|
|
377
|
+
}
|
|
378
|
+
this.waiters.clear();
|
|
379
|
+
}
|
|
380
|
+
|
|
381
|
+
suspendLifecycle(): void {
|
|
382
|
+
if (this.lifecycleSuspended) return;
|
|
383
|
+
this.lifecycleSuspended = true;
|
|
384
|
+
this.unsubscribeLifecycle();
|
|
385
|
+
}
|
|
386
|
+
|
|
387
|
+
resumeLifecycle(): void {
|
|
388
|
+
if (!this.lifecycleSuspended) return;
|
|
389
|
+
this.lifecycleSuspended = false;
|
|
390
|
+
this.unsubscribeLifecycle = registerLifecycleListener(this.events, (eventName, data) => {
|
|
391
|
+
this.onLifecycle(eventName, data);
|
|
392
|
+
});
|
|
393
|
+
}
|
|
394
|
+
|
|
395
|
+
/** Stop owned agents while Pi replaces the active session-tree branch. */
|
|
396
|
+
async quiesceForBranchChange(): Promise<WorkflowQuiesceResult> {
|
|
397
|
+
if (this.branchQuiescePromise) return this.branchQuiescePromise;
|
|
398
|
+
this.branchChanging = true;
|
|
399
|
+
this.clearJournalRetries();
|
|
400
|
+
this.lateDispatchTargets.clear();
|
|
401
|
+
this.lateDispatchQuiescence.clear();
|
|
402
|
+
this.branchEpoch += 1;
|
|
403
|
+
this.branchEventsClosed = false;
|
|
404
|
+
this.branchQuiescePromise = (async () => {
|
|
405
|
+
const operations: Array<Promise<{ runId: string; result: WorkflowQuiesceResult }>> = [];
|
|
406
|
+
for (const run of this.runs.values()) {
|
|
407
|
+
if (isTerminalWorkflow(run.status)) continue;
|
|
408
|
+
const activeNodes = Object.entries(run.taskStatus)
|
|
409
|
+
.filter(([, status]) => activeStatus(status))
|
|
410
|
+
.map(([nodeId]) => ({ nodeId, agentId: run.agentIds[nodeId] }))
|
|
411
|
+
.filter((item): item is { nodeId: string; agentId: string } => Boolean(item.agentId));
|
|
412
|
+
const agentIds = activeNodes.map(({ agentId }) => agentId);
|
|
413
|
+
const owners = activeNodes.map(({ nodeId }) => this.ownerForNode(run, nodeId));
|
|
414
|
+
const hasDispatches = (this.dispatches.get(run.runId)?.size ?? 0) > 0;
|
|
415
|
+
if (run.synthesisAgentId && !run.synthesisResult) {
|
|
416
|
+
agentIds.push(run.synthesisAgentId);
|
|
417
|
+
owners.push(this.ownerForNode(run, SYNTHESIS_NODE_ID));
|
|
418
|
+
}
|
|
419
|
+
if (agentIds.length === 0 && !hasDispatches) continue;
|
|
420
|
+
for (const agentId of agentIds) this.quarantineAgent(agentId);
|
|
421
|
+
const includeDispatches = (
|
|
422
|
+
result: WorkflowQuiesceResult,
|
|
423
|
+
waitForDispatches: boolean,
|
|
424
|
+
): Promise<{ runId: string; result: WorkflowQuiesceResult }> => {
|
|
425
|
+
const settled = waitForDispatches
|
|
426
|
+
? this.waitForDispatches(run.runId, BRANCH_QUIESCE_TIMEOUT_MS)
|
|
427
|
+
: Promise.resolve(true);
|
|
428
|
+
return settled.then((dispatchSettled) => {
|
|
429
|
+
const latePending = dispatchSettled ? this.takeLateDispatchQuiescence(run.runId) : [];
|
|
430
|
+
const diagnostic = [
|
|
431
|
+
result.diagnostic,
|
|
432
|
+
latePending.length > 0 ? `late spawn quiescence incomplete: ${latePending.join(", ")}` : undefined,
|
|
433
|
+
]
|
|
434
|
+
.filter((value): value is string => Boolean(value))
|
|
435
|
+
.join("; ");
|
|
436
|
+
return {
|
|
437
|
+
runId: run.runId,
|
|
438
|
+
result: {
|
|
439
|
+
settled: result.settled && dispatchSettled && latePending.length === 0,
|
|
440
|
+
pending: [
|
|
441
|
+
...new Set([
|
|
442
|
+
...result.pending,
|
|
443
|
+
...latePending,
|
|
444
|
+
...(dispatchSettled ? [] : [`${run.runId}:dispatch`]),
|
|
445
|
+
]),
|
|
446
|
+
].slice(0, 256),
|
|
447
|
+
...(diagnostic ? { diagnostic } : {}),
|
|
448
|
+
},
|
|
449
|
+
};
|
|
450
|
+
});
|
|
451
|
+
};
|
|
452
|
+
if (this.client.quiesceOwned && (agentIds.length > 0 || hasDispatches)) {
|
|
453
|
+
operations.push(
|
|
454
|
+
this.client
|
|
455
|
+
.quiesceOwned(run.runId, agentIds, BRANCH_QUIESCE_TIMEOUT_MS, owners)
|
|
456
|
+
.then((result) =>
|
|
457
|
+
includeDispatches(
|
|
458
|
+
{
|
|
459
|
+
settled: result.settled && result.pending.length === 0,
|
|
460
|
+
pending: [...new Set(result.pending)].slice(0, 256),
|
|
461
|
+
},
|
|
462
|
+
true,
|
|
463
|
+
),
|
|
464
|
+
)
|
|
465
|
+
.catch((error: unknown) =>
|
|
466
|
+
includeDispatches(
|
|
467
|
+
{
|
|
468
|
+
settled: false,
|
|
469
|
+
pending: agentIds.slice(0, 256),
|
|
470
|
+
diagnostic: `owned quiescence RPC failed: ${boundedError(error) ?? "unknown error"}`,
|
|
471
|
+
},
|
|
472
|
+
true,
|
|
473
|
+
),
|
|
474
|
+
),
|
|
475
|
+
);
|
|
476
|
+
} else {
|
|
477
|
+
const stopOperations = activeNodes.map(({ nodeId, agentId }) => this.stopOwned(run, nodeId, agentId));
|
|
478
|
+
if (run.synthesisAgentId && !run.synthesisResult) {
|
|
479
|
+
stopOperations.push(this.stopOwned(run, SYNTHESIS_NODE_ID, run.synthesisAgentId));
|
|
480
|
+
}
|
|
481
|
+
operations.push(
|
|
482
|
+
Promise.allSettled(stopOperations).then(() =>
|
|
483
|
+
includeDispatches(
|
|
484
|
+
{
|
|
485
|
+
settled: agentIds.length === 0 && !hasDispatches,
|
|
486
|
+
pending: [...agentIds, ...(hasDispatches ? [`${run.runId}:dispatch`] : [])].slice(0, 256),
|
|
487
|
+
diagnostic: hasDispatches
|
|
488
|
+
? "dispatch quiescence is unavailable; pending spawn calls were quarantined conservatively"
|
|
489
|
+
: agentIds.length > 0
|
|
490
|
+
? "owned quiescence is unavailable; stopped agents were quarantined conservatively"
|
|
491
|
+
: undefined,
|
|
492
|
+
},
|
|
493
|
+
true,
|
|
494
|
+
),
|
|
495
|
+
),
|
|
496
|
+
);
|
|
497
|
+
}
|
|
498
|
+
}
|
|
499
|
+
|
|
500
|
+
const observations = await Promise.all(operations);
|
|
501
|
+
const pending = [...new Set(observations.flatMap(({ result }) => result.pending))].slice(0, 256);
|
|
502
|
+
const diagnostics: string[] = [];
|
|
503
|
+
for (const { runId, result } of observations) {
|
|
504
|
+
if (result.diagnostic) diagnostics.push(result.diagnostic);
|
|
505
|
+
if (result.settled) continue;
|
|
506
|
+
const run = this.runs.get(runId);
|
|
507
|
+
if (!run || isTerminalWorkflow(run.status)) continue;
|
|
508
|
+
const diagnostic = `branch quiescence incomplete for workflow ${runId}: ${
|
|
509
|
+
result.pending.length > 0 ? `pending agents ${result.pending.join(", ")}` : "terminal state was not confirmed"
|
|
510
|
+
}${result.diagnostic ? ` (${result.diagnostic})` : ""}`;
|
|
511
|
+
diagnostics.push(diagnostic);
|
|
512
|
+
try {
|
|
513
|
+
this.setWorkflowStatus(run, run.status, boundedError(diagnostic) ?? diagnostic);
|
|
514
|
+
} catch (error: unknown) {
|
|
515
|
+
diagnostics.push(
|
|
516
|
+
`workflow quiescence diagnostic was not journaled: ${boundedError(error) ?? "unknown error"}`,
|
|
517
|
+
);
|
|
518
|
+
}
|
|
519
|
+
}
|
|
520
|
+
const diagnostic = boundedError(diagnostics.join("; ")) ?? undefined;
|
|
521
|
+
const result: WorkflowQuiesceResult = {
|
|
522
|
+
settled: pending.length === 0 && observations.every(({ result: item }) => item.settled),
|
|
523
|
+
pending,
|
|
524
|
+
...(diagnostic ? { diagnostic } : {}),
|
|
525
|
+
};
|
|
526
|
+
// After this point no lifecycle event from the old branch is accepted.
|
|
527
|
+
// Events are already ignored while branchChanging; the subagents manager
|
|
528
|
+
// detaches timed-out records before returning, and the generation gate
|
|
529
|
+
// below also protects fixtures and legacy RPC peers without quiescence.
|
|
530
|
+
this.branchEventsClosed = true;
|
|
531
|
+
this.suspendLifecycle();
|
|
532
|
+
return result;
|
|
533
|
+
})();
|
|
534
|
+
return this.branchQuiescePromise;
|
|
535
|
+
}
|
|
536
|
+
|
|
537
|
+
/**
|
|
538
|
+
* Restore a session-tree branch. Reusing the same branch generation is
|
|
539
|
+
* intentionally idempotent: a stale snapshot may be replayed while Pi is
|
|
540
|
+
* still delivering tree events. Callers that replace the branch must pass a
|
|
541
|
+
* new generation (the no-argument path also advances after quiescence).
|
|
542
|
+
*/
|
|
543
|
+
restore(entries: readonly SessionEntryLike[], branchGeneration?: number): void {
|
|
544
|
+
const snapshot = this.captureRestoreSnapshot();
|
|
545
|
+
try {
|
|
546
|
+
this.restoreInternal(entries, branchGeneration);
|
|
547
|
+
this.restoreFailureSnapshot = undefined;
|
|
548
|
+
} catch (error: unknown) {
|
|
549
|
+
// Recovery is a transaction from the engine's point of view. In
|
|
550
|
+
// particular, an append failure must not leave a half-rotated run in
|
|
551
|
+
// memory or advance the branch namespace without its durable event.
|
|
552
|
+
const rollback = this.restoreFailureSnapshot ?? snapshot;
|
|
553
|
+
this.restoreFailureSnapshot = undefined;
|
|
554
|
+
this.restoreAfterFailure(rollback);
|
|
555
|
+
throw error;
|
|
556
|
+
}
|
|
557
|
+
}
|
|
558
|
+
|
|
559
|
+
private captureRestoreSnapshot(): EngineRestoreSnapshot {
|
|
560
|
+
return {
|
|
561
|
+
runs: new Map([...this.runs].map(([runId, run]) => [runId, snapshotRun(run)])),
|
|
562
|
+
recoveryMemo: new Map(
|
|
563
|
+
[...this.recoveryMemo].map(([key, memo]) => [
|
|
564
|
+
key,
|
|
565
|
+
{
|
|
566
|
+
...memo,
|
|
567
|
+
event: structuredClone(memo.event),
|
|
568
|
+
},
|
|
569
|
+
]),
|
|
570
|
+
),
|
|
571
|
+
terminalRecoveryMemo: new Map(
|
|
572
|
+
[...this.terminalRecoveryMemo].map(([key, memo]) => [
|
|
573
|
+
key,
|
|
574
|
+
{
|
|
575
|
+
...memo,
|
|
576
|
+
event: structuredClone(memo.event),
|
|
577
|
+
},
|
|
578
|
+
]),
|
|
579
|
+
),
|
|
580
|
+
recoveryBranchGeneration: this.recoveryBranchGeneration,
|
|
581
|
+
branchEpoch: this.branchEpoch,
|
|
582
|
+
branchChanging: this.branchChanging,
|
|
583
|
+
branchEventsClosed: this.branchEventsClosed,
|
|
584
|
+
lifecycleSuspended: this.lifecycleSuspended,
|
|
585
|
+
quarantinedAgentIds: new Set(this.quarantinedAgentIds),
|
|
586
|
+
};
|
|
587
|
+
}
|
|
588
|
+
|
|
589
|
+
private restoreAfterFailure(snapshot: EngineRestoreSnapshot): void {
|
|
590
|
+
const invalidationEpoch = this.branchEpoch + 1;
|
|
591
|
+
const lifecycleWasSuspended = this.lifecycleSuspended;
|
|
592
|
+
this.dispatches.clear();
|
|
593
|
+
this.lateDispatchTargets.clear();
|
|
594
|
+
this.pumping.clear();
|
|
595
|
+
this.settling.clear();
|
|
596
|
+
this.runs.clear();
|
|
597
|
+
for (const [runId, run] of snapshot.runs) this.runs.set(runId, run);
|
|
598
|
+
this.recoveryMemo.clear();
|
|
599
|
+
for (const [key, memo] of snapshot.recoveryMemo) this.recoveryMemo.set(key, memo);
|
|
600
|
+
this.terminalRecoveryMemo.clear();
|
|
601
|
+
for (const [key, memo] of snapshot.terminalRecoveryMemo) this.terminalRecoveryMemo.set(key, memo);
|
|
602
|
+
this.recoveryBranchGeneration = snapshot.recoveryBranchGeneration;
|
|
603
|
+
this.branchEpoch = Math.max(invalidationEpoch, snapshot.branchEpoch + 1);
|
|
604
|
+
this.branchChanging = snapshot.branchChanging;
|
|
605
|
+
this.branchEventsClosed = snapshot.branchEventsClosed;
|
|
606
|
+
if (snapshot.lifecycleSuspended && !lifecycleWasSuspended) this.suspendLifecycle();
|
|
607
|
+
else if (!snapshot.lifecycleSuspended && lifecycleWasSuspended) this.resumeLifecycle();
|
|
608
|
+
this.quarantinedAgentIds.clear();
|
|
609
|
+
for (const agentId of snapshot.quarantinedAgentIds) this.quarantinedAgentIds.add(agentId);
|
|
610
|
+
this.branchQuiescePromise = undefined;
|
|
611
|
+
}
|
|
612
|
+
|
|
613
|
+
private restoreInternal(entries: readonly SessionEntryLike[], branchGeneration?: number): void {
|
|
614
|
+
const branchReplacing = this.branchChanging;
|
|
615
|
+
this.prepareRecoveryNamespace(branchGeneration, branchReplacing);
|
|
616
|
+
|
|
617
|
+
// Invalidate dispatch promises from the replaced branch before clearing the
|
|
618
|
+
// run map. A managed RPC reply arriving after this point must be stopped,
|
|
619
|
+
// never journaled into the new branch.
|
|
620
|
+
this.branchEpoch += 1;
|
|
621
|
+
this.branchChanging = false;
|
|
622
|
+
this.branchEventsClosed = false;
|
|
623
|
+
this.branchQuiescePromise = undefined;
|
|
624
|
+
this.dispatches.clear();
|
|
625
|
+
this.lateDispatchTargets.clear();
|
|
626
|
+
this.pumping.clear();
|
|
627
|
+
this.settling.clear();
|
|
628
|
+
this.runs.clear();
|
|
629
|
+
for (const [runId, run] of replayJournal(entries, {
|
|
630
|
+
onInvalid: (diagnostic) => console.warn(`[pi-workflows] ${diagnostic}`),
|
|
631
|
+
}))
|
|
632
|
+
this.runs.set(runId, run);
|
|
633
|
+
this.restoreFailureSnapshot = this.captureRestoreSnapshot();
|
|
634
|
+
|
|
635
|
+
// Persisted subagent records are the terminal source of truth when a
|
|
636
|
+
// session was interrupted between lifecycle delivery and workflow
|
|
637
|
+
// journaling. Interrupted records are deferred: recovery must append one
|
|
638
|
+
// complete event before changing the in-memory attempt generation.
|
|
639
|
+
const recoveredNodes = new Map<string, DeferredInterruptedRecord>();
|
|
640
|
+
const terminalRecords = new Map<string, DeferredTerminalRecord>();
|
|
641
|
+
const recoveryKey = (runId: string, nodeId: string): string => `${runId}\u0000${nodeId}`;
|
|
642
|
+
for (const entry of entries) {
|
|
643
|
+
const candidates = [this.reconcileSubagentRecord(entry), this.reconcileManagedSpawnTombstone(entry)];
|
|
644
|
+
for (const reconciled of candidates) {
|
|
645
|
+
if (!reconciled) continue;
|
|
646
|
+
const key = recoveryKey(reconciled.runId, reconciled.nodeId);
|
|
647
|
+
if (reconciled.kind === "interrupted") recoveredNodes.set(key, reconciled);
|
|
648
|
+
else if (!terminalRecords.has(key)) terminalRecords.set(key, reconciled);
|
|
649
|
+
}
|
|
650
|
+
}
|
|
651
|
+
|
|
652
|
+
for (const run of this.runs.values()) {
|
|
653
|
+
const wasActive =
|
|
654
|
+
run.status === "running" ||
|
|
655
|
+
run.status === "pausing" ||
|
|
656
|
+
run.status === "synthesizing" ||
|
|
657
|
+
run.status === "stopping";
|
|
658
|
+
const runTerminalRecords = [...terminalRecords.values()].filter((record) => record.runId === run.runId);
|
|
659
|
+
if (!wasActive && runTerminalRecords.length > 0) {
|
|
660
|
+
// Non-active legacy/paused runs retain the existing reconciliation
|
|
661
|
+
// behavior. Active terminal-intent runs are handled below so each
|
|
662
|
+
// recovered terminal fact is journaled before the final transition.
|
|
663
|
+
this.commitDeferredTerminals(run, runTerminalRecords);
|
|
664
|
+
}
|
|
665
|
+
if (wasActive) {
|
|
666
|
+
if (run.terminalIntent === "stop" || run.terminalIntent === "failure") {
|
|
667
|
+
// A user stop/failure is an immutable terminal fact, never an
|
|
668
|
+
// interrupted run that can be resumed after reload. Finish every
|
|
669
|
+
// active node first; otherwise replay can leave a terminal workflow
|
|
670
|
+
// with live task statuses and a later restore can resurrect them.
|
|
671
|
+
this.recoverTerminalIntent(run, runTerminalRecords);
|
|
672
|
+
} else {
|
|
673
|
+
// A run-level event owns the workflow interruption and every active
|
|
674
|
+
// task/synthesis rotation. append() happens before any mutation, so
|
|
675
|
+
// an injected failure leaves this replayed run untouched and an
|
|
676
|
+
// exact retry can safely emit the same generation once.
|
|
677
|
+
this.recoverInterruptedRun(run, this.recoveryPlan(run, recoveredNodes, terminalRecords), runTerminalRecords);
|
|
678
|
+
}
|
|
679
|
+
} else if (run.status === "interrupted") {
|
|
680
|
+
// A legacy sequence could have persisted workflow_transition before
|
|
681
|
+
// its task_attempt. Migrate each still-active node independently;
|
|
682
|
+
// every attempt_recovery event is valid on its own prefix.
|
|
683
|
+
for (const item of this.recoveryPlan(run, recoveredNodes, terminalRecords)) {
|
|
684
|
+
this.recoverInterruptedNode(run, item.nodeId, item.oldAgentId, false, true);
|
|
685
|
+
}
|
|
686
|
+
}
|
|
687
|
+
|
|
688
|
+
if (isTerminalWorkflow(run.status)) continue;
|
|
689
|
+
|
|
690
|
+
// A terminal record can arrive without its lifecycle result having been
|
|
691
|
+
// journaled. Preserve the existing reconciliation behavior for that
|
|
692
|
+
// record; it is not part of the generation rotation itself.
|
|
693
|
+
for (const task of run.definition.tasks) {
|
|
694
|
+
const status = run.taskStatus[task.id];
|
|
695
|
+
if (run.taskResults[task.id] && !isTerminalTask(status)) {
|
|
696
|
+
this.setTaskStatus(run, task.id, run.taskResults[task.id].status, run.agentIds[task.id]);
|
|
697
|
+
}
|
|
698
|
+
}
|
|
699
|
+
}
|
|
700
|
+
for (const waiters of this.waiters.values()) {
|
|
701
|
+
for (const waiter of waiters) {
|
|
702
|
+
if (waiter.settled) continue;
|
|
703
|
+
waiter.settled = true;
|
|
704
|
+
waiter.cleanup();
|
|
705
|
+
waiter.reject(new Error("workflow session branch changed; the old run is no longer active"));
|
|
706
|
+
}
|
|
707
|
+
}
|
|
708
|
+
this.waiters.clear();
|
|
709
|
+
}
|
|
710
|
+
|
|
711
|
+
private prepareRecoveryNamespace(branchGeneration: number | undefined, branchReplacing: boolean): void {
|
|
712
|
+
if (branchGeneration !== undefined && (!Number.isSafeInteger(branchGeneration) || branchGeneration < 0)) {
|
|
713
|
+
throw new Error("workflow branch generation must be a non-negative safe integer");
|
|
714
|
+
}
|
|
715
|
+
const nextGeneration =
|
|
716
|
+
branchGeneration === undefined
|
|
717
|
+
? branchReplacing
|
|
718
|
+
? this.recoveryBranchGeneration + 1
|
|
719
|
+
: this.recoveryBranchGeneration
|
|
720
|
+
: branchGeneration;
|
|
721
|
+
if (nextGeneration === this.recoveryBranchGeneration) {
|
|
722
|
+
if (branchReplacing) {
|
|
723
|
+
this.recoveryMemo.clear();
|
|
724
|
+
this.terminalRecoveryMemo.clear();
|
|
725
|
+
}
|
|
726
|
+
return;
|
|
727
|
+
}
|
|
728
|
+
this.recoveryMemo.clear();
|
|
729
|
+
this.terminalRecoveryMemo.clear();
|
|
730
|
+
this.recoveryBranchGeneration = nextGeneration;
|
|
731
|
+
}
|
|
732
|
+
|
|
733
|
+
private recoveryMemoKey(runId: string, kind: RecoveryEvent["kind"], sourceKey: string): string {
|
|
734
|
+
return JSON.stringify([this.recoveryBranchGeneration, runId, kind, sourceKey]);
|
|
735
|
+
}
|
|
736
|
+
|
|
737
|
+
private sourceAttemptId(run: WorkflowRun, nodeId: string): string {
|
|
738
|
+
return run.attemptIds[nodeId] ?? `${run.runId}/${nodeId}/attempt-${run.attempts[nodeId] ?? 1}`;
|
|
739
|
+
}
|
|
740
|
+
|
|
741
|
+
private recoveryPlan(
|
|
742
|
+
run: WorkflowRun,
|
|
743
|
+
deferred: ReadonlyMap<string, DeferredInterruptedRecord>,
|
|
744
|
+
terminalRecords: ReadonlyMap<string, DeferredTerminalRecord> = new Map(),
|
|
745
|
+
): RecoveryPlanItem[] {
|
|
746
|
+
const activeWorkflow =
|
|
747
|
+
run.status === "running" ||
|
|
748
|
+
run.status === "pausing" ||
|
|
749
|
+
run.status === "synthesizing" ||
|
|
750
|
+
run.status === "stopping";
|
|
751
|
+
const legacyInterrupted = run.status === "interrupted";
|
|
752
|
+
if ((!activeWorkflow && !legacyInterrupted) || run.terminalIntent) return [];
|
|
753
|
+
const plan: RecoveryPlanItem[] = [];
|
|
754
|
+
for (const task of run.definition.tasks) {
|
|
755
|
+
const status = run.taskStatus[task.id];
|
|
756
|
+
const key = `${run.runId}\u0000${task.id}`;
|
|
757
|
+
const deferredRecord = deferred.get(key);
|
|
758
|
+
if (run.taskResults[task.id] || isTerminalTask(status) || terminalRecords.has(key)) continue;
|
|
759
|
+
if (!activeStatus(status) && deferredRecord === undefined) continue;
|
|
760
|
+
plan.push({
|
|
761
|
+
nodeId: task.id,
|
|
762
|
+
sourceAttemptId: this.sourceAttemptId(run, task.id),
|
|
763
|
+
sourceGeneration: run.attempts[task.id],
|
|
764
|
+
sourceStatus: status,
|
|
765
|
+
oldAgentId: deferredRecord?.oldAgentId ?? run.agentIds[task.id] ?? "",
|
|
766
|
+
});
|
|
767
|
+
}
|
|
768
|
+
if ((run.status === "synthesizing" || (legacyInterrupted && run.synthesisAgentId)) && !run.synthesisResult) {
|
|
769
|
+
const synthesisKey = `${run.runId}\u0000${SYNTHESIS_NODE_ID}`;
|
|
770
|
+
const deferredRecord = deferred.get(synthesisKey);
|
|
771
|
+
if (terminalRecords.has(synthesisKey)) return plan;
|
|
772
|
+
plan.push({
|
|
773
|
+
nodeId: SYNTHESIS_NODE_ID,
|
|
774
|
+
sourceAttemptId: this.sourceAttemptId(run, SYNTHESIS_NODE_ID),
|
|
775
|
+
sourceGeneration: run.attempts[SYNTHESIS_NODE_ID],
|
|
776
|
+
sourceStatus: "synthesizing",
|
|
777
|
+
oldAgentId: deferredRecord?.oldAgentId ?? run.synthesisAgentId ?? "",
|
|
778
|
+
});
|
|
779
|
+
}
|
|
780
|
+
return plan;
|
|
781
|
+
}
|
|
782
|
+
|
|
783
|
+
private recoverySourceKey(
|
|
784
|
+
run: WorkflowRun,
|
|
785
|
+
plan: readonly RecoveryPlanItem[],
|
|
786
|
+
terminalResults: readonly DeferredTerminalRecord[] = [],
|
|
787
|
+
): string {
|
|
788
|
+
return JSON.stringify({
|
|
789
|
+
status: run.status,
|
|
790
|
+
terminalIntent: run.terminalIntent,
|
|
791
|
+
rotations: plan.map((item) => [
|
|
792
|
+
item.nodeId,
|
|
793
|
+
item.sourceAttemptId,
|
|
794
|
+
item.sourceGeneration,
|
|
795
|
+
item.sourceStatus,
|
|
796
|
+
item.oldAgentId,
|
|
797
|
+
]),
|
|
798
|
+
terminalResults: terminalResults.map((item) => [
|
|
799
|
+
item.nodeId,
|
|
800
|
+
item.attemptId,
|
|
801
|
+
item.result.status,
|
|
802
|
+
item.result.agentId,
|
|
803
|
+
item.result.text,
|
|
804
|
+
item.result.error,
|
|
805
|
+
item.result.outputFile,
|
|
806
|
+
item.result.tokenCount,
|
|
807
|
+
item.result.compactionCount,
|
|
808
|
+
item.result.truncated,
|
|
809
|
+
]),
|
|
810
|
+
});
|
|
811
|
+
}
|
|
812
|
+
|
|
813
|
+
private rememberRecovery(memo: RecoveryMemo): void {
|
|
814
|
+
const key = this.recoveryMemoKey(memo.runId, memo.event.kind, memo.key);
|
|
815
|
+
this.recoveryMemo.delete(key);
|
|
816
|
+
this.recoveryMemo.set(key, memo);
|
|
817
|
+
while (this.recoveryMemo.size > MAX_RECOVERY_MEMO_ENTRIES) {
|
|
818
|
+
const oldest = this.recoveryMemo.keys().next().value;
|
|
819
|
+
if (typeof oldest !== "string") break;
|
|
820
|
+
this.recoveryMemo.delete(oldest);
|
|
821
|
+
}
|
|
822
|
+
}
|
|
823
|
+
|
|
824
|
+
private memoizedRecovery(runId: string, kind: RecoveryEvent["kind"], sourceKey: string): RecoveryEvent | undefined {
|
|
825
|
+
return this.recoveryMemo.get(this.recoveryMemoKey(runId, kind, sourceKey))?.event;
|
|
826
|
+
}
|
|
827
|
+
|
|
828
|
+
list(): WorkflowSummary[] {
|
|
829
|
+
return [...this.runs.values()].sort((a, b) => b.startedAt - a.startedAt).map((run) => this.summary(run));
|
|
830
|
+
}
|
|
831
|
+
|
|
832
|
+
get(runId: string): WorkflowSummary | undefined {
|
|
833
|
+
const run = this.runs.get(runId);
|
|
834
|
+
return run ? this.summary(run) : undefined;
|
|
835
|
+
}
|
|
836
|
+
|
|
837
|
+
getRun(runId: string): WorkflowRun | undefined {
|
|
838
|
+
return this.runs.get(runId);
|
|
839
|
+
}
|
|
840
|
+
|
|
841
|
+
async start(definition: WorkflowDefinition, signal?: AbortSignal): Promise<WorkflowStartResult>;
|
|
842
|
+
/** @deprecated The context argument is ignored; pass the foreground signal as the second argument. */
|
|
843
|
+
async start(
|
|
844
|
+
definition: WorkflowDefinition,
|
|
845
|
+
_legacyContext: unknown,
|
|
846
|
+
signal?: AbortSignal,
|
|
847
|
+
): Promise<WorkflowStartResult>;
|
|
848
|
+
async start(
|
|
849
|
+
definition: WorkflowDefinition,
|
|
850
|
+
signalOrLegacyContext?: AbortSignal | unknown,
|
|
851
|
+
legacySignal?: AbortSignal,
|
|
852
|
+
): Promise<WorkflowStartResult> {
|
|
853
|
+
const signal = legacySignal ?? (isAbortSignal(signalOrLegacyContext) ? signalOrLegacyContext : undefined);
|
|
854
|
+
const now = Date.now();
|
|
855
|
+
const runId = randomUUID();
|
|
856
|
+
const run: WorkflowRun = {
|
|
857
|
+
runId,
|
|
858
|
+
schemaVersion: JOURNAL_SCHEMA_VERSION,
|
|
859
|
+
definition,
|
|
860
|
+
status: "pending",
|
|
861
|
+
taskStatus: Object.fromEntries(definition.tasks.map((task) => [task.id, "pending" as const])),
|
|
862
|
+
agentIds: {},
|
|
863
|
+
attempts: Object.fromEntries([
|
|
864
|
+
...definition.tasks.map((task) => [task.id, 1] as const),
|
|
865
|
+
[SYNTHESIS_NODE_ID, 1] as const,
|
|
866
|
+
]),
|
|
867
|
+
attemptIds: Object.fromEntries([
|
|
868
|
+
...definition.tasks.map((task) => [task.id, `${runId}/${task.id}/attempt-1`] as const),
|
|
869
|
+
[SYNTHESIS_NODE_ID, `${runId}/${SYNTHESIS_NODE_ID}/attempt-1`] as const,
|
|
870
|
+
]),
|
|
871
|
+
attemptTracking: true,
|
|
872
|
+
taskResults: {},
|
|
873
|
+
compactions: {},
|
|
874
|
+
startedAt: now,
|
|
875
|
+
updatedAt: now,
|
|
876
|
+
};
|
|
877
|
+
this.write({
|
|
878
|
+
kind: "run_created",
|
|
879
|
+
schemaVersion: JOURNAL_SCHEMA_VERSION,
|
|
880
|
+
runId: run.runId,
|
|
881
|
+
definition,
|
|
882
|
+
attempts: run.attempts,
|
|
883
|
+
attemptIds: run.attemptIds,
|
|
884
|
+
timestamp: now,
|
|
885
|
+
});
|
|
886
|
+
this.setWorkflowStatus(run, "running");
|
|
887
|
+
this.runs.set(run.runId, run);
|
|
888
|
+
void this.pump(run);
|
|
889
|
+
if (definition.background) return { runId: run.runId, status: run.status, background: true };
|
|
890
|
+
try {
|
|
891
|
+
const settled = await this.waitFor(run.runId, signal);
|
|
892
|
+
return {
|
|
893
|
+
runId: run.runId,
|
|
894
|
+
status: settled.status,
|
|
895
|
+
background: false,
|
|
896
|
+
...(settled.synthesisResult?.text ? { result: settled.synthesisResult.text } : {}),
|
|
897
|
+
...(settled.error ? { error: settled.error } : {}),
|
|
898
|
+
};
|
|
899
|
+
} catch (error: unknown) {
|
|
900
|
+
if (!(error instanceof WorkflowWaitAbortedError)) throw error;
|
|
901
|
+
const current = this.runs.get(run.runId) ?? run;
|
|
902
|
+
return {
|
|
903
|
+
runId: current.runId,
|
|
904
|
+
status: current.status,
|
|
905
|
+
background: false,
|
|
906
|
+
waitAborted: true,
|
|
907
|
+
error: error.message,
|
|
908
|
+
};
|
|
909
|
+
}
|
|
910
|
+
}
|
|
911
|
+
|
|
912
|
+
async control(action: "list" | "get" | "pause" | "resume" | "stop", runId?: string): Promise<WorkflowControlResult> {
|
|
913
|
+
if (action === "list") return { action, runs: this.list() };
|
|
914
|
+
if (!runId) throw new Error(`${action} requires runId`);
|
|
915
|
+
const run = this.runs.get(runId);
|
|
916
|
+
if (!run) throw new Error(`workflow run not found: ${runId}`);
|
|
917
|
+
if (action === "get") return { action, run: this.summary(run, true) };
|
|
918
|
+
if (action === "pause") {
|
|
919
|
+
if (run.status === "running") {
|
|
920
|
+
this.setWorkflowStatus(run, "pausing");
|
|
921
|
+
this.maybePause(run);
|
|
922
|
+
} else if (run.status !== "paused") {
|
|
923
|
+
throw new Error(`cannot pause workflow in state ${run.status}`);
|
|
924
|
+
}
|
|
925
|
+
} else if (action === "resume") {
|
|
926
|
+
if (run.nonResumable || run.status === "stopped") throw new Error("workflow run is non-resumable after stop");
|
|
927
|
+
if (run.status !== "paused" && run.status !== "interrupted") {
|
|
928
|
+
throw new Error(`cannot resume workflow in state ${run.status}`);
|
|
929
|
+
}
|
|
930
|
+
this.setWorkflowStatus(run, "running");
|
|
931
|
+
void this.pump(run);
|
|
932
|
+
} else if (action === "stop") {
|
|
933
|
+
await this.stop(run);
|
|
934
|
+
}
|
|
935
|
+
return { action, run: this.summary(run) };
|
|
936
|
+
}
|
|
937
|
+
|
|
938
|
+
async waitFor(runId: string, signal?: AbortSignal): Promise<WorkflowRun> {
|
|
939
|
+
const run = this.runs.get(runId);
|
|
940
|
+
if (!run) throw new Error(`workflow run not found: ${runId}`);
|
|
941
|
+
if (isTerminalWorkflow(run.status)) return run;
|
|
942
|
+
const all = this.waiters.get(runId) ?? new Set<Waiter>();
|
|
943
|
+
if (all.size >= MAX_WAITERS) throw new Error("too many workflow waiters");
|
|
944
|
+
this.waiters.set(runId, all);
|
|
945
|
+
return new Promise<WorkflowRun>((resolve, reject) => {
|
|
946
|
+
let abort = () => {};
|
|
947
|
+
const waiter: Waiter = {
|
|
948
|
+
resolve,
|
|
949
|
+
reject,
|
|
950
|
+
settled: false,
|
|
951
|
+
cleanup: () => signal?.removeEventListener("abort", abort),
|
|
952
|
+
};
|
|
953
|
+
abort = () => {
|
|
954
|
+
if (waiter.settled) return;
|
|
955
|
+
waiter.settled = true;
|
|
956
|
+
all.delete(waiter);
|
|
957
|
+
if (all.size === 0) this.waiters.delete(runId);
|
|
958
|
+
waiter.cleanup();
|
|
959
|
+
reject(new WorkflowWaitAbortedError());
|
|
960
|
+
};
|
|
961
|
+
all.add(waiter);
|
|
962
|
+
if (signal) {
|
|
963
|
+
if (signal.aborted) {
|
|
964
|
+
abort();
|
|
965
|
+
return;
|
|
966
|
+
}
|
|
967
|
+
signal.addEventListener("abort", abort, { once: true });
|
|
968
|
+
}
|
|
969
|
+
if (isTerminalWorkflow(run.status)) {
|
|
970
|
+
all.delete(waiter);
|
|
971
|
+
if (all.size === 0) this.waiters.delete(runId);
|
|
972
|
+
waiter.settled = true;
|
|
973
|
+
waiter.cleanup();
|
|
974
|
+
resolve(run);
|
|
975
|
+
}
|
|
976
|
+
});
|
|
977
|
+
}
|
|
978
|
+
|
|
979
|
+
private trackDispatch(runId: string, dispatch: Promise<void>): void {
|
|
980
|
+
const pending = this.dispatches.get(runId) ?? new Set<Promise<void>>();
|
|
981
|
+
pending.add(dispatch);
|
|
982
|
+
this.dispatches.set(runId, pending);
|
|
983
|
+
// Use both settlement branches so a rejected dispatch never creates an
|
|
984
|
+
// unhandled rejection while its bookkeeping is removed.
|
|
985
|
+
void dispatch.then(
|
|
986
|
+
() => this.untrackDispatch(runId, dispatch),
|
|
987
|
+
() => this.untrackDispatch(runId, dispatch),
|
|
988
|
+
);
|
|
989
|
+
}
|
|
990
|
+
|
|
991
|
+
private untrackDispatch(runId: string, dispatch: Promise<void>): void {
|
|
992
|
+
const pending = this.dispatches.get(runId);
|
|
993
|
+
if (!pending) return;
|
|
994
|
+
pending.delete(dispatch);
|
|
995
|
+
if (pending.size === 0) this.dispatches.delete(runId);
|
|
996
|
+
}
|
|
997
|
+
|
|
998
|
+
private async waitForDispatches(runId: string, timeoutMs: number): Promise<boolean> {
|
|
999
|
+
const pending = this.dispatches.get(runId);
|
|
1000
|
+
if (!pending || pending.size === 0) return true;
|
|
1001
|
+
let timer: ReturnType<typeof setTimeout> | undefined;
|
|
1002
|
+
const allSettled = Promise.allSettled([...pending]).then(() => true);
|
|
1003
|
+
const timedOut = new Promise<boolean>((resolve) => {
|
|
1004
|
+
timer = setTimeout(() => resolve(false), timeoutMs);
|
|
1005
|
+
});
|
|
1006
|
+
const settled = await Promise.race([allSettled, timedOut]);
|
|
1007
|
+
if (timer) clearTimeout(timer);
|
|
1008
|
+
return settled;
|
|
1009
|
+
}
|
|
1010
|
+
|
|
1011
|
+
private quarantineAgent(agentId: string): void {
|
|
1012
|
+
this.quarantinedAgentIds.add(agentId);
|
|
1013
|
+
if (this.quarantinedAgentIds.size > MAX_QUARANTINED_AGENT_IDS) {
|
|
1014
|
+
const oldest = this.quarantinedAgentIds.values().next().value;
|
|
1015
|
+
if (typeof oldest === "string") this.quarantinedAgentIds.delete(oldest);
|
|
1016
|
+
}
|
|
1017
|
+
}
|
|
1018
|
+
|
|
1019
|
+
private stopOwned(run: WorkflowRun, nodeId: string, agentId: string, attemptId?: string): Promise<void> {
|
|
1020
|
+
const owner = {
|
|
1021
|
+
...this.ownerForNode(run, nodeId),
|
|
1022
|
+
...(attemptId === undefined ? {} : { attemptId }),
|
|
1023
|
+
};
|
|
1024
|
+
if (!this.client.stopOwned) {
|
|
1025
|
+
return Promise.reject(new Error("workflow owner-scoped stop is unavailable; refusing unowned stop fallback"));
|
|
1026
|
+
}
|
|
1027
|
+
return this.client.stopOwned(agentId, owner);
|
|
1028
|
+
}
|
|
1029
|
+
|
|
1030
|
+
private rememberLateDispatchTarget(
|
|
1031
|
+
run: WorkflowRun,
|
|
1032
|
+
nodeId: string,
|
|
1033
|
+
agentId: string,
|
|
1034
|
+
attemptId: string | undefined,
|
|
1035
|
+
): void {
|
|
1036
|
+
const owner: WorkflowOwner = {
|
|
1037
|
+
...this.ownerForNode(run, nodeId),
|
|
1038
|
+
...(attemptId === undefined ? {} : { attemptId }),
|
|
1039
|
+
};
|
|
1040
|
+
const targets = this.lateDispatchTargets.get(run.runId) ?? [];
|
|
1041
|
+
if (
|
|
1042
|
+
!targets.some(
|
|
1043
|
+
(target) =>
|
|
1044
|
+
target.agentId === agentId &&
|
|
1045
|
+
target.owner.nodeId === owner.nodeId &&
|
|
1046
|
+
target.owner.attemptId === owner.attemptId,
|
|
1047
|
+
)
|
|
1048
|
+
) {
|
|
1049
|
+
targets.push({ agentId, owner, fallbackStopRequested: false });
|
|
1050
|
+
}
|
|
1051
|
+
this.lateDispatchTargets.set(run.runId, targets);
|
|
1052
|
+
}
|
|
1053
|
+
|
|
1054
|
+
private takeLateDispatchTargets(run: WorkflowRun): TerminalCleanupTarget[] {
|
|
1055
|
+
const targets = this.lateDispatchTargets.get(run.runId);
|
|
1056
|
+
if (!targets) return [];
|
|
1057
|
+
this.lateDispatchTargets.delete(run.runId);
|
|
1058
|
+
return targets;
|
|
1059
|
+
}
|
|
1060
|
+
|
|
1061
|
+
private rememberLateDispatchQuiescence(runId: string, agentId: string): void {
|
|
1062
|
+
const pending = this.lateDispatchQuiescence.get(runId) ?? [];
|
|
1063
|
+
if (!pending.includes(agentId)) pending.push(agentId);
|
|
1064
|
+
this.lateDispatchQuiescence.set(runId, pending.slice(0, 256));
|
|
1065
|
+
}
|
|
1066
|
+
|
|
1067
|
+
private takeLateDispatchQuiescence(runId: string): string[] {
|
|
1068
|
+
const pending = this.lateDispatchQuiescence.get(runId) ?? [];
|
|
1069
|
+
this.lateDispatchQuiescence.delete(runId);
|
|
1070
|
+
return pending;
|
|
1071
|
+
}
|
|
1072
|
+
|
|
1073
|
+
private initialTerminalCleanupTargets(run: WorkflowRun): TerminalCleanupTarget[] {
|
|
1074
|
+
const targets: TerminalCleanupTarget[] = [];
|
|
1075
|
+
for (const [nodeId, status] of Object.entries(run.taskStatus)) {
|
|
1076
|
+
if (!activeStatus(status)) continue;
|
|
1077
|
+
const agentId = run.agentIds[nodeId];
|
|
1078
|
+
if (!agentId) continue;
|
|
1079
|
+
targets.push({ agentId, owner: this.ownerForNode(run, nodeId), fallbackStopRequested: false });
|
|
1080
|
+
}
|
|
1081
|
+
if (run.synthesisAgentId && !run.synthesisResult) {
|
|
1082
|
+
targets.push({
|
|
1083
|
+
agentId: run.synthesisAgentId,
|
|
1084
|
+
owner: this.ownerForNode(run, SYNTHESIS_NODE_ID),
|
|
1085
|
+
fallbackStopRequested: false,
|
|
1086
|
+
});
|
|
1087
|
+
}
|
|
1088
|
+
return targets;
|
|
1089
|
+
}
|
|
1090
|
+
|
|
1091
|
+
private async stopAndQuiesceOwned(
|
|
1092
|
+
run: WorkflowRun,
|
|
1093
|
+
nodeId: string,
|
|
1094
|
+
agentId: string,
|
|
1095
|
+
attemptId: string | undefined,
|
|
1096
|
+
branchEpoch: number,
|
|
1097
|
+
quiesce = true,
|
|
1098
|
+
allowStaleBranch = false,
|
|
1099
|
+
): Promise<boolean> {
|
|
1100
|
+
const isCurrent = (): boolean => {
|
|
1101
|
+
if (this.runs.get(run.runId) !== run) return false;
|
|
1102
|
+
return allowStaleBranch || (!this.branchChanging && branchEpoch === this.branchEpoch);
|
|
1103
|
+
};
|
|
1104
|
+
await this.stopOwned(run, nodeId, agentId, attemptId).catch(() => {});
|
|
1105
|
+
if (!quiesce) return true;
|
|
1106
|
+
if (!this.client.quiesceOwned) return false;
|
|
1107
|
+
const owner = {
|
|
1108
|
+
...this.ownerForNode(run, nodeId),
|
|
1109
|
+
...(attemptId === undefined ? {} : { attemptId }),
|
|
1110
|
+
};
|
|
1111
|
+
while (isCurrent()) {
|
|
1112
|
+
const result = await this.client
|
|
1113
|
+
.quiesceOwned(run.runId, [agentId], BRANCH_QUIESCE_TIMEOUT_MS, [owner])
|
|
1114
|
+
.catch((): WorkflowQuiesceResult => ({ settled: false, pending: [agentId] }));
|
|
1115
|
+
if (result.settled && result.pending.length === 0) return true;
|
|
1116
|
+
if (allowStaleBranch) return false;
|
|
1117
|
+
if (!isCurrent()) return false;
|
|
1118
|
+
await new Promise<void>((resolve) => setTimeout(resolve, 100));
|
|
1119
|
+
}
|
|
1120
|
+
return false;
|
|
1121
|
+
}
|
|
1122
|
+
|
|
1123
|
+
private clearJournalRetries(): void {
|
|
1124
|
+
for (const retry of this.journalRetries.values()) {
|
|
1125
|
+
if (retry.timer) clearTimeout(retry.timer);
|
|
1126
|
+
}
|
|
1127
|
+
this.journalRetries.clear();
|
|
1128
|
+
this.journalBatchProgress.clear();
|
|
1129
|
+
this.journalBlockedRuns.clear();
|
|
1130
|
+
}
|
|
1131
|
+
|
|
1132
|
+
private hasPendingJournalWork(runId: string): boolean {
|
|
1133
|
+
if (this.journalBlockedRuns.has(runId)) return true;
|
|
1134
|
+
const retryMarker = `\\u0000${runId}\\u0000`;
|
|
1135
|
+
if ([...this.journalRetries.keys()].some((key) => key.includes(retryMarker))) return true;
|
|
1136
|
+
const batchMarker = `${runId}:`;
|
|
1137
|
+
return [...this.journalBatchProgress.keys()].some((key) => key.startsWith(batchMarker));
|
|
1138
|
+
}
|
|
1139
|
+
|
|
1140
|
+
/**
|
|
1141
|
+
* A Pi lifecycle event is delivered once. If its durable append is transiently
|
|
1142
|
+
* unavailable, retain the immutable mutation and retry it without requiring
|
|
1143
|
+
* the subagents peer to emit a second terminal callback.
|
|
1144
|
+
*/
|
|
1145
|
+
private scheduleJournalRetry(run: WorkflowRun, key: string, mutation: () => void): void {
|
|
1146
|
+
const epoch = this.branchEpoch;
|
|
1147
|
+
const retryKey = `${epoch}\\u0000${run.runId}\\u0000${key}`;
|
|
1148
|
+
if (this.journalRetries.has(retryKey)) return;
|
|
1149
|
+
const retry = { attempt: 0, timer: undefined as ReturnType<typeof setTimeout> | undefined };
|
|
1150
|
+
const attempt = (): void => {
|
|
1151
|
+
const current = this.journalRetries.get(retryKey);
|
|
1152
|
+
if (!current) return;
|
|
1153
|
+
current.timer = undefined;
|
|
1154
|
+
if (
|
|
1155
|
+
this.branchEpoch !== epoch ||
|
|
1156
|
+
this.branchChanging ||
|
|
1157
|
+
this.lifecycleSuspended ||
|
|
1158
|
+
this.runs.get(run.runId) !== run
|
|
1159
|
+
) {
|
|
1160
|
+
this.journalRetries.delete(retryKey);
|
|
1161
|
+
return;
|
|
1162
|
+
}
|
|
1163
|
+
const previousKey = this.journalMutationKey;
|
|
1164
|
+
this.journalMutationKey = `${run.runId}:${key}`;
|
|
1165
|
+
try {
|
|
1166
|
+
mutation();
|
|
1167
|
+
this.journalRetries.delete(retryKey);
|
|
1168
|
+
} catch (error: unknown) {
|
|
1169
|
+
if (!(error instanceof JournalAppendError)) {
|
|
1170
|
+
this.journalRetries.delete(retryKey);
|
|
1171
|
+
console.warn(`[pi-workflows] lifecycle retry abandoned: ${boundedError(error) ?? "unknown error"}`);
|
|
1172
|
+
return;
|
|
1173
|
+
}
|
|
1174
|
+
current.attempt += 1;
|
|
1175
|
+
if (current.attempt >= 8) {
|
|
1176
|
+
this.journalRetries.delete(retryKey);
|
|
1177
|
+
this.journalBlockedRuns.add(run.runId);
|
|
1178
|
+
console.warn(`[pi-workflows] lifecycle journal retry exhausted: ${error.message}`);
|
|
1179
|
+
return;
|
|
1180
|
+
}
|
|
1181
|
+
const delay = Math.min(
|
|
1182
|
+
JOURNAL_RETRY_MAX_DELAY_MS,
|
|
1183
|
+
JOURNAL_RETRY_INITIAL_DELAY_MS * 2 ** Math.min(current.attempt - 1, 6),
|
|
1184
|
+
);
|
|
1185
|
+
current.timer = setTimeout(attempt, delay);
|
|
1186
|
+
} finally {
|
|
1187
|
+
this.journalMutationKey = previousKey;
|
|
1188
|
+
}
|
|
1189
|
+
};
|
|
1190
|
+
this.journalRetries.set(retryKey, retry);
|
|
1191
|
+
queueMicrotask(attempt);
|
|
1192
|
+
}
|
|
1193
|
+
|
|
1194
|
+
private tryLifecycleMutation(run: WorkflowRun, key: string, mutation: () => void): boolean {
|
|
1195
|
+
const previousKey = this.journalMutationKey;
|
|
1196
|
+
this.journalMutationKey = `${run.runId}:${key}`;
|
|
1197
|
+
try {
|
|
1198
|
+
mutation();
|
|
1199
|
+
return true;
|
|
1200
|
+
} catch (error: unknown) {
|
|
1201
|
+
if (!(error instanceof JournalAppendError)) throw error;
|
|
1202
|
+
this.scheduleJournalRetry(run, key, mutation);
|
|
1203
|
+
return false;
|
|
1204
|
+
} finally {
|
|
1205
|
+
this.journalMutationKey = previousKey;
|
|
1206
|
+
}
|
|
1207
|
+
}
|
|
1208
|
+
|
|
1209
|
+
private write(event: JournalEvent): void {
|
|
1210
|
+
try {
|
|
1211
|
+
this.journal.append(event);
|
|
1212
|
+
} catch (error: unknown) {
|
|
1213
|
+
throw new JournalAppendError(event.kind, error);
|
|
1214
|
+
}
|
|
1215
|
+
}
|
|
1216
|
+
|
|
1217
|
+
private appendEvents(events: readonly JournalEvent[]): void {
|
|
1218
|
+
const key = this.journalMutationKey;
|
|
1219
|
+
const serialized = key ? JSON.stringify(events) : undefined;
|
|
1220
|
+
let start = 0;
|
|
1221
|
+
if (key && serialized !== undefined) {
|
|
1222
|
+
const prior = this.journalBatchProgress.get(key);
|
|
1223
|
+
if (prior && prior.serialized === serialized) {
|
|
1224
|
+
start = prior.nextIndex;
|
|
1225
|
+
} else {
|
|
1226
|
+
this.journalBatchProgress.set(key, { serialized, nextIndex: 0 });
|
|
1227
|
+
}
|
|
1228
|
+
}
|
|
1229
|
+
for (let index = start; index < events.length; index += 1) {
|
|
1230
|
+
const event = events[index];
|
|
1231
|
+
if (!event) continue;
|
|
1232
|
+
this.write(event);
|
|
1233
|
+
if (key) {
|
|
1234
|
+
const progress = this.journalBatchProgress.get(key);
|
|
1235
|
+
if (progress) progress.nextIndex = index + 1;
|
|
1236
|
+
}
|
|
1237
|
+
}
|
|
1238
|
+
if (key) this.journalBatchProgress.delete(key);
|
|
1239
|
+
}
|
|
1240
|
+
|
|
1241
|
+
private ownerForNode(run: WorkflowRun, nodeId: string): WorkflowOwner {
|
|
1242
|
+
return {
|
|
1243
|
+
extension: "pi-workflows",
|
|
1244
|
+
runId: run.runId,
|
|
1245
|
+
nodeId,
|
|
1246
|
+
...(run.attemptIds[nodeId] ? { attemptId: run.attemptIds[nodeId] } : {}),
|
|
1247
|
+
};
|
|
1248
|
+
}
|
|
1249
|
+
|
|
1250
|
+
private setWorkflowStatus(
|
|
1251
|
+
run: WorkflowRun,
|
|
1252
|
+
status: WorkflowStatus,
|
|
1253
|
+
error?: string,
|
|
1254
|
+
terminalIntent: WorkflowRun["terminalIntent"] = run.terminalIntent,
|
|
1255
|
+
): void {
|
|
1256
|
+
const nextStatus = transitionWorkflow(run.status, status);
|
|
1257
|
+
const timestamp = Date.now();
|
|
1258
|
+
const event: JournalEvent = {
|
|
1259
|
+
kind: "workflow_transition",
|
|
1260
|
+
schemaVersion: JOURNAL_SCHEMA_VERSION,
|
|
1261
|
+
runId: run.runId,
|
|
1262
|
+
status: nextStatus,
|
|
1263
|
+
timestamp,
|
|
1264
|
+
...(error === undefined ? {} : { error }),
|
|
1265
|
+
...(terminalIntent === undefined ? {} : { terminalIntent }),
|
|
1266
|
+
};
|
|
1267
|
+
// Do not advance the aggregate before the durable prefix accepts this
|
|
1268
|
+
// event. A caller can retry the same transition after append() throws.
|
|
1269
|
+
this.write(event);
|
|
1270
|
+
run.status = nextStatus;
|
|
1271
|
+
run.updatedAt = timestamp;
|
|
1272
|
+
if (error !== undefined) run.error = error;
|
|
1273
|
+
if (terminalIntent !== undefined) run.terminalIntent = terminalIntent;
|
|
1274
|
+
if (isTerminalWorkflow(nextStatus)) this.resolveWaiters(run);
|
|
1275
|
+
}
|
|
1276
|
+
|
|
1277
|
+
private setTaskStatus(run: WorkflowRun, nodeId: string, status: TaskStatus, agentId?: string): void {
|
|
1278
|
+
const previous = run.taskStatus[nodeId];
|
|
1279
|
+
if (!previous) return;
|
|
1280
|
+
if (!canTransitionTask(previous, status)) {
|
|
1281
|
+
if (isTerminalTask(previous)) return;
|
|
1282
|
+
throw new Error(`invalid task transition: ${previous} -> ${status}`);
|
|
1283
|
+
}
|
|
1284
|
+
const nextStatus = transitionTask(previous, status);
|
|
1285
|
+
const timestamp = Date.now();
|
|
1286
|
+
const event: JournalEvent = {
|
|
1287
|
+
kind: "task_transition",
|
|
1288
|
+
schemaVersion: JOURNAL_SCHEMA_VERSION,
|
|
1289
|
+
runId: run.runId,
|
|
1290
|
+
nodeId,
|
|
1291
|
+
status: nextStatus,
|
|
1292
|
+
...(agentId ? { agentId } : {}),
|
|
1293
|
+
...(run.attemptIds[nodeId] ? { attemptId: run.attemptIds[nodeId] } : {}),
|
|
1294
|
+
owner: this.ownerForNode(run, nodeId),
|
|
1295
|
+
timestamp,
|
|
1296
|
+
};
|
|
1297
|
+
this.write(event);
|
|
1298
|
+
run.taskStatus[nodeId] = nextStatus;
|
|
1299
|
+
run.updatedAt = timestamp;
|
|
1300
|
+
if (agentId) run.agentIds[nodeId] = agentId;
|
|
1301
|
+
}
|
|
1302
|
+
|
|
1303
|
+
private blockedDependentIds(run: WorkflowRun, failedNodeId: string): string[] {
|
|
1304
|
+
const queue = [failedNodeId];
|
|
1305
|
+
const blocked = new Set<string>();
|
|
1306
|
+
while (queue.length > 0) {
|
|
1307
|
+
const dependency = queue.shift();
|
|
1308
|
+
if (!dependency) continue;
|
|
1309
|
+
for (const task of run.definition.tasks) {
|
|
1310
|
+
if (!task.depends_on.includes(dependency) || blocked.has(task.id)) continue;
|
|
1311
|
+
const status = run.taskStatus[task.id];
|
|
1312
|
+
if (status !== "pending" && status !== "ready") continue;
|
|
1313
|
+
blocked.add(task.id);
|
|
1314
|
+
queue.push(task.id);
|
|
1315
|
+
}
|
|
1316
|
+
}
|
|
1317
|
+
return [...blocked];
|
|
1318
|
+
}
|
|
1319
|
+
|
|
1320
|
+
private commitTaskTerminal(
|
|
1321
|
+
run: WorkflowRun,
|
|
1322
|
+
nodeId: string,
|
|
1323
|
+
result: WorkflowRun["taskResults"][string],
|
|
1324
|
+
status: "completed" | "failed" | "stopped",
|
|
1325
|
+
failureMessage?: string,
|
|
1326
|
+
blockDependents = true,
|
|
1327
|
+
): void {
|
|
1328
|
+
const previous = run.taskStatus[nodeId];
|
|
1329
|
+
if (!previous || run.taskResults[nodeId] || isTerminalTask(previous)) return;
|
|
1330
|
+
if (!canTransitionTask(previous, status)) {
|
|
1331
|
+
throw new Error(`invalid task transition: ${previous} -> ${status}`);
|
|
1332
|
+
}
|
|
1333
|
+
const timestamp = result.updatedAt;
|
|
1334
|
+
const resultEvent: JournalEvent = {
|
|
1335
|
+
kind: "task_result",
|
|
1336
|
+
schemaVersion: JOURNAL_SCHEMA_VERSION,
|
|
1337
|
+
runId: run.runId,
|
|
1338
|
+
nodeId,
|
|
1339
|
+
result,
|
|
1340
|
+
attemptId: run.attemptIds[nodeId],
|
|
1341
|
+
owner: this.ownerForNode(run, nodeId),
|
|
1342
|
+
timestamp,
|
|
1343
|
+
};
|
|
1344
|
+
const transitionEvent: JournalEvent = {
|
|
1345
|
+
kind: "task_transition",
|
|
1346
|
+
schemaVersion: JOURNAL_SCHEMA_VERSION,
|
|
1347
|
+
runId: run.runId,
|
|
1348
|
+
nodeId,
|
|
1349
|
+
status,
|
|
1350
|
+
...(result.agentId ? { agentId: result.agentId } : {}),
|
|
1351
|
+
...(run.attemptIds[nodeId] ? { attemptId: run.attemptIds[nodeId] } : {}),
|
|
1352
|
+
owner: this.ownerForNode(run, nodeId),
|
|
1353
|
+
timestamp,
|
|
1354
|
+
};
|
|
1355
|
+
const blockedIds = blockDependents && status !== "completed" ? this.blockedDependentIds(run, nodeId) : [];
|
|
1356
|
+
const blockedEvents: JournalEvent[] = blockedIds.map((blockedNodeId) => ({
|
|
1357
|
+
kind: "task_transition",
|
|
1358
|
+
schemaVersion: JOURNAL_SCHEMA_VERSION,
|
|
1359
|
+
runId: run.runId,
|
|
1360
|
+
nodeId: blockedNodeId,
|
|
1361
|
+
status: "blocked",
|
|
1362
|
+
...(run.attemptIds[blockedNodeId] ? { attemptId: run.attemptIds[blockedNodeId] } : {}),
|
|
1363
|
+
owner: this.ownerForNode(run, blockedNodeId),
|
|
1364
|
+
timestamp,
|
|
1365
|
+
}));
|
|
1366
|
+
const message = failureMessage ?? result.error ?? `workflow task ${nodeId} ${status}`;
|
|
1367
|
+
const needsFailureCleanup =
|
|
1368
|
+
status !== "completed" &&
|
|
1369
|
+
run.terminalIntent !== "stop" &&
|
|
1370
|
+
(run.status !== "stopping" || run.terminalIntent !== "failure" || run.error !== message);
|
|
1371
|
+
const workflowEvent: JournalEvent | undefined = needsFailureCleanup
|
|
1372
|
+
? {
|
|
1373
|
+
kind: "workflow_transition",
|
|
1374
|
+
schemaVersion: JOURNAL_SCHEMA_VERSION,
|
|
1375
|
+
runId: run.runId,
|
|
1376
|
+
status: "stopping",
|
|
1377
|
+
terminalIntent: "failure",
|
|
1378
|
+
error: message,
|
|
1379
|
+
timestamp,
|
|
1380
|
+
}
|
|
1381
|
+
: undefined;
|
|
1382
|
+
this.appendEvents([resultEvent, transitionEvent, ...blockedEvents, ...(workflowEvent ? [workflowEvent] : [])]);
|
|
1383
|
+
|
|
1384
|
+
run.taskResults[nodeId] = result;
|
|
1385
|
+
run.taskStatus[nodeId] = status;
|
|
1386
|
+
if (result.agentId) run.agentIds[nodeId] = result.agentId;
|
|
1387
|
+
run.compactions[nodeId] = Math.max(run.compactions[nodeId] ?? 0, result.compactionCount);
|
|
1388
|
+
run.updatedAt = timestamp;
|
|
1389
|
+
for (const blockedNodeId of blockedIds) {
|
|
1390
|
+
run.taskStatus[blockedNodeId] = "blocked";
|
|
1391
|
+
run.updatedAt = timestamp;
|
|
1392
|
+
}
|
|
1393
|
+
if (workflowEvent) {
|
|
1394
|
+
run.status = "stopping";
|
|
1395
|
+
run.terminalIntent = "failure";
|
|
1396
|
+
run.error = message;
|
|
1397
|
+
run.updatedAt = timestamp;
|
|
1398
|
+
this.startFailureCleanup(run, message);
|
|
1399
|
+
}
|
|
1400
|
+
}
|
|
1401
|
+
|
|
1402
|
+
private commitSynthesisTerminal(
|
|
1403
|
+
run: WorkflowRun,
|
|
1404
|
+
result: WorkflowRun["synthesisResult"],
|
|
1405
|
+
finalStatus: "completed" | "failed" | "stopped",
|
|
1406
|
+
): void {
|
|
1407
|
+
if (run.synthesisResult || !result) return;
|
|
1408
|
+
const timestamp = result.updatedAt;
|
|
1409
|
+
const resultEvent: JournalEvent = {
|
|
1410
|
+
kind: "synthesis_result",
|
|
1411
|
+
schemaVersion: JOURNAL_SCHEMA_VERSION,
|
|
1412
|
+
runId: run.runId,
|
|
1413
|
+
result,
|
|
1414
|
+
attemptId: run.attemptIds[SYNTHESIS_NODE_ID],
|
|
1415
|
+
owner: this.ownerForNode(run, SYNTHESIS_NODE_ID),
|
|
1416
|
+
timestamp,
|
|
1417
|
+
};
|
|
1418
|
+
const settling = run.status === "stopping" || this.settling.has(run.runId);
|
|
1419
|
+
const workflowEvent: JournalEvent | undefined = settling
|
|
1420
|
+
? undefined
|
|
1421
|
+
: {
|
|
1422
|
+
kind: "workflow_transition",
|
|
1423
|
+
schemaVersion: JOURNAL_SCHEMA_VERSION,
|
|
1424
|
+
runId: run.runId,
|
|
1425
|
+
status: finalStatus,
|
|
1426
|
+
...(result.error ? { error: result.error } : {}),
|
|
1427
|
+
...(run.terminalIntent ? { terminalIntent: run.terminalIntent } : {}),
|
|
1428
|
+
timestamp,
|
|
1429
|
+
};
|
|
1430
|
+
this.appendEvents([resultEvent, ...(workflowEvent ? [workflowEvent] : [])]);
|
|
1431
|
+
|
|
1432
|
+
run.synthesisResult = result;
|
|
1433
|
+
if (result.agentId) run.synthesisAgentId = result.agentId;
|
|
1434
|
+
run.compactions[SYNTHESIS_NODE_ID] = Math.max(run.compactions[SYNTHESIS_NODE_ID] ?? 0, result.compactionCount);
|
|
1435
|
+
run.updatedAt = timestamp;
|
|
1436
|
+
if (workflowEvent) {
|
|
1437
|
+
run.status = finalStatus;
|
|
1438
|
+
run.updatedAt = timestamp;
|
|
1439
|
+
if (result.error) run.error = result.error;
|
|
1440
|
+
if (isTerminalWorkflow(finalStatus)) this.resolveWaiters(run);
|
|
1441
|
+
}
|
|
1442
|
+
}
|
|
1443
|
+
|
|
1444
|
+
private commitDeferredTerminals(run: WorkflowRun, records: readonly DeferredTerminalRecord[]): void {
|
|
1445
|
+
const pending = records.filter((record) =>
|
|
1446
|
+
record.nodeId === SYNTHESIS_NODE_ID
|
|
1447
|
+
? run.synthesisResult === undefined
|
|
1448
|
+
: run.taskResults[record.nodeId] === undefined && !isTerminalTask(run.taskStatus[record.nodeId]),
|
|
1449
|
+
);
|
|
1450
|
+
if (pending.length === 0) return;
|
|
1451
|
+
const events: JournalEvent[] = pending.map((record) =>
|
|
1452
|
+
record.nodeId === SYNTHESIS_NODE_ID
|
|
1453
|
+
? {
|
|
1454
|
+
kind: "synthesis_result",
|
|
1455
|
+
schemaVersion: JOURNAL_SCHEMA_VERSION,
|
|
1456
|
+
runId: run.runId,
|
|
1457
|
+
result: record.result,
|
|
1458
|
+
attemptId: run.attemptIds[SYNTHESIS_NODE_ID],
|
|
1459
|
+
owner: this.ownerForNode(run, SYNTHESIS_NODE_ID),
|
|
1460
|
+
timestamp: Math.max(run.updatedAt, record.result.updatedAt),
|
|
1461
|
+
}
|
|
1462
|
+
: {
|
|
1463
|
+
kind: "task_result",
|
|
1464
|
+
schemaVersion: JOURNAL_SCHEMA_VERSION,
|
|
1465
|
+
runId: run.runId,
|
|
1466
|
+
nodeId: record.nodeId,
|
|
1467
|
+
result: record.result,
|
|
1468
|
+
attemptId: run.attemptIds[record.nodeId],
|
|
1469
|
+
owner: this.ownerForNode(run, record.nodeId),
|
|
1470
|
+
timestamp: Math.max(run.updatedAt, record.result.updatedAt),
|
|
1471
|
+
},
|
|
1472
|
+
);
|
|
1473
|
+
const transitions: JournalEvent[] = pending
|
|
1474
|
+
.filter((record) => record.nodeId !== SYNTHESIS_NODE_ID)
|
|
1475
|
+
.map((record) => ({
|
|
1476
|
+
kind: "task_transition",
|
|
1477
|
+
schemaVersion: JOURNAL_SCHEMA_VERSION,
|
|
1478
|
+
runId: run.runId,
|
|
1479
|
+
nodeId: record.nodeId,
|
|
1480
|
+
status: record.result.status,
|
|
1481
|
+
...(record.result.agentId ? { agentId: record.result.agentId } : {}),
|
|
1482
|
+
attemptId: run.attemptIds[record.nodeId],
|
|
1483
|
+
owner: this.ownerForNode(run, record.nodeId),
|
|
1484
|
+
timestamp: Math.max(run.updatedAt, record.result.updatedAt),
|
|
1485
|
+
}));
|
|
1486
|
+
this.appendEvents([...events, ...transitions]);
|
|
1487
|
+
for (const record of pending) {
|
|
1488
|
+
if (record.nodeId === SYNTHESIS_NODE_ID) {
|
|
1489
|
+
run.synthesisResult = record.result;
|
|
1490
|
+
if (record.result.agentId) run.synthesisAgentId = record.result.agentId;
|
|
1491
|
+
} else {
|
|
1492
|
+
run.taskResults[record.nodeId] = record.result;
|
|
1493
|
+
run.taskStatus[record.nodeId] = record.result.status;
|
|
1494
|
+
if (record.result.agentId) run.agentIds[record.nodeId] = record.result.agentId;
|
|
1495
|
+
}
|
|
1496
|
+
run.compactions[record.nodeId] = Math.max(run.compactions[record.nodeId] ?? 0, record.result.compactionCount);
|
|
1497
|
+
run.updatedAt = Math.max(run.updatedAt, record.result.updatedAt);
|
|
1498
|
+
}
|
|
1499
|
+
}
|
|
1500
|
+
private async pump(run: WorkflowRun): Promise<void> {
|
|
1501
|
+
if (this.branchChanging || this.pumping.has(run.runId) || run.status !== "running") return;
|
|
1502
|
+
this.pumping.add(run.runId);
|
|
1503
|
+
const branchEpoch = this.branchEpoch;
|
|
1504
|
+
const isCurrent = (): boolean =>
|
|
1505
|
+
!this.branchChanging && branchEpoch === this.branchEpoch && this.runs.get(run.runId) === run;
|
|
1506
|
+
try {
|
|
1507
|
+
if (!isCurrent()) return;
|
|
1508
|
+
const completed = new Set(
|
|
1509
|
+
Object.entries(run.taskStatus)
|
|
1510
|
+
.filter(([, status]) => status === "completed")
|
|
1511
|
+
.map(([nodeId]) => nodeId),
|
|
1512
|
+
);
|
|
1513
|
+
const ready = readyTaskIds(run.definition, completed);
|
|
1514
|
+
for (const nodeId of ready) {
|
|
1515
|
+
if (!isCurrent() || run.status !== "running") break;
|
|
1516
|
+
if (run.taskStatus[nodeId] !== "pending" && run.taskStatus[nodeId] !== "ready") continue;
|
|
1517
|
+
if (run.taskStatus[nodeId] === "pending") this.setTaskStatus(run, nodeId, "ready");
|
|
1518
|
+
this.setTaskStatus(run, nodeId, "dispatching");
|
|
1519
|
+
const dispatch = this.dispatchTask(run, nodeId, branchEpoch);
|
|
1520
|
+
this.trackDispatch(run.runId, dispatch);
|
|
1521
|
+
}
|
|
1522
|
+
if (!isCurrent() || run.status !== "running") return;
|
|
1523
|
+
const statuses = Object.values(run.taskStatus);
|
|
1524
|
+
if (statuses.some((status) => status === "failed" || status === "blocked" || status === "stopped")) {
|
|
1525
|
+
if (statuses.some((status) => status === "stopped") && run.terminalIntent === "stop") {
|
|
1526
|
+
if (!isCurrent()) return;
|
|
1527
|
+
await this.stop(run);
|
|
1528
|
+
} else {
|
|
1529
|
+
if (!isCurrent()) return;
|
|
1530
|
+
await this.failRun(
|
|
1531
|
+
run,
|
|
1532
|
+
statuses.some((status) => status === "stopped")
|
|
1533
|
+
? "one or more workflow tasks stopped unexpectedly"
|
|
1534
|
+
: "one or more workflow tasks failed",
|
|
1535
|
+
);
|
|
1536
|
+
}
|
|
1537
|
+
return;
|
|
1538
|
+
}
|
|
1539
|
+
if (!isCurrent() || statuses.some((status) => !isTerminalTask(status))) return;
|
|
1540
|
+
if (run.definition.synthesis && !run.synthesisResult && (!run.synthesisAgentId || run.status === "running")) {
|
|
1541
|
+
if (!isCurrent()) return;
|
|
1542
|
+
this.setWorkflowStatus(run, "synthesizing");
|
|
1543
|
+
const dispatch = this.dispatchSynthesis(run);
|
|
1544
|
+
this.trackDispatch(run.runId, dispatch);
|
|
1545
|
+
return;
|
|
1546
|
+
}
|
|
1547
|
+
if (!isCurrent()) return;
|
|
1548
|
+
if (!run.definition.synthesis) {
|
|
1549
|
+
this.setWorkflowStatus(run, "completed");
|
|
1550
|
+
} else if (run.synthesisResult) {
|
|
1551
|
+
const synthesisStatus =
|
|
1552
|
+
run.synthesisResult.status === "completed"
|
|
1553
|
+
? "completed"
|
|
1554
|
+
: run.synthesisResult.status === "stopped"
|
|
1555
|
+
? "stopped"
|
|
1556
|
+
: "failed";
|
|
1557
|
+
this.setWorkflowStatus(run, synthesisStatus, run.synthesisResult.error);
|
|
1558
|
+
}
|
|
1559
|
+
} finally {
|
|
1560
|
+
if (this.runs.get(run.runId) === run && this.branchEpoch === branchEpoch) {
|
|
1561
|
+
this.pumping.delete(run.runId);
|
|
1562
|
+
}
|
|
1563
|
+
}
|
|
1564
|
+
}
|
|
1565
|
+
|
|
1566
|
+
private recoverInterruptedRun(
|
|
1567
|
+
run: WorkflowRun,
|
|
1568
|
+
plan: readonly RecoveryPlanItem[],
|
|
1569
|
+
terminalResults: readonly DeferredTerminalRecord[],
|
|
1570
|
+
): void {
|
|
1571
|
+
const sourceKey = this.recoverySourceKey(run, plan, terminalResults);
|
|
1572
|
+
const memo = this.memoizedRecovery(run.runId, "run_recovery", sourceKey);
|
|
1573
|
+
if (memo?.kind === "run_recovery") {
|
|
1574
|
+
// The durable event belongs to this branch and source snapshot. Reapply
|
|
1575
|
+
// it to the freshly replayed run without another append.
|
|
1576
|
+
applyRecoveryEvent(run, memo);
|
|
1577
|
+
return;
|
|
1578
|
+
}
|
|
1579
|
+
|
|
1580
|
+
const exhausted = plan.find((item) => item.sourceGeneration + 1 > MAX_ATTEMPTS_PER_NODE);
|
|
1581
|
+
if (exhausted) {
|
|
1582
|
+
const message = `workflow node ${exhausted.nodeId} exceeded interrupted-attempt limit`;
|
|
1583
|
+
if (exhausted.nodeId === SYNTHESIS_NODE_ID) {
|
|
1584
|
+
const result = resultFromLifecycle(
|
|
1585
|
+
"failed",
|
|
1586
|
+
run.synthesisAgentId,
|
|
1587
|
+
undefined,
|
|
1588
|
+
message,
|
|
1589
|
+
run.compactions[SYNTHESIS_NODE_ID] ?? 0,
|
|
1590
|
+
);
|
|
1591
|
+
result.attemptId = run.attemptIds[SYNTHESIS_NODE_ID];
|
|
1592
|
+
this.commitSynthesisTerminal(run, result, "failed");
|
|
1593
|
+
} else {
|
|
1594
|
+
this.failTask(run, exhausted.nodeId, message);
|
|
1595
|
+
}
|
|
1596
|
+
return;
|
|
1597
|
+
}
|
|
1598
|
+
|
|
1599
|
+
const rotations: RecoveryRotation[] = plan.map((item) => {
|
|
1600
|
+
const generation = item.sourceGeneration + 1;
|
|
1601
|
+
const attemptId = `${run.runId}/${item.nodeId}/attempt-${generation}`;
|
|
1602
|
+
return {
|
|
1603
|
+
nodeId: item.nodeId,
|
|
1604
|
+
sourceAttemptId: item.sourceAttemptId,
|
|
1605
|
+
sourceGeneration: item.sourceGeneration,
|
|
1606
|
+
sourceStatus: item.sourceStatus,
|
|
1607
|
+
attemptId,
|
|
1608
|
+
generation,
|
|
1609
|
+
owner: { extension: "pi-workflows", runId: run.runId, nodeId: item.nodeId, attemptId },
|
|
1610
|
+
...(item.oldAgentId ? { supersededAgentId: item.oldAgentId } : {}),
|
|
1611
|
+
};
|
|
1612
|
+
});
|
|
1613
|
+
const recoveryBase = {
|
|
1614
|
+
kind: "run_recovery" as const,
|
|
1615
|
+
schemaVersion: JOURNAL_SCHEMA_VERSION as 2,
|
|
1616
|
+
runId: run.runId,
|
|
1617
|
+
status: "interrupted" as const,
|
|
1618
|
+
branchGeneration: this.recoveryBranchGeneration,
|
|
1619
|
+
rotations,
|
|
1620
|
+
...(terminalResults.length > 0
|
|
1621
|
+
? {
|
|
1622
|
+
terminalResults: terminalResults.map(
|
|
1623
|
+
({ kind: _kind, runId: _runId, nodeId, attemptId, result, owner }) => ({
|
|
1624
|
+
nodeId,
|
|
1625
|
+
attemptId,
|
|
1626
|
+
result: { ...result, updatedAt: run.updatedAt },
|
|
1627
|
+
owner,
|
|
1628
|
+
}),
|
|
1629
|
+
),
|
|
1630
|
+
}
|
|
1631
|
+
: {}),
|
|
1632
|
+
};
|
|
1633
|
+
const event: RunRecoveryEvent = {
|
|
1634
|
+
...recoveryBase,
|
|
1635
|
+
recoveryId: deriveRecoveryId(recoveryBase),
|
|
1636
|
+
timestamp: Date.now(),
|
|
1637
|
+
};
|
|
1638
|
+
// This is the only persistence boundary for the complete recovery. Do not
|
|
1639
|
+
// mutate the run or memoize until append returns successfully.
|
|
1640
|
+
this.write(event);
|
|
1641
|
+
applyRecoveryEvent(run, event);
|
|
1642
|
+
this.rememberRecovery({
|
|
1643
|
+
branchGeneration: this.recoveryBranchGeneration,
|
|
1644
|
+
runId: run.runId,
|
|
1645
|
+
key: sourceKey,
|
|
1646
|
+
event,
|
|
1647
|
+
});
|
|
1648
|
+
}
|
|
1649
|
+
|
|
1650
|
+
private recoverInterruptedNode(
|
|
1651
|
+
run: WorkflowRun,
|
|
1652
|
+
nodeId: string,
|
|
1653
|
+
oldAgentId: string,
|
|
1654
|
+
resumeImmediately = true,
|
|
1655
|
+
legacyMigration = false,
|
|
1656
|
+
): void {
|
|
1657
|
+
const sourceGeneration = run.attempts[nodeId] ?? 1;
|
|
1658
|
+
const sourceAttemptId = this.sourceAttemptId(run, nodeId);
|
|
1659
|
+
const sourceStatus: RecoverySourceStatus = nodeId === SYNTHESIS_NODE_ID ? "synthesizing" : run.taskStatus[nodeId];
|
|
1660
|
+
if (nodeId !== SYNTHESIS_NODE_ID && isTerminalTask(run.taskStatus[nodeId])) return;
|
|
1661
|
+
const plan: RecoveryPlanItem = {
|
|
1662
|
+
nodeId,
|
|
1663
|
+
sourceAttemptId,
|
|
1664
|
+
sourceGeneration,
|
|
1665
|
+
sourceStatus,
|
|
1666
|
+
oldAgentId,
|
|
1667
|
+
};
|
|
1668
|
+
const sourceKey = this.recoverySourceKey(run, [plan]);
|
|
1669
|
+
const memo = this.memoizedRecovery(run.runId, "attempt_recovery", sourceKey);
|
|
1670
|
+
if (memo?.kind === "attempt_recovery") {
|
|
1671
|
+
applyRecoveryEvent(run, memo);
|
|
1672
|
+
this.resumeRecoveredNode(run, nodeId, resumeImmediately);
|
|
1673
|
+
return;
|
|
1674
|
+
}
|
|
1675
|
+
|
|
1676
|
+
const generation = sourceGeneration + 1;
|
|
1677
|
+
if (generation > MAX_ATTEMPTS_PER_NODE) {
|
|
1678
|
+
const message = `workflow node ${nodeId} exceeded interrupted-attempt limit`;
|
|
1679
|
+
if (nodeId === SYNTHESIS_NODE_ID) {
|
|
1680
|
+
const result = resultFromLifecycle(
|
|
1681
|
+
"failed",
|
|
1682
|
+
run.synthesisAgentId,
|
|
1683
|
+
undefined,
|
|
1684
|
+
message,
|
|
1685
|
+
run.compactions[nodeId] ?? 0,
|
|
1686
|
+
);
|
|
1687
|
+
result.attemptId = run.attemptIds[nodeId];
|
|
1688
|
+
this.commitSynthesisTerminal(run, result, "failed");
|
|
1689
|
+
} else {
|
|
1690
|
+
this.failTask(run, nodeId, message);
|
|
1691
|
+
}
|
|
1692
|
+
return;
|
|
1693
|
+
}
|
|
1694
|
+
|
|
1695
|
+
const attemptId = `${run.runId}/${nodeId}/attempt-${generation}`;
|
|
1696
|
+
const rotation: RecoveryRotation = {
|
|
1697
|
+
nodeId,
|
|
1698
|
+
sourceAttemptId,
|
|
1699
|
+
sourceGeneration,
|
|
1700
|
+
sourceStatus,
|
|
1701
|
+
attemptId,
|
|
1702
|
+
generation,
|
|
1703
|
+
owner: { extension: "pi-workflows", runId: run.runId, nodeId, attemptId },
|
|
1704
|
+
...(oldAgentId ? { supersededAgentId: oldAgentId } : {}),
|
|
1705
|
+
};
|
|
1706
|
+
const event: AttemptRecoveryEvent = {
|
|
1707
|
+
kind: "attempt_recovery",
|
|
1708
|
+
schemaVersion: JOURNAL_SCHEMA_VERSION,
|
|
1709
|
+
runId: run.runId,
|
|
1710
|
+
nodeId,
|
|
1711
|
+
branchGeneration: this.recoveryBranchGeneration,
|
|
1712
|
+
rotation,
|
|
1713
|
+
...(legacyMigration ? { legacyMigration: true } : {}),
|
|
1714
|
+
timestamp: Date.now(),
|
|
1715
|
+
};
|
|
1716
|
+
this.write(event);
|
|
1717
|
+
applyRecoveryEvent(run, event);
|
|
1718
|
+
this.rememberRecovery({
|
|
1719
|
+
branchGeneration: this.recoveryBranchGeneration,
|
|
1720
|
+
runId: run.runId,
|
|
1721
|
+
key: sourceKey,
|
|
1722
|
+
event,
|
|
1723
|
+
});
|
|
1724
|
+
this.resumeRecoveredNode(run, nodeId, resumeImmediately);
|
|
1725
|
+
}
|
|
1726
|
+
|
|
1727
|
+
private resumeRecoveredNode(run: WorkflowRun, nodeId: string, resumeImmediately: boolean): void {
|
|
1728
|
+
if (!resumeImmediately) return;
|
|
1729
|
+
if (nodeId === SYNTHESIS_NODE_ID && run.status === "synthesizing") {
|
|
1730
|
+
const dispatch = this.dispatchSynthesis(run);
|
|
1731
|
+
this.trackDispatch(run.runId, dispatch);
|
|
1732
|
+
} else if (nodeId !== SYNTHESIS_NODE_ID && run.status === "running") {
|
|
1733
|
+
void this.pump(run);
|
|
1734
|
+
}
|
|
1735
|
+
}
|
|
1736
|
+
|
|
1737
|
+
private applyManagedTerminal(run: WorkflowRun, nodeId: string, response: ManagedSpawnResponse): void {
|
|
1738
|
+
const snapshot = response.terminal;
|
|
1739
|
+
if (!snapshot) return;
|
|
1740
|
+
if (snapshot.status === "interrupted") {
|
|
1741
|
+
this.recoverInterruptedNode(run, nodeId, response.id);
|
|
1742
|
+
return;
|
|
1743
|
+
}
|
|
1744
|
+
const status = managedTerminalStatus(snapshot);
|
|
1745
|
+
const compactionCount = Math.max(run.compactions[nodeId] ?? 0, snapshot.compactionCount);
|
|
1746
|
+
const result = resultFromLifecycle(
|
|
1747
|
+
status,
|
|
1748
|
+
response.id,
|
|
1749
|
+
snapshot.result,
|
|
1750
|
+
snapshot.error,
|
|
1751
|
+
compactionCount,
|
|
1752
|
+
snapshot.completedAt,
|
|
1753
|
+
snapshot.outputFile,
|
|
1754
|
+
snapshot.tokenCount,
|
|
1755
|
+
);
|
|
1756
|
+
result.attemptId = run.attemptIds[nodeId];
|
|
1757
|
+
|
|
1758
|
+
if (nodeId === SYNTHESIS_NODE_ID) {
|
|
1759
|
+
this.commitSynthesisTerminal(run, result, status);
|
|
1760
|
+
return;
|
|
1761
|
+
}
|
|
1762
|
+
if (!run.taskStatus[nodeId] || run.taskResults[nodeId] || isTerminalTask(run.taskStatus[nodeId])) return;
|
|
1763
|
+
this.commitTaskTerminal(run, nodeId, result, status);
|
|
1764
|
+
if (status === "completed" && run.status !== "stopping" && !this.settling.has(run.runId)) {
|
|
1765
|
+
this.maybePause(run);
|
|
1766
|
+
if (run.status === "running") void this.pump(run);
|
|
1767
|
+
}
|
|
1768
|
+
}
|
|
1769
|
+
|
|
1770
|
+
private async dispatchTask(run: WorkflowRun, nodeId: string, branchEpoch: number): Promise<void> {
|
|
1771
|
+
const task = run.definition.tasks.find((candidate) => candidate.id === nodeId);
|
|
1772
|
+
if (
|
|
1773
|
+
!task ||
|
|
1774
|
+
this.branchChanging ||
|
|
1775
|
+
branchEpoch !== this.branchEpoch ||
|
|
1776
|
+
this.runs.get(run.runId) !== run ||
|
|
1777
|
+
run.status !== "running"
|
|
1778
|
+
)
|
|
1779
|
+
return;
|
|
1780
|
+
const attemptId = run.attemptIds[nodeId];
|
|
1781
|
+
const dispatchTaskDefinition =
|
|
1782
|
+
task.tier === undefined && run.definition.tier !== undefined ? { ...task, tier: run.definition.tier } : task;
|
|
1783
|
+
let response: ManagedSpawnResponse;
|
|
1784
|
+
try {
|
|
1785
|
+
response = normalizeManagedSpawnResponse(
|
|
1786
|
+
await spawnManaged(this.client, dispatchTaskDefinition, run.runId, nodeId, attemptId),
|
|
1787
|
+
);
|
|
1788
|
+
} catch (error: unknown) {
|
|
1789
|
+
if (branchEpoch !== this.branchEpoch || this.runs.get(run.runId) !== run) return;
|
|
1790
|
+
const message = boundedError(error) ?? "managed spawn failed";
|
|
1791
|
+
if ((run.status as WorkflowStatus) === "stopping") {
|
|
1792
|
+
if (!isTerminalTask(run.taskStatus[nodeId])) this.setTaskStatus(run, nodeId, "stopped", run.agentIds[nodeId]);
|
|
1793
|
+
return;
|
|
1794
|
+
}
|
|
1795
|
+
this.failTask(run, nodeId, message);
|
|
1796
|
+
return;
|
|
1797
|
+
}
|
|
1798
|
+
|
|
1799
|
+
if (
|
|
1800
|
+
this.branchChanging ||
|
|
1801
|
+
branchEpoch !== this.branchEpoch ||
|
|
1802
|
+
this.runs.get(run.runId) !== run ||
|
|
1803
|
+
run.status !== "running"
|
|
1804
|
+
) {
|
|
1805
|
+
if (
|
|
1806
|
+
!this.branchChanging &&
|
|
1807
|
+
branchEpoch === this.branchEpoch &&
|
|
1808
|
+
this.runs.get(run.runId) === run &&
|
|
1809
|
+
(run.status as WorkflowStatus) === "stopping"
|
|
1810
|
+
) {
|
|
1811
|
+
run.agentIds[nodeId] = response.id;
|
|
1812
|
+
this.rememberLateDispatchTarget(run, nodeId, response.id, attemptId);
|
|
1813
|
+
return;
|
|
1814
|
+
}
|
|
1815
|
+
this.quarantineAgent(response.id);
|
|
1816
|
+
const allowStaleBranch = this.branchChanging || branchEpoch !== this.branchEpoch;
|
|
1817
|
+
const quiesced = await this.stopAndQuiesceOwned(
|
|
1818
|
+
run,
|
|
1819
|
+
nodeId,
|
|
1820
|
+
response.id,
|
|
1821
|
+
attemptId,
|
|
1822
|
+
branchEpoch,
|
|
1823
|
+
!response.terminal,
|
|
1824
|
+
allowStaleBranch,
|
|
1825
|
+
);
|
|
1826
|
+
if (allowStaleBranch && !quiesced) this.rememberLateDispatchQuiescence(run.runId, response.id);
|
|
1827
|
+
return;
|
|
1828
|
+
}
|
|
1829
|
+
if (response.terminal) {
|
|
1830
|
+
// Keep terminal persistence failures outside the spawn-error handler. A
|
|
1831
|
+
// journal append failure must leave the current attempt untouched rather
|
|
1832
|
+
// than turning an already durable managed result into a new task failure.
|
|
1833
|
+
this.tryLifecycleMutation(run, `managed-terminal:${nodeId}:${attemptId}`, () => {
|
|
1834
|
+
this.applyManagedTerminal(run, nodeId, response);
|
|
1835
|
+
});
|
|
1836
|
+
return;
|
|
1837
|
+
}
|
|
1838
|
+
if (
|
|
1839
|
+
this.branchChanging ||
|
|
1840
|
+
branchEpoch !== this.branchEpoch ||
|
|
1841
|
+
this.runs.get(run.runId) !== run ||
|
|
1842
|
+
run.status !== "running"
|
|
1843
|
+
) {
|
|
1844
|
+
this.quarantineAgent(response.id);
|
|
1845
|
+
const allowStaleBranch = this.branchChanging || branchEpoch !== this.branchEpoch;
|
|
1846
|
+
const quiesced = await this.stopAndQuiesceOwned(
|
|
1847
|
+
run,
|
|
1848
|
+
nodeId,
|
|
1849
|
+
response.id,
|
|
1850
|
+
attemptId,
|
|
1851
|
+
branchEpoch,
|
|
1852
|
+
!response.terminal,
|
|
1853
|
+
allowStaleBranch,
|
|
1854
|
+
);
|
|
1855
|
+
if (allowStaleBranch && !quiesced) this.rememberLateDispatchQuiescence(run.runId, response.id);
|
|
1856
|
+
return;
|
|
1857
|
+
}
|
|
1858
|
+
const taskStatus = run.taskStatus[nodeId];
|
|
1859
|
+
if (response.state === "running" && (taskStatus === "dispatching" || taskStatus === "queued")) {
|
|
1860
|
+
this.setTaskStatus(run, nodeId, "running", response.id);
|
|
1861
|
+
} else if (taskStatus === "dispatching") {
|
|
1862
|
+
this.setTaskStatus(run, nodeId, "queued", response.id);
|
|
1863
|
+
}
|
|
1864
|
+
}
|
|
1865
|
+
|
|
1866
|
+
private async dispatchSynthesis(run: WorkflowRun, branchEpoch = this.branchEpoch): Promise<void> {
|
|
1867
|
+
const synthesis = run.definition.synthesis;
|
|
1868
|
+
if (
|
|
1869
|
+
!synthesis ||
|
|
1870
|
+
this.branchChanging ||
|
|
1871
|
+
branchEpoch !== this.branchEpoch ||
|
|
1872
|
+
this.runs.get(run.runId) !== run ||
|
|
1873
|
+
run.status !== "synthesizing"
|
|
1874
|
+
)
|
|
1875
|
+
return;
|
|
1876
|
+
const prompt = buildSynthesisPrompt(run.definition, run);
|
|
1877
|
+
const task: WorkflowTask = {
|
|
1878
|
+
id: SYNTHESIS_NODE_ID,
|
|
1879
|
+
subagent_type: synthesis.subagent_type,
|
|
1880
|
+
tier: synthesis.tier ?? run.definition.tier,
|
|
1881
|
+
description: "Synthesize workflow results",
|
|
1882
|
+
prompt,
|
|
1883
|
+
depends_on: [],
|
|
1884
|
+
};
|
|
1885
|
+
const attemptId = run.attemptIds[SYNTHESIS_NODE_ID];
|
|
1886
|
+
let response: ManagedSpawnResponse;
|
|
1887
|
+
try {
|
|
1888
|
+
response = normalizeManagedSpawnResponse(
|
|
1889
|
+
await spawnManaged(this.client, task, run.runId, SYNTHESIS_NODE_ID, attemptId),
|
|
1890
|
+
);
|
|
1891
|
+
} catch (error: unknown) {
|
|
1892
|
+
if (branchEpoch !== this.branchEpoch || this.runs.get(run.runId) !== run) return;
|
|
1893
|
+
const result = resultFromLifecycle("failed", run.synthesisAgentId, undefined, boundedError(error), 0);
|
|
1894
|
+
result.attemptId = run.attemptIds[SYNTHESIS_NODE_ID];
|
|
1895
|
+
this.commitSynthesisTerminal(run, result, "failed");
|
|
1896
|
+
return;
|
|
1897
|
+
}
|
|
1898
|
+
|
|
1899
|
+
if (
|
|
1900
|
+
this.branchChanging ||
|
|
1901
|
+
branchEpoch !== this.branchEpoch ||
|
|
1902
|
+
this.runs.get(run.runId) !== run ||
|
|
1903
|
+
run.status !== "synthesizing"
|
|
1904
|
+
) {
|
|
1905
|
+
if (
|
|
1906
|
+
!this.branchChanging &&
|
|
1907
|
+
branchEpoch === this.branchEpoch &&
|
|
1908
|
+
this.runs.get(run.runId) === run &&
|
|
1909
|
+
(run.status as WorkflowStatus) === "stopping"
|
|
1910
|
+
) {
|
|
1911
|
+
run.synthesisAgentId = response.id;
|
|
1912
|
+
this.rememberLateDispatchTarget(run, SYNTHESIS_NODE_ID, response.id, attemptId);
|
|
1913
|
+
return;
|
|
1914
|
+
}
|
|
1915
|
+
this.quarantineAgent(response.id);
|
|
1916
|
+
const allowStaleBranch = this.branchChanging || branchEpoch !== this.branchEpoch;
|
|
1917
|
+
const quiesced = await this.stopAndQuiesceOwned(
|
|
1918
|
+
run,
|
|
1919
|
+
SYNTHESIS_NODE_ID,
|
|
1920
|
+
response.id,
|
|
1921
|
+
attemptId,
|
|
1922
|
+
branchEpoch,
|
|
1923
|
+
!response.terminal,
|
|
1924
|
+
allowStaleBranch,
|
|
1925
|
+
);
|
|
1926
|
+
if (allowStaleBranch && !quiesced) this.rememberLateDispatchQuiescence(run.runId, response.id);
|
|
1927
|
+
return;
|
|
1928
|
+
}
|
|
1929
|
+
if (response.terminal) {
|
|
1930
|
+
// Do not route a terminal journal failure through the RPC error path; the
|
|
1931
|
+
// managed terminal fact remains retryable/recoverable as this attempt.
|
|
1932
|
+
this.tryLifecycleMutation(run, `managed-terminal:${SYNTHESIS_NODE_ID}:${attemptId}`, () => {
|
|
1933
|
+
this.applyManagedTerminal(run, SYNTHESIS_NODE_ID, response);
|
|
1934
|
+
});
|
|
1935
|
+
return;
|
|
1936
|
+
}
|
|
1937
|
+
run.synthesisAgentId ??= response.id;
|
|
1938
|
+
if (
|
|
1939
|
+
this.branchChanging ||
|
|
1940
|
+
branchEpoch !== this.branchEpoch ||
|
|
1941
|
+
this.runs.get(run.runId) !== run ||
|
|
1942
|
+
run.status !== "synthesizing"
|
|
1943
|
+
) {
|
|
1944
|
+
this.quarantineAgent(response.id);
|
|
1945
|
+
const allowStaleBranch = this.branchChanging || branchEpoch !== this.branchEpoch;
|
|
1946
|
+
const quiesced = await this.stopAndQuiesceOwned(
|
|
1947
|
+
run,
|
|
1948
|
+
SYNTHESIS_NODE_ID,
|
|
1949
|
+
response.id,
|
|
1950
|
+
attemptId,
|
|
1951
|
+
branchEpoch,
|
|
1952
|
+
!response.terminal,
|
|
1953
|
+
allowStaleBranch,
|
|
1954
|
+
);
|
|
1955
|
+
if (allowStaleBranch && !quiesced) this.rememberLateDispatchQuiescence(run.runId, response.id);
|
|
1956
|
+
return;
|
|
1957
|
+
}
|
|
1958
|
+
}
|
|
1959
|
+
/**
|
|
1960
|
+
* Gate lifecycle observations at the workflow boundary. Attempt-aware runs
|
|
1961
|
+
* must carry the exact current generation and agent identity; only a created
|
|
1962
|
+
* event may establish an agent id when dispatch has not returned one yet.
|
|
1963
|
+
* Legacy journal runs intentionally retain the pre-v2 permissive behavior.
|
|
1964
|
+
*/
|
|
1965
|
+
private acceptsLifecycleEvent(
|
|
1966
|
+
run: WorkflowRun,
|
|
1967
|
+
nodeId: string,
|
|
1968
|
+
eventName: string,
|
|
1969
|
+
ownerAttemptId: string | undefined,
|
|
1970
|
+
agentId: string | undefined,
|
|
1971
|
+
): boolean {
|
|
1972
|
+
const expectedAttemptId = run.attemptIds[nodeId];
|
|
1973
|
+
if (run.attemptTracking) {
|
|
1974
|
+
if (!expectedAttemptId || ownerAttemptId !== expectedAttemptId || !agentId) return false;
|
|
1975
|
+
}
|
|
1976
|
+
|
|
1977
|
+
const expectedAgentId = nodeId === SYNTHESIS_NODE_ID ? run.synthesisAgentId : run.agentIds[nodeId];
|
|
1978
|
+
if (agentId && expectedAgentId && agentId !== expectedAgentId) return false;
|
|
1979
|
+
if (run.attemptTracking && !expectedAgentId && eventName !== "subagents:created") return false;
|
|
1980
|
+
return true;
|
|
1981
|
+
}
|
|
1982
|
+
|
|
1983
|
+
private onLifecycle(eventName: string, data: Record<string, unknown>): void {
|
|
1984
|
+
if (this.lifecycleSuspended || this.branchChanging || this.branchEventsClosed) return;
|
|
1985
|
+
const owner = ownerFor(data);
|
|
1986
|
+
if (!owner) return;
|
|
1987
|
+
const run = this.runs.get(owner.runId);
|
|
1988
|
+
if (!run) return;
|
|
1989
|
+
const agentId = typeof data.id === "string" ? data.id : undefined;
|
|
1990
|
+
if (agentId && this.quarantinedAgentIds.has(agentId)) return;
|
|
1991
|
+
if (owner.nodeId !== SYNTHESIS_NODE_ID && !(owner.nodeId in run.taskStatus)) return;
|
|
1992
|
+
|
|
1993
|
+
const settling = run.status === "stopping" || this.settling.has(run.runId);
|
|
1994
|
+
if (isTerminalWorkflow(run.status)) return;
|
|
1995
|
+
if (!this.acceptsLifecycleEvent(run, owner.nodeId, eventName, owner.attemptId, agentId)) return;
|
|
1996
|
+
const terminalEvent = eventName === "subagents:completed" || eventName === "subagents:failed";
|
|
1997
|
+
if (run.status === "stopping" && !terminalEvent) return;
|
|
1998
|
+
if (
|
|
1999
|
+
owner.nodeId === SYNTHESIS_NODE_ID
|
|
2000
|
+
? run.synthesisResult !== undefined
|
|
2001
|
+
: run.taskResults[owner.nodeId] !== undefined || isTerminalTask(run.taskStatus[owner.nodeId])
|
|
2002
|
+
)
|
|
2003
|
+
return;
|
|
2004
|
+
|
|
2005
|
+
if (eventName === "subagents:compacted") {
|
|
2006
|
+
const previousCount = run.compactions[owner.nodeId] ?? 0;
|
|
2007
|
+
const observedCount = data.compactionCount;
|
|
2008
|
+
const count =
|
|
2009
|
+
typeof observedCount === "number" && Number.isSafeInteger(observedCount) && observedCount >= 0
|
|
2010
|
+
? Math.max(previousCount, observedCount)
|
|
2011
|
+
: previousCount + 1;
|
|
2012
|
+
const timestamp = Date.now();
|
|
2013
|
+
const event: JournalEvent = {
|
|
2014
|
+
kind: "task_compacted",
|
|
2015
|
+
schemaVersion: JOURNAL_SCHEMA_VERSION,
|
|
2016
|
+
runId: run.runId,
|
|
2017
|
+
nodeId: owner.nodeId,
|
|
2018
|
+
compactionCount: count,
|
|
2019
|
+
attemptId: run.attemptIds[owner.nodeId],
|
|
2020
|
+
owner: this.ownerForNode(run, owner.nodeId),
|
|
2021
|
+
timestamp,
|
|
2022
|
+
};
|
|
2023
|
+
this.tryLifecycleMutation(run, `compacted:${owner.nodeId}:${count}`, () => {
|
|
2024
|
+
this.write(event);
|
|
2025
|
+
run.compactions[owner.nodeId] = Math.max(run.compactions[owner.nodeId] ?? 0, count);
|
|
2026
|
+
run.updatedAt = timestamp;
|
|
2027
|
+
});
|
|
2028
|
+
return;
|
|
2029
|
+
}
|
|
2030
|
+
|
|
2031
|
+
if (owner.nodeId === SYNTHESIS_NODE_ID) {
|
|
2032
|
+
if (eventName === "subagents:created" && agentId) run.synthesisAgentId ??= agentId;
|
|
2033
|
+
if (eventName === "subagents:started" || !terminalEvent) return;
|
|
2034
|
+
const status = statusFromLifecycle(eventName, data);
|
|
2035
|
+
const timestamp = Date.now();
|
|
2036
|
+
const result = resultFromLifecycle(
|
|
2037
|
+
status,
|
|
2038
|
+
agentId ?? run.synthesisAgentId,
|
|
2039
|
+
data.result,
|
|
2040
|
+
data.error,
|
|
2041
|
+
run.compactions[SYNTHESIS_NODE_ID] ?? 0,
|
|
2042
|
+
timestamp,
|
|
2043
|
+
data.outputFile,
|
|
2044
|
+
tokenCountFrom(data.tokens),
|
|
2045
|
+
);
|
|
2046
|
+
result.attemptId = run.attemptIds[SYNTHESIS_NODE_ID];
|
|
2047
|
+
this.tryLifecycleMutation(run, `terminal:${owner.nodeId}:${owner.attemptId ?? "legacy"}`, () => {
|
|
2048
|
+
this.commitSynthesisTerminal(run, result, status);
|
|
2049
|
+
});
|
|
2050
|
+
return;
|
|
2051
|
+
}
|
|
2052
|
+
|
|
2053
|
+
if (eventName === "subagents:created") {
|
|
2054
|
+
if (run.taskStatus[owner.nodeId] === "dispatching") {
|
|
2055
|
+
this.tryLifecycleMutation(run, `created:${owner.nodeId}:${owner.attemptId ?? "legacy"}`, () => {
|
|
2056
|
+
this.setTaskStatus(run, owner.nodeId, "queued", agentId);
|
|
2057
|
+
});
|
|
2058
|
+
}
|
|
2059
|
+
return;
|
|
2060
|
+
}
|
|
2061
|
+
if (eventName === "subagents:started") {
|
|
2062
|
+
if (run.taskStatus[owner.nodeId] === "dispatching" || run.taskStatus[owner.nodeId] === "queued") {
|
|
2063
|
+
this.tryLifecycleMutation(run, `started:${owner.nodeId}:${owner.attemptId ?? "legacy"}`, () => {
|
|
2064
|
+
this.setTaskStatus(run, owner.nodeId, "running", agentId);
|
|
2065
|
+
});
|
|
2066
|
+
}
|
|
2067
|
+
return;
|
|
2068
|
+
}
|
|
2069
|
+
if (!terminalEvent) return;
|
|
2070
|
+
|
|
2071
|
+
const status = statusFromLifecycle(eventName, data);
|
|
2072
|
+
const timestamp = Date.now();
|
|
2073
|
+
const result = resultFromLifecycle(
|
|
2074
|
+
status,
|
|
2075
|
+
agentId ?? run.agentIds[owner.nodeId],
|
|
2076
|
+
data.result,
|
|
2077
|
+
data.error,
|
|
2078
|
+
run.compactions[owner.nodeId] ?? 0,
|
|
2079
|
+
timestamp,
|
|
2080
|
+
data.outputFile,
|
|
2081
|
+
tokenCountFrom(data.tokens),
|
|
2082
|
+
);
|
|
2083
|
+
result.attemptId = run.attemptIds[owner.nodeId];
|
|
2084
|
+
this.tryLifecycleMutation(run, `terminal:${owner.nodeId}:${owner.attemptId ?? "legacy"}`, () => {
|
|
2085
|
+
this.commitTaskTerminal(run, owner.nodeId, result, status);
|
|
2086
|
+
if (status === "completed" && !settling && run.status !== "stopping") {
|
|
2087
|
+
this.maybePause(run);
|
|
2088
|
+
if (run.status === "running") void this.pump(run);
|
|
2089
|
+
}
|
|
2090
|
+
});
|
|
2091
|
+
}
|
|
2092
|
+
|
|
2093
|
+
private blockDependents(run: WorkflowRun, failedNodeId: string): void {
|
|
2094
|
+
const blockedIds = this.blockedDependentIds(run, failedNodeId);
|
|
2095
|
+
if (blockedIds.length === 0) return;
|
|
2096
|
+
const timestamp = Date.now();
|
|
2097
|
+
const events: JournalEvent[] = blockedIds.map((nodeId) => ({
|
|
2098
|
+
kind: "task_transition",
|
|
2099
|
+
schemaVersion: JOURNAL_SCHEMA_VERSION,
|
|
2100
|
+
runId: run.runId,
|
|
2101
|
+
nodeId,
|
|
2102
|
+
status: "blocked",
|
|
2103
|
+
...(run.attemptIds[nodeId] ? { attemptId: run.attemptIds[nodeId] } : {}),
|
|
2104
|
+
owner: this.ownerForNode(run, nodeId),
|
|
2105
|
+
timestamp,
|
|
2106
|
+
}));
|
|
2107
|
+
this.appendEvents(events);
|
|
2108
|
+
for (const nodeId of blockedIds) run.taskStatus[nodeId] = "blocked";
|
|
2109
|
+
run.updatedAt = timestamp;
|
|
2110
|
+
}
|
|
2111
|
+
|
|
2112
|
+
private failTask(run: WorkflowRun, nodeId: string, message: string): void {
|
|
2113
|
+
if (isTerminalWorkflow(run.status) || run.terminalIntent === "stop") return;
|
|
2114
|
+
const result = resultFromLifecycle(
|
|
2115
|
+
"failed",
|
|
2116
|
+
run.agentIds[nodeId],
|
|
2117
|
+
undefined,
|
|
2118
|
+
message,
|
|
2119
|
+
run.compactions[nodeId] ?? 0,
|
|
2120
|
+
);
|
|
2121
|
+
result.attemptId = run.attemptIds[nodeId];
|
|
2122
|
+
this.commitTaskTerminal(run, nodeId, result, "failed", message);
|
|
2123
|
+
}
|
|
2124
|
+
|
|
2125
|
+
/**
|
|
2126
|
+
* Failure policy: fail-fast for scheduling, but never abandon sibling agents.
|
|
2127
|
+
* Mark the intent, stop every remaining owned agent, consume/journal terminal
|
|
2128
|
+
* callbacks, then settle the workflow as failed with dependents blocked.
|
|
2129
|
+
*/
|
|
2130
|
+
private beginFailureCleanup(run: WorkflowRun, message: string): void {
|
|
2131
|
+
if (isTerminalWorkflow(run.status) || run.terminalIntent === "stop") return;
|
|
2132
|
+
if (run.terminalIntent !== "failure" || run.status !== "stopping" || run.error !== message) {
|
|
2133
|
+
this.setWorkflowStatus(run, "stopping", message, "failure");
|
|
2134
|
+
}
|
|
2135
|
+
this.startFailureCleanup(run, message);
|
|
2136
|
+
}
|
|
2137
|
+
|
|
2138
|
+
private startFailureCleanup(run: WorkflowRun, message: string): void {
|
|
2139
|
+
if (isTerminalWorkflow(run.status) || run.terminalIntent !== "failure") return;
|
|
2140
|
+
if (this.settling.has(run.runId)) return;
|
|
2141
|
+
const cleanup = this.settleTerminalCleanup(run, message);
|
|
2142
|
+
this.settling.set(run.runId, cleanup);
|
|
2143
|
+
void cleanup.then(
|
|
2144
|
+
() => {
|
|
2145
|
+
if (this.settling.get(run.runId) === cleanup) this.settling.delete(run.runId);
|
|
2146
|
+
},
|
|
2147
|
+
() => {
|
|
2148
|
+
if (this.settling.get(run.runId) === cleanup) this.settling.delete(run.runId);
|
|
2149
|
+
},
|
|
2150
|
+
);
|
|
2151
|
+
}
|
|
2152
|
+
|
|
2153
|
+
private terminalRecoveryKey(runId: string, sourceKey: string): string {
|
|
2154
|
+
return JSON.stringify([this.recoveryBranchGeneration, runId, "terminal_recovery", sourceKey]);
|
|
2155
|
+
}
|
|
2156
|
+
|
|
2157
|
+
private applyTerminalRecoveryMemo(run: WorkflowRun, memo: TerminalRecoveryMemo): void {
|
|
2158
|
+
applyTerminalRecoveryEvent(run, memo.event);
|
|
2159
|
+
}
|
|
2160
|
+
|
|
2161
|
+
private rememberTerminalRecovery(memo: TerminalRecoveryMemo): void {
|
|
2162
|
+
const key = this.terminalRecoveryKey(memo.runId, memo.key);
|
|
2163
|
+
this.terminalRecoveryMemo.delete(key);
|
|
2164
|
+
this.terminalRecoveryMemo.set(key, memo);
|
|
2165
|
+
while (this.terminalRecoveryMemo.size > MAX_RECOVERY_MEMO_ENTRIES) {
|
|
2166
|
+
const oldest = this.terminalRecoveryMemo.keys().next().value;
|
|
2167
|
+
if (typeof oldest !== "string") break;
|
|
2168
|
+
this.terminalRecoveryMemo.delete(oldest);
|
|
2169
|
+
}
|
|
2170
|
+
}
|
|
2171
|
+
|
|
2172
|
+
/**
|
|
2173
|
+
* Complete a terminal-intent run recovered from a session branch with one
|
|
2174
|
+
* durable state transition. The event is built entirely from an immutable
|
|
2175
|
+
* source snapshot; append is the only boundary before applying it.
|
|
2176
|
+
*/
|
|
2177
|
+
private recoverTerminalIntent(run: WorkflowRun, terminalRecords: readonly DeferredTerminalRecord[]): void {
|
|
2178
|
+
const source = snapshotRun(run);
|
|
2179
|
+
const terminalRecordsForKey = [...terminalRecords]
|
|
2180
|
+
.sort((left, right) => left.nodeId.localeCompare(right.nodeId) || left.attemptId.localeCompare(right.attemptId))
|
|
2181
|
+
.map((record) => ({
|
|
2182
|
+
nodeId: record.nodeId,
|
|
2183
|
+
attemptId: record.attemptId,
|
|
2184
|
+
owner: record.owner,
|
|
2185
|
+
result: { ...record.result, updatedAt: 0 },
|
|
2186
|
+
}));
|
|
2187
|
+
const sourceKey = JSON.stringify({ source, terminalRecords: terminalRecordsForKey });
|
|
2188
|
+
const memo = this.terminalRecoveryMemo.get(this.terminalRecoveryKey(run.runId, sourceKey));
|
|
2189
|
+
if (memo) {
|
|
2190
|
+
this.applyTerminalRecoveryMemo(run, memo);
|
|
2191
|
+
return;
|
|
2192
|
+
}
|
|
2193
|
+
|
|
2194
|
+
const terminalIntent = source.terminalIntent;
|
|
2195
|
+
if (terminalIntent === undefined) return;
|
|
2196
|
+
const fallbackError =
|
|
2197
|
+
source.error ?? (terminalIntent === "failure" ? "workflow failure cleanup" : "workflow stopped");
|
|
2198
|
+
const recoveredByNode = new Map(terminalRecords.map((record) => [record.nodeId, record]));
|
|
2199
|
+
const finalStatuses: Record<string, TaskStatus> = { ...source.taskStatus };
|
|
2200
|
+
const resultForNode = (nodeId: string): WorkflowRun["taskResults"][string] | undefined => {
|
|
2201
|
+
const sourceResult = source.taskResults[nodeId];
|
|
2202
|
+
if (sourceResult) {
|
|
2203
|
+
return { ...sourceResult, attemptId: source.attemptIds[nodeId] };
|
|
2204
|
+
}
|
|
2205
|
+
const recovered = recoveredByNode.get(nodeId)?.result;
|
|
2206
|
+
if (!recovered) return undefined;
|
|
2207
|
+
return { ...recovered, attemptId: source.attemptIds[nodeId], updatedAt: source.updatedAt };
|
|
2208
|
+
};
|
|
2209
|
+
for (const task of source.definition.tasks) {
|
|
2210
|
+
const observed = resultForNode(task.id);
|
|
2211
|
+
if (observed && !isTerminalTask(source.taskStatus[task.id])) finalStatuses[task.id] = observed.status;
|
|
2212
|
+
}
|
|
2213
|
+
|
|
2214
|
+
const blockedNodeIds = new Set(
|
|
2215
|
+
source.definition.tasks.filter((task) => source.taskStatus[task.id] === "blocked").map((task) => task.id),
|
|
2216
|
+
);
|
|
2217
|
+
if (terminalIntent === "failure") {
|
|
2218
|
+
const failedOrBlocked = new Set(
|
|
2219
|
+
source.definition.tasks
|
|
2220
|
+
.filter((task) => finalStatuses[task.id] === "failed" || finalStatuses[task.id] === "blocked")
|
|
2221
|
+
.map((task) => task.id),
|
|
2222
|
+
);
|
|
2223
|
+
let changed = true;
|
|
2224
|
+
while (changed) {
|
|
2225
|
+
changed = false;
|
|
2226
|
+
for (const task of source.definition.tasks) {
|
|
2227
|
+
if (isTerminalTask(finalStatuses[task.id])) continue;
|
|
2228
|
+
if (!task.depends_on.some((dependency) => failedOrBlocked.has(dependency))) continue;
|
|
2229
|
+
finalStatuses[task.id] = "blocked";
|
|
2230
|
+
failedOrBlocked.add(task.id);
|
|
2231
|
+
blockedNodeIds.add(task.id);
|
|
2232
|
+
changed = true;
|
|
2233
|
+
}
|
|
2234
|
+
}
|
|
2235
|
+
}
|
|
2236
|
+
|
|
2237
|
+
const terminalResults: RecoveryTerminalResult[] = [];
|
|
2238
|
+
const terminalFact = (nodeId: string, result: WorkflowRun["taskResults"][string]): RecoveryTerminalResult => {
|
|
2239
|
+
const attemptId = source.attemptIds[nodeId];
|
|
2240
|
+
return {
|
|
2241
|
+
nodeId,
|
|
2242
|
+
attemptId,
|
|
2243
|
+
result: { ...result, attemptId },
|
|
2244
|
+
owner: { extension: "pi-workflows", runId: source.runId, nodeId, attemptId },
|
|
2245
|
+
};
|
|
2246
|
+
};
|
|
2247
|
+
for (const task of source.definition.tasks) {
|
|
2248
|
+
const sourceStatus = source.taskStatus[task.id];
|
|
2249
|
+
if (isTerminalTask(sourceStatus)) continue;
|
|
2250
|
+
if (blockedNodeIds.has(task.id)) continue;
|
|
2251
|
+
const result =
|
|
2252
|
+
resultForNode(task.id) ??
|
|
2253
|
+
resultFromLifecycle(
|
|
2254
|
+
"stopped",
|
|
2255
|
+
source.agentIds[task.id],
|
|
2256
|
+
undefined,
|
|
2257
|
+
fallbackError,
|
|
2258
|
+
source.compactions[task.id] ?? 0,
|
|
2259
|
+
source.updatedAt,
|
|
2260
|
+
);
|
|
2261
|
+
terminalResults.push(terminalFact(task.id, result));
|
|
2262
|
+
}
|
|
2263
|
+
|
|
2264
|
+
if (source.definition.synthesis && !source.synthesisResult) {
|
|
2265
|
+
const observed = source.synthesisResult ?? recoveredByNode.get(SYNTHESIS_NODE_ID)?.result;
|
|
2266
|
+
const result =
|
|
2267
|
+
observed ??
|
|
2268
|
+
resultFromLifecycle(
|
|
2269
|
+
"stopped",
|
|
2270
|
+
source.synthesisAgentId,
|
|
2271
|
+
undefined,
|
|
2272
|
+
fallbackError,
|
|
2273
|
+
source.compactions[SYNTHESIS_NODE_ID] ?? 0,
|
|
2274
|
+
source.updatedAt,
|
|
2275
|
+
);
|
|
2276
|
+
terminalResults.push(terminalFact(SYNTHESIS_NODE_ID, result));
|
|
2277
|
+
}
|
|
2278
|
+
|
|
2279
|
+
const recoveryBase = {
|
|
2280
|
+
kind: "terminal_recovery" as const,
|
|
2281
|
+
schemaVersion: JOURNAL_SCHEMA_VERSION as 2,
|
|
2282
|
+
runId: source.runId,
|
|
2283
|
+
status: terminalIntent === "failure" ? ("failed" as const) : ("stopped" as const),
|
|
2284
|
+
terminalIntent,
|
|
2285
|
+
branchGeneration: this.recoveryBranchGeneration,
|
|
2286
|
+
terminalResults,
|
|
2287
|
+
blockedNodeIds: [...blockedNodeIds].sort(),
|
|
2288
|
+
error: fallbackError,
|
|
2289
|
+
};
|
|
2290
|
+
const event: TerminalRecoveryEvent = {
|
|
2291
|
+
...recoveryBase,
|
|
2292
|
+
recoveryId: deriveRecoveryId(recoveryBase),
|
|
2293
|
+
timestamp: source.updatedAt,
|
|
2294
|
+
};
|
|
2295
|
+
// Do not mutate the replayed run or memoize until this single append succeeds.
|
|
2296
|
+
this.write(event);
|
|
2297
|
+
applyTerminalRecoveryEvent(run, event);
|
|
2298
|
+
this.rememberTerminalRecovery({
|
|
2299
|
+
branchGeneration: this.recoveryBranchGeneration,
|
|
2300
|
+
runId: run.runId,
|
|
2301
|
+
key: sourceKey,
|
|
2302
|
+
event,
|
|
2303
|
+
});
|
|
2304
|
+
}
|
|
2305
|
+
|
|
2306
|
+
private async settleTerminalCleanup(
|
|
2307
|
+
run: WorkflowRun,
|
|
2308
|
+
message: string,
|
|
2309
|
+
cleanupEpoch = this.branchEpoch,
|
|
2310
|
+
cleanupTargets?: TerminalCleanupTarget[],
|
|
2311
|
+
): Promise<void> {
|
|
2312
|
+
const isCurrent = (): boolean =>
|
|
2313
|
+
!this.branchChanging && cleanupEpoch === this.branchEpoch && this.runs.get(run.runId) === run;
|
|
2314
|
+
// Capture exact owners before the first yield. A pending lifecycle callback
|
|
2315
|
+
// may make a target terminal before the cleanup RPC is allowed to run.
|
|
2316
|
+
let pendingTargets = cleanupTargets ? [...cleanupTargets] : this.initialTerminalCleanupTargets(run);
|
|
2317
|
+
// Let beginFailureCleanup/stop publish the promise in `settling` before a
|
|
2318
|
+
// synchronous stop RPC can emit lifecycle callbacks.
|
|
2319
|
+
await Promise.resolve();
|
|
2320
|
+
if (!isCurrent()) return;
|
|
2321
|
+
const addTarget = (target: TerminalCleanupTarget): void => {
|
|
2322
|
+
if (
|
|
2323
|
+
pendingTargets.some(
|
|
2324
|
+
(existing) =>
|
|
2325
|
+
existing.agentId === target.agentId &&
|
|
2326
|
+
existing.owner.nodeId === target.owner.nodeId &&
|
|
2327
|
+
existing.owner.attemptId === target.owner.attemptId,
|
|
2328
|
+
)
|
|
2329
|
+
)
|
|
2330
|
+
return;
|
|
2331
|
+
pendingTargets.push(target);
|
|
2332
|
+
};
|
|
2333
|
+
for (const target of this.takeLateDispatchTargets(run)) addTarget(target);
|
|
2334
|
+
|
|
2335
|
+
let agentsSettled = pendingTargets.length === 0;
|
|
2336
|
+
if (pendingTargets.length > 0) {
|
|
2337
|
+
if (this.client.quiesceOwned) {
|
|
2338
|
+
const agentIds = pendingTargets.map(({ agentId }) => agentId);
|
|
2339
|
+
const owners = pendingTargets.map(({ owner }) => owner);
|
|
2340
|
+
const result = await this.client.quiesceOwned(run.runId, agentIds, BRANCH_QUIESCE_TIMEOUT_MS, owners).catch(
|
|
2341
|
+
(error: unknown): WorkflowQuiesceResult => ({
|
|
2342
|
+
settled: false,
|
|
2343
|
+
pending: agentIds,
|
|
2344
|
+
diagnostic: `owned quiescence failed: ${boundedError(error) ?? "unknown error"}`,
|
|
2345
|
+
}),
|
|
2346
|
+
);
|
|
2347
|
+
const pendingIds = new Set(result.pending);
|
|
2348
|
+
if (result.settled && result.pending.length === 0) {
|
|
2349
|
+
pendingTargets = [];
|
|
2350
|
+
} else {
|
|
2351
|
+
const retained = pendingTargets.filter((target) => pendingIds.has(target.agentId));
|
|
2352
|
+
// A false result without a matching pending id does not prove that
|
|
2353
|
+
// any target settled. Keep the exact targets and retry fail-closed.
|
|
2354
|
+
pendingTargets = result.pending.length === 0 || retained.length === 0 ? pendingTargets : retained;
|
|
2355
|
+
}
|
|
2356
|
+
agentsSettled = pendingTargets.length === 0;
|
|
2357
|
+
} else {
|
|
2358
|
+
// An owner-scoped stop only requests cancellation; without the v3
|
|
2359
|
+
// quiescence proof, the AgentSession may still own provider work.
|
|
2360
|
+
const stopPromises = pendingTargets
|
|
2361
|
+
.filter((target) => !target.fallbackStopRequested)
|
|
2362
|
+
.map((target) => {
|
|
2363
|
+
target.fallbackStopRequested = true;
|
|
2364
|
+
return this.stopOwned(run, target.owner.nodeId, target.agentId, target.owner.attemptId).catch(() => {});
|
|
2365
|
+
});
|
|
2366
|
+
await Promise.allSettled(stopPromises);
|
|
2367
|
+
agentsSettled = false;
|
|
2368
|
+
}
|
|
2369
|
+
}
|
|
2370
|
+
|
|
2371
|
+
const dispatchesSettled = await this.waitForDispatches(run.runId, BRANCH_QUIESCE_TIMEOUT_MS);
|
|
2372
|
+
if (!isCurrent()) return;
|
|
2373
|
+
let lateTargetsAdded = false;
|
|
2374
|
+
for (const target of this.takeLateDispatchTargets(run)) {
|
|
2375
|
+
const previousCount = pendingTargets.length;
|
|
2376
|
+
addTarget(target);
|
|
2377
|
+
lateTargetsAdded ||= pendingTargets.length > previousCount;
|
|
2378
|
+
}
|
|
2379
|
+
agentsSettled = pendingTargets.length === 0;
|
|
2380
|
+
if (lateTargetsAdded) {
|
|
2381
|
+
if (isCurrent()) await this.settleTerminalCleanup(run, message, cleanupEpoch, pendingTargets);
|
|
2382
|
+
return;
|
|
2383
|
+
}
|
|
2384
|
+
if (!agentsSettled || !dispatchesSettled) {
|
|
2385
|
+
// A stop acknowledgement only requests cancellation; the AgentSession
|
|
2386
|
+
// still owns provider work and its pool slot until its promise settles.
|
|
2387
|
+
// Stay in `stopping` and retry instead of publishing a false terminal run.
|
|
2388
|
+
await new Promise<void>((resolve) => setTimeout(resolve, 100));
|
|
2389
|
+
if (isCurrent()) await this.settleTerminalCleanup(run, message, cleanupEpoch, pendingTargets);
|
|
2390
|
+
return;
|
|
2391
|
+
}
|
|
2392
|
+
|
|
2393
|
+
if (this.hasPendingJournalWork(run.runId)) {
|
|
2394
|
+
// Do not synthesize a stopped/failed fact over a lifecycle callback whose
|
|
2395
|
+
// durable prefix is still retrying (or has become permanently blocked).
|
|
2396
|
+
// A retry may commit the observed terminal result and make the synthetic
|
|
2397
|
+
// fact unnecessary; a blocked journal must remain nonterminal fail-closed.
|
|
2398
|
+
if (this.journalBlockedRuns.has(run.runId)) return;
|
|
2399
|
+
await new Promise<void>((resolve) => setTimeout(resolve, 100));
|
|
2400
|
+
if (isCurrent()) await this.settleTerminalCleanup(run, message, cleanupEpoch, pendingTargets);
|
|
2401
|
+
return;
|
|
2402
|
+
}
|
|
2403
|
+
|
|
2404
|
+
for (const [nodeId, status] of Object.entries(run.taskStatus)) {
|
|
2405
|
+
if (isTerminalTask(status)) continue;
|
|
2406
|
+
const result =
|
|
2407
|
+
run.taskResults[nodeId] ??
|
|
2408
|
+
resultFromLifecycle(
|
|
2409
|
+
"stopped",
|
|
2410
|
+
run.agentIds[nodeId],
|
|
2411
|
+
undefined,
|
|
2412
|
+
run.terminalIntent === "failure" ? "workflow failure cleanup" : message,
|
|
2413
|
+
run.compactions[nodeId] ?? 0,
|
|
2414
|
+
);
|
|
2415
|
+
result.attemptId ??= run.attemptIds[nodeId];
|
|
2416
|
+
this.commitTaskTerminal(run, nodeId, result, "stopped", undefined, false);
|
|
2417
|
+
}
|
|
2418
|
+
|
|
2419
|
+
if (run.synthesisAgentId && !run.synthesisResult) {
|
|
2420
|
+
const result = resultFromLifecycle(
|
|
2421
|
+
"stopped",
|
|
2422
|
+
run.synthesisAgentId,
|
|
2423
|
+
undefined,
|
|
2424
|
+
run.terminalIntent === "failure" ? "workflow failure cleanup" : message,
|
|
2425
|
+
run.compactions[SYNTHESIS_NODE_ID] ?? 0,
|
|
2426
|
+
);
|
|
2427
|
+
result.attemptId = run.attemptIds[SYNTHESIS_NODE_ID];
|
|
2428
|
+
this.commitSynthesisTerminal(run, result, "stopped");
|
|
2429
|
+
}
|
|
2430
|
+
|
|
2431
|
+
if (run.status === "stopping" && isCurrent()) {
|
|
2432
|
+
this.setWorkflowStatus(run, run.terminalIntent === "failure" ? "failed" : "stopped", message);
|
|
2433
|
+
this.lateDispatchTargets.delete(run.runId);
|
|
2434
|
+
}
|
|
2435
|
+
}
|
|
2436
|
+
|
|
2437
|
+
private async failRun(run: WorkflowRun, message: string): Promise<void> {
|
|
2438
|
+
if (isTerminalWorkflow(run.status)) return;
|
|
2439
|
+
for (const task of run.definition.tasks) {
|
|
2440
|
+
if (run.taskStatus[task.id] === "failed") this.blockDependents(run, task.id);
|
|
2441
|
+
}
|
|
2442
|
+
this.beginFailureCleanup(run, message);
|
|
2443
|
+
const cleanup = this.settling.get(run.runId);
|
|
2444
|
+
if (cleanup) await cleanup;
|
|
2445
|
+
}
|
|
2446
|
+
|
|
2447
|
+
private maybePause(run: WorkflowRun): void {
|
|
2448
|
+
if (run.status !== "pausing") return;
|
|
2449
|
+
if (!Object.values(run.taskStatus).some(activeStatus) && !run.synthesisAgentId) {
|
|
2450
|
+
this.setWorkflowStatus(run, "paused");
|
|
2451
|
+
}
|
|
2452
|
+
}
|
|
2453
|
+
|
|
2454
|
+
private async stop(run: WorkflowRun): Promise<void> {
|
|
2455
|
+
if (isTerminalWorkflow(run.status)) return;
|
|
2456
|
+
if (run.status !== "stopping" || run.terminalIntent !== "stop" || run.error !== "workflow stopped") {
|
|
2457
|
+
this.setWorkflowStatus(run, "stopping", "workflow stopped", "stop");
|
|
2458
|
+
}
|
|
2459
|
+
run.nonResumable = true;
|
|
2460
|
+
const existing = this.settling.get(run.runId);
|
|
2461
|
+
if (existing) {
|
|
2462
|
+
await existing;
|
|
2463
|
+
return;
|
|
2464
|
+
}
|
|
2465
|
+
const cleanup = this.settleTerminalCleanup(run, "workflow stopped");
|
|
2466
|
+
this.settling.set(run.runId, cleanup);
|
|
2467
|
+
try {
|
|
2468
|
+
await cleanup;
|
|
2469
|
+
} finally {
|
|
2470
|
+
if (this.settling.get(run.runId) === cleanup) this.settling.delete(run.runId);
|
|
2471
|
+
}
|
|
2472
|
+
}
|
|
2473
|
+
|
|
2474
|
+
private reconcileManagedSpawnTombstone(entry: SessionEntryLike): DeferredTerminalRecord | undefined {
|
|
2475
|
+
if (entry.type !== "custom" || entry.customType !== "subagents:managed-spawn" || !isRecord(entry.data)) return;
|
|
2476
|
+
const data = entry.data;
|
|
2477
|
+
const allowedKeys = new Set([
|
|
2478
|
+
"schemaVersion",
|
|
2479
|
+
"spawnKey",
|
|
2480
|
+
"fingerprint",
|
|
2481
|
+
"id",
|
|
2482
|
+
"requestId",
|
|
2483
|
+
"type",
|
|
2484
|
+
"description",
|
|
2485
|
+
"owner",
|
|
2486
|
+
"tier",
|
|
2487
|
+
"tierSnapshot",
|
|
2488
|
+
"state",
|
|
2489
|
+
"createdAt",
|
|
2490
|
+
"updatedAt",
|
|
2491
|
+
"compactionCount",
|
|
2492
|
+
"terminal",
|
|
2493
|
+
]);
|
|
2494
|
+
if (Object.keys(data).some((key) => !allowedKeys.has(key))) return;
|
|
2495
|
+
if (data.schemaVersion !== 1 || (data.state !== "completed" && data.state !== "failed" && data.state !== "stopped"))
|
|
2496
|
+
return;
|
|
2497
|
+
const tier = data.tier;
|
|
2498
|
+
if (tier !== undefined && (typeof tier !== "string" || !WORKFLOW_TIER_VALUES.has(tier))) return;
|
|
2499
|
+
if (data.tierSnapshot !== undefined && (typeof tier !== "string" || !isValidTierSnapshot(data.tierSnapshot, tier)))
|
|
2500
|
+
return;
|
|
2501
|
+
const bounded = (value: unknown, max: number): string | undefined =>
|
|
2502
|
+
typeof value === "string" && value.length > 0 && value.length <= max && value.trim() === value
|
|
2503
|
+
? value
|
|
2504
|
+
: undefined;
|
|
2505
|
+
const spawnKey = bounded(data.spawnKey, 256);
|
|
2506
|
+
const id = bounded(data.id, 128);
|
|
2507
|
+
const fingerprint = bounded(data.fingerprint, 128);
|
|
2508
|
+
const requestId = bounded(data.requestId, 128);
|
|
2509
|
+
const type = bounded(data.type, 128);
|
|
2510
|
+
const description = bounded(data.description, 512);
|
|
2511
|
+
if (!spawnKey || !id || !fingerprint || !requestId || !type || !description || !isRecord(data.owner)) return;
|
|
2512
|
+
const ownerKeys = new Set(["extension", "runId", "nodeId", "attemptId"]);
|
|
2513
|
+
if (Object.keys(data.owner).some((key) => !ownerKeys.has(key))) return;
|
|
2514
|
+
const ownerRunId = bounded(data.owner.runId, 256);
|
|
2515
|
+
const ownerNodeId = bounded(data.owner.nodeId, 256);
|
|
2516
|
+
const ownerAttemptId = bounded(data.owner.attemptId, 256);
|
|
2517
|
+
if (data.owner.extension !== "pi-workflows" || !ownerRunId || !ownerNodeId || !ownerAttemptId) return;
|
|
2518
|
+
const run = this.runs.get(ownerRunId);
|
|
2519
|
+
if (!run || isTerminalWorkflow(run.status)) return;
|
|
2520
|
+
const isSynthesis = ownerNodeId === SYNTHESIS_NODE_ID;
|
|
2521
|
+
if (!isSynthesis && !(ownerNodeId in run.taskStatus)) return;
|
|
2522
|
+
if (isSynthesis && !run.definition.synthesis) return;
|
|
2523
|
+
const currentAttemptId = run.attemptIds[ownerNodeId];
|
|
2524
|
+
const generation = run.attempts[ownerNodeId];
|
|
2525
|
+
if (!currentAttemptId || !Number.isSafeInteger(generation) || generation < 1 || ownerAttemptId !== currentAttemptId)
|
|
2526
|
+
return;
|
|
2527
|
+
if (currentAttemptId !== `${run.runId}/${ownerNodeId}/attempt-${generation}`) return;
|
|
2528
|
+
const expectedSpawnKeys = new Set([
|
|
2529
|
+
currentAttemptId,
|
|
2530
|
+
`${run.runId}/${ownerNodeId}/${currentAttemptId}`,
|
|
2531
|
+
`${run.runId}/${ownerNodeId}/attempt-${generation}`,
|
|
2532
|
+
`${run.runId}:${ownerNodeId}:${currentAttemptId}`,
|
|
2533
|
+
]);
|
|
2534
|
+
if (!expectedSpawnKeys.has(spawnKey)) return;
|
|
2535
|
+
const currentAgentId = isSynthesis ? run.synthesisAgentId : run.agentIds[ownerNodeId];
|
|
2536
|
+
if (currentAgentId && currentAgentId !== id) return;
|
|
2537
|
+
if (isSynthesis ? run.synthesisResult : run.taskResults[ownerNodeId]) return;
|
|
2538
|
+
if (!isSynthesis && isTerminalTask(run.taskStatus[ownerNodeId])) return;
|
|
2539
|
+
|
|
2540
|
+
const terminal = data.terminal;
|
|
2541
|
+
if (!isRecord(terminal)) return;
|
|
2542
|
+
const terminalKeys = new Set([
|
|
2543
|
+
"status",
|
|
2544
|
+
"result",
|
|
2545
|
+
"error",
|
|
2546
|
+
"outputFile",
|
|
2547
|
+
"tokenCount",
|
|
2548
|
+
"compactionCount",
|
|
2549
|
+
"completedAt",
|
|
2550
|
+
]);
|
|
2551
|
+
if (Object.keys(terminal).some((key) => !terminalKeys.has(key))) return;
|
|
2552
|
+
if (terminal.status !== data.state) return;
|
|
2553
|
+
const resultText = terminal.result === undefined ? undefined : bounded(terminal.result, 8_000);
|
|
2554
|
+
const errorText = terminal.error === undefined ? undefined : bounded(terminal.error, 2_000);
|
|
2555
|
+
const outputFile = terminal.outputFile === undefined ? undefined : bounded(terminal.outputFile, 2_000);
|
|
2556
|
+
if (
|
|
2557
|
+
(terminal.result !== undefined && !resultText) ||
|
|
2558
|
+
(terminal.error !== undefined && !errorText) ||
|
|
2559
|
+
(terminal.outputFile !== undefined && !outputFile)
|
|
2560
|
+
)
|
|
2561
|
+
return;
|
|
2562
|
+
const compactionCount = terminal.compactionCount;
|
|
2563
|
+
const persistedCompactionCount = data.compactionCount;
|
|
2564
|
+
const completedAt = terminal.completedAt;
|
|
2565
|
+
const createdAt = data.createdAt;
|
|
2566
|
+
const updatedAt = data.updatedAt;
|
|
2567
|
+
if (
|
|
2568
|
+
typeof compactionCount !== "number" ||
|
|
2569
|
+
!Number.isSafeInteger(compactionCount) ||
|
|
2570
|
+
compactionCount < 0 ||
|
|
2571
|
+
typeof persistedCompactionCount !== "number" ||
|
|
2572
|
+
!Number.isSafeInteger(persistedCompactionCount) ||
|
|
2573
|
+
persistedCompactionCount !== compactionCount ||
|
|
2574
|
+
typeof completedAt !== "number" ||
|
|
2575
|
+
!Number.isFinite(completedAt) ||
|
|
2576
|
+
completedAt < 0 ||
|
|
2577
|
+
typeof createdAt !== "number" ||
|
|
2578
|
+
!Number.isFinite(createdAt) ||
|
|
2579
|
+
createdAt < 0 ||
|
|
2580
|
+
typeof updatedAt !== "number" ||
|
|
2581
|
+
!Number.isFinite(updatedAt) ||
|
|
2582
|
+
updatedAt < createdAt ||
|
|
2583
|
+
completedAt < createdAt ||
|
|
2584
|
+
completedAt > updatedAt
|
|
2585
|
+
)
|
|
2586
|
+
return;
|
|
2587
|
+
const tokenCount = terminal.tokenCount;
|
|
2588
|
+
if (
|
|
2589
|
+
tokenCount !== undefined &&
|
|
2590
|
+
(typeof tokenCount !== "number" || !Number.isSafeInteger(tokenCount) || tokenCount < 0)
|
|
2591
|
+
)
|
|
2592
|
+
return;
|
|
2593
|
+
const status = data.state === "completed" ? "completed" : data.state === "stopped" ? "stopped" : "failed";
|
|
2594
|
+
const result = resultFromLifecycle(
|
|
2595
|
+
status,
|
|
2596
|
+
id,
|
|
2597
|
+
resultText,
|
|
2598
|
+
errorText,
|
|
2599
|
+
compactionCount,
|
|
2600
|
+
completedAt,
|
|
2601
|
+
outputFile,
|
|
2602
|
+
tokenCount,
|
|
2603
|
+
);
|
|
2604
|
+
result.attemptId = currentAttemptId;
|
|
2605
|
+
return {
|
|
2606
|
+
kind: "terminal",
|
|
2607
|
+
runId: run.runId,
|
|
2608
|
+
nodeId: ownerNodeId,
|
|
2609
|
+
attemptId: currentAttemptId,
|
|
2610
|
+
result,
|
|
2611
|
+
owner: { extension: "pi-workflows", runId: run.runId, nodeId: ownerNodeId, attemptId: currentAttemptId },
|
|
2612
|
+
};
|
|
2613
|
+
}
|
|
2614
|
+
private reconcileSubagentRecord(
|
|
2615
|
+
entry: SessionEntryLike,
|
|
2616
|
+
): DeferredInterruptedRecord | DeferredTerminalRecord | undefined {
|
|
2617
|
+
if (entry.type !== "custom" || entry.customType !== "subagents:record" || !isRecord(entry.data)) return;
|
|
2618
|
+
const owner = ownerFor(entry.data);
|
|
2619
|
+
if (!owner) return;
|
|
2620
|
+
const run = this.runs.get(owner.runId);
|
|
2621
|
+
if (!run) return;
|
|
2622
|
+
if (isTerminalWorkflow(run.status)) return;
|
|
2623
|
+
if (owner.nodeId !== SYNTHESIS_NODE_ID && !(owner.nodeId in run.taskStatus)) return;
|
|
2624
|
+
|
|
2625
|
+
const currentAttemptId = run.attemptIds[owner.nodeId];
|
|
2626
|
+
const recordAgentId = typeof entry.data.id === "string" ? entry.data.id : undefined;
|
|
2627
|
+
if (!this.acceptsLifecycleEvent(run, owner.nodeId, "subagents:record", owner.attemptId, recordAgentId)) return;
|
|
2628
|
+
const currentAgentId = owner.nodeId === SYNTHESIS_NODE_ID ? run.synthesisAgentId : run.agentIds[owner.nodeId];
|
|
2629
|
+
const status = entry.data.status;
|
|
2630
|
+
if (
|
|
2631
|
+
status !== "completed" &&
|
|
2632
|
+
status !== "steered" &&
|
|
2633
|
+
status !== "error" &&
|
|
2634
|
+
status !== "stopped" &&
|
|
2635
|
+
status !== "aborted" &&
|
|
2636
|
+
status !== "interrupted"
|
|
2637
|
+
)
|
|
2638
|
+
return;
|
|
2639
|
+
|
|
2640
|
+
// An interrupted record is not a failed task result. It means the previous
|
|
2641
|
+
// AgentSession disappeared during reload. Legacy records without an
|
|
2642
|
+
// attemptId are accepted only while their agent id is still the current
|
|
2643
|
+
// journaled agent; after a recovery event clears that id, the same old
|
|
2644
|
+
// record is ignored on every later restore.
|
|
2645
|
+
if (status === "interrupted") {
|
|
2646
|
+
if (run.terminalIntent) return;
|
|
2647
|
+
if (!owner.attemptId && (!recordAgentId || !currentAgentId || recordAgentId !== currentAgentId)) return;
|
|
2648
|
+
if (owner.nodeId === SYNTHESIS_NODE_ID ? run.synthesisResult : run.taskResults[owner.nodeId]) return;
|
|
2649
|
+
return {
|
|
2650
|
+
kind: "interrupted",
|
|
2651
|
+
runId: run.runId,
|
|
2652
|
+
nodeId: owner.nodeId,
|
|
2653
|
+
oldAgentId: recordAgentId ?? currentAgentId ?? "",
|
|
2654
|
+
};
|
|
2655
|
+
}
|
|
2656
|
+
|
|
2657
|
+
const resultStatus =
|
|
2658
|
+
status === "completed" || status === "steered" ? "completed" : status === "stopped" ? "stopped" : "failed";
|
|
2659
|
+
if (owner.nodeId === SYNTHESIS_NODE_ID ? run.synthesisResult : run.taskResults[owner.nodeId]) return;
|
|
2660
|
+
if (!currentAttemptId) return;
|
|
2661
|
+
const result = resultFromLifecycle(
|
|
2662
|
+
resultStatus,
|
|
2663
|
+
recordAgentId ?? currentAgentId,
|
|
2664
|
+
entry.data.result,
|
|
2665
|
+
entry.data.error,
|
|
2666
|
+
run.compactions[owner.nodeId] ?? 0,
|
|
2667
|
+
Date.now(),
|
|
2668
|
+
entry.data.outputFile,
|
|
2669
|
+
typeof entry.data.tokens === "number" ? entry.data.tokens : undefined,
|
|
2670
|
+
);
|
|
2671
|
+
result.attemptId = currentAttemptId;
|
|
2672
|
+
return {
|
|
2673
|
+
kind: "terminal",
|
|
2674
|
+
runId: run.runId,
|
|
2675
|
+
nodeId: owner.nodeId,
|
|
2676
|
+
attemptId: currentAttemptId,
|
|
2677
|
+
result,
|
|
2678
|
+
owner: {
|
|
2679
|
+
extension: "pi-workflows",
|
|
2680
|
+
runId: run.runId,
|
|
2681
|
+
nodeId: owner.nodeId,
|
|
2682
|
+
attemptId: currentAttemptId,
|
|
2683
|
+
},
|
|
2684
|
+
};
|
|
2685
|
+
}
|
|
2686
|
+
|
|
2687
|
+
private resolveWaiters(run: WorkflowRun): void {
|
|
2688
|
+
const waiters = this.waiters.get(run.runId);
|
|
2689
|
+
if (!waiters) return;
|
|
2690
|
+
this.waiters.delete(run.runId);
|
|
2691
|
+
for (const waiter of waiters) {
|
|
2692
|
+
if (waiter.settled) continue;
|
|
2693
|
+
waiter.settled = true;
|
|
2694
|
+
waiter.cleanup();
|
|
2695
|
+
waiter.resolve(run);
|
|
2696
|
+
}
|
|
2697
|
+
}
|
|
2698
|
+
|
|
2699
|
+
private summary(run: WorkflowRun, includeDetails = false): WorkflowSummary {
|
|
2700
|
+
const statuses = Object.values(run.taskStatus);
|
|
2701
|
+
const completedTasks = statuses.filter((status) => status === "completed").length;
|
|
2702
|
+
const failedTasks = statuses.filter(
|
|
2703
|
+
(status) => status === "failed" || status === "blocked" || status === "stopped",
|
|
2704
|
+
).length;
|
|
2705
|
+
const activeTasks = statuses.filter(activeStatus).length;
|
|
2706
|
+
const phaseCompletion = run.definition.phases.map((phase) => {
|
|
2707
|
+
const phaseTasks = run.definition.tasks.filter((task) => task.phase === phase.id);
|
|
2708
|
+
return phaseTasks.length > 0 && phaseTasks.every((task) => run.taskStatus[task.id] === "completed");
|
|
2709
|
+
});
|
|
2710
|
+
const completedPhases = phaseCompletion.filter(Boolean).length;
|
|
2711
|
+
const tokenValues = [...Object.values(run.taskResults), ...(run.synthesisResult ? [run.synthesisResult] : [])]
|
|
2712
|
+
.map((result) => result.tokenCount)
|
|
2713
|
+
.filter((value): value is number => value !== undefined);
|
|
2714
|
+
const tokenCount = tokenValues.length > 0 ? tokenValues.reduce((total, value) => total + value, 0) : undefined;
|
|
2715
|
+
const taskDetails = includeDetails
|
|
2716
|
+
? run.definition.tasks.map((task) => ({
|
|
2717
|
+
id: task.id,
|
|
2718
|
+
status: run.taskStatus[task.id] ?? "pending",
|
|
2719
|
+
...(run.agentIds[task.id] ? { agentId: run.agentIds[task.id] } : {}),
|
|
2720
|
+
compactions: run.compactions[task.id] ?? 0,
|
|
2721
|
+
...resultPreview(run.taskResults[task.id]),
|
|
2722
|
+
}))
|
|
2723
|
+
: undefined;
|
|
2724
|
+
const synthesisDetails =
|
|
2725
|
+
includeDetails && run.synthesisResult
|
|
2726
|
+
? {
|
|
2727
|
+
status: run.synthesisResult.status,
|
|
2728
|
+
...(run.synthesisAgentId ? { agentId: run.synthesisAgentId } : {}),
|
|
2729
|
+
compactions: run.compactions[SYNTHESIS_NODE_ID] ?? 0,
|
|
2730
|
+
...resultPreview(run.synthesisResult),
|
|
2731
|
+
}
|
|
2732
|
+
: undefined;
|
|
2733
|
+
return {
|
|
2734
|
+
runId: run.runId,
|
|
2735
|
+
name: run.definition.name,
|
|
2736
|
+
...(run.definition.description ? { description: run.definition.description } : {}),
|
|
2737
|
+
status: run.status,
|
|
2738
|
+
startedAt: run.startedAt,
|
|
2739
|
+
updatedAt: run.updatedAt,
|
|
2740
|
+
elapsedMs: Math.max(
|
|
2741
|
+
0,
|
|
2742
|
+
(run.status === "completed" || run.status === "failed" || run.status === "stopped"
|
|
2743
|
+
? run.updatedAt
|
|
2744
|
+
: Date.now()) - run.startedAt,
|
|
2745
|
+
),
|
|
2746
|
+
taskCount: run.definition.tasks.length,
|
|
2747
|
+
phaseCount: run.definition.phases.length,
|
|
2748
|
+
completedPhases,
|
|
2749
|
+
completedTasks,
|
|
2750
|
+
failedTasks,
|
|
2751
|
+
activeTasks,
|
|
2752
|
+
...(tokenCount === undefined ? {} : { tokenCount }),
|
|
2753
|
+
...(run.synthesisResult?.text ? { synthesisPreview: run.synthesisResult.text.slice(0, 500) } : {}),
|
|
2754
|
+
...(run.error ? { error: run.error.slice(0, 2_000) } : {}),
|
|
2755
|
+
...(run.nonResumable ? { nonResumable: true } : {}),
|
|
2756
|
+
...(taskDetails ? { tasks: taskDetails } : {}),
|
|
2757
|
+
...(synthesisDetails ? { synthesis: synthesisDetails } : {}),
|
|
2758
|
+
};
|
|
2759
|
+
}
|
|
2760
|
+
}
|