@signalridge/pi-workflows 1.2.0 → 1.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +45 -0
- package/README.md +142 -17
- package/package.json +14 -8
- package/skills/workflow-authoring/SKILL.md +30 -0
- package/skills/workflow-authoring/examples/adversarial-verification.js +63 -0
- package/skills/workflow-authoring/examples/bounded-semantic-retry.js +56 -0
- package/skills/workflow-authoring/examples/classify-and-act.js +67 -0
- package/skills/workflow-authoring/examples/defensive-json-parsing.js +63 -0
- package/skills/workflow-authoring/examples/fan-out-and-synthesize.js +47 -0
- package/skills/workflow-authoring/examples/generate-and-filter.js +68 -0
- package/skills/workflow-authoring/examples/loop-until-done.js +68 -0
- package/skills/workflow-authoring/examples/phased-budgets.js +75 -0
- package/skills/workflow-authoring/examples/saved-nested-workflows.js +44 -0
- package/skills/workflow-authoring/examples/structured-output.js +37 -0
- package/skills/workflow-authoring/examples/tournament.js +81 -0
- package/skills/workflow-authoring/examples/validated-gate.js +63 -0
- package/skills/workflow-authoring/references/capabilities.md +44 -0
- package/skills/workflow-authoring/references/capability-details.md +294 -0
- package/skills/workflow-authoring/references/common-helpers.md +4 -0
- package/skills/workflow-authoring/references/debugging.md +29 -0
- package/skills/workflow-authoring/references/focused-recipes.md +12 -0
- package/skills/workflow-authoring/references/helpers.md +9 -0
- package/skills/workflow-authoring/references/lifecycle.md +37 -0
- package/skills/workflow-authoring/references/pattern-selection.md +15 -0
- package/skills/workflow-authoring/references/quality-helpers.md +8 -0
- package/skills/workflow-authoring/references/registry-ownership.md +17 -0
- package/skills/workflow-authoring/references/retry-helper.md +5 -0
- package/skills/workflow-authoring/references/review.md +43 -0
- package/skills/workflow-authoring/references/runtime.md +29 -0
- package/skills/workflow-authoring/references/specialized-helpers.md +19 -0
- package/skills/workflow-authoring/references/versions.md +13 -0
- package/skills/workflow-patterns/SKILL.md +51 -0
- package/src/arming.ts +69 -0
- package/src/builtins.ts +331 -0
- package/src/capabilities.ts +318 -0
- package/src/code-review-scope.ts +144 -0
- package/src/engine.ts +1158 -2561
- package/src/errors.ts +136 -0
- package/src/index.ts +750 -103
- package/src/journal.ts +594 -1001
- package/src/rpc-client.ts +95 -24
- package/src/runtime.ts +1708 -0
- package/src/saved-workflows.ts +121 -0
- package/src/shared-store.ts +132 -0
- package/src/ui.ts +77 -40
- package/src/workflow-settings.ts +100 -0
- package/src/schema.ts +0 -215
- package/src/state-machine.ts +0 -79
- package/src/synthesis.ts +0 -140
package/src/engine.ts
CHANGED
|
@@ -1,936 +1,596 @@
|
|
|
1
|
-
|
|
1
|
+
/**
|
|
2
|
+
* engine.ts — script-driven workflow engine.
|
|
3
|
+
*
|
|
4
|
+
* A workflow is a raw JavaScript module (see runtime.ts). This engine owns the
|
|
5
|
+
* durable side: it journals the script (`run_created` with script text, hash,
|
|
6
|
+
* and meta), dispatches each live `agent()` call through the spawn-managed
|
|
7
|
+
* protocol with A4 generation-rotated spawn keys, persists every completed
|
|
8
|
+
* call as a `call_result` fact (the resume journal source), and exposes
|
|
9
|
+
* list/get/pause/resume/stop control plus branch-change quiescence and
|
|
10
|
+
* session-tree restore.
|
|
11
|
+
*
|
|
12
|
+
* Resume semantics: the engine rebuilds the runtime's resume journal from
|
|
13
|
+
* journaled call_result events, re-runs the script with the SAME runId, and
|
|
14
|
+
* the runtime replays the longest unchanged prefix from cache while calls at
|
|
15
|
+
* or after the first hash miss run live. Live dispatches of a previously
|
|
16
|
+
* journaled call index rotate the spawnKey generation (A4), so an edited
|
|
17
|
+
* script never collides with pi-subagents' fingerprint tombstone.
|
|
18
|
+
*/
|
|
19
|
+
|
|
20
|
+
import { createHash, randomUUID } from "node:crypto";
|
|
21
|
+
import type { ManagedSpawnResponse } from "@signalridge/pi-subagents-protocol";
|
|
22
|
+
import { WorkflowError, WorkflowErrorCode, wrapError } from "./errors.js";
|
|
2
23
|
import {
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
applyTerminalRecoveryEvent,
|
|
6
|
-
boundedError,
|
|
24
|
+
buildResumeJournal,
|
|
25
|
+
type CallResult,
|
|
7
26
|
deriveRecoveryId,
|
|
27
|
+
JOURNAL_ENTRY_TYPE,
|
|
8
28
|
JOURNAL_SCHEMA_VERSION,
|
|
9
29
|
type JournalEvent,
|
|
10
30
|
type JournalWriter,
|
|
11
|
-
type RecoveryRotation,
|
|
12
|
-
type RecoverySourceStatus,
|
|
13
31
|
type RecoveryTerminalResult,
|
|
14
|
-
type RunRecoveryEvent,
|
|
15
32
|
replayJournal,
|
|
33
|
+
type ScriptRun,
|
|
16
34
|
type SessionEntryLike,
|
|
17
35
|
snapshotRun,
|
|
18
36
|
type TerminalRecoveryEvent,
|
|
19
37
|
type WorkflowOwner,
|
|
20
|
-
type WorkflowRun,
|
|
21
38
|
} from "./journal.js";
|
|
22
|
-
import type {
|
|
23
|
-
ManagedSpawnClient,
|
|
24
|
-
ManagedSpawnResponse,
|
|
25
|
-
ManagedTerminalSnapshot,
|
|
26
|
-
WorkflowEventBus,
|
|
27
|
-
} from "./rpc-client.js";
|
|
28
|
-
import { registerLifecycleListener } from "./rpc-client.js";
|
|
29
|
-
import { readyTaskIds, type WorkflowDefinition, type WorkflowTask } from "./schema.js";
|
|
30
39
|
import {
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
type
|
|
35
|
-
|
|
36
|
-
transitionWorkflow,
|
|
37
|
-
type WorkflowStatus,
|
|
38
|
-
} from "./state-machine.js";
|
|
40
|
+
createManagedSpawnClient,
|
|
41
|
+
type DispatchTask,
|
|
42
|
+
type ManagedSpawnClient,
|
|
43
|
+
type WorkflowEventBus,
|
|
44
|
+
} from "./rpc-client.js";
|
|
39
45
|
import {
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
const SYNTHESIS_NODE_ID = "__synthesis__";
|
|
46
|
+
type AgentRunOptions,
|
|
47
|
+
type AgentUsage,
|
|
48
|
+
type JournalEntry,
|
|
49
|
+
parseWorkflowScript,
|
|
50
|
+
runWorkflow,
|
|
51
|
+
type WorkflowAgentRunner,
|
|
52
|
+
type WorkflowRunResult,
|
|
53
|
+
} from "./runtime.js";
|
|
54
|
+
|
|
55
|
+
export const MAX_ATTEMPTS_PER_NODE = 3;
|
|
56
|
+
|
|
57
|
+
const BRANCH_QUIESCE_TIMEOUT_MS = 8_000;
|
|
53
58
|
const MAX_WAITERS = 256;
|
|
54
|
-
const
|
|
55
|
-
const
|
|
56
|
-
const
|
|
57
|
-
const
|
|
58
|
-
|
|
59
|
-
export interface WorkflowQuiesceResult {
|
|
60
|
-
settled: boolean;
|
|
61
|
-
pending: string[];
|
|
62
|
-
diagnostic?: string;
|
|
63
|
-
}
|
|
64
|
-
|
|
65
|
-
export interface WorkflowSummary {
|
|
66
|
-
runId: string;
|
|
67
|
-
name: string;
|
|
68
|
-
description?: string;
|
|
69
|
-
status: WorkflowStatus;
|
|
70
|
-
startedAt: number;
|
|
71
|
-
updatedAt: number;
|
|
72
|
-
elapsedMs: number;
|
|
73
|
-
taskCount: number;
|
|
74
|
-
phaseCount: number;
|
|
75
|
-
completedPhases: number;
|
|
76
|
-
completedTasks: number;
|
|
77
|
-
failedTasks: number;
|
|
78
|
-
activeTasks: number;
|
|
79
|
-
tokenCount?: number;
|
|
80
|
-
synthesisPreview?: string;
|
|
81
|
-
error?: string;
|
|
82
|
-
nonResumable?: boolean;
|
|
83
|
-
tasks?: WorkflowTaskDetail[];
|
|
84
|
-
synthesis?: WorkflowSynthesisDetail;
|
|
85
|
-
}
|
|
86
|
-
|
|
87
|
-
export interface WorkflowTaskDetail {
|
|
88
|
-
id: string;
|
|
89
|
-
status: TaskStatus;
|
|
90
|
-
agentId?: string;
|
|
91
|
-
compactions: number;
|
|
92
|
-
resultPreview?: string;
|
|
93
|
-
errorPreview?: string;
|
|
94
|
-
}
|
|
59
|
+
const JOURNAL_RETRY_INITIAL_DELAY_MS = 25;
|
|
60
|
+
const JOURNAL_RETRY_MAX_DELAY_MS = 2_000;
|
|
61
|
+
const JOURNAL_RETRY_MAX_ATTEMPTS = 8;
|
|
62
|
+
const PROVIDER_RETRY_DEFAULT_MS = 60_000;
|
|
63
|
+
const PROVIDER_RETRY_MAX_MS = 24 * 60 * 60 * 1_000;
|
|
95
64
|
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
errorPreview?: string;
|
|
65
|
+
class JournalAppendError extends Error {
|
|
66
|
+
constructor(kind: string, cause: unknown) {
|
|
67
|
+
super(`journal append failed for ${kind}: ${cause instanceof Error ? cause.message : String(cause)}`);
|
|
68
|
+
this.name = "JournalAppendError";
|
|
69
|
+
}
|
|
102
70
|
}
|
|
103
71
|
|
|
104
|
-
export
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
72
|
+
export type WorkflowQuiesceResult = { settled: boolean; pending: string[]; diagnostic?: string };
|
|
73
|
+
|
|
74
|
+
export interface ScriptStartOptions {
|
|
75
|
+
args?: unknown;
|
|
76
|
+
background?: boolean;
|
|
77
|
+
maxAgents?: number;
|
|
78
|
+
concurrency?: number;
|
|
79
|
+
agentRetries?: number;
|
|
80
|
+
tokenBudget?: number | null;
|
|
81
|
+
agentTimeoutMs?: number | null;
|
|
82
|
+
toolset?: string;
|
|
83
|
+
excludeTools?: string[];
|
|
84
|
+
signal?: AbortSignal;
|
|
85
|
+
/** Foreground human confirmation for checkpoint() — absent in headless runs. */
|
|
86
|
+
confirm?: (
|
|
87
|
+
promptText: string,
|
|
88
|
+
options: {
|
|
89
|
+
default?: unknown;
|
|
90
|
+
headless?: "default" | "abort";
|
|
91
|
+
kind?: "confirm" | "input" | "select";
|
|
92
|
+
choices?: string[];
|
|
93
|
+
timeoutMs?: number;
|
|
94
|
+
},
|
|
95
|
+
) => Promise<unknown>;
|
|
96
|
+
loadSavedWorkflow?: (name: string) => string | undefined;
|
|
97
|
+
mainModel?: string;
|
|
108
98
|
}
|
|
109
99
|
|
|
110
|
-
export interface
|
|
100
|
+
export interface ScriptStartResult {
|
|
111
101
|
runId: string;
|
|
112
|
-
status:
|
|
102
|
+
status: string;
|
|
113
103
|
background: boolean;
|
|
114
104
|
result?: string;
|
|
115
105
|
error?: string;
|
|
116
106
|
waitAborted?: boolean;
|
|
117
107
|
}
|
|
118
108
|
|
|
119
|
-
interface
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
}
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
}
|
|
156
|
-
|
|
157
|
-
interface DeferredTerminalRecord extends RecoveryTerminalResult {
|
|
158
|
-
kind: "terminal";
|
|
159
|
-
runId: string;
|
|
160
|
-
}
|
|
161
|
-
|
|
162
|
-
interface TerminalCleanupTarget {
|
|
163
|
-
agentId: string;
|
|
164
|
-
owner: WorkflowOwner;
|
|
165
|
-
fallbackStopRequested: boolean;
|
|
166
|
-
}
|
|
167
|
-
|
|
168
|
-
interface EngineRestoreSnapshot {
|
|
169
|
-
runs: Map<string, WorkflowRun>;
|
|
170
|
-
recoveryMemo: Map<string, RecoveryMemo>;
|
|
171
|
-
terminalRecoveryMemo: Map<string, TerminalRecoveryMemo>;
|
|
172
|
-
recoveryBranchGeneration: number;
|
|
173
|
-
branchEpoch: number;
|
|
174
|
-
branchChanging: boolean;
|
|
175
|
-
branchEventsClosed: boolean;
|
|
109
|
+
export interface ScriptRunState {
|
|
110
|
+
run: ScriptRun;
|
|
111
|
+
/** Frozen run parameters — resume reuses the original values, ignoring new budget/toolset overrides. */
|
|
112
|
+
frozenOptions: Pick<
|
|
113
|
+
ScriptStartOptions,
|
|
114
|
+
"tokenBudget" | "maxAgents" | "concurrency" | "agentRetries" | "agentTimeoutMs" | "toolset" | "excludeTools"
|
|
115
|
+
>;
|
|
116
|
+
/** Abort controller for this run's script execution. */
|
|
117
|
+
controller: AbortController;
|
|
118
|
+
/** The in-flight runWorkflow promise, or undefined before start/after settle. */
|
|
119
|
+
execution?: Promise<WorkflowRunResult<unknown>>;
|
|
120
|
+
/** Monotonic execution generation; stale pause/resume continuations cannot mutate a newer run. */
|
|
121
|
+
executionGeneration: number;
|
|
122
|
+
/** The script's final return value, captured when the run settles. */
|
|
123
|
+
result?: unknown;
|
|
124
|
+
/** Per-call-index generation base for A4 spawnKey rotation (seeded from journal). */
|
|
125
|
+
generations: Map<string, number>;
|
|
126
|
+
/** Live dispatch waiters keyed by agent id. */
|
|
127
|
+
agentWaiters: Map<
|
|
128
|
+
string,
|
|
129
|
+
{
|
|
130
|
+
callIndex: string;
|
|
131
|
+
generation: number;
|
|
132
|
+
onUsage?: (usage: AgentUsage) => void;
|
|
133
|
+
onHistory?: (history: unknown[]) => void;
|
|
134
|
+
executionGeneration: number;
|
|
135
|
+
resolve: (r: ManagedSpawnResponse) => void;
|
|
136
|
+
reject: (e: unknown) => void;
|
|
137
|
+
settled: boolean;
|
|
138
|
+
}
|
|
139
|
+
>;
|
|
140
|
+
/** Allocations whose managed-spawn reply has not arrived yet. */
|
|
141
|
+
pendingSpawns: Map<string, { spawnKey: string; owner: WorkflowOwner }>;
|
|
142
|
+
/** Journal retry bookkeeping. */
|
|
143
|
+
pendingJournal: Set<string>;
|
|
144
|
+
/** Whether a lifecycle suspension is in effect (branch change). */
|
|
176
145
|
lifecycleSuspended: boolean;
|
|
177
|
-
quarantinedAgentIds: Set<string>;
|
|
178
|
-
}
|
|
179
|
-
export class WorkflowWaitAbortedError extends Error {
|
|
180
|
-
constructor() {
|
|
181
|
-
super("foreground workflow wait aborted; the workflow run continues");
|
|
182
|
-
this.name = "WorkflowWaitAbortedError";
|
|
183
|
-
}
|
|
184
|
-
}
|
|
185
|
-
|
|
186
|
-
function isRecord(value: unknown): value is Record<string, unknown> {
|
|
187
|
-
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
188
|
-
}
|
|
189
|
-
|
|
190
|
-
const WORKFLOW_TIER_VALUES = new Set(["small", "medium", "large"]);
|
|
191
|
-
const WORKFLOW_THINKING_LEVELS = new Set(["minimal", "low", "medium", "high", "xhigh", "max"]);
|
|
192
|
-
const WORKFLOW_CONFIGURED_THINKING = new Set([...WORKFLOW_THINKING_LEVELS, "inherit"]);
|
|
193
|
-
const TIER_SNAPSHOT_KEYS = new Set([
|
|
194
|
-
"tier",
|
|
195
|
-
"model",
|
|
196
|
-
"thinking",
|
|
197
|
-
"configuredModel",
|
|
198
|
-
"configuredThinking",
|
|
199
|
-
"requestedThinking",
|
|
200
|
-
"modelSource",
|
|
201
|
-
"thinkingSource",
|
|
202
|
-
"clamped",
|
|
203
|
-
"diagnostic",
|
|
204
|
-
]);
|
|
205
|
-
|
|
206
|
-
function isValidTierSnapshot(value: unknown, tier: string): boolean {
|
|
207
|
-
if (!isRecord(value) || value.tier !== tier || !WORKFLOW_TIER_VALUES.has(tier)) return false;
|
|
208
|
-
if (Object.keys(value).some((key) => !TIER_SNAPSHOT_KEYS.has(key))) return false;
|
|
209
|
-
for (const key of ["model", "configuredModel"] as const) {
|
|
210
|
-
const field = value[key];
|
|
211
|
-
if (field !== undefined && (typeof field !== "string" || !field.trim() || field.length > 2_000)) return false;
|
|
212
|
-
if (key === "model" && typeof field === "string" && !field.includes("/")) return false;
|
|
213
|
-
}
|
|
214
|
-
for (const key of ["thinking", "requestedThinking"] as const) {
|
|
215
|
-
const field = value[key];
|
|
216
|
-
if (field !== undefined && (typeof field !== "string" || !WORKFLOW_THINKING_LEVELS.has(field))) return false;
|
|
217
|
-
}
|
|
218
|
-
const configuredThinking = value.configuredThinking;
|
|
219
|
-
if (
|
|
220
|
-
configuredThinking !== undefined &&
|
|
221
|
-
(typeof configuredThinking !== "string" || !WORKFLOW_CONFIGURED_THINKING.has(configuredThinking))
|
|
222
|
-
)
|
|
223
|
-
return false;
|
|
224
|
-
if (value.modelSource !== "frontmatter" && value.modelSource !== "tier" && value.modelSource !== "parent")
|
|
225
|
-
return false;
|
|
226
|
-
if (value.thinkingSource !== "frontmatter" && value.thinkingSource !== "tier" && value.thinkingSource !== "parent")
|
|
227
|
-
return false;
|
|
228
|
-
if (value.clamped !== undefined && typeof value.clamped !== "boolean") return false;
|
|
229
|
-
if (value.diagnostic !== undefined && (typeof value.diagnostic !== "string" || value.diagnostic.length > 2_000))
|
|
230
|
-
return false;
|
|
231
|
-
return true;
|
|
232
|
-
}
|
|
233
|
-
|
|
234
|
-
function isAbortSignal(value: unknown): value is AbortSignal {
|
|
235
|
-
return (
|
|
236
|
-
isRecord(value) &&
|
|
237
|
-
typeof value.aborted === "boolean" &&
|
|
238
|
-
typeof value.addEventListener === "function" &&
|
|
239
|
-
typeof value.removeEventListener === "function"
|
|
240
|
-
);
|
|
241
|
-
}
|
|
242
|
-
|
|
243
|
-
function ownerFor(data: Record<string, unknown>): { runId: string; nodeId: string; attemptId?: string } | undefined {
|
|
244
|
-
const owner = data.owner;
|
|
245
|
-
if (!isRecord(owner) || owner.extension !== "pi-workflows") return undefined;
|
|
246
|
-
const validId = (value: unknown): value is string =>
|
|
247
|
-
typeof value === "string" && value.length > 0 && value.length <= 256;
|
|
248
|
-
if (!validId(owner.runId) || !validId(owner.nodeId)) return undefined;
|
|
249
|
-
if (owner.attemptId !== undefined && !validId(owner.attemptId)) return undefined;
|
|
250
|
-
return {
|
|
251
|
-
runId: owner.runId,
|
|
252
|
-
nodeId: owner.nodeId,
|
|
253
|
-
...(typeof owner.attemptId === "string" ? { attemptId: owner.attemptId } : {}),
|
|
254
|
-
};
|
|
255
146
|
}
|
|
256
147
|
|
|
257
|
-
function
|
|
258
|
-
|
|
259
|
-
const status = data.status;
|
|
260
|
-
return status === "stopped" ? "stopped" : "failed";
|
|
148
|
+
function isTerminalWorkflow(status: string): boolean {
|
|
149
|
+
return status === "completed" || status === "failed" || status === "stopped";
|
|
261
150
|
}
|
|
262
151
|
|
|
263
|
-
function
|
|
264
|
-
return status === "
|
|
152
|
+
function managedTerminalStatus(snapshot: { status: string }): "completed" | "failed" | "stopped" {
|
|
153
|
+
return snapshot.status === "completed" ? "completed" : snapshot.status === "failed" ? "failed" : "stopped";
|
|
265
154
|
}
|
|
266
155
|
|
|
267
|
-
function
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
156
|
+
function resultFromLifecycle(
|
|
157
|
+
status: "completed" | "failed" | "stopped",
|
|
158
|
+
agentId: string | undefined,
|
|
159
|
+
result: string | undefined,
|
|
160
|
+
error: string | undefined,
|
|
161
|
+
compactionCount: number,
|
|
162
|
+
tokenCount?: number,
|
|
163
|
+
): CallResult {
|
|
271
164
|
return {
|
|
272
|
-
|
|
273
|
-
...(
|
|
165
|
+
status,
|
|
166
|
+
...(agentId ? { agentId } : {}),
|
|
167
|
+
...(result !== undefined && result.length > 0 ? { result } : {}),
|
|
168
|
+
...(error !== undefined && error.length > 0 ? { error } : {}),
|
|
169
|
+
...(tokenCount !== undefined ? { tokenCount } : {}),
|
|
170
|
+
compactionCount,
|
|
171
|
+
updatedAt: Date.now(),
|
|
274
172
|
};
|
|
275
173
|
}
|
|
276
174
|
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
return
|
|
282
|
-
}
|
|
283
|
-
|
|
284
|
-
function normalizeManagedSpawnResponse(response: ManagedSpawnResponse | string): ManagedSpawnResponse {
|
|
285
|
-
return typeof response === "string"
|
|
286
|
-
? { id: response, state: "running", created: true }
|
|
287
|
-
: { ...response, created: response.created ?? true };
|
|
288
|
-
}
|
|
289
|
-
|
|
290
|
-
function managedTerminalStatus(snapshot: ManagedTerminalSnapshot): "completed" | "failed" | "stopped" {
|
|
291
|
-
if (snapshot.status === "completed") return "completed";
|
|
292
|
-
if (snapshot.status === "stopped") return "stopped";
|
|
293
|
-
return "failed";
|
|
294
|
-
}
|
|
295
|
-
|
|
296
|
-
class JournalAppendError extends Error {
|
|
297
|
-
constructor(eventKind: JournalEvent["kind"], cause: unknown) {
|
|
298
|
-
super(`${eventKind} append failed: ${cause instanceof Error ? cause.message : String(cause)}`);
|
|
299
|
-
this.name = "JournalAppendError";
|
|
300
|
-
}
|
|
301
|
-
}
|
|
302
|
-
|
|
303
|
-
const JOURNAL_RETRY_INITIAL_DELAY_MS = 25;
|
|
304
|
-
const JOURNAL_RETRY_MAX_DELAY_MS = 2_000;
|
|
305
|
-
|
|
306
|
-
function spawnManaged(
|
|
307
|
-
client: ManagedSpawnClient,
|
|
308
|
-
task: WorkflowTask,
|
|
309
|
-
runId: string,
|
|
310
|
-
nodeId: string,
|
|
311
|
-
attemptId: string | undefined,
|
|
312
|
-
): Promise<ManagedSpawnResponse | string> {
|
|
313
|
-
// Protocol-v3 clients receive the attempt as the fourth argument. Keep the
|
|
314
|
-
// old in-process fixture shape working without passing an ExtensionContext
|
|
315
|
-
// back into the engine.
|
|
316
|
-
if (client.spawn.length >= 5) {
|
|
317
|
-
return client.spawn(task, runId, nodeId, undefined, attemptId);
|
|
318
|
-
}
|
|
319
|
-
return client.spawn(task, runId, nodeId, attemptId);
|
|
175
|
+
/** Parse a journaled attemptId of the form `<runId>/<nodeId>/attempt-<generation>`. */
|
|
176
|
+
function generationFromAttemptId(attemptId: string | undefined): number {
|
|
177
|
+
if (!attemptId) return 0;
|
|
178
|
+
const match = /\/attempt-(\d+)$/u.exec(attemptId);
|
|
179
|
+
return match ? Number(match[1]) : 0;
|
|
320
180
|
}
|
|
321
181
|
|
|
322
182
|
export class WorkflowEngine {
|
|
323
|
-
private readonly runs = new Map<string,
|
|
324
|
-
/**
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
private readonly waiters = new Map<string, Set<Waiter>>();
|
|
331
|
-
private readonly pumping = new Set<string>();
|
|
332
|
-
private readonly settling = new Map<string, Promise<void>>();
|
|
183
|
+
private readonly runs = new Map<string, ScriptRunState>();
|
|
184
|
+
/** Fired when a run settles terminal; wired by the UI surface for widget refresh. */
|
|
185
|
+
onRunSettled: (() => void) | undefined;
|
|
186
|
+
private readonly waiters = new Map<
|
|
187
|
+
string,
|
|
188
|
+
Set<{ resolve: (run: ScriptRun) => void; reject: (e: unknown) => void; settled: boolean; cleanup: () => void }>
|
|
189
|
+
>();
|
|
333
190
|
private unsubscribeLifecycle: () => void;
|
|
334
|
-
private
|
|
335
|
-
private branchChanging = false;
|
|
336
|
-
private branchEpoch = 0;
|
|
337
|
-
private branchEventsClosed = false;
|
|
191
|
+
private lifecyclePaused = false;
|
|
338
192
|
private branchQuiescePromise: Promise<WorkflowQuiesceResult> | undefined;
|
|
339
|
-
/** Agent ids quarantined across a branch replacement; never accept late events. */
|
|
340
|
-
private readonly quarantinedAgentIds = new Set<string>();
|
|
341
193
|
private recoveryBranchGeneration = 0;
|
|
342
|
-
/**
|
|
343
|
-
* Durable recovery rotations already emitted for this engine/branch. The
|
|
344
|
-
* key is semantic (branch, run, node, source attempt), never a journal
|
|
345
|
-
* payload hash. A bounded map prevents repeated reloads from retaining
|
|
346
|
-
* unbounded history.
|
|
347
|
-
*/
|
|
348
|
-
private readonly recoveryMemo = new Map<string, RecoveryMemo>();
|
|
349
|
-
/** Terminal cleanup facts already emitted for a stale branch snapshot. */
|
|
350
|
-
private readonly terminalRecoveryMemo = new Map<string, TerminalRecoveryMemo>();
|
|
351
|
-
/** Candidate replay state used to roll back a failed recovery append. */
|
|
352
|
-
private restoreFailureSnapshot: EngineRestoreSnapshot | undefined;
|
|
353
|
-
/** Lifecycle journal writes that failed transiently and are waiting to retry. */
|
|
354
194
|
private readonly journalRetries = new Map<string, { timer?: ReturnType<typeof setTimeout>; attempt: number }>();
|
|
355
|
-
/** Prefixes durably written before a lifecycle batch append failed. */
|
|
356
|
-
private readonly journalBatchProgress = new Map<string, { serialized: string; nextIndex: number }>();
|
|
357
|
-
/** Runs whose lifecycle journal cannot be made durable after bounded retries. */
|
|
358
195
|
private readonly journalBlockedRuns = new Set<string>();
|
|
196
|
+
private readonly providerResumeTimers = new Map<string, ReturnType<typeof setTimeout>>();
|
|
359
197
|
private journalMutationKey: string | undefined;
|
|
198
|
+
private readonly quarantinedRunIds = new Set<string>();
|
|
360
199
|
|
|
361
200
|
constructor(
|
|
362
201
|
readonly events: WorkflowEventBus,
|
|
363
202
|
private readonly client: ManagedSpawnClient,
|
|
364
203
|
private readonly journal: JournalWriter,
|
|
204
|
+
private readonly readEntries: () => readonly SessionEntryLike[] = () => [],
|
|
365
205
|
) {
|
|
366
|
-
this.unsubscribeLifecycle =
|
|
367
|
-
|
|
368
|
-
|
|
206
|
+
this.unsubscribeLifecycle = this.attachLifecycle();
|
|
207
|
+
}
|
|
208
|
+
|
|
209
|
+
private attachLifecycle(): () => void {
|
|
210
|
+
const unsubs = [
|
|
211
|
+
"subagents:created",
|
|
212
|
+
"subagents:started",
|
|
213
|
+
"subagents:completed",
|
|
214
|
+
"subagents:failed",
|
|
215
|
+
"subagents:compacted",
|
|
216
|
+
].map((eventName) =>
|
|
217
|
+
this.events.on(eventName, (raw: unknown) => {
|
|
218
|
+
this.onLifecycle(eventName, raw);
|
|
219
|
+
}),
|
|
220
|
+
);
|
|
221
|
+
return () => {
|
|
222
|
+
for (const unsubscribe of unsubs) unsubscribe();
|
|
223
|
+
};
|
|
369
224
|
}
|
|
370
225
|
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
226
|
+
private onLifecycle(eventName: string, raw: unknown): void {
|
|
227
|
+
if (!raw || typeof raw !== "object") return;
|
|
228
|
+
const data = raw as Record<string, unknown>;
|
|
229
|
+
const agentId = typeof data.id === "string" ? data.id : typeof data.agentId === "string" ? data.agentId : undefined;
|
|
230
|
+
if (!agentId) return;
|
|
231
|
+
|
|
232
|
+
// Resolve a live dispatch waiter on terminal lifecycle events.
|
|
233
|
+
if (eventName === "subagents:completed" || eventName === "subagents:failed") {
|
|
234
|
+
const lifecycleStatus =
|
|
235
|
+
eventName === "subagents:completed"
|
|
236
|
+
? "completed"
|
|
237
|
+
: data.status === "stopped" || data.status === "interrupted"
|
|
238
|
+
? "stopped"
|
|
239
|
+
: "failed";
|
|
240
|
+
for (const state of this.runs.values()) {
|
|
241
|
+
const waiter = state.agentWaiters.get(agentId);
|
|
242
|
+
if (!waiter || waiter.settled || waiter.executionGeneration !== state.executionGeneration) continue;
|
|
243
|
+
const owner = data.owner;
|
|
244
|
+
if (!owner || typeof owner !== "object" || Array.isArray(owner)) continue;
|
|
245
|
+
const ownerRecord = owner as Record<string, unknown>;
|
|
246
|
+
if (
|
|
247
|
+
ownerRecord.extension !== "pi-workflows" ||
|
|
248
|
+
ownerRecord.runId !== state.run.runId ||
|
|
249
|
+
ownerRecord.nodeId !== `call-${waiter.callIndex}` ||
|
|
250
|
+
ownerRecord.attemptId !== `attempt-${waiter.generation}`
|
|
251
|
+
)
|
|
252
|
+
continue;
|
|
253
|
+
const tokenCount =
|
|
254
|
+
typeof data.tokenCount === "number"
|
|
255
|
+
? data.tokenCount
|
|
256
|
+
: data.tokens &&
|
|
257
|
+
typeof data.tokens === "object" &&
|
|
258
|
+
typeof (data.tokens as Record<string, unknown>).total === "number"
|
|
259
|
+
? ((data.tokens as Record<string, unknown>).total as number)
|
|
260
|
+
: undefined;
|
|
384
261
|
waiter.settled = true;
|
|
385
|
-
|
|
386
|
-
|
|
262
|
+
const terminal: ManagedSpawnResponse = {
|
|
263
|
+
id: agentId,
|
|
264
|
+
terminal: {
|
|
265
|
+
status: lifecycleStatus,
|
|
266
|
+
...(typeof data.result === "string" ? { result: data.result } : {}),
|
|
267
|
+
...(typeof data.error === "string" ? { error: data.error } : {}),
|
|
268
|
+
...(typeof data.outputFile === "string" ? { outputFile: data.outputFile } : {}),
|
|
269
|
+
...(tokenCount === undefined ? {} : { tokenCount }),
|
|
270
|
+
compactionCount: typeof data.compactionCount === "number" ? data.compactionCount : 0,
|
|
271
|
+
completedAt: typeof data.completedAt === "number" ? data.completedAt : Date.now(),
|
|
272
|
+
},
|
|
273
|
+
};
|
|
274
|
+
state.agentWaiters.delete(agentId);
|
|
275
|
+
if (tokenCount !== undefined)
|
|
276
|
+
waiter.onUsage?.({ input: 0, output: tokenCount, total: tokenCount, cost: 0, cacheRead: 0, cacheWrite: 0 });
|
|
277
|
+
if (Array.isArray(data.history)) waiter.onHistory?.(data.history);
|
|
278
|
+
waiter.resolve(terminal);
|
|
387
279
|
}
|
|
388
280
|
}
|
|
389
|
-
|
|
281
|
+
}
|
|
282
|
+
|
|
283
|
+
dispose(): void {
|
|
284
|
+
this.unsubscribeLifecycle();
|
|
285
|
+
this.quarantinedRunIds.clear();
|
|
286
|
+
for (const state of this.runs.values()) {
|
|
287
|
+
this.cleanupAgents(state.run.runId);
|
|
288
|
+
state.controller.abort(new DOMException("Workflow engine disposed", "AbortError"));
|
|
289
|
+
}
|
|
290
|
+
this.rejectWaiters(new WorkflowWaitAbortedError());
|
|
291
|
+
this.runs.clear();
|
|
292
|
+
for (const timer of this.providerResumeTimers.values()) clearTimeout(timer);
|
|
293
|
+
this.providerResumeTimers.clear();
|
|
294
|
+
this.branchQuiescePromise = undefined;
|
|
390
295
|
}
|
|
391
296
|
|
|
392
297
|
suspendLifecycle(): void {
|
|
393
|
-
if (this.
|
|
394
|
-
this.
|
|
298
|
+
if (this.lifecyclePaused) return;
|
|
299
|
+
this.lifecyclePaused = true;
|
|
395
300
|
this.unsubscribeLifecycle();
|
|
396
301
|
}
|
|
397
302
|
|
|
398
303
|
resumeLifecycle(): void {
|
|
399
|
-
if (!this.
|
|
400
|
-
this.
|
|
401
|
-
this.unsubscribeLifecycle =
|
|
402
|
-
this.onLifecycle(eventName, data);
|
|
403
|
-
});
|
|
304
|
+
if (!this.lifecyclePaused) return;
|
|
305
|
+
this.lifecyclePaused = false;
|
|
306
|
+
this.unsubscribeLifecycle = this.attachLifecycle();
|
|
404
307
|
}
|
|
405
308
|
|
|
406
309
|
/** Stop owned agents while Pi replaces the active session-tree branch. */
|
|
407
310
|
async quiesceForBranchChange(): Promise<WorkflowQuiesceResult> {
|
|
408
311
|
if (this.branchQuiescePromise) return this.branchQuiescePromise;
|
|
409
|
-
this.branchChanging = true;
|
|
410
312
|
this.clearJournalRetries();
|
|
411
|
-
this.lateDispatchTargets.clear();
|
|
412
|
-
this.lateDispatchQuiescence.clear();
|
|
413
|
-
this.branchEpoch += 1;
|
|
414
|
-
this.branchEventsClosed = false;
|
|
415
|
-
this.branchQuiescePromise = (async () => {
|
|
416
|
-
const operations: Array<Promise<{ runId: string; result: WorkflowQuiesceResult }>> = [];
|
|
417
|
-
for (const run of this.runs.values()) {
|
|
418
|
-
if (isTerminalWorkflow(run.status)) continue;
|
|
419
|
-
const activeNodes = Object.entries(run.taskStatus)
|
|
420
|
-
.filter(([, status]) => activeStatus(status))
|
|
421
|
-
.map(([nodeId]) => ({ nodeId, agentId: run.agentIds[nodeId] }))
|
|
422
|
-
.filter((item): item is { nodeId: string; agentId: string } => Boolean(item.agentId));
|
|
423
|
-
const agentIds = activeNodes.map(({ agentId }) => agentId);
|
|
424
|
-
const owners = activeNodes.map(({ nodeId }) => this.ownerForNode(run, nodeId));
|
|
425
|
-
const hasDispatches = (this.dispatches.get(run.runId)?.size ?? 0) > 0;
|
|
426
|
-
if (run.synthesisAgentId && !run.synthesisResult) {
|
|
427
|
-
agentIds.push(run.synthesisAgentId);
|
|
428
|
-
owners.push(this.ownerForNode(run, SYNTHESIS_NODE_ID));
|
|
429
|
-
}
|
|
430
|
-
if (agentIds.length === 0 && !hasDispatches) continue;
|
|
431
|
-
for (const agentId of agentIds) this.quarantineAgent(agentId);
|
|
432
|
-
const includeDispatches = (
|
|
433
|
-
result: WorkflowQuiesceResult,
|
|
434
|
-
waitForDispatches: boolean,
|
|
435
|
-
): Promise<{ runId: string; result: WorkflowQuiesceResult }> => {
|
|
436
|
-
const settled = waitForDispatches
|
|
437
|
-
? this.waitForDispatches(run.runId, BRANCH_QUIESCE_TIMEOUT_MS)
|
|
438
|
-
: Promise.resolve(true);
|
|
439
|
-
return settled.then((dispatchSettled) => {
|
|
440
|
-
const latePending = dispatchSettled ? this.takeLateDispatchQuiescence(run.runId) : [];
|
|
441
|
-
const diagnostic = [
|
|
442
|
-
result.diagnostic,
|
|
443
|
-
latePending.length > 0 ? `late spawn quiescence incomplete: ${latePending.join(", ")}` : undefined,
|
|
444
|
-
]
|
|
445
|
-
.filter((value): value is string => Boolean(value))
|
|
446
|
-
.join("; ");
|
|
447
|
-
return {
|
|
448
|
-
runId: run.runId,
|
|
449
|
-
result: {
|
|
450
|
-
settled: result.settled && dispatchSettled && latePending.length === 0,
|
|
451
|
-
pending: [
|
|
452
|
-
...new Set([
|
|
453
|
-
...result.pending,
|
|
454
|
-
...latePending,
|
|
455
|
-
...(dispatchSettled ? [] : [`${run.runId}:dispatch`]),
|
|
456
|
-
]),
|
|
457
|
-
].slice(0, 256),
|
|
458
|
-
...(diagnostic ? { diagnostic } : {}),
|
|
459
|
-
},
|
|
460
|
-
};
|
|
461
|
-
});
|
|
462
|
-
};
|
|
463
|
-
if (this.client.quiesceOwned && (agentIds.length > 0 || hasDispatches)) {
|
|
464
|
-
operations.push(
|
|
465
|
-
this.client
|
|
466
|
-
.quiesceOwned(run.runId, agentIds, BRANCH_QUIESCE_TIMEOUT_MS, owners)
|
|
467
|
-
.then((result) =>
|
|
468
|
-
includeDispatches(
|
|
469
|
-
{
|
|
470
|
-
settled: result.settled && result.pending.length === 0,
|
|
471
|
-
pending: [...new Set(result.pending)].slice(0, 256),
|
|
472
|
-
},
|
|
473
|
-
true,
|
|
474
|
-
),
|
|
475
|
-
)
|
|
476
|
-
.catch((error: unknown) =>
|
|
477
|
-
includeDispatches(
|
|
478
|
-
{
|
|
479
|
-
settled: false,
|
|
480
|
-
pending: agentIds.slice(0, 256),
|
|
481
|
-
diagnostic: `owned quiescence RPC failed: ${boundedError(error) ?? "unknown error"}`,
|
|
482
|
-
},
|
|
483
|
-
true,
|
|
484
|
-
),
|
|
485
|
-
),
|
|
486
|
-
);
|
|
487
|
-
} else {
|
|
488
|
-
const stopOperations = activeNodes.map(({ nodeId, agentId }) => this.stopOwned(run, nodeId, agentId));
|
|
489
|
-
if (run.synthesisAgentId && !run.synthesisResult) {
|
|
490
|
-
stopOperations.push(this.stopOwned(run, SYNTHESIS_NODE_ID, run.synthesisAgentId));
|
|
491
|
-
}
|
|
492
|
-
operations.push(
|
|
493
|
-
Promise.allSettled(stopOperations).then(() =>
|
|
494
|
-
includeDispatches(
|
|
495
|
-
{
|
|
496
|
-
settled: agentIds.length === 0 && !hasDispatches,
|
|
497
|
-
pending: [...agentIds, ...(hasDispatches ? [`${run.runId}:dispatch`] : [])].slice(0, 256),
|
|
498
|
-
diagnostic: hasDispatches
|
|
499
|
-
? "dispatch quiescence is unavailable; pending spawn calls were quarantined conservatively"
|
|
500
|
-
: agentIds.length > 0
|
|
501
|
-
? "owned quiescence is unavailable; stopped agents were quarantined conservatively"
|
|
502
|
-
: undefined,
|
|
503
|
-
},
|
|
504
|
-
true,
|
|
505
|
-
),
|
|
506
|
-
),
|
|
507
|
-
);
|
|
508
|
-
}
|
|
509
|
-
}
|
|
510
313
|
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
const
|
|
520
|
-
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
}
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
|
|
314
|
+
// Suspend and abort synchronously, before any await can let a stale runtime
|
|
315
|
+
// dispatch another call or let a provider-limit continuation recreate an old
|
|
316
|
+
// branch agent. `restore()` creates fresh states with suspension cleared.
|
|
317
|
+
const targets = new Map<string, { agentIds: string[]; owners: WorkflowOwner[] }>();
|
|
318
|
+
for (const state of this.runs.values()) {
|
|
319
|
+
if (isTerminalWorkflow(state.run.status)) continue;
|
|
320
|
+
const agentIds = [...state.agentWaiters.keys()];
|
|
321
|
+
const owners = agentIds.map((agentId) => {
|
|
322
|
+
const waiter = state.agentWaiters.get(agentId);
|
|
323
|
+
return {
|
|
324
|
+
extension: "pi-workflows" as const,
|
|
325
|
+
runId: state.run.runId,
|
|
326
|
+
nodeId: waiter ? `call-${waiter.callIndex}` : "",
|
|
327
|
+
attemptId: `attempt-${waiter?.generation ?? 1}`,
|
|
328
|
+
};
|
|
329
|
+
});
|
|
330
|
+
targets.set(state.run.runId, { agentIds, owners });
|
|
331
|
+
state.lifecycleSuspended = true;
|
|
332
|
+
state.controller.abort(new DOMException("Workflow branch change", "AbortError"));
|
|
333
|
+
}
|
|
334
|
+
|
|
335
|
+
const operation = (async () => {
|
|
336
|
+
const operations: Array<Promise<WorkflowQuiesceResult>> = [];
|
|
337
|
+
for (const state of this.runs.values()) {
|
|
338
|
+
const run = state.run;
|
|
339
|
+
const target = targets.get(run.runId);
|
|
340
|
+
if (isTerminalWorkflow(run.status)) {
|
|
341
|
+
this.reconcilePendingSpawns(state);
|
|
342
|
+
continue;
|
|
529
343
|
}
|
|
344
|
+
if (state.pendingSpawns.size > 0) operations.push(this.quiescePendingSpawns(state));
|
|
345
|
+
const activeAgentIds = target?.agentIds ?? [];
|
|
346
|
+
if (activeAgentIds.length === 0) continue;
|
|
347
|
+
operations.push(
|
|
348
|
+
this.client
|
|
349
|
+
.quiesceOwned?.(run.runId, activeAgentIds, BRANCH_QUIESCE_TIMEOUT_MS, target?.owners)
|
|
350
|
+
.then((result) => ({
|
|
351
|
+
settled: result.settled && result.pending.length === 0,
|
|
352
|
+
pending: [...new Set(result.pending)].slice(0, 256),
|
|
353
|
+
}))
|
|
354
|
+
.catch((error: unknown) => ({
|
|
355
|
+
settled: false,
|
|
356
|
+
pending: activeAgentIds.slice(0, 256),
|
|
357
|
+
diagnostic: error instanceof Error ? error.message : String(error),
|
|
358
|
+
})) ?? Promise.resolve({ settled: false, pending: activeAgentIds.slice(0, 256) }),
|
|
359
|
+
);
|
|
530
360
|
}
|
|
531
|
-
const
|
|
532
|
-
const
|
|
533
|
-
|
|
361
|
+
const results = await Promise.all(operations);
|
|
362
|
+
const pending = [...new Set(results.flatMap((r) => r.pending))].slice(0, 256);
|
|
363
|
+
const diagnostics = results.map((r) => r.diagnostic).filter((d): d is string => Boolean(d));
|
|
364
|
+
return {
|
|
365
|
+
settled: results.every((r) => r.settled) && pending.length === 0,
|
|
534
366
|
pending,
|
|
535
|
-
...(
|
|
367
|
+
...(diagnostics.length > 0 ? { diagnostic: diagnostics.join("; ") } : {}),
|
|
536
368
|
};
|
|
537
|
-
// After this point no lifecycle event from the old branch is accepted.
|
|
538
|
-
// Events are already ignored while branchChanging; the subagents manager
|
|
539
|
-
// detaches timed-out records before returning, and the generation gate
|
|
540
|
-
// below also protects fixtures and legacy RPC peers without quiescence.
|
|
541
|
-
this.branchEventsClosed = true;
|
|
542
|
-
this.suspendLifecycle();
|
|
543
|
-
return result;
|
|
544
369
|
})();
|
|
545
|
-
|
|
370
|
+
const tracked = operation.then(
|
|
371
|
+
(result) => {
|
|
372
|
+
if (this.branchQuiescePromise === tracked) this.branchQuiescePromise = undefined;
|
|
373
|
+
return result;
|
|
374
|
+
},
|
|
375
|
+
(error: unknown) => {
|
|
376
|
+
if (this.branchQuiescePromise === tracked) this.branchQuiescePromise = undefined;
|
|
377
|
+
throw error;
|
|
378
|
+
},
|
|
379
|
+
);
|
|
380
|
+
this.branchQuiescePromise = tracked;
|
|
381
|
+
return tracked;
|
|
546
382
|
}
|
|
547
383
|
|
|
384
|
+
// ── start / control ────────────────────────────────────────────────────────
|
|
385
|
+
|
|
548
386
|
/**
|
|
549
|
-
*
|
|
550
|
-
*
|
|
551
|
-
*
|
|
552
|
-
* new generation (the no-argument path also advances after quiescence).
|
|
387
|
+
* Start a script run. The script is parsed once here for the journal's meta;
|
|
388
|
+
* the runtime re-parses it inside the sandbox (single source of truth for
|
|
389
|
+
* the body it executes).
|
|
553
390
|
*/
|
|
554
|
-
|
|
555
|
-
const
|
|
556
|
-
try {
|
|
557
|
-
this.restoreInternal(entries, branchGeneration);
|
|
558
|
-
this.restoreFailureSnapshot = undefined;
|
|
559
|
-
} catch (error: unknown) {
|
|
560
|
-
// Recovery is a transaction from the engine's point of view. In
|
|
561
|
-
// particular, an append failure must not leave a half-rotated run in
|
|
562
|
-
// memory or advance the branch namespace without its durable event.
|
|
563
|
-
const rollback = this.restoreFailureSnapshot ?? snapshot;
|
|
564
|
-
this.restoreFailureSnapshot = undefined;
|
|
565
|
-
this.restoreAfterFailure(rollback);
|
|
566
|
-
throw error;
|
|
567
|
-
}
|
|
568
|
-
}
|
|
569
|
-
|
|
570
|
-
private captureRestoreSnapshot(): EngineRestoreSnapshot {
|
|
571
|
-
return {
|
|
572
|
-
runs: new Map([...this.runs].map(([runId, run]) => [runId, snapshotRun(run)])),
|
|
573
|
-
recoveryMemo: new Map(
|
|
574
|
-
[...this.recoveryMemo].map(([key, memo]) => [
|
|
575
|
-
key,
|
|
576
|
-
{
|
|
577
|
-
...memo,
|
|
578
|
-
event: structuredClone(memo.event),
|
|
579
|
-
},
|
|
580
|
-
]),
|
|
581
|
-
),
|
|
582
|
-
terminalRecoveryMemo: new Map(
|
|
583
|
-
[...this.terminalRecoveryMemo].map(([key, memo]) => [
|
|
584
|
-
key,
|
|
585
|
-
{
|
|
586
|
-
...memo,
|
|
587
|
-
event: structuredClone(memo.event),
|
|
588
|
-
},
|
|
589
|
-
]),
|
|
590
|
-
),
|
|
591
|
-
recoveryBranchGeneration: this.recoveryBranchGeneration,
|
|
592
|
-
branchEpoch: this.branchEpoch,
|
|
593
|
-
branchChanging: this.branchChanging,
|
|
594
|
-
branchEventsClosed: this.branchEventsClosed,
|
|
595
|
-
lifecycleSuspended: this.lifecycleSuspended,
|
|
596
|
-
quarantinedAgentIds: new Set(this.quarantinedAgentIds),
|
|
597
|
-
};
|
|
598
|
-
}
|
|
599
|
-
|
|
600
|
-
private restoreAfterFailure(snapshot: EngineRestoreSnapshot): void {
|
|
601
|
-
const invalidationEpoch = this.branchEpoch + 1;
|
|
602
|
-
const lifecycleWasSuspended = this.lifecycleSuspended;
|
|
603
|
-
this.dispatches.clear();
|
|
604
|
-
this.lateDispatchTargets.clear();
|
|
605
|
-
this.pumping.clear();
|
|
606
|
-
this.settling.clear();
|
|
607
|
-
this.runs.clear();
|
|
608
|
-
for (const [runId, run] of snapshot.runs) this.runs.set(runId, run);
|
|
609
|
-
this.recoveryMemo.clear();
|
|
610
|
-
for (const [key, memo] of snapshot.recoveryMemo) this.recoveryMemo.set(key, memo);
|
|
611
|
-
this.terminalRecoveryMemo.clear();
|
|
612
|
-
for (const [key, memo] of snapshot.terminalRecoveryMemo) this.terminalRecoveryMemo.set(key, memo);
|
|
613
|
-
this.recoveryBranchGeneration = snapshot.recoveryBranchGeneration;
|
|
614
|
-
this.branchEpoch = Math.max(invalidationEpoch, snapshot.branchEpoch + 1);
|
|
615
|
-
this.branchChanging = snapshot.branchChanging;
|
|
616
|
-
this.branchEventsClosed = snapshot.branchEventsClosed;
|
|
617
|
-
if (snapshot.lifecycleSuspended && !lifecycleWasSuspended) this.suspendLifecycle();
|
|
618
|
-
else if (!snapshot.lifecycleSuspended && lifecycleWasSuspended) this.resumeLifecycle();
|
|
619
|
-
this.quarantinedAgentIds.clear();
|
|
620
|
-
for (const agentId of snapshot.quarantinedAgentIds) this.quarantinedAgentIds.add(agentId);
|
|
621
|
-
this.branchQuiescePromise = undefined;
|
|
622
|
-
}
|
|
623
|
-
|
|
624
|
-
private restoreInternal(entries: readonly SessionEntryLike[], branchGeneration?: number): void {
|
|
625
|
-
const branchReplacing = this.branchChanging;
|
|
626
|
-
this.prepareRecoveryNamespace(branchGeneration, branchReplacing);
|
|
627
|
-
|
|
628
|
-
// Invalidate dispatch promises from the replaced branch before clearing the
|
|
629
|
-
// run map. A managed RPC reply arriving after this point must be stopped,
|
|
630
|
-
// never journaled into the new branch.
|
|
631
|
-
this.branchEpoch += 1;
|
|
632
|
-
this.branchChanging = false;
|
|
633
|
-
this.branchEventsClosed = false;
|
|
634
|
-
this.branchQuiescePromise = undefined;
|
|
635
|
-
this.dispatches.clear();
|
|
636
|
-
this.lateDispatchTargets.clear();
|
|
637
|
-
this.pumping.clear();
|
|
638
|
-
this.settling.clear();
|
|
639
|
-
this.runs.clear();
|
|
640
|
-
for (const [runId, run] of replayJournal(entries, {
|
|
641
|
-
onInvalid: (diagnostic) => console.warn(`[pi-workflows] ${diagnostic}`),
|
|
642
|
-
}))
|
|
643
|
-
this.runs.set(runId, run);
|
|
644
|
-
this.restoreFailureSnapshot = this.captureRestoreSnapshot();
|
|
645
|
-
|
|
646
|
-
// Persisted subagent records are the terminal source of truth when a
|
|
647
|
-
// session was interrupted between lifecycle delivery and workflow
|
|
648
|
-
// journaling. Interrupted records are deferred: recovery must append one
|
|
649
|
-
// complete event before changing the in-memory attempt generation.
|
|
650
|
-
const recoveredNodes = new Map<string, DeferredInterruptedRecord>();
|
|
651
|
-
const terminalRecords = new Map<string, DeferredTerminalRecord>();
|
|
652
|
-
const recoveryKey = (runId: string, nodeId: string): string => `${runId}\u0000${nodeId}`;
|
|
653
|
-
for (const entry of entries) {
|
|
654
|
-
const candidates = [this.reconcileSubagentRecord(entry), this.reconcileManagedSpawnTombstone(entry)];
|
|
655
|
-
for (const reconciled of candidates) {
|
|
656
|
-
if (!reconciled) continue;
|
|
657
|
-
const key = recoveryKey(reconciled.runId, reconciled.nodeId);
|
|
658
|
-
if (reconciled.kind === "interrupted") recoveredNodes.set(key, reconciled);
|
|
659
|
-
else if (!terminalRecords.has(key)) terminalRecords.set(key, reconciled);
|
|
660
|
-
}
|
|
661
|
-
}
|
|
662
|
-
|
|
663
|
-
for (const run of this.runs.values()) {
|
|
664
|
-
const wasActive =
|
|
665
|
-
run.status === "running" ||
|
|
666
|
-
run.status === "pausing" ||
|
|
667
|
-
run.status === "synthesizing" ||
|
|
668
|
-
run.status === "stopping";
|
|
669
|
-
const runTerminalRecords = [...terminalRecords.values()].filter((record) => record.runId === run.runId);
|
|
670
|
-
if (!wasActive && runTerminalRecords.length > 0) {
|
|
671
|
-
// Non-active legacy/paused runs retain the existing reconciliation
|
|
672
|
-
// behavior. Active terminal-intent runs are handled below so each
|
|
673
|
-
// recovered terminal fact is journaled before the final transition.
|
|
674
|
-
this.commitDeferredTerminals(run, runTerminalRecords);
|
|
675
|
-
}
|
|
676
|
-
if (wasActive) {
|
|
677
|
-
if (run.terminalIntent === "stop" || run.terminalIntent === "failure") {
|
|
678
|
-
// A user stop/failure is an immutable terminal fact, never an
|
|
679
|
-
// interrupted run that can be resumed after reload. Finish every
|
|
680
|
-
// active node first; otherwise replay can leave a terminal workflow
|
|
681
|
-
// with live task statuses and a later restore can resurrect them.
|
|
682
|
-
this.recoverTerminalIntent(run, runTerminalRecords);
|
|
683
|
-
} else {
|
|
684
|
-
// A run-level event owns the workflow interruption and every active
|
|
685
|
-
// task/synthesis rotation. append() happens before any mutation, so
|
|
686
|
-
// an injected failure leaves this replayed run untouched and an
|
|
687
|
-
// exact retry can safely emit the same generation once.
|
|
688
|
-
this.recoverInterruptedRun(run, this.recoveryPlan(run, recoveredNodes, terminalRecords), runTerminalRecords);
|
|
689
|
-
}
|
|
690
|
-
} else if (run.status === "interrupted") {
|
|
691
|
-
// A legacy sequence could have persisted workflow_transition before
|
|
692
|
-
// its task_attempt. Migrate each still-active node independently;
|
|
693
|
-
// every attempt_recovery event is valid on its own prefix.
|
|
694
|
-
for (const item of this.recoveryPlan(run, recoveredNodes, terminalRecords)) {
|
|
695
|
-
this.recoverInterruptedNode(run, item.nodeId, item.oldAgentId, false, true);
|
|
696
|
-
}
|
|
697
|
-
}
|
|
698
|
-
|
|
699
|
-
if (isTerminalWorkflow(run.status)) continue;
|
|
700
|
-
|
|
701
|
-
// A terminal record can arrive without its lifecycle result having been
|
|
702
|
-
// journaled. Preserve the existing reconciliation behavior for that
|
|
703
|
-
// record; it is not part of the generation rotation itself.
|
|
704
|
-
for (const task of run.definition.tasks) {
|
|
705
|
-
const status = run.taskStatus[task.id];
|
|
706
|
-
if (run.taskResults[task.id] && !isTerminalTask(status)) {
|
|
707
|
-
this.setTaskStatus(run, task.id, run.taskResults[task.id].status, run.agentIds[task.id]);
|
|
708
|
-
}
|
|
709
|
-
}
|
|
710
|
-
}
|
|
711
|
-
for (const waiters of this.waiters.values()) {
|
|
712
|
-
for (const waiter of waiters) {
|
|
713
|
-
if (waiter.settled) continue;
|
|
714
|
-
waiter.settled = true;
|
|
715
|
-
waiter.cleanup();
|
|
716
|
-
waiter.reject(new Error("workflow session branch changed; the old run is no longer active"));
|
|
717
|
-
}
|
|
718
|
-
}
|
|
719
|
-
this.waiters.clear();
|
|
720
|
-
}
|
|
721
|
-
|
|
722
|
-
private prepareRecoveryNamespace(branchGeneration: number | undefined, branchReplacing: boolean): void {
|
|
723
|
-
if (branchGeneration !== undefined && (!Number.isSafeInteger(branchGeneration) || branchGeneration < 0)) {
|
|
724
|
-
throw new Error("workflow branch generation must be a non-negative safe integer");
|
|
725
|
-
}
|
|
726
|
-
const nextGeneration =
|
|
727
|
-
branchGeneration === undefined
|
|
728
|
-
? branchReplacing
|
|
729
|
-
? this.recoveryBranchGeneration + 1
|
|
730
|
-
: this.recoveryBranchGeneration
|
|
731
|
-
: branchGeneration;
|
|
732
|
-
if (nextGeneration === this.recoveryBranchGeneration) {
|
|
733
|
-
if (branchReplacing) {
|
|
734
|
-
this.recoveryMemo.clear();
|
|
735
|
-
this.terminalRecoveryMemo.clear();
|
|
736
|
-
}
|
|
737
|
-
return;
|
|
738
|
-
}
|
|
739
|
-
this.recoveryMemo.clear();
|
|
740
|
-
this.terminalRecoveryMemo.clear();
|
|
741
|
-
this.recoveryBranchGeneration = nextGeneration;
|
|
742
|
-
}
|
|
743
|
-
|
|
744
|
-
private recoveryMemoKey(runId: string, kind: RecoveryEvent["kind"], sourceKey: string): string {
|
|
745
|
-
return JSON.stringify([this.recoveryBranchGeneration, runId, kind, sourceKey]);
|
|
746
|
-
}
|
|
747
|
-
|
|
748
|
-
private sourceAttemptId(run: WorkflowRun, nodeId: string): string {
|
|
749
|
-
return run.attemptIds[nodeId] ?? `${run.runId}/${nodeId}/attempt-${run.attempts[nodeId] ?? 1}`;
|
|
750
|
-
}
|
|
751
|
-
|
|
752
|
-
private recoveryPlan(
|
|
753
|
-
run: WorkflowRun,
|
|
754
|
-
deferred: ReadonlyMap<string, DeferredInterruptedRecord>,
|
|
755
|
-
terminalRecords: ReadonlyMap<string, DeferredTerminalRecord> = new Map(),
|
|
756
|
-
): RecoveryPlanItem[] {
|
|
757
|
-
const activeWorkflow =
|
|
758
|
-
run.status === "running" ||
|
|
759
|
-
run.status === "pausing" ||
|
|
760
|
-
run.status === "synthesizing" ||
|
|
761
|
-
run.status === "stopping";
|
|
762
|
-
const legacyInterrupted = run.status === "interrupted";
|
|
763
|
-
if ((!activeWorkflow && !legacyInterrupted) || run.terminalIntent) return [];
|
|
764
|
-
const plan: RecoveryPlanItem[] = [];
|
|
765
|
-
for (const task of run.definition.tasks) {
|
|
766
|
-
const status = run.taskStatus[task.id];
|
|
767
|
-
const key = `${run.runId}\u0000${task.id}`;
|
|
768
|
-
const deferredRecord = deferred.get(key);
|
|
769
|
-
if (run.taskResults[task.id] || isTerminalTask(status) || terminalRecords.has(key)) continue;
|
|
770
|
-
if (!activeStatus(status) && deferredRecord === undefined) continue;
|
|
771
|
-
plan.push({
|
|
772
|
-
nodeId: task.id,
|
|
773
|
-
sourceAttemptId: this.sourceAttemptId(run, task.id),
|
|
774
|
-
sourceGeneration: run.attempts[task.id],
|
|
775
|
-
sourceStatus: status,
|
|
776
|
-
oldAgentId: deferredRecord?.oldAgentId ?? run.agentIds[task.id] ?? "",
|
|
777
|
-
});
|
|
778
|
-
}
|
|
779
|
-
if ((run.status === "synthesizing" || (legacyInterrupted && run.synthesisAgentId)) && !run.synthesisResult) {
|
|
780
|
-
const synthesisKey = `${run.runId}\u0000${SYNTHESIS_NODE_ID}`;
|
|
781
|
-
const deferredRecord = deferred.get(synthesisKey);
|
|
782
|
-
if (terminalRecords.has(synthesisKey)) return plan;
|
|
783
|
-
plan.push({
|
|
784
|
-
nodeId: SYNTHESIS_NODE_ID,
|
|
785
|
-
sourceAttemptId: this.sourceAttemptId(run, SYNTHESIS_NODE_ID),
|
|
786
|
-
sourceGeneration: run.attempts[SYNTHESIS_NODE_ID],
|
|
787
|
-
sourceStatus: "synthesizing",
|
|
788
|
-
oldAgentId: deferredRecord?.oldAgentId ?? run.synthesisAgentId ?? "",
|
|
789
|
-
});
|
|
790
|
-
}
|
|
791
|
-
return plan;
|
|
792
|
-
}
|
|
793
|
-
|
|
794
|
-
private recoverySourceKey(
|
|
795
|
-
run: WorkflowRun,
|
|
796
|
-
plan: readonly RecoveryPlanItem[],
|
|
797
|
-
terminalResults: readonly DeferredTerminalRecord[] = [],
|
|
798
|
-
): string {
|
|
799
|
-
return JSON.stringify({
|
|
800
|
-
status: run.status,
|
|
801
|
-
terminalIntent: run.terminalIntent,
|
|
802
|
-
rotations: plan.map((item) => [
|
|
803
|
-
item.nodeId,
|
|
804
|
-
item.sourceAttemptId,
|
|
805
|
-
item.sourceGeneration,
|
|
806
|
-
item.sourceStatus,
|
|
807
|
-
item.oldAgentId,
|
|
808
|
-
]),
|
|
809
|
-
terminalResults: terminalResults.map((item) => [
|
|
810
|
-
item.nodeId,
|
|
811
|
-
item.attemptId,
|
|
812
|
-
item.result.status,
|
|
813
|
-
item.result.agentId,
|
|
814
|
-
item.result.text,
|
|
815
|
-
item.result.error,
|
|
816
|
-
item.result.outputFile,
|
|
817
|
-
item.result.tokenCount,
|
|
818
|
-
item.result.compactionCount,
|
|
819
|
-
item.result.truncated,
|
|
820
|
-
]),
|
|
821
|
-
});
|
|
822
|
-
}
|
|
823
|
-
|
|
824
|
-
private rememberRecovery(memo: RecoveryMemo): void {
|
|
825
|
-
const key = this.recoveryMemoKey(memo.runId, memo.event.kind, memo.key);
|
|
826
|
-
this.recoveryMemo.delete(key);
|
|
827
|
-
this.recoveryMemo.set(key, memo);
|
|
828
|
-
while (this.recoveryMemo.size > MAX_RECOVERY_MEMO_ENTRIES) {
|
|
829
|
-
const oldest = this.recoveryMemo.keys().next().value;
|
|
830
|
-
if (typeof oldest !== "string") break;
|
|
831
|
-
this.recoveryMemo.delete(oldest);
|
|
832
|
-
}
|
|
833
|
-
}
|
|
834
|
-
|
|
835
|
-
private memoizedRecovery(runId: string, kind: RecoveryEvent["kind"], sourceKey: string): RecoveryEvent | undefined {
|
|
836
|
-
return this.recoveryMemo.get(this.recoveryMemoKey(runId, kind, sourceKey))?.event;
|
|
837
|
-
}
|
|
838
|
-
|
|
839
|
-
list(): WorkflowSummary[] {
|
|
840
|
-
return [...this.runs.values()].sort((a, b) => b.startedAt - a.startedAt).map((run) => this.summary(run));
|
|
841
|
-
}
|
|
842
|
-
|
|
843
|
-
get(runId: string): WorkflowSummary | undefined {
|
|
844
|
-
const run = this.runs.get(runId);
|
|
845
|
-
return run ? this.summary(run) : undefined;
|
|
846
|
-
}
|
|
847
|
-
|
|
848
|
-
getRun(runId: string): WorkflowRun | undefined {
|
|
849
|
-
return this.runs.get(runId);
|
|
850
|
-
}
|
|
851
|
-
|
|
852
|
-
async start(definition: WorkflowDefinition, signal?: AbortSignal): Promise<WorkflowStartResult>;
|
|
853
|
-
/** @deprecated The context argument is ignored; pass the foreground signal as the second argument. */
|
|
854
|
-
async start(
|
|
855
|
-
definition: WorkflowDefinition,
|
|
856
|
-
_legacyContext: unknown,
|
|
857
|
-
signal?: AbortSignal,
|
|
858
|
-
): Promise<WorkflowStartResult>;
|
|
859
|
-
async start(
|
|
860
|
-
definition: WorkflowDefinition,
|
|
861
|
-
signalOrLegacyContext?: AbortSignal | unknown,
|
|
862
|
-
legacySignal?: AbortSignal,
|
|
863
|
-
): Promise<WorkflowStartResult> {
|
|
864
|
-
const signal = legacySignal ?? (isAbortSignal(signalOrLegacyContext) ? signalOrLegacyContext : undefined);
|
|
391
|
+
async start(script: string, options: ScriptStartOptions = {}): Promise<ScriptStartResult> {
|
|
392
|
+
const { meta } = parseWorkflowScript(script);
|
|
865
393
|
const now = Date.now();
|
|
866
394
|
const runId = randomUUID();
|
|
867
|
-
const
|
|
395
|
+
const scriptHash = hashScript(script);
|
|
396
|
+
const toolset = options.toolset;
|
|
397
|
+
const run: ScriptRun = {
|
|
868
398
|
runId,
|
|
869
399
|
schemaVersion: JOURNAL_SCHEMA_VERSION,
|
|
870
|
-
|
|
400
|
+
script,
|
|
401
|
+
scriptHash,
|
|
402
|
+
meta,
|
|
403
|
+
...(toolset ? { toolset } : {}),
|
|
404
|
+
...(options.maxAgents !== undefined ? { frozenMaxAgents: options.maxAgents } : {}),
|
|
405
|
+
...(options.concurrency !== undefined ? { frozenConcurrency: options.concurrency } : {}),
|
|
406
|
+
...(options.agentRetries !== undefined ? { frozenAgentRetries: options.agentRetries } : {}),
|
|
407
|
+
...(options.tokenBudget !== undefined ? { frozenTokenBudget: options.tokenBudget } : {}),
|
|
408
|
+
...(options.agentTimeoutMs !== undefined ? { frozenAgentTimeoutMs: options.agentTimeoutMs } : {}),
|
|
409
|
+
...(options.excludeTools ? { frozenExcludeTools: [...options.excludeTools] } : {}),
|
|
871
410
|
status: "pending",
|
|
872
|
-
|
|
411
|
+
callStatus: {},
|
|
873
412
|
agentIds: {},
|
|
874
|
-
attempts:
|
|
875
|
-
|
|
876
|
-
|
|
877
|
-
]),
|
|
878
|
-
attemptIds: Object.fromEntries([
|
|
879
|
-
...definition.tasks.map((task) => [task.id, `${runId}/${task.id}/attempt-1`] as const),
|
|
880
|
-
[SYNTHESIS_NODE_ID, `${runId}/${SYNTHESIS_NODE_ID}/attempt-1`] as const,
|
|
881
|
-
]),
|
|
882
|
-
attemptTracking: true,
|
|
883
|
-
taskResults: {},
|
|
413
|
+
attempts: {},
|
|
414
|
+
attemptIds: {},
|
|
415
|
+
callResults: {},
|
|
884
416
|
compactions: {},
|
|
885
417
|
startedAt: now,
|
|
886
418
|
updatedAt: now,
|
|
887
419
|
};
|
|
888
|
-
this.
|
|
420
|
+
this.appendRequired({
|
|
889
421
|
kind: "run_created",
|
|
890
422
|
schemaVersion: JOURNAL_SCHEMA_VERSION,
|
|
891
|
-
runId
|
|
892
|
-
|
|
893
|
-
|
|
894
|
-
|
|
423
|
+
runId,
|
|
424
|
+
script,
|
|
425
|
+
scriptHash,
|
|
426
|
+
meta,
|
|
427
|
+
...(toolset ? { toolset } : {}),
|
|
428
|
+
...(options.maxAgents !== undefined ? { frozenMaxAgents: options.maxAgents } : {}),
|
|
429
|
+
...(options.concurrency !== undefined ? { frozenConcurrency: options.concurrency } : {}),
|
|
430
|
+
...(options.agentRetries !== undefined ? { frozenAgentRetries: options.agentRetries } : {}),
|
|
431
|
+
...(options.tokenBudget !== undefined ? { frozenTokenBudget: options.tokenBudget } : {}),
|
|
432
|
+
...(options.agentTimeoutMs !== undefined ? { frozenAgentTimeoutMs: options.agentTimeoutMs } : {}),
|
|
433
|
+
...(options.excludeTools ? { frozenExcludeTools: [...options.excludeTools] } : {}),
|
|
895
434
|
timestamp: now,
|
|
896
435
|
});
|
|
436
|
+
const state: ScriptRunState = {
|
|
437
|
+
run,
|
|
438
|
+
frozenOptions: {
|
|
439
|
+
...(options.tokenBudget !== undefined ? { tokenBudget: options.tokenBudget } : {}),
|
|
440
|
+
...(options.maxAgents !== undefined ? { maxAgents: options.maxAgents } : {}),
|
|
441
|
+
...(options.concurrency !== undefined ? { concurrency: options.concurrency } : {}),
|
|
442
|
+
...(options.agentRetries !== undefined ? { agentRetries: options.agentRetries } : {}),
|
|
443
|
+
...(options.agentTimeoutMs !== undefined ? { agentTimeoutMs: options.agentTimeoutMs } : {}),
|
|
444
|
+
...(options.toolset ? { toolset: options.toolset } : {}),
|
|
445
|
+
...(options.excludeTools ? { excludeTools: [...options.excludeTools] } : {}),
|
|
446
|
+
},
|
|
447
|
+
controller: new AbortController(),
|
|
448
|
+
generations: new Map(),
|
|
449
|
+
agentWaiters: new Map(),
|
|
450
|
+
pendingSpawns: new Map(),
|
|
451
|
+
pendingJournal: new Set(),
|
|
452
|
+
lifecycleSuspended: false,
|
|
453
|
+
executionGeneration: 0,
|
|
454
|
+
};
|
|
455
|
+
this.runs.set(runId, state);
|
|
897
456
|
this.setWorkflowStatus(run, "running");
|
|
898
|
-
|
|
899
|
-
|
|
900
|
-
|
|
457
|
+
state.executionGeneration += 1;
|
|
458
|
+
state.execution = this.execute(runId, script, options, undefined, state.executionGeneration);
|
|
459
|
+
// A run-fatal abort/stop already journals its terminal state; never let a
|
|
460
|
+
// leftover rejection become an unhandled promise rejection.
|
|
461
|
+
void state.execution.catch(() => {});
|
|
462
|
+
if (options.background) return { runId, status: run.status, background: true };
|
|
901
463
|
try {
|
|
902
|
-
const settled = await this.waitFor(
|
|
464
|
+
const settled = await this.waitFor(runId, options.signal);
|
|
465
|
+
const state = this.runs.get(runId);
|
|
466
|
+
const scriptResult = state?.result;
|
|
903
467
|
return {
|
|
904
|
-
runId
|
|
468
|
+
runId,
|
|
905
469
|
status: settled.status,
|
|
906
470
|
background: false,
|
|
907
|
-
...(
|
|
471
|
+
...(scriptResult !== undefined ? { result: String(scriptResult) } : {}),
|
|
908
472
|
...(settled.error ? { error: settled.error } : {}),
|
|
909
473
|
};
|
|
910
474
|
} catch (error: unknown) {
|
|
911
475
|
if (!(error instanceof WorkflowWaitAbortedError)) throw error;
|
|
912
|
-
const current = this.runs.get(
|
|
913
|
-
return {
|
|
914
|
-
|
|
915
|
-
|
|
916
|
-
|
|
917
|
-
|
|
918
|
-
|
|
919
|
-
|
|
476
|
+
const current = this.runs.get(runId)?.run ?? run;
|
|
477
|
+
return { runId: current.runId, status: current.status, background: false, waitAborted: true };
|
|
478
|
+
}
|
|
479
|
+
}
|
|
480
|
+
|
|
481
|
+
/**
|
|
482
|
+
* Resume an interrupted/paused run: rebuild the resume journal from journaled
|
|
483
|
+
* call_result events and re-run the same script under the same runId. The
|
|
484
|
+
* runtime replays the longest unchanged prefix; every live dispatch of a
|
|
485
|
+
* previously-journaled call index rotates the spawnKey generation.
|
|
486
|
+
*/
|
|
487
|
+
async resume(
|
|
488
|
+
runId: string,
|
|
489
|
+
entries: readonly SessionEntryLike[],
|
|
490
|
+
options: ScriptStartOptions = {},
|
|
491
|
+
replacementScript?: string,
|
|
492
|
+
): Promise<ScriptStartResult | undefined> {
|
|
493
|
+
const state = this.runs.get(runId);
|
|
494
|
+
if (!state) return undefined;
|
|
495
|
+
if (state.lifecycleSuspended) return undefined;
|
|
496
|
+
const run = state.run;
|
|
497
|
+
const providerTimer = this.providerResumeTimers.get(runId);
|
|
498
|
+
if (providerTimer) {
|
|
499
|
+
clearTimeout(providerTimer);
|
|
500
|
+
this.providerResumeTimers.delete(runId);
|
|
501
|
+
}
|
|
502
|
+
if (run.nonResumable || (isTerminalWorkflow(run.status) && run.status !== "failed" && run.status !== "stopped"))
|
|
503
|
+
return undefined;
|
|
504
|
+
if (replacementScript !== undefined) {
|
|
505
|
+
const parsed = parseWorkflowScript(replacementScript);
|
|
506
|
+
const revision = (run.revision ?? 0) + 1;
|
|
507
|
+
run.script = replacementScript;
|
|
508
|
+
run.scriptHash = hashScript(replacementScript);
|
|
509
|
+
run.meta = parsed.meta;
|
|
510
|
+
run.revision = revision;
|
|
511
|
+
this.persist(runId, `revision:${revision}`, () =>
|
|
512
|
+
this.write({
|
|
513
|
+
kind: "run_revision",
|
|
514
|
+
schemaVersion: JOURNAL_SCHEMA_VERSION,
|
|
515
|
+
runId,
|
|
516
|
+
revision,
|
|
517
|
+
script: replacementScript,
|
|
518
|
+
scriptHash: run.scriptHash,
|
|
519
|
+
meta: parsed.meta,
|
|
520
|
+
timestamp: Date.now(),
|
|
521
|
+
}),
|
|
522
|
+
);
|
|
523
|
+
}
|
|
524
|
+
const resumeJournal = buildResumeJournal(entries);
|
|
525
|
+
// Seed generation bases from journaled attemptIds so a live re-dispatch of a
|
|
526
|
+
// previously-journaled call never reuses a spawnKey (A4).
|
|
527
|
+
for (const [callIndex, attemptId] of Object.entries(run.attemptIds)) {
|
|
528
|
+
state.generations.set(callIndex, generationFromAttemptId(attemptId));
|
|
529
|
+
}
|
|
530
|
+
this.setWorkflowStatus(run, "running");
|
|
531
|
+
state.controller = new AbortController();
|
|
532
|
+
// Freeze semantics: original run parameters win over new options.
|
|
533
|
+
// Only signal/confirm/args/background/loadSavedWorkflow/mainModel are per-resume context;
|
|
534
|
+
// budget/scale/toolset are frozen from the original start.
|
|
535
|
+
const frozen = state.frozenOptions;
|
|
536
|
+
const resumeOptions: ScriptStartOptions = {
|
|
537
|
+
...options,
|
|
538
|
+
tokenBudget: frozen.tokenBudget,
|
|
539
|
+
maxAgents: frozen.maxAgents,
|
|
540
|
+
concurrency: frozen.concurrency,
|
|
541
|
+
agentRetries: frozen.agentRetries,
|
|
542
|
+
agentTimeoutMs: frozen.agentTimeoutMs,
|
|
543
|
+
toolset: frozen.toolset,
|
|
544
|
+
excludeTools: frozen.excludeTools ? [...frozen.excludeTools] : undefined,
|
|
545
|
+
};
|
|
546
|
+
state.executionGeneration += 1;
|
|
547
|
+
state.execution = this.execute(runId, run.script, resumeOptions, resumeJournal, state.executionGeneration);
|
|
548
|
+
void state.execution.catch(() => {});
|
|
549
|
+
if (options.background) return { runId, status: run.status, background: true };
|
|
550
|
+
try {
|
|
551
|
+
const settled = await this.waitFor(runId, options.signal);
|
|
552
|
+
return { runId, status: settled.status, background: false };
|
|
553
|
+
} catch (error: unknown) {
|
|
554
|
+
if (!(error instanceof WorkflowWaitAbortedError)) throw error;
|
|
555
|
+
return { runId, status: run.status, background: false, waitAborted: true };
|
|
920
556
|
}
|
|
921
557
|
}
|
|
922
558
|
|
|
923
|
-
async control(
|
|
559
|
+
async control(
|
|
560
|
+
action: "list" | "get" | "pause" | "resume" | "stop" | "rm",
|
|
561
|
+
runId?: string,
|
|
562
|
+
entries?: readonly SessionEntryLike[],
|
|
563
|
+
): Promise<{ action: string; runs?: unknown[]; run?: unknown }> {
|
|
924
564
|
if (action === "list") return { action, runs: this.list() };
|
|
925
565
|
if (!runId) throw new Error(`${action} requires runId`);
|
|
926
|
-
const
|
|
927
|
-
if (!
|
|
566
|
+
const state = this.runs.get(runId);
|
|
567
|
+
if (!state) throw new Error(`workflow run not found: ${runId}`);
|
|
568
|
+
const run = state.run;
|
|
928
569
|
if (action === "get") return { action, run: this.summary(run, true) };
|
|
929
570
|
if (action === "pause") {
|
|
930
571
|
if (run.status === "running") {
|
|
931
572
|
this.setWorkflowStatus(run, "pausing");
|
|
932
|
-
this.
|
|
933
|
-
|
|
573
|
+
this.reconcilePendingSpawns(state);
|
|
574
|
+
// Abort the script execution; the run settles as interrupted and can be
|
|
575
|
+
// resumed from the journal. The script's in-flight agents are stopped by
|
|
576
|
+
// the runtime's abort signal via the dispatch waiters.
|
|
577
|
+
state.controller.abort();
|
|
578
|
+
this.persist(runId, "pause-recovery", () =>
|
|
579
|
+
this.write({
|
|
580
|
+
kind: "run_recovery",
|
|
581
|
+
schemaVersion: JOURNAL_SCHEMA_VERSION,
|
|
582
|
+
runId,
|
|
583
|
+
status: "interrupted",
|
|
584
|
+
branchGeneration: this.recoveryBranchGeneration,
|
|
585
|
+
rotations: [],
|
|
586
|
+
recoveryId: deriveRecoveryId({ runId, rotations: [], timestamp: Date.now() }),
|
|
587
|
+
timestamp: Date.now(),
|
|
588
|
+
}),
|
|
589
|
+
);
|
|
590
|
+
run.status = "interrupted";
|
|
591
|
+
run.updatedAt = Date.now();
|
|
592
|
+
this.settleWaiters(runId, run);
|
|
593
|
+
} else if (run.status !== "paused" && run.status !== "interrupted") {
|
|
934
594
|
throw new Error(`cannot pause workflow in state ${run.status}`);
|
|
935
595
|
}
|
|
936
596
|
} else if (action === "resume") {
|
|
@@ -938,24 +598,45 @@ export class WorkflowEngine {
|
|
|
938
598
|
if (run.status !== "paused" && run.status !== "interrupted") {
|
|
939
599
|
throw new Error(`cannot resume workflow in state ${run.status}`);
|
|
940
600
|
}
|
|
941
|
-
this.
|
|
942
|
-
|
|
601
|
+
const sessionEntries = entries ?? this.readEntries();
|
|
602
|
+
const resumed = await this.resume(runId, sessionEntries, {
|
|
603
|
+
background: true,
|
|
604
|
+
mainModel: undefined,
|
|
605
|
+
});
|
|
606
|
+
if (!resumed) throw new Error(`cannot resume workflow run: ${runId}`);
|
|
607
|
+
return { action, run: this.summary(this.runs.get(runId)?.run ?? run) };
|
|
943
608
|
} else if (action === "stop") {
|
|
944
|
-
await this.stop(
|
|
609
|
+
await this.stop(state);
|
|
610
|
+
} else if (action === "rm") {
|
|
611
|
+
if (!isTerminalWorkflow(run.status)) throw new Error("cannot remove a running workflow");
|
|
612
|
+
this.persist(runId, "run-removed", () =>
|
|
613
|
+
this.write({ kind: "run_removed", schemaVersion: JOURNAL_SCHEMA_VERSION, runId, timestamp: Date.now() }),
|
|
614
|
+
);
|
|
615
|
+
this.runs.delete(runId);
|
|
616
|
+
this.waiters.delete(runId);
|
|
617
|
+
return { action, run: this.summary(run) };
|
|
945
618
|
}
|
|
946
619
|
return { action, run: this.summary(run) };
|
|
947
620
|
}
|
|
948
621
|
|
|
949
|
-
async waitFor(runId: string, signal?: AbortSignal): Promise<
|
|
950
|
-
const
|
|
951
|
-
if (!
|
|
952
|
-
if (isTerminalWorkflow(run.status)
|
|
953
|
-
|
|
622
|
+
async waitFor(runId: string, signal?: AbortSignal): Promise<ScriptRun> {
|
|
623
|
+
const state = this.runs.get(runId);
|
|
624
|
+
if (!state) throw new Error(`workflow run not found: ${runId}`);
|
|
625
|
+
if (isTerminalWorkflow(state.run.status) || state.run.status === "paused" || state.run.status === "interrupted")
|
|
626
|
+
return state.run;
|
|
627
|
+
const all =
|
|
628
|
+
this.waiters.get(runId) ??
|
|
629
|
+
new Set<{
|
|
630
|
+
resolve: (run: ScriptRun) => void;
|
|
631
|
+
reject: (e: unknown) => void;
|
|
632
|
+
settled: boolean;
|
|
633
|
+
cleanup: () => void;
|
|
634
|
+
}>();
|
|
954
635
|
if (all.size >= MAX_WAITERS) throw new Error("too many workflow waiters");
|
|
955
636
|
this.waiters.set(runId, all);
|
|
956
|
-
return new Promise<
|
|
637
|
+
return new Promise<ScriptRun>((resolve, reject) => {
|
|
957
638
|
let abort = () => {};
|
|
958
|
-
const waiter
|
|
639
|
+
const waiter = {
|
|
959
640
|
resolve,
|
|
960
641
|
reject,
|
|
961
642
|
settled: false,
|
|
@@ -970,223 +651,518 @@ export class WorkflowEngine {
|
|
|
970
651
|
reject(new WorkflowWaitAbortedError());
|
|
971
652
|
};
|
|
972
653
|
all.add(waiter);
|
|
973
|
-
|
|
974
|
-
if (signal.aborted) {
|
|
975
|
-
abort();
|
|
976
|
-
return;
|
|
977
|
-
}
|
|
978
|
-
signal.addEventListener("abort", abort, { once: true });
|
|
979
|
-
}
|
|
980
|
-
if (isTerminalWorkflow(run.status)) {
|
|
981
|
-
all.delete(waiter);
|
|
982
|
-
if (all.size === 0) this.waiters.delete(runId);
|
|
983
|
-
waiter.settled = true;
|
|
984
|
-
waiter.cleanup();
|
|
985
|
-
resolve(run);
|
|
986
|
-
}
|
|
987
|
-
});
|
|
988
|
-
}
|
|
989
|
-
|
|
990
|
-
private trackDispatch(runId: string, dispatch: Promise<void>): void {
|
|
991
|
-
const pending = this.dispatches.get(runId) ?? new Set<Promise<void>>();
|
|
992
|
-
pending.add(dispatch);
|
|
993
|
-
this.dispatches.set(runId, pending);
|
|
994
|
-
// Use both settlement branches so a rejected dispatch never creates an
|
|
995
|
-
// unhandled rejection while its bookkeeping is removed.
|
|
996
|
-
void dispatch.then(
|
|
997
|
-
() => this.untrackDispatch(runId, dispatch),
|
|
998
|
-
() => this.untrackDispatch(runId, dispatch),
|
|
999
|
-
);
|
|
1000
|
-
}
|
|
1001
|
-
|
|
1002
|
-
private untrackDispatch(runId: string, dispatch: Promise<void>): void {
|
|
1003
|
-
const pending = this.dispatches.get(runId);
|
|
1004
|
-
if (!pending) return;
|
|
1005
|
-
pending.delete(dispatch);
|
|
1006
|
-
if (pending.size === 0) this.dispatches.delete(runId);
|
|
1007
|
-
}
|
|
1008
|
-
|
|
1009
|
-
private async waitForDispatches(runId: string, timeoutMs: number): Promise<boolean> {
|
|
1010
|
-
const pending = this.dispatches.get(runId);
|
|
1011
|
-
if (!pending || pending.size === 0) return true;
|
|
1012
|
-
let timer: ReturnType<typeof setTimeout> | undefined;
|
|
1013
|
-
const allSettled = Promise.allSettled([...pending]).then(() => true);
|
|
1014
|
-
const timedOut = new Promise<boolean>((resolve) => {
|
|
1015
|
-
timer = setTimeout(() => resolve(false), timeoutMs);
|
|
654
|
+
signal?.addEventListener("abort", abort, { once: true });
|
|
1016
655
|
});
|
|
1017
|
-
const settled = await Promise.race([allSettled, timedOut]);
|
|
1018
|
-
if (timer) clearTimeout(timer);
|
|
1019
|
-
return settled;
|
|
1020
|
-
}
|
|
1021
|
-
|
|
1022
|
-
private quarantineAgent(agentId: string): void {
|
|
1023
|
-
this.quarantinedAgentIds.add(agentId);
|
|
1024
|
-
if (this.quarantinedAgentIds.size > MAX_QUARANTINED_AGENT_IDS) {
|
|
1025
|
-
const oldest = this.quarantinedAgentIds.values().next().value;
|
|
1026
|
-
if (typeof oldest === "string") this.quarantinedAgentIds.delete(oldest);
|
|
1027
|
-
}
|
|
1028
656
|
}
|
|
1029
657
|
|
|
1030
|
-
private
|
|
1031
|
-
const
|
|
1032
|
-
|
|
1033
|
-
|
|
1034
|
-
|
|
1035
|
-
|
|
1036
|
-
|
|
658
|
+
private rejectWaiters(error: Error): void {
|
|
659
|
+
for (const all of this.waiters.values()) {
|
|
660
|
+
for (const waiter of [...all]) {
|
|
661
|
+
if (waiter.settled) continue;
|
|
662
|
+
waiter.settled = true;
|
|
663
|
+
all.delete(waiter);
|
|
664
|
+
waiter.cleanup();
|
|
665
|
+
waiter.reject(error);
|
|
666
|
+
}
|
|
1037
667
|
}
|
|
1038
|
-
|
|
668
|
+
this.waiters.clear();
|
|
1039
669
|
}
|
|
1040
670
|
|
|
1041
|
-
private
|
|
1042
|
-
|
|
1043
|
-
|
|
1044
|
-
|
|
1045
|
-
|
|
1046
|
-
|
|
1047
|
-
|
|
1048
|
-
|
|
1049
|
-
|
|
1050
|
-
};
|
|
1051
|
-
const targets = this.lateDispatchTargets.get(run.runId) ?? [];
|
|
1052
|
-
if (
|
|
1053
|
-
!targets.some(
|
|
1054
|
-
(target) =>
|
|
1055
|
-
target.agentId === agentId &&
|
|
1056
|
-
target.owner.nodeId === owner.nodeId &&
|
|
1057
|
-
target.owner.attemptId === owner.attemptId,
|
|
1058
|
-
)
|
|
1059
|
-
) {
|
|
1060
|
-
targets.push({ agentId, owner, fallbackStopRequested: false });
|
|
671
|
+
private settleWaiters(runId: string, run: ScriptRun): void {
|
|
672
|
+
const all = this.waiters.get(runId);
|
|
673
|
+
if (!all) return;
|
|
674
|
+
for (const waiter of [...all]) {
|
|
675
|
+
if (waiter.settled) continue;
|
|
676
|
+
waiter.settled = true;
|
|
677
|
+
all.delete(waiter);
|
|
678
|
+
waiter.cleanup();
|
|
679
|
+
waiter.resolve(run);
|
|
1061
680
|
}
|
|
1062
|
-
this.
|
|
681
|
+
if (all.size === 0) this.waiters.delete(runId);
|
|
1063
682
|
}
|
|
1064
683
|
|
|
1065
|
-
|
|
1066
|
-
const targets = this.lateDispatchTargets.get(run.runId);
|
|
1067
|
-
if (!targets) return [];
|
|
1068
|
-
this.lateDispatchTargets.delete(run.runId);
|
|
1069
|
-
return targets;
|
|
1070
|
-
}
|
|
684
|
+
// ── execution ──────────────────────────────────────────────────────────────
|
|
1071
685
|
|
|
1072
|
-
private
|
|
1073
|
-
|
|
1074
|
-
|
|
1075
|
-
|
|
1076
|
-
|
|
686
|
+
private async execute(
|
|
687
|
+
runId: string,
|
|
688
|
+
script: string,
|
|
689
|
+
options: ScriptStartOptions,
|
|
690
|
+
resumeJournal?: Map<string, JournalEntry>,
|
|
691
|
+
executionGeneration?: number,
|
|
692
|
+
): Promise<WorkflowRunResult<unknown>> {
|
|
693
|
+
const state = this.runs.get(runId);
|
|
694
|
+
if (!state) throw new Error(`workflow run not found: ${runId}`);
|
|
695
|
+
const generation = executionGeneration ?? state.executionGeneration;
|
|
1077
696
|
|
|
1078
|
-
|
|
1079
|
-
|
|
1080
|
-
|
|
1081
|
-
|
|
1082
|
-
|
|
697
|
+
const runner: WorkflowAgentRunner = {
|
|
698
|
+
run: async (prompt: string, runOptions: AgentRunOptions = {}) => {
|
|
699
|
+
return this.dispatchAgent(runId, prompt, runOptions);
|
|
700
|
+
},
|
|
701
|
+
};
|
|
1083
702
|
|
|
1084
|
-
|
|
1085
|
-
|
|
1086
|
-
|
|
1087
|
-
|
|
1088
|
-
|
|
1089
|
-
|
|
1090
|
-
|
|
1091
|
-
|
|
1092
|
-
|
|
1093
|
-
|
|
1094
|
-
|
|
1095
|
-
|
|
1096
|
-
|
|
703
|
+
try {
|
|
704
|
+
const result = await runWorkflow(script, {
|
|
705
|
+
args: options.args,
|
|
706
|
+
agent: runner,
|
|
707
|
+
mainModel: options.mainModel,
|
|
708
|
+
concurrency: options.concurrency,
|
|
709
|
+
agentRetries: options.agentRetries,
|
|
710
|
+
tokenBudget: options.tokenBudget,
|
|
711
|
+
agentTimeoutMs: options.agentTimeoutMs,
|
|
712
|
+
signal: state.controller.signal,
|
|
713
|
+
maxAgents: options.maxAgents,
|
|
714
|
+
runId,
|
|
715
|
+
resumeJournal,
|
|
716
|
+
toolset: options.toolset,
|
|
717
|
+
excludeTools: options.excludeTools,
|
|
718
|
+
confirm: options.confirm,
|
|
719
|
+
loadSavedWorkflow: options.loadSavedWorkflow,
|
|
720
|
+
onAgentJournal: (entry) => {
|
|
721
|
+
// Nested workflow frames use their own run ids. They are intentionally
|
|
722
|
+
// not persisted as top-level call facts until a namespaced journal exists;
|
|
723
|
+
// mixing them into the parent would quarantine the run on replay.
|
|
724
|
+
if (entry.runId === undefined || entry.runId === runId) this.journalCallResult(runId, entry);
|
|
725
|
+
},
|
|
726
|
+
onRuntimeEvent: () => {},
|
|
1097
727
|
});
|
|
728
|
+
const live = this.runs.get(runId);
|
|
729
|
+
if (live && state.executionGeneration === generation) live.result = result.result;
|
|
730
|
+
const run = live?.run;
|
|
731
|
+
if (run && state.executionGeneration === generation && !isTerminalWorkflow(run.status)) {
|
|
732
|
+
this.setWorkflowStatus(run, "completed");
|
|
733
|
+
}
|
|
734
|
+
return result;
|
|
735
|
+
} catch (error: unknown) {
|
|
736
|
+
const run = this.runs.get(runId)?.run;
|
|
737
|
+
if (run && state.executionGeneration === generation && !isTerminalWorkflow(run.status)) {
|
|
738
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
739
|
+
const workflowError = error instanceof WorkflowError ? error : wrapError(error);
|
|
740
|
+
if (workflowError.code === WorkflowErrorCode.PROVIDER_USAGE_LIMIT) {
|
|
741
|
+
this.setWorkflowStatus(run, "paused", message);
|
|
742
|
+
this.cleanupAgents(runId);
|
|
743
|
+
this.scheduleProviderResume(runId, workflowError.resetHint);
|
|
744
|
+
} else if (workflowError.code === WorkflowErrorCode.WORKFLOW_ABORTED && state.controller.signal.aborted) {
|
|
745
|
+
this.setWorkflowStatus(run, "interrupted");
|
|
746
|
+
} else {
|
|
747
|
+
this.setWorkflowStatus(run, "failed", message);
|
|
748
|
+
this.cleanupAgents(runId);
|
|
749
|
+
}
|
|
750
|
+
}
|
|
751
|
+
throw error;
|
|
752
|
+
} finally {
|
|
753
|
+
// Reject any waiters that never settled (agents left in flight).
|
|
754
|
+
const run = this.runs.get(runId)?.run;
|
|
755
|
+
if (run && state.executionGeneration === generation) this.settleWaiters(runId, run);
|
|
1098
756
|
}
|
|
1099
|
-
return targets;
|
|
1100
757
|
}
|
|
1101
758
|
|
|
1102
|
-
|
|
1103
|
-
|
|
1104
|
-
|
|
1105
|
-
|
|
1106
|
-
|
|
1107
|
-
|
|
1108
|
-
|
|
1109
|
-
|
|
1110
|
-
|
|
1111
|
-
|
|
1112
|
-
|
|
1113
|
-
|
|
759
|
+
/**
|
|
760
|
+
* Dispatch one live agent() call through spawn-managed with an A4-rotated
|
|
761
|
+
* spawnKey: `${runId}/call-${callIndex}/attempt-${generation}`. The generation
|
|
762
|
+
* combines the journaled base (so a resume of an edited call lands on a fresh
|
|
763
|
+
* key) with the runtime's per-attempt counter.
|
|
764
|
+
*/
|
|
765
|
+
private async dispatchAgent(runId: string, prompt: string, runOptions: AgentRunOptions): Promise<unknown> {
|
|
766
|
+
const state = this.runs.get(runId);
|
|
767
|
+
if (!state) throw new Error(`workflow run not found: ${runId}`);
|
|
768
|
+
if (state.lifecycleSuspended || state.controller.signal.aborted) {
|
|
769
|
+
throw new WorkflowError("workflow branch is quiescing", WorkflowErrorCode.WORKFLOW_ABORTED, {
|
|
770
|
+
recoverable: true,
|
|
771
|
+
});
|
|
772
|
+
}
|
|
773
|
+
const run = state.run;
|
|
774
|
+
const executionGeneration = state.executionGeneration;
|
|
775
|
+
const callIndex = runOptions.callIndex ?? state.generations.size;
|
|
776
|
+
// The spawnKey identity carries the `call-` prefix (A4):
|
|
777
|
+
// spawnKey = `${runId}/call-${callIndex}/attempt-${generation}`
|
|
778
|
+
// The journal node identity is the bare call index, and the journal
|
|
779
|
+
// attemptId is the full `${runId}/${nodeId}/attempt-${generation}` path the
|
|
780
|
+
// journal replay validates.
|
|
781
|
+
const spawnNodeId = `call-${callIndex}`;
|
|
782
|
+
const nodeId = String(callIndex);
|
|
783
|
+
const base = state.generations.get(nodeId) ?? 0;
|
|
784
|
+
const generation = base + (runOptions.attempt ?? 1);
|
|
785
|
+
state.generations.set(nodeId, generation);
|
|
786
|
+
const spawnAttemptId = `attempt-${generation}`;
|
|
787
|
+
const attemptId = `${runId}/${nodeId}/attempt-${generation}`;
|
|
788
|
+
|
|
789
|
+
const type = runOptions.agentType ?? "general-purpose";
|
|
790
|
+
const tier =
|
|
791
|
+
runOptions.tier === "small" || runOptions.tier === "medium" || runOptions.tier === "large"
|
|
792
|
+
? runOptions.tier
|
|
793
|
+
: undefined;
|
|
794
|
+
const task: DispatchTask = {
|
|
795
|
+
subagent_type: type,
|
|
796
|
+
prompt,
|
|
797
|
+
description: runOptions.label ?? `workflow call ${callIndex}`,
|
|
798
|
+
...(tier === undefined ? {} : { tier }),
|
|
799
|
+
...(runOptions.model === undefined ? {} : { model: runOptions.model }),
|
|
800
|
+
...(runOptions.thread === undefined ? {} : { thread: runOptions.thread }),
|
|
801
|
+
...(runOptions.toolset === undefined ? {} : { toolset: runOptions.toolset }),
|
|
802
|
+
...(runOptions.excludeTools === undefined ? {} : { excludeTools: runOptions.excludeTools }),
|
|
803
|
+
...(runOptions.isolation === undefined ? {} : { isolation: runOptions.isolation }),
|
|
804
|
+
};
|
|
805
|
+
|
|
806
|
+
// Journal the attempt rotation so a later resume knows the generation base.
|
|
807
|
+
this.persist(runId, `call-attempt:${nodeId}`, () =>
|
|
808
|
+
this.write({
|
|
809
|
+
kind: "call_attempt",
|
|
810
|
+
schemaVersion: JOURNAL_SCHEMA_VERSION,
|
|
811
|
+
runId,
|
|
812
|
+
nodeId,
|
|
813
|
+
attemptId,
|
|
814
|
+
generation,
|
|
815
|
+
owner: { extension: "pi-workflows", runId, nodeId, attemptId },
|
|
816
|
+
timestamp: Date.now(),
|
|
817
|
+
}),
|
|
818
|
+
);
|
|
819
|
+
run.callStatus[nodeId] = "running";
|
|
820
|
+
run.attempts[nodeId] = generation;
|
|
821
|
+
run.attemptIds[nodeId] = attemptId;
|
|
822
|
+
run.updatedAt = Date.now();
|
|
823
|
+
|
|
824
|
+
// Abort linkage: an external signal (pause/stop/Esc) cancels the wait and
|
|
825
|
+
// stops the owned agent. Track the spawn key before the RPC reply arrives so
|
|
826
|
+
// a lost reply can still be reconciled during shutdown or branch replacement.
|
|
827
|
+
const externalSignal = runOptions.signal ?? state.controller.signal;
|
|
828
|
+
const spawnKey = `${runId}/${spawnNodeId}/${spawnAttemptId}`;
|
|
829
|
+
const owner: WorkflowOwner = {
|
|
830
|
+
extension: "pi-workflows",
|
|
831
|
+
runId,
|
|
832
|
+
nodeId: spawnNodeId,
|
|
833
|
+
attemptId: spawnAttemptId,
|
|
1114
834
|
};
|
|
1115
|
-
|
|
1116
|
-
|
|
1117
|
-
|
|
1118
|
-
|
|
1119
|
-
|
|
1120
|
-
|
|
835
|
+
const pendingKey = `${nodeId}\u0000${generation}`;
|
|
836
|
+
state.pendingSpawns.set(pendingKey, { spawnKey, owner });
|
|
837
|
+
const reconcilePending = () => {
|
|
838
|
+
const reconciliation = this.client.reconcileManaged?.(spawnKey, owner);
|
|
839
|
+
void reconciliation?.catch(() => {});
|
|
840
|
+
};
|
|
841
|
+
const onPendingAbort = () => reconcilePending();
|
|
842
|
+
externalSignal.addEventListener("abort", onPendingAbort, { once: true });
|
|
843
|
+
if (externalSignal.aborted) onPendingAbort();
|
|
844
|
+
|
|
845
|
+
let pendingId = "";
|
|
846
|
+
let response: ManagedSpawnResponse | string;
|
|
847
|
+
try {
|
|
848
|
+
response = await this.client.spawn(task, runId, spawnNodeId, spawnAttemptId, undefined, externalSignal);
|
|
849
|
+
} finally {
|
|
850
|
+
externalSignal.removeEventListener("abort", onPendingAbort);
|
|
851
|
+
state.pendingSpawns.delete(pendingKey);
|
|
852
|
+
}
|
|
853
|
+
const responseId = typeof response === "string" ? response : response.id;
|
|
854
|
+
if (
|
|
855
|
+
state.executionGeneration !== executionGeneration ||
|
|
856
|
+
state.controller.signal.aborted ||
|
|
857
|
+
externalSignal.aborted
|
|
858
|
+
) {
|
|
859
|
+
await this.client
|
|
860
|
+
.stopOwned?.(responseId, { extension: "pi-workflows", runId, nodeId: spawnNodeId, attemptId: spawnAttemptId })
|
|
861
|
+
.catch(() => {});
|
|
862
|
+
throw new WorkflowError(
|
|
863
|
+
"workflow dispatch became stale during pause/resume",
|
|
864
|
+
WorkflowErrorCode.WORKFLOW_ABORTED,
|
|
865
|
+
{ recoverable: true },
|
|
866
|
+
);
|
|
867
|
+
}
|
|
868
|
+
if (typeof response === "string") {
|
|
869
|
+
pendingId = response;
|
|
870
|
+
} else if (response.terminal) {
|
|
871
|
+
// The spawn response already carries a terminal snapshot (fast completion
|
|
872
|
+
// or interrupted replay). The runtime journals successful calls itself
|
|
873
|
+
// (onAgentJournal); a terminal snapshot that is NOT a success is
|
|
874
|
+
// journaled here so the resume journal reflects the failure.
|
|
875
|
+
const terminal = response.terminal;
|
|
876
|
+
if (typeof terminal.tokenCount === "number")
|
|
877
|
+
runOptions.onUsage?.({
|
|
878
|
+
input: 0,
|
|
879
|
+
output: terminal.tokenCount,
|
|
880
|
+
total: terminal.tokenCount,
|
|
881
|
+
cost: 0,
|
|
882
|
+
cacheRead: 0,
|
|
883
|
+
cacheWrite: 0,
|
|
884
|
+
});
|
|
885
|
+
const status = managedTerminalStatus(terminal);
|
|
886
|
+
if (status !== "completed") {
|
|
887
|
+
const result = resultFromLifecycle(
|
|
888
|
+
status,
|
|
889
|
+
response.id,
|
|
890
|
+
terminal.result,
|
|
891
|
+
terminal.error,
|
|
892
|
+
terminal.compactionCount,
|
|
893
|
+
terminal.tokenCount,
|
|
894
|
+
);
|
|
895
|
+
result.attemptId = attemptId;
|
|
896
|
+
this.commitCallTerminal(runId, nodeId, result);
|
|
897
|
+
if (status === "failed" && terminal.error) {
|
|
898
|
+
throw wrapError(new Error(terminal.error), { agentLabel: runOptions.label });
|
|
899
|
+
}
|
|
900
|
+
}
|
|
901
|
+
return terminal.result ?? "";
|
|
902
|
+
} else {
|
|
903
|
+
pendingId = response.id;
|
|
904
|
+
}
|
|
905
|
+
|
|
906
|
+
const waitAbort = () => {
|
|
907
|
+
const waiter = state.agentWaiters.get(pendingId);
|
|
908
|
+
if (waiter && !waiter.settled) {
|
|
909
|
+
waiter.settled = true;
|
|
910
|
+
state.agentWaiters.delete(pendingId);
|
|
911
|
+
void this.client
|
|
912
|
+
.stopOwned?.(pendingId, {
|
|
913
|
+
extension: "pi-workflows",
|
|
914
|
+
runId,
|
|
915
|
+
nodeId: spawnNodeId,
|
|
916
|
+
attemptId: spawnAttemptId,
|
|
917
|
+
})
|
|
918
|
+
.catch(() => {});
|
|
919
|
+
waiter.reject(new WorkflowError("workflow aborted", WorkflowErrorCode.WORKFLOW_ABORTED, { recoverable: true }));
|
|
920
|
+
}
|
|
1121
921
|
};
|
|
1122
|
-
|
|
1123
|
-
|
|
1124
|
-
|
|
1125
|
-
|
|
1126
|
-
|
|
1127
|
-
|
|
1128
|
-
|
|
1129
|
-
|
|
922
|
+
|
|
923
|
+
// Wait for the terminal lifecycle event.
|
|
924
|
+
const terminal = await new Promise<ManagedSpawnResponse>((resolve, reject) => {
|
|
925
|
+
const waiter = {
|
|
926
|
+
callIndex: String(callIndex),
|
|
927
|
+
generation,
|
|
928
|
+
executionGeneration,
|
|
929
|
+
onUsage: runOptions.onUsage,
|
|
930
|
+
onHistory: runOptions.onHistory,
|
|
931
|
+
settled: false,
|
|
932
|
+
resolve,
|
|
933
|
+
reject,
|
|
934
|
+
};
|
|
935
|
+
state.agentWaiters.set(pendingId, waiter);
|
|
936
|
+
externalSignal.addEventListener("abort", waitAbort, { once: true });
|
|
937
|
+
if (externalSignal.aborted) waitAbort();
|
|
938
|
+
}).finally(() => {
|
|
939
|
+
externalSignal.removeEventListener("abort", waitAbort);
|
|
940
|
+
});
|
|
941
|
+
|
|
942
|
+
const snapshot = terminal.terminal;
|
|
943
|
+
if (!snapshot)
|
|
944
|
+
throw new WorkflowError("subagent settled without a terminal snapshot", WorkflowErrorCode.AGENT_EXECUTION_ERROR, {
|
|
945
|
+
recoverable: true,
|
|
946
|
+
});
|
|
947
|
+
const status = managedTerminalStatus(snapshot);
|
|
948
|
+
// Successful calls are journaled by the runtime's onAgentJournal (the
|
|
949
|
+
// single writer for completed results); only failures settle here so the
|
|
950
|
+
// resume journal reflects them.
|
|
951
|
+
if (status !== "completed") {
|
|
952
|
+
const result = resultFromLifecycle(
|
|
953
|
+
status,
|
|
954
|
+
terminal.id,
|
|
955
|
+
snapshot.result,
|
|
956
|
+
snapshot.error,
|
|
957
|
+
snapshot.compactionCount,
|
|
958
|
+
snapshot.tokenCount,
|
|
959
|
+
);
|
|
960
|
+
result.attemptId = attemptId;
|
|
961
|
+
this.commitCallTerminal(runId, nodeId, result);
|
|
962
|
+
if (status === "failed" && snapshot.error) {
|
|
963
|
+
throw wrapError(new Error(snapshot.error), { agentLabel: runOptions.label });
|
|
964
|
+
}
|
|
1130
965
|
}
|
|
1131
|
-
return
|
|
966
|
+
return snapshot.result ?? "";
|
|
1132
967
|
}
|
|
1133
968
|
|
|
1134
|
-
private
|
|
1135
|
-
|
|
1136
|
-
|
|
969
|
+
private journalCallResult(runId: string, entry: JournalEntry): void {
|
|
970
|
+
const state = this.runs.get(runId);
|
|
971
|
+
if (!state) return;
|
|
972
|
+
const run = state.run;
|
|
973
|
+
const nodeId = String(entry.index);
|
|
974
|
+
const result = entry.result;
|
|
975
|
+
const callResult: CallResult = {
|
|
976
|
+
status: "completed",
|
|
977
|
+
...(typeof result === "string" && result.length === 0 ? {} : { result }),
|
|
978
|
+
compactionCount: 0,
|
|
979
|
+
updatedAt: Date.now(),
|
|
980
|
+
};
|
|
981
|
+
const attemptId = run.attemptIds[nodeId];
|
|
982
|
+
if (attemptId) callResult.attemptId = attemptId;
|
|
983
|
+
const event: JournalEvent = {
|
|
984
|
+
kind: "call_result",
|
|
985
|
+
schemaVersion: JOURNAL_SCHEMA_VERSION,
|
|
986
|
+
runId,
|
|
987
|
+
nodeId,
|
|
988
|
+
result: callResult,
|
|
989
|
+
callHash: entry.hash,
|
|
990
|
+
...(entry.storeDelta !== undefined && Object.keys(entry.storeDelta).length > 0
|
|
991
|
+
? { storeDelta: entry.storeDelta }
|
|
992
|
+
: {}),
|
|
993
|
+
...(attemptId ? { attemptId } : {}),
|
|
994
|
+
...(attemptId ? { owner: { extension: "pi-workflows", runId, nodeId, attemptId } } : {}),
|
|
995
|
+
timestamp: Date.now(),
|
|
996
|
+
};
|
|
997
|
+
this.persist(runId, `call-result:${nodeId}`, () => this.write(event));
|
|
998
|
+
run.callResults[nodeId] = callResult;
|
|
999
|
+
run.callStatus[nodeId] = "completed";
|
|
1000
|
+
run.updatedAt = Date.now();
|
|
1001
|
+
}
|
|
1002
|
+
|
|
1003
|
+
private commitCallTerminal(runId: string, nodeId: string, result: CallResult): void {
|
|
1004
|
+
const state = this.runs.get(runId);
|
|
1005
|
+
if (!state) return;
|
|
1006
|
+
const run = state.run;
|
|
1007
|
+
const attemptId = result.attemptId ?? run.attemptIds[nodeId];
|
|
1008
|
+
const event: JournalEvent = {
|
|
1009
|
+
kind: "call_result",
|
|
1010
|
+
schemaVersion: JOURNAL_SCHEMA_VERSION,
|
|
1011
|
+
runId,
|
|
1012
|
+
nodeId,
|
|
1013
|
+
result,
|
|
1014
|
+
...(attemptId ? { attemptId } : {}),
|
|
1015
|
+
...(attemptId ? { owner: { extension: "pi-workflows", runId, nodeId, attemptId } } : {}),
|
|
1016
|
+
timestamp: Date.now(),
|
|
1017
|
+
};
|
|
1018
|
+
this.persist(runId, `call-terminal:${nodeId}`, () => this.write(event));
|
|
1019
|
+
run.callResults[nodeId] = result;
|
|
1020
|
+
run.callStatus[nodeId] = result.status;
|
|
1021
|
+
if (result.agentId) run.agentIds[nodeId] = result.agentId;
|
|
1022
|
+
run.compactions[nodeId] = Math.max(run.compactions[nodeId] ?? 0, result.compactionCount);
|
|
1023
|
+
run.updatedAt = Date.now();
|
|
1024
|
+
// A recoverable child failure is a value for runtime retry/handling. The
|
|
1025
|
+
// script, not this lifecycle callback, decides whether the whole run fails.
|
|
1026
|
+
}
|
|
1027
|
+
|
|
1028
|
+
private reconcilePendingSpawns(state: ScriptRunState): void {
|
|
1029
|
+
for (const pending of state.pendingSpawns.values()) {
|
|
1030
|
+
const reconciliation = this.client.reconcileManaged?.(pending.spawnKey, pending.owner);
|
|
1031
|
+
void reconciliation?.catch(() => {});
|
|
1032
|
+
}
|
|
1033
|
+
state.pendingSpawns.clear();
|
|
1034
|
+
}
|
|
1035
|
+
|
|
1036
|
+
private async quiescePendingSpawns(state: ScriptRunState): Promise<WorkflowQuiesceResult> {
|
|
1037
|
+
const pending = [...state.pendingSpawns.values()];
|
|
1038
|
+
if (pending.length === 0) return { settled: true, pending: [] };
|
|
1039
|
+
if (!this.client.reconcileManaged) {
|
|
1040
|
+
return { settled: false, pending: pending.map(({ spawnKey }) => spawnKey).slice(0, 256) };
|
|
1041
|
+
}
|
|
1042
|
+
const failed: string[] = [];
|
|
1043
|
+
await Promise.all(
|
|
1044
|
+
pending.map(async ({ spawnKey, owner }) => {
|
|
1045
|
+
try {
|
|
1046
|
+
const result = await this.client.reconcileManaged?.(spawnKey, owner);
|
|
1047
|
+
if (result === undefined) failed.push(spawnKey);
|
|
1048
|
+
} catch {
|
|
1049
|
+
failed.push(spawnKey);
|
|
1050
|
+
}
|
|
1051
|
+
}),
|
|
1052
|
+
);
|
|
1053
|
+
return { settled: failed.length === 0, pending: failed.slice(0, 256) };
|
|
1054
|
+
}
|
|
1055
|
+
|
|
1056
|
+
private cleanupAgents(runId: string): void {
|
|
1057
|
+
const state = this.runs.get(runId);
|
|
1058
|
+
if (!state) return;
|
|
1059
|
+
this.reconcilePendingSpawns(state);
|
|
1060
|
+
for (const [agentId, waiter] of [...state.agentWaiters]) {
|
|
1061
|
+
if (waiter.settled) continue;
|
|
1062
|
+
waiter.settled = true;
|
|
1063
|
+
state.agentWaiters.delete(agentId);
|
|
1064
|
+
waiter.reject(
|
|
1065
|
+
new WorkflowError("workflow failed while agent in flight", WorkflowErrorCode.WORKFLOW_ABORTED, {
|
|
1066
|
+
recoverable: true,
|
|
1067
|
+
}),
|
|
1068
|
+
);
|
|
1069
|
+
void this.client
|
|
1070
|
+
.stopOwned?.(agentId, {
|
|
1071
|
+
extension: "pi-workflows",
|
|
1072
|
+
runId,
|
|
1073
|
+
nodeId: `call-${waiter.callIndex}`,
|
|
1074
|
+
attemptId: `attempt-${waiter.generation}`,
|
|
1075
|
+
})
|
|
1076
|
+
.catch(() => {});
|
|
1137
1077
|
}
|
|
1138
|
-
this.journalRetries.clear();
|
|
1139
|
-
this.journalBatchProgress.clear();
|
|
1140
|
-
this.journalBlockedRuns.clear();
|
|
1141
1078
|
}
|
|
1142
1079
|
|
|
1143
|
-
private
|
|
1144
|
-
|
|
1145
|
-
|
|
1146
|
-
|
|
1147
|
-
|
|
1148
|
-
|
|
1080
|
+
private async stop(state: ScriptRunState): Promise<void> {
|
|
1081
|
+
const run = state.run;
|
|
1082
|
+
if (isTerminalWorkflow(run.status)) return;
|
|
1083
|
+
this.setWorkflowStatus(run, "stopping", "workflow stopped", "stop");
|
|
1084
|
+
run.nonResumable = true;
|
|
1085
|
+
state.controller.abort();
|
|
1086
|
+
this.cleanupAgents(run.runId);
|
|
1087
|
+
const terminalResults: RecoveryTerminalResult[] = [];
|
|
1088
|
+
for (const [nodeId, status] of Object.entries(run.callStatus)) {
|
|
1089
|
+
if (status !== "completed") {
|
|
1090
|
+
const result = resultFromLifecycle(
|
|
1091
|
+
"stopped",
|
|
1092
|
+
run.agentIds[nodeId],
|
|
1093
|
+
undefined,
|
|
1094
|
+
"workflow stopped",
|
|
1095
|
+
run.compactions[nodeId] ?? 0,
|
|
1096
|
+
);
|
|
1097
|
+
result.attemptId = run.attemptIds[nodeId];
|
|
1098
|
+
terminalResults.push({
|
|
1099
|
+
nodeId,
|
|
1100
|
+
attemptId: run.attemptIds[nodeId] ?? "",
|
|
1101
|
+
result,
|
|
1102
|
+
owner: { extension: "pi-workflows", runId: run.runId, nodeId, attemptId: run.attemptIds[nodeId] ?? "" },
|
|
1103
|
+
});
|
|
1104
|
+
run.callResults[nodeId] = result;
|
|
1105
|
+
run.callStatus[nodeId] = "stopped";
|
|
1106
|
+
}
|
|
1107
|
+
}
|
|
1108
|
+
const event: TerminalRecoveryEvent = {
|
|
1109
|
+
kind: "terminal_recovery",
|
|
1110
|
+
schemaVersion: JOURNAL_SCHEMA_VERSION,
|
|
1111
|
+
runId: run.runId,
|
|
1112
|
+
status: "stopped",
|
|
1113
|
+
terminalIntent: "stop",
|
|
1114
|
+
branchGeneration: this.recoveryBranchGeneration,
|
|
1115
|
+
terminalResults,
|
|
1116
|
+
blockedNodeIds: [],
|
|
1117
|
+
error: "workflow stopped",
|
|
1118
|
+
recoveryId: deriveRecoveryId({ runId: run.runId, rotations: [], timestamp: Date.now() }),
|
|
1119
|
+
timestamp: Date.now(),
|
|
1120
|
+
};
|
|
1121
|
+
this.persist(run.runId, "stop-terminal", () => this.write(event));
|
|
1122
|
+
run.status = "stopped";
|
|
1123
|
+
run.updatedAt = Date.now();
|
|
1124
|
+
this.settleWaiters(run.runId, run);
|
|
1149
1125
|
}
|
|
1150
1126
|
|
|
1127
|
+
// ── journal writes ─────────────────────────────────────────────────────────
|
|
1128
|
+
|
|
1151
1129
|
/**
|
|
1152
1130
|
* A Pi lifecycle event is delivered once. If its durable append is transiently
|
|
1153
|
-
* unavailable, retain the
|
|
1154
|
-
* the
|
|
1131
|
+
* unavailable, retain the mutation in memory and retry the append with
|
|
1132
|
+
* backoff without requiring the peer to re-emit. The in-memory run state is
|
|
1133
|
+
* authoritative; the journal is best-effort with bounded retries, and a run
|
|
1134
|
+
* whose appends keep failing is marked blocked (reported once, not spammed).
|
|
1155
1135
|
*/
|
|
1156
|
-
private scheduleJournalRetry(
|
|
1157
|
-
const
|
|
1158
|
-
const retryKey = `${epoch}\\u0000${run.runId}\\u0000${key}`;
|
|
1136
|
+
private scheduleJournalRetry(runId: string, key: string, mutation: () => void): void {
|
|
1137
|
+
const retryKey = `${runId}\u0000${key}`;
|
|
1159
1138
|
if (this.journalRetries.has(retryKey)) return;
|
|
1160
1139
|
const retry = { attempt: 0, timer: undefined as ReturnType<typeof setTimeout> | undefined };
|
|
1161
1140
|
const attempt = (): void => {
|
|
1162
1141
|
const current = this.journalRetries.get(retryKey);
|
|
1163
1142
|
if (!current) return;
|
|
1164
1143
|
current.timer = undefined;
|
|
1165
|
-
if (
|
|
1166
|
-
this.branchEpoch !== epoch ||
|
|
1167
|
-
this.branchChanging ||
|
|
1168
|
-
this.lifecycleSuspended ||
|
|
1169
|
-
this.runs.get(run.runId) !== run
|
|
1170
|
-
) {
|
|
1144
|
+
if (this.lifecyclePaused || !this.runs.has(runId)) {
|
|
1171
1145
|
this.journalRetries.delete(retryKey);
|
|
1172
1146
|
return;
|
|
1173
1147
|
}
|
|
1174
1148
|
const previousKey = this.journalMutationKey;
|
|
1175
|
-
this.journalMutationKey = `${
|
|
1149
|
+
this.journalMutationKey = `${runId}:${key}`;
|
|
1176
1150
|
try {
|
|
1177
1151
|
mutation();
|
|
1178
1152
|
this.journalRetries.delete(retryKey);
|
|
1179
1153
|
} catch (error: unknown) {
|
|
1180
1154
|
if (!(error instanceof JournalAppendError)) {
|
|
1181
1155
|
this.journalRetries.delete(retryKey);
|
|
1182
|
-
console.warn(
|
|
1156
|
+
console.warn(
|
|
1157
|
+
`[pi-workflows] journal retry abandoned: ${error instanceof Error ? error.message : String(error)}`,
|
|
1158
|
+
);
|
|
1183
1159
|
return;
|
|
1184
1160
|
}
|
|
1185
1161
|
current.attempt += 1;
|
|
1186
|
-
if (current.attempt >=
|
|
1162
|
+
if (current.attempt >= JOURNAL_RETRY_MAX_ATTEMPTS) {
|
|
1187
1163
|
this.journalRetries.delete(retryKey);
|
|
1188
|
-
this.journalBlockedRuns.add(
|
|
1189
|
-
console.warn(`[pi-workflows]
|
|
1164
|
+
this.journalBlockedRuns.add(runId);
|
|
1165
|
+
console.warn(`[pi-workflows] journal retry exhausted for ${runId}: ${error.message}`);
|
|
1190
1166
|
return;
|
|
1191
1167
|
}
|
|
1192
1168
|
const delay = Math.min(
|
|
@@ -1202,1611 +1178,232 @@ export class WorkflowEngine {
|
|
|
1202
1178
|
queueMicrotask(attempt);
|
|
1203
1179
|
}
|
|
1204
1180
|
|
|
1205
|
-
private
|
|
1206
|
-
const previousKey = this.journalMutationKey;
|
|
1207
|
-
this.journalMutationKey = `${run.runId}:${key}`;
|
|
1181
|
+
private appendRequired(event: JournalEvent): void {
|
|
1208
1182
|
try {
|
|
1209
|
-
|
|
1210
|
-
return true;
|
|
1183
|
+
this.journal.append(event);
|
|
1211
1184
|
} catch (error: unknown) {
|
|
1212
|
-
|
|
1213
|
-
|
|
1214
|
-
|
|
1215
|
-
|
|
1216
|
-
|
|
1185
|
+
throw new WorkflowError(
|
|
1186
|
+
`Could not persist workflow start: ${error instanceof Error ? error.message : String(error)}`,
|
|
1187
|
+
WorkflowErrorCode.PERSISTENCE_ERROR,
|
|
1188
|
+
{ recoverable: true, details: error },
|
|
1189
|
+
);
|
|
1217
1190
|
}
|
|
1218
1191
|
}
|
|
1219
1192
|
|
|
1220
1193
|
private write(event: JournalEvent): void {
|
|
1194
|
+
// The in-memory mutation is applied by the caller BEFORE calling write;
|
|
1195
|
+
// here we only persist. A transient failure schedules a retry of this exact
|
|
1196
|
+
// append. Terminal bookkeeping already happened, so a retry that never
|
|
1197
|
+
// succeeds degrades to a warning rather than corrupting run state.
|
|
1221
1198
|
try {
|
|
1222
1199
|
this.journal.append(event);
|
|
1223
1200
|
} catch (error: unknown) {
|
|
1224
|
-
|
|
1201
|
+
if (this.journalMutationKey) {
|
|
1202
|
+
throw new JournalAppendError(event.kind, error);
|
|
1203
|
+
}
|
|
1204
|
+
console.warn(`[pi-workflows] journal append failed: ${error instanceof Error ? error.message : String(error)}`);
|
|
1225
1205
|
}
|
|
1226
1206
|
}
|
|
1227
1207
|
|
|
1228
|
-
|
|
1229
|
-
|
|
1230
|
-
|
|
1231
|
-
|
|
1232
|
-
|
|
1233
|
-
|
|
1234
|
-
|
|
1235
|
-
|
|
1236
|
-
|
|
1237
|
-
|
|
1238
|
-
|
|
1239
|
-
|
|
1240
|
-
|
|
1241
|
-
|
|
1242
|
-
if (!event) continue;
|
|
1243
|
-
this.write(event);
|
|
1244
|
-
if (key) {
|
|
1245
|
-
const progress = this.journalBatchProgress.get(key);
|
|
1246
|
-
if (progress) progress.nextIndex = index + 1;
|
|
1208
|
+
/**
|
|
1209
|
+
* Persist one journal event with bounded retry. The caller applies the
|
|
1210
|
+
* in-memory mutation itself; `persist` only makes the append durable and, on
|
|
1211
|
+
* a transient failure, schedules a retry of the exact same append.
|
|
1212
|
+
*/
|
|
1213
|
+
private persist(runId: string, key: string, mutation: () => void): void {
|
|
1214
|
+
const previousKey = this.journalMutationKey;
|
|
1215
|
+
this.journalMutationKey = `${runId}:${key}`;
|
|
1216
|
+
try {
|
|
1217
|
+
mutation();
|
|
1218
|
+
} catch (error: unknown) {
|
|
1219
|
+
if (error instanceof JournalAppendError) {
|
|
1220
|
+
this.scheduleJournalRetry(runId, key, mutation);
|
|
1221
|
+
return;
|
|
1247
1222
|
}
|
|
1223
|
+
console.warn(`[pi-workflows] journal append failed: ${error instanceof Error ? error.message : String(error)}`);
|
|
1224
|
+
} finally {
|
|
1225
|
+
this.journalMutationKey = previousKey;
|
|
1248
1226
|
}
|
|
1249
|
-
if (key) this.journalBatchProgress.delete(key);
|
|
1250
1227
|
}
|
|
1251
1228
|
|
|
1252
|
-
private
|
|
1253
|
-
return
|
|
1254
|
-
|
|
1255
|
-
|
|
1256
|
-
|
|
1257
|
-
|
|
1258
|
-
|
|
1229
|
+
private scheduleProviderResume(runId: string, hint?: string): void {
|
|
1230
|
+
if (this.providerResumeTimers.has(runId)) return;
|
|
1231
|
+
const match = hint ? /(\d+(?:\.\d+)?)\s*(seconds?|minutes?|hours?|days?)/i.exec(hint) : undefined;
|
|
1232
|
+
const unit = match?.[2]?.toLowerCase();
|
|
1233
|
+
const multiplier = unit?.startsWith("second")
|
|
1234
|
+
? 1_000
|
|
1235
|
+
: unit?.startsWith("minute")
|
|
1236
|
+
? 60_000
|
|
1237
|
+
: unit?.startsWith("hour")
|
|
1238
|
+
? 3_600_000
|
|
1239
|
+
: unit?.startsWith("day")
|
|
1240
|
+
? 86_400_000
|
|
1241
|
+
: undefined;
|
|
1242
|
+
const delay = Math.min(
|
|
1243
|
+
PROVIDER_RETRY_MAX_MS,
|
|
1244
|
+
match && multiplier ? Math.max(1_000, Number(match[1]) * multiplier) : PROVIDER_RETRY_DEFAULT_MS,
|
|
1245
|
+
);
|
|
1246
|
+
const timer = setTimeout(() => {
|
|
1247
|
+
this.providerResumeTimers.delete(runId);
|
|
1248
|
+
const state = this.runs.get(runId);
|
|
1249
|
+
if (state?.run.status !== "paused") return;
|
|
1250
|
+
void this.resume(runId, this.readEntries(), { background: true }).catch((error: unknown) => {
|
|
1251
|
+
console.warn(
|
|
1252
|
+
`[pi-workflows] automatic provider-limit resume failed: ${error instanceof Error ? error.message : String(error)}`,
|
|
1253
|
+
);
|
|
1254
|
+
});
|
|
1255
|
+
}, delay);
|
|
1256
|
+
timer.unref?.();
|
|
1257
|
+
this.providerResumeTimers.set(runId, timer);
|
|
1258
|
+
}
|
|
1259
|
+
private clearJournalRetries(): void {
|
|
1260
|
+
for (const { timer } of this.journalRetries.values()) {
|
|
1261
|
+
if (timer) clearTimeout(timer);
|
|
1262
|
+
}
|
|
1263
|
+
this.journalRetries.clear();
|
|
1264
|
+
this.journalBlockedRuns.clear();
|
|
1265
|
+
for (const timer of this.providerResumeTimers.values()) clearTimeout(timer);
|
|
1266
|
+
this.providerResumeTimers.clear();
|
|
1259
1267
|
}
|
|
1260
1268
|
|
|
1261
1269
|
private setWorkflowStatus(
|
|
1262
|
-
run:
|
|
1263
|
-
status:
|
|
1270
|
+
run: ScriptRun,
|
|
1271
|
+
status: ScriptRun["status"],
|
|
1264
1272
|
error?: string,
|
|
1265
|
-
terminalIntent
|
|
1273
|
+
terminalIntent?: "stop" | "failure",
|
|
1266
1274
|
): void {
|
|
1267
|
-
|
|
1275
|
+
if (isTerminalWorkflow(run.status) && !isTerminalWorkflow(status)) return;
|
|
1268
1276
|
const timestamp = Date.now();
|
|
1269
|
-
|
|
1270
|
-
|
|
1271
|
-
|
|
1272
|
-
|
|
1273
|
-
|
|
1274
|
-
|
|
1275
|
-
|
|
1276
|
-
|
|
1277
|
-
|
|
1278
|
-
|
|
1279
|
-
|
|
1280
|
-
|
|
1281
|
-
run.status = nextStatus;
|
|
1282
|
-
run.updatedAt = timestamp;
|
|
1277
|
+
this.persist(run.runId, `transition:${status}`, () =>
|
|
1278
|
+
this.write({
|
|
1279
|
+
kind: "workflow_transition",
|
|
1280
|
+
schemaVersion: JOURNAL_SCHEMA_VERSION,
|
|
1281
|
+
runId: run.runId,
|
|
1282
|
+
status,
|
|
1283
|
+
...(error !== undefined ? { error } : {}),
|
|
1284
|
+
...(terminalIntent !== undefined ? { terminalIntent } : {}),
|
|
1285
|
+
timestamp,
|
|
1286
|
+
}),
|
|
1287
|
+
);
|
|
1288
|
+
run.status = status;
|
|
1283
1289
|
if (error !== undefined) run.error = error;
|
|
1284
1290
|
if (terminalIntent !== undefined) run.terminalIntent = terminalIntent;
|
|
1285
|
-
if (isTerminalWorkflow(nextStatus)) this.resolveWaiters(run);
|
|
1286
|
-
}
|
|
1287
|
-
|
|
1288
|
-
private setTaskStatus(run: WorkflowRun, nodeId: string, status: TaskStatus, agentId?: string): void {
|
|
1289
|
-
const previous = run.taskStatus[nodeId];
|
|
1290
|
-
if (!previous) return;
|
|
1291
|
-
if (!canTransitionTask(previous, status)) {
|
|
1292
|
-
if (isTerminalTask(previous)) return;
|
|
1293
|
-
throw new Error(`invalid task transition: ${previous} -> ${status}`);
|
|
1294
|
-
}
|
|
1295
|
-
const nextStatus = transitionTask(previous, status);
|
|
1296
|
-
const timestamp = Date.now();
|
|
1297
|
-
const event: JournalEvent = {
|
|
1298
|
-
kind: "task_transition",
|
|
1299
|
-
schemaVersion: JOURNAL_SCHEMA_VERSION,
|
|
1300
|
-
runId: run.runId,
|
|
1301
|
-
nodeId,
|
|
1302
|
-
status: nextStatus,
|
|
1303
|
-
...(agentId ? { agentId } : {}),
|
|
1304
|
-
...(run.attemptIds[nodeId] ? { attemptId: run.attemptIds[nodeId] } : {}),
|
|
1305
|
-
owner: this.ownerForNode(run, nodeId),
|
|
1306
|
-
timestamp,
|
|
1307
|
-
};
|
|
1308
|
-
this.write(event);
|
|
1309
|
-
run.taskStatus[nodeId] = nextStatus;
|
|
1310
1291
|
run.updatedAt = timestamp;
|
|
1311
|
-
if (
|
|
1312
|
-
|
|
1313
|
-
|
|
1314
|
-
private blockedDependentIds(run: WorkflowRun, failedNodeId: string): string[] {
|
|
1315
|
-
const queue = [failedNodeId];
|
|
1316
|
-
const blocked = new Set<string>();
|
|
1317
|
-
while (queue.length > 0) {
|
|
1318
|
-
const dependency = queue.shift();
|
|
1319
|
-
if (!dependency) continue;
|
|
1320
|
-
for (const task of run.definition.tasks) {
|
|
1321
|
-
if (!task.depends_on.includes(dependency) || blocked.has(task.id)) continue;
|
|
1322
|
-
const status = run.taskStatus[task.id];
|
|
1323
|
-
if (status !== "pending" && status !== "ready") continue;
|
|
1324
|
-
blocked.add(task.id);
|
|
1325
|
-
queue.push(task.id);
|
|
1326
|
-
}
|
|
1292
|
+
if (isTerminalWorkflow(status)) {
|
|
1293
|
+
this.settleWaiters(run.runId, run);
|
|
1294
|
+
this.onRunSettled?.();
|
|
1327
1295
|
}
|
|
1328
|
-
return [...blocked];
|
|
1329
1296
|
}
|
|
1330
1297
|
|
|
1331
|
-
|
|
1332
|
-
run: WorkflowRun,
|
|
1333
|
-
nodeId: string,
|
|
1334
|
-
result: WorkflowRun["taskResults"][string],
|
|
1335
|
-
status: "completed" | "failed" | "stopped",
|
|
1336
|
-
failureMessage?: string,
|
|
1337
|
-
blockDependents = true,
|
|
1338
|
-
): void {
|
|
1339
|
-
const previous = run.taskStatus[nodeId];
|
|
1340
|
-
if (!previous || run.taskResults[nodeId] || isTerminalTask(previous)) return;
|
|
1341
|
-
if (!canTransitionTask(previous, status)) {
|
|
1342
|
-
throw new Error(`invalid task transition: ${previous} -> ${status}`);
|
|
1343
|
-
}
|
|
1344
|
-
const timestamp = result.updatedAt;
|
|
1345
|
-
const resultEvent: JournalEvent = {
|
|
1346
|
-
kind: "task_result",
|
|
1347
|
-
schemaVersion: JOURNAL_SCHEMA_VERSION,
|
|
1348
|
-
runId: run.runId,
|
|
1349
|
-
nodeId,
|
|
1350
|
-
result,
|
|
1351
|
-
attemptId: run.attemptIds[nodeId],
|
|
1352
|
-
owner: this.ownerForNode(run, nodeId),
|
|
1353
|
-
timestamp,
|
|
1354
|
-
};
|
|
1355
|
-
const transitionEvent: JournalEvent = {
|
|
1356
|
-
kind: "task_transition",
|
|
1357
|
-
schemaVersion: JOURNAL_SCHEMA_VERSION,
|
|
1358
|
-
runId: run.runId,
|
|
1359
|
-
nodeId,
|
|
1360
|
-
status,
|
|
1361
|
-
...(result.agentId ? { agentId: result.agentId } : {}),
|
|
1362
|
-
...(run.attemptIds[nodeId] ? { attemptId: run.attemptIds[nodeId] } : {}),
|
|
1363
|
-
owner: this.ownerForNode(run, nodeId),
|
|
1364
|
-
timestamp,
|
|
1365
|
-
};
|
|
1366
|
-
const blockedIds = blockDependents && status !== "completed" ? this.blockedDependentIds(run, nodeId) : [];
|
|
1367
|
-
const blockedEvents: JournalEvent[] = blockedIds.map((blockedNodeId) => ({
|
|
1368
|
-
kind: "task_transition",
|
|
1369
|
-
schemaVersion: JOURNAL_SCHEMA_VERSION,
|
|
1370
|
-
runId: run.runId,
|
|
1371
|
-
nodeId: blockedNodeId,
|
|
1372
|
-
status: "blocked",
|
|
1373
|
-
...(run.attemptIds[blockedNodeId] ? { attemptId: run.attemptIds[blockedNodeId] } : {}),
|
|
1374
|
-
owner: this.ownerForNode(run, blockedNodeId),
|
|
1375
|
-
timestamp,
|
|
1376
|
-
}));
|
|
1377
|
-
const message = failureMessage ?? result.error ?? `workflow task ${nodeId} ${status}`;
|
|
1378
|
-
const needsFailureCleanup =
|
|
1379
|
-
status !== "completed" &&
|
|
1380
|
-
run.terminalIntent !== "stop" &&
|
|
1381
|
-
(run.status !== "stopping" || run.terminalIntent !== "failure" || run.error !== message);
|
|
1382
|
-
const workflowEvent: JournalEvent | undefined = needsFailureCleanup
|
|
1383
|
-
? {
|
|
1384
|
-
kind: "workflow_transition",
|
|
1385
|
-
schemaVersion: JOURNAL_SCHEMA_VERSION,
|
|
1386
|
-
runId: run.runId,
|
|
1387
|
-
status: "stopping",
|
|
1388
|
-
terminalIntent: "failure",
|
|
1389
|
-
error: message,
|
|
1390
|
-
timestamp,
|
|
1391
|
-
}
|
|
1392
|
-
: undefined;
|
|
1393
|
-
this.appendEvents([resultEvent, transitionEvent, ...blockedEvents, ...(workflowEvent ? [workflowEvent] : [])]);
|
|
1298
|
+
// ── restore / summary ──────────────────────────────────────────────────────
|
|
1394
1299
|
|
|
1395
|
-
|
|
1396
|
-
|
|
1397
|
-
|
|
1398
|
-
|
|
1399
|
-
|
|
1400
|
-
|
|
1401
|
-
|
|
1402
|
-
run.updatedAt = timestamp;
|
|
1300
|
+
restore(entries: readonly SessionEntryLike[], branchGeneration = 0): void {
|
|
1301
|
+
// A branch replay is a replacement, not a merge. Runs absent from the new
|
|
1302
|
+
// branch must not remain controllable or append facts into it.
|
|
1303
|
+
this.clearJournalRetries();
|
|
1304
|
+
for (const state of this.runs.values()) {
|
|
1305
|
+
this.cleanupAgents(state.run.runId);
|
|
1306
|
+
state.controller.abort(new DOMException("Workflow branch replaced", "AbortError"));
|
|
1403
1307
|
}
|
|
1404
|
-
|
|
1405
|
-
|
|
1406
|
-
|
|
1407
|
-
|
|
1408
|
-
|
|
1409
|
-
|
|
1308
|
+
this.rejectWaiters(new WorkflowWaitAbortedError());
|
|
1309
|
+
this.runs.clear();
|
|
1310
|
+
this.recoveryBranchGeneration = branchGeneration;
|
|
1311
|
+
const runs = replayJournal(entries, {
|
|
1312
|
+
onInvalid: (diagnostic) => console.warn(`[pi-workflows] ${diagnostic}`),
|
|
1313
|
+
});
|
|
1314
|
+
for (const run of runs.values()) {
|
|
1315
|
+
const state: ScriptRunState = {
|
|
1316
|
+
run,
|
|
1317
|
+
frozenOptions: {
|
|
1318
|
+
...(run.frozenTokenBudget !== undefined ? { tokenBudget: run.frozenTokenBudget } : {}),
|
|
1319
|
+
...(run.frozenMaxAgents !== undefined ? { maxAgents: run.frozenMaxAgents } : {}),
|
|
1320
|
+
...(run.frozenConcurrency !== undefined ? { concurrency: run.frozenConcurrency } : {}),
|
|
1321
|
+
...(run.frozenAgentRetries !== undefined ? { agentRetries: run.frozenAgentRetries } : {}),
|
|
1322
|
+
...(run.frozenAgentTimeoutMs !== undefined ? { agentTimeoutMs: run.frozenAgentTimeoutMs } : {}),
|
|
1323
|
+
...(run.toolset ? { toolset: run.toolset } : {}),
|
|
1324
|
+
...(run.frozenExcludeTools ? { excludeTools: [...run.frozenExcludeTools] } : {}),
|
|
1325
|
+
},
|
|
1326
|
+
controller: new AbortController(),
|
|
1327
|
+
generations: new Map(),
|
|
1328
|
+
agentWaiters: new Map(),
|
|
1329
|
+
pendingSpawns: new Map(),
|
|
1330
|
+
pendingJournal: new Set(),
|
|
1331
|
+
lifecycleSuspended: false,
|
|
1332
|
+
executionGeneration: 0,
|
|
1333
|
+
};
|
|
1334
|
+
for (const [callIndex, attemptId] of Object.entries(run.attemptIds)) {
|
|
1335
|
+
state.generations.set(callIndex, generationFromAttemptId(attemptId));
|
|
1336
|
+
}
|
|
1337
|
+
// Only non-terminal runs can be resumed; terminal runs are inert history.
|
|
1338
|
+
if (isTerminalWorkflow(run.status)) {
|
|
1339
|
+
this.runs.set(run.runId, state);
|
|
1340
|
+
continue;
|
|
1341
|
+
}
|
|
1342
|
+
// A restored run that never settled was cut off mid-execution (branch
|
|
1343
|
+
// change, reload, or crash). Mark it interrupted so it is visibly
|
|
1344
|
+
// resumable rather than looking live-but-frozen.
|
|
1345
|
+
run.status = "interrupted";
|
|
1346
|
+
run.updatedAt = Date.now();
|
|
1347
|
+
this.runs.set(run.runId, state);
|
|
1410
1348
|
}
|
|
1411
1349
|
}
|
|
1412
1350
|
|
|
1413
|
-
|
|
1414
|
-
|
|
1415
|
-
|
|
1416
|
-
|
|
1417
|
-
): void {
|
|
1418
|
-
if (run.synthesisResult || !result) return;
|
|
1419
|
-
const timestamp = result.updatedAt;
|
|
1420
|
-
const resultEvent: JournalEvent = {
|
|
1421
|
-
kind: "synthesis_result",
|
|
1422
|
-
schemaVersion: JOURNAL_SCHEMA_VERSION,
|
|
1423
|
-
runId: run.runId,
|
|
1424
|
-
result,
|
|
1425
|
-
attemptId: run.attemptIds[SYNTHESIS_NODE_ID],
|
|
1426
|
-
owner: this.ownerForNode(run, SYNTHESIS_NODE_ID),
|
|
1427
|
-
timestamp,
|
|
1428
|
-
};
|
|
1429
|
-
const settling = run.status === "stopping" || this.settling.has(run.runId);
|
|
1430
|
-
const workflowEvent: JournalEvent | undefined = settling
|
|
1431
|
-
? undefined
|
|
1432
|
-
: {
|
|
1433
|
-
kind: "workflow_transition",
|
|
1434
|
-
schemaVersion: JOURNAL_SCHEMA_VERSION,
|
|
1435
|
-
runId: run.runId,
|
|
1436
|
-
status: finalStatus,
|
|
1437
|
-
...(result.error ? { error: result.error } : {}),
|
|
1438
|
-
...(run.terminalIntent ? { terminalIntent: run.terminalIntent } : {}),
|
|
1439
|
-
timestamp,
|
|
1440
|
-
};
|
|
1441
|
-
this.appendEvents([resultEvent, ...(workflowEvent ? [workflowEvent] : [])]);
|
|
1442
|
-
|
|
1443
|
-
run.synthesisResult = result;
|
|
1444
|
-
if (result.agentId) run.synthesisAgentId = result.agentId;
|
|
1445
|
-
run.compactions[SYNTHESIS_NODE_ID] = Math.max(run.compactions[SYNTHESIS_NODE_ID] ?? 0, result.compactionCount);
|
|
1446
|
-
run.updatedAt = timestamp;
|
|
1447
|
-
if (workflowEvent) {
|
|
1448
|
-
run.status = finalStatus;
|
|
1449
|
-
run.updatedAt = timestamp;
|
|
1450
|
-
if (result.error) run.error = result.error;
|
|
1451
|
-
if (isTerminalWorkflow(finalStatus)) this.resolveWaiters(run);
|
|
1452
|
-
}
|
|
1351
|
+
list(): Array<Record<string, unknown>> {
|
|
1352
|
+
return [...this.runs.values()]
|
|
1353
|
+
.sort((a, b) => b.run.startedAt - a.run.startedAt)
|
|
1354
|
+
.map((state) => this.summary(state.run));
|
|
1453
1355
|
}
|
|
1454
1356
|
|
|
1455
|
-
|
|
1456
|
-
|
|
1457
|
-
|
|
1458
|
-
? run.synthesisResult === undefined
|
|
1459
|
-
: run.taskResults[record.nodeId] === undefined && !isTerminalTask(run.taskStatus[record.nodeId]),
|
|
1460
|
-
);
|
|
1461
|
-
if (pending.length === 0) return;
|
|
1462
|
-
const events: JournalEvent[] = pending.map((record) =>
|
|
1463
|
-
record.nodeId === SYNTHESIS_NODE_ID
|
|
1464
|
-
? {
|
|
1465
|
-
kind: "synthesis_result",
|
|
1466
|
-
schemaVersion: JOURNAL_SCHEMA_VERSION,
|
|
1467
|
-
runId: run.runId,
|
|
1468
|
-
result: record.result,
|
|
1469
|
-
attemptId: run.attemptIds[SYNTHESIS_NODE_ID],
|
|
1470
|
-
owner: this.ownerForNode(run, SYNTHESIS_NODE_ID),
|
|
1471
|
-
timestamp: Math.max(run.updatedAt, record.result.updatedAt),
|
|
1472
|
-
}
|
|
1473
|
-
: {
|
|
1474
|
-
kind: "task_result",
|
|
1475
|
-
schemaVersion: JOURNAL_SCHEMA_VERSION,
|
|
1476
|
-
runId: run.runId,
|
|
1477
|
-
nodeId: record.nodeId,
|
|
1478
|
-
result: record.result,
|
|
1479
|
-
attemptId: run.attemptIds[record.nodeId],
|
|
1480
|
-
owner: this.ownerForNode(run, record.nodeId),
|
|
1481
|
-
timestamp: Math.max(run.updatedAt, record.result.updatedAt),
|
|
1482
|
-
},
|
|
1483
|
-
);
|
|
1484
|
-
const transitions: JournalEvent[] = pending
|
|
1485
|
-
.filter((record) => record.nodeId !== SYNTHESIS_NODE_ID)
|
|
1486
|
-
.map((record) => ({
|
|
1487
|
-
kind: "task_transition",
|
|
1488
|
-
schemaVersion: JOURNAL_SCHEMA_VERSION,
|
|
1489
|
-
runId: run.runId,
|
|
1490
|
-
nodeId: record.nodeId,
|
|
1491
|
-
status: record.result.status,
|
|
1492
|
-
...(record.result.agentId ? { agentId: record.result.agentId } : {}),
|
|
1493
|
-
attemptId: run.attemptIds[record.nodeId],
|
|
1494
|
-
owner: this.ownerForNode(run, record.nodeId),
|
|
1495
|
-
timestamp: Math.max(run.updatedAt, record.result.updatedAt),
|
|
1496
|
-
}));
|
|
1497
|
-
this.appendEvents([...events, ...transitions]);
|
|
1498
|
-
for (const record of pending) {
|
|
1499
|
-
if (record.nodeId === SYNTHESIS_NODE_ID) {
|
|
1500
|
-
run.synthesisResult = record.result;
|
|
1501
|
-
if (record.result.agentId) run.synthesisAgentId = record.result.agentId;
|
|
1502
|
-
} else {
|
|
1503
|
-
run.taskResults[record.nodeId] = record.result;
|
|
1504
|
-
run.taskStatus[record.nodeId] = record.result.status;
|
|
1505
|
-
if (record.result.agentId) run.agentIds[record.nodeId] = record.result.agentId;
|
|
1506
|
-
}
|
|
1507
|
-
run.compactions[record.nodeId] = Math.max(run.compactions[record.nodeId] ?? 0, record.result.compactionCount);
|
|
1508
|
-
run.updatedAt = Math.max(run.updatedAt, record.result.updatedAt);
|
|
1509
|
-
}
|
|
1510
|
-
}
|
|
1511
|
-
private async pump(run: WorkflowRun): Promise<void> {
|
|
1512
|
-
if (this.branchChanging || this.pumping.has(run.runId) || run.status !== "running") return;
|
|
1513
|
-
this.pumping.add(run.runId);
|
|
1514
|
-
const branchEpoch = this.branchEpoch;
|
|
1515
|
-
const isCurrent = (): boolean =>
|
|
1516
|
-
!this.branchChanging && branchEpoch === this.branchEpoch && this.runs.get(run.runId) === run;
|
|
1517
|
-
try {
|
|
1518
|
-
if (!isCurrent()) return;
|
|
1519
|
-
const completed = new Set(
|
|
1520
|
-
Object.entries(run.taskStatus)
|
|
1521
|
-
.filter(([, status]) => status === "completed")
|
|
1522
|
-
.map(([nodeId]) => nodeId),
|
|
1523
|
-
);
|
|
1524
|
-
const ready = readyTaskIds(run.definition, completed);
|
|
1525
|
-
for (const nodeId of ready) {
|
|
1526
|
-
if (!isCurrent() || run.status !== "running") break;
|
|
1527
|
-
if (run.taskStatus[nodeId] !== "pending" && run.taskStatus[nodeId] !== "ready") continue;
|
|
1528
|
-
if (run.taskStatus[nodeId] === "pending") this.setTaskStatus(run, nodeId, "ready");
|
|
1529
|
-
this.setTaskStatus(run, nodeId, "dispatching");
|
|
1530
|
-
const dispatch = this.dispatchTask(run, nodeId, branchEpoch);
|
|
1531
|
-
this.trackDispatch(run.runId, dispatch);
|
|
1532
|
-
}
|
|
1533
|
-
if (!isCurrent() || run.status !== "running") return;
|
|
1534
|
-
const statuses = Object.values(run.taskStatus);
|
|
1535
|
-
if (statuses.some((status) => status === "failed" || status === "blocked" || status === "stopped")) {
|
|
1536
|
-
if (statuses.some((status) => status === "stopped") && run.terminalIntent === "stop") {
|
|
1537
|
-
if (!isCurrent()) return;
|
|
1538
|
-
await this.stop(run);
|
|
1539
|
-
} else {
|
|
1540
|
-
if (!isCurrent()) return;
|
|
1541
|
-
await this.failRun(
|
|
1542
|
-
run,
|
|
1543
|
-
statuses.some((status) => status === "stopped")
|
|
1544
|
-
? "one or more workflow tasks stopped unexpectedly"
|
|
1545
|
-
: "one or more workflow tasks failed",
|
|
1546
|
-
);
|
|
1547
|
-
}
|
|
1548
|
-
return;
|
|
1549
|
-
}
|
|
1550
|
-
if (!isCurrent() || statuses.some((status) => !isTerminalTask(status))) return;
|
|
1551
|
-
if (run.definition.synthesis && !run.synthesisResult && (!run.synthesisAgentId || run.status === "running")) {
|
|
1552
|
-
if (!isCurrent()) return;
|
|
1553
|
-
this.setWorkflowStatus(run, "synthesizing");
|
|
1554
|
-
const dispatch = this.dispatchSynthesis(run);
|
|
1555
|
-
this.trackDispatch(run.runId, dispatch);
|
|
1556
|
-
return;
|
|
1557
|
-
}
|
|
1558
|
-
if (!isCurrent()) return;
|
|
1559
|
-
if (!run.definition.synthesis) {
|
|
1560
|
-
this.setWorkflowStatus(run, "completed");
|
|
1561
|
-
} else if (run.synthesisResult) {
|
|
1562
|
-
const synthesisStatus =
|
|
1563
|
-
run.synthesisResult.status === "completed"
|
|
1564
|
-
? "completed"
|
|
1565
|
-
: run.synthesisResult.status === "stopped"
|
|
1566
|
-
? "stopped"
|
|
1567
|
-
: "failed";
|
|
1568
|
-
this.setWorkflowStatus(run, synthesisStatus, run.synthesisResult.error);
|
|
1569
|
-
}
|
|
1570
|
-
} finally {
|
|
1571
|
-
if (this.runs.get(run.runId) === run && this.branchEpoch === branchEpoch) {
|
|
1572
|
-
this.pumping.delete(run.runId);
|
|
1573
|
-
}
|
|
1574
|
-
}
|
|
1357
|
+
/** Raw run list for the navigator (keeps live ScriptRun state). */
|
|
1358
|
+
getState(runId: string): ScriptRunState | undefined {
|
|
1359
|
+
return this.runs.get(runId);
|
|
1575
1360
|
}
|
|
1576
1361
|
|
|
1577
|
-
|
|
1578
|
-
run
|
|
1579
|
-
|
|
1580
|
-
terminalResults: readonly DeferredTerminalRecord[],
|
|
1581
|
-
): void {
|
|
1582
|
-
const sourceKey = this.recoverySourceKey(run, plan, terminalResults);
|
|
1583
|
-
const memo = this.memoizedRecovery(run.runId, "run_recovery", sourceKey);
|
|
1584
|
-
if (memo?.kind === "run_recovery") {
|
|
1585
|
-
// The durable event belongs to this branch and source snapshot. Reapply
|
|
1586
|
-
// it to the freshly replayed run without another append.
|
|
1587
|
-
applyRecoveryEvent(run, memo);
|
|
1588
|
-
return;
|
|
1589
|
-
}
|
|
1590
|
-
|
|
1591
|
-
const exhausted = plan.find((item) => item.sourceGeneration + 1 > MAX_ATTEMPTS_PER_NODE);
|
|
1592
|
-
if (exhausted) {
|
|
1593
|
-
const message = `workflow node ${exhausted.nodeId} exceeded interrupted-attempt limit`;
|
|
1594
|
-
if (exhausted.nodeId === SYNTHESIS_NODE_ID) {
|
|
1595
|
-
const result = resultFromLifecycle(
|
|
1596
|
-
"failed",
|
|
1597
|
-
run.synthesisAgentId,
|
|
1598
|
-
undefined,
|
|
1599
|
-
message,
|
|
1600
|
-
run.compactions[SYNTHESIS_NODE_ID] ?? 0,
|
|
1601
|
-
);
|
|
1602
|
-
result.attemptId = run.attemptIds[SYNTHESIS_NODE_ID];
|
|
1603
|
-
this.commitSynthesisTerminal(run, result, "failed");
|
|
1604
|
-
} else {
|
|
1605
|
-
this.failTask(run, exhausted.nodeId, message);
|
|
1606
|
-
}
|
|
1607
|
-
return;
|
|
1608
|
-
}
|
|
1362
|
+
getRun(runId: string): ScriptRun | undefined {
|
|
1363
|
+
return this.runs.get(runId)?.run;
|
|
1364
|
+
}
|
|
1609
1365
|
|
|
1610
|
-
|
|
1611
|
-
|
|
1612
|
-
const attemptId = `${run.runId}/${item.nodeId}/attempt-${generation}`;
|
|
1613
|
-
return {
|
|
1614
|
-
nodeId: item.nodeId,
|
|
1615
|
-
sourceAttemptId: item.sourceAttemptId,
|
|
1616
|
-
sourceGeneration: item.sourceGeneration,
|
|
1617
|
-
sourceStatus: item.sourceStatus,
|
|
1618
|
-
attemptId,
|
|
1619
|
-
generation,
|
|
1620
|
-
owner: { extension: "pi-workflows", runId: run.runId, nodeId: item.nodeId, attemptId },
|
|
1621
|
-
...(item.oldAgentId ? { supersededAgentId: item.oldAgentId } : {}),
|
|
1622
|
-
};
|
|
1623
|
-
});
|
|
1624
|
-
const recoveryBase = {
|
|
1625
|
-
kind: "run_recovery" as const,
|
|
1626
|
-
schemaVersion: JOURNAL_SCHEMA_VERSION as 2,
|
|
1366
|
+
summary(run: ScriptRun, detailed = false): Record<string, unknown> {
|
|
1367
|
+
const base: Record<string, unknown> = {
|
|
1627
1368
|
runId: run.runId,
|
|
1628
|
-
status:
|
|
1629
|
-
|
|
1630
|
-
|
|
1631
|
-
|
|
1632
|
-
|
|
1633
|
-
|
|
1634
|
-
|
|
1635
|
-
|
|
1636
|
-
attemptId,
|
|
1637
|
-
result: { ...result, updatedAt: run.updatedAt },
|
|
1638
|
-
owner,
|
|
1639
|
-
}),
|
|
1640
|
-
),
|
|
1641
|
-
}
|
|
1642
|
-
: {}),
|
|
1643
|
-
};
|
|
1644
|
-
const event: RunRecoveryEvent = {
|
|
1645
|
-
...recoveryBase,
|
|
1646
|
-
recoveryId: deriveRecoveryId(recoveryBase),
|
|
1647
|
-
timestamp: Date.now(),
|
|
1369
|
+
status: run.status,
|
|
1370
|
+
name: run.meta.name,
|
|
1371
|
+
description: run.meta.description.slice(0, 500),
|
|
1372
|
+
agentCount: Object.keys(run.callResults).length,
|
|
1373
|
+
phases: run.meta.phases?.map((p) => p.title) ?? [],
|
|
1374
|
+
startedAt: run.startedAt,
|
|
1375
|
+
updatedAt: run.updatedAt,
|
|
1376
|
+
...(run.error ? { error: run.error.slice(0, 2_000) } : {}),
|
|
1648
1377
|
};
|
|
1649
|
-
|
|
1650
|
-
|
|
1651
|
-
|
|
1652
|
-
|
|
1653
|
-
|
|
1654
|
-
|
|
1655
|
-
|
|
1656
|
-
|
|
1657
|
-
|
|
1658
|
-
|
|
1378
|
+
if (!detailed) return base;
|
|
1379
|
+
const calls = Object.entries(run.callStatus)
|
|
1380
|
+
.sort(([a], [b]) => Number(a) - Number(b))
|
|
1381
|
+
.map(([nodeId, status]) => {
|
|
1382
|
+
const result = run.callResults[nodeId];
|
|
1383
|
+
return {
|
|
1384
|
+
index: Number(nodeId),
|
|
1385
|
+
status,
|
|
1386
|
+
...(result?.result !== undefined ? { resultPreview: String(result.result).slice(0, 300) } : {}),
|
|
1387
|
+
...(result?.error ? { error: result.error.slice(0, 300) } : {}),
|
|
1388
|
+
...(result?.tokenCount !== undefined ? { tokenCount: result.tokenCount } : {}),
|
|
1389
|
+
...(run.agentIds[nodeId] ? { agentId: run.agentIds[nodeId] } : {}),
|
|
1390
|
+
attemptId: run.attemptIds[nodeId],
|
|
1391
|
+
};
|
|
1392
|
+
});
|
|
1393
|
+
return { ...base, calls };
|
|
1659
1394
|
}
|
|
1395
|
+
}
|
|
1660
1396
|
|
|
1661
|
-
|
|
1662
|
-
|
|
1663
|
-
|
|
1664
|
-
oldAgentId: string,
|
|
1665
|
-
resumeImmediately = true,
|
|
1666
|
-
legacyMigration = false,
|
|
1667
|
-
): void {
|
|
1668
|
-
const sourceGeneration = run.attempts[nodeId] ?? 1;
|
|
1669
|
-
const sourceAttemptId = this.sourceAttemptId(run, nodeId);
|
|
1670
|
-
const sourceStatus: RecoverySourceStatus = nodeId === SYNTHESIS_NODE_ID ? "synthesizing" : run.taskStatus[nodeId];
|
|
1671
|
-
if (nodeId !== SYNTHESIS_NODE_ID && isTerminalTask(run.taskStatus[nodeId])) return;
|
|
1672
|
-
const plan: RecoveryPlanItem = {
|
|
1673
|
-
nodeId,
|
|
1674
|
-
sourceAttemptId,
|
|
1675
|
-
sourceGeneration,
|
|
1676
|
-
sourceStatus,
|
|
1677
|
-
oldAgentId,
|
|
1678
|
-
};
|
|
1679
|
-
const sourceKey = this.recoverySourceKey(run, [plan]);
|
|
1680
|
-
const memo = this.memoizedRecovery(run.runId, "attempt_recovery", sourceKey);
|
|
1681
|
-
if (memo?.kind === "attempt_recovery") {
|
|
1682
|
-
applyRecoveryEvent(run, memo);
|
|
1683
|
-
this.resumeRecoveredNode(run, nodeId, resumeImmediately);
|
|
1684
|
-
return;
|
|
1685
|
-
}
|
|
1397
|
+
function hashScript(script: string): string {
|
|
1398
|
+
return createHash("sha256").update(script).digest("hex");
|
|
1399
|
+
}
|
|
1686
1400
|
|
|
1687
|
-
|
|
1688
|
-
|
|
1689
|
-
|
|
1690
|
-
|
|
1691
|
-
const result = resultFromLifecycle(
|
|
1692
|
-
"failed",
|
|
1693
|
-
run.synthesisAgentId,
|
|
1694
|
-
undefined,
|
|
1695
|
-
message,
|
|
1696
|
-
run.compactions[nodeId] ?? 0,
|
|
1697
|
-
);
|
|
1698
|
-
result.attemptId = run.attemptIds[nodeId];
|
|
1699
|
-
this.commitSynthesisTerminal(run, result, "failed");
|
|
1700
|
-
} else {
|
|
1701
|
-
this.failTask(run, nodeId, message);
|
|
1702
|
-
}
|
|
1703
|
-
return;
|
|
1704
|
-
}
|
|
1705
|
-
|
|
1706
|
-
const attemptId = `${run.runId}/${nodeId}/attempt-${generation}`;
|
|
1707
|
-
const rotation: RecoveryRotation = {
|
|
1708
|
-
nodeId,
|
|
1709
|
-
sourceAttemptId,
|
|
1710
|
-
sourceGeneration,
|
|
1711
|
-
sourceStatus,
|
|
1712
|
-
attemptId,
|
|
1713
|
-
generation,
|
|
1714
|
-
owner: { extension: "pi-workflows", runId: run.runId, nodeId, attemptId },
|
|
1715
|
-
...(oldAgentId ? { supersededAgentId: oldAgentId } : {}),
|
|
1716
|
-
};
|
|
1717
|
-
const event: AttemptRecoveryEvent = {
|
|
1718
|
-
kind: "attempt_recovery",
|
|
1719
|
-
schemaVersion: JOURNAL_SCHEMA_VERSION,
|
|
1720
|
-
runId: run.runId,
|
|
1721
|
-
nodeId,
|
|
1722
|
-
branchGeneration: this.recoveryBranchGeneration,
|
|
1723
|
-
rotation,
|
|
1724
|
-
...(legacyMigration ? { legacyMigration: true } : {}),
|
|
1725
|
-
timestamp: Date.now(),
|
|
1726
|
-
};
|
|
1727
|
-
this.write(event);
|
|
1728
|
-
applyRecoveryEvent(run, event);
|
|
1729
|
-
this.rememberRecovery({
|
|
1730
|
-
branchGeneration: this.recoveryBranchGeneration,
|
|
1731
|
-
runId: run.runId,
|
|
1732
|
-
key: sourceKey,
|
|
1733
|
-
event,
|
|
1734
|
-
});
|
|
1735
|
-
this.resumeRecoveredNode(run, nodeId, resumeImmediately);
|
|
1736
|
-
}
|
|
1737
|
-
|
|
1738
|
-
private resumeRecoveredNode(run: WorkflowRun, nodeId: string, resumeImmediately: boolean): void {
|
|
1739
|
-
if (!resumeImmediately) return;
|
|
1740
|
-
if (nodeId === SYNTHESIS_NODE_ID && run.status === "synthesizing") {
|
|
1741
|
-
const dispatch = this.dispatchSynthesis(run);
|
|
1742
|
-
this.trackDispatch(run.runId, dispatch);
|
|
1743
|
-
} else if (nodeId !== SYNTHESIS_NODE_ID && run.status === "running") {
|
|
1744
|
-
void this.pump(run);
|
|
1745
|
-
}
|
|
1746
|
-
}
|
|
1747
|
-
|
|
1748
|
-
private applyManagedTerminal(run: WorkflowRun, nodeId: string, response: ManagedSpawnResponse): void {
|
|
1749
|
-
const snapshot = response.terminal;
|
|
1750
|
-
if (!snapshot) return;
|
|
1751
|
-
if (snapshot.status === "interrupted") {
|
|
1752
|
-
this.recoverInterruptedNode(run, nodeId, response.id);
|
|
1753
|
-
return;
|
|
1754
|
-
}
|
|
1755
|
-
const status = managedTerminalStatus(snapshot);
|
|
1756
|
-
const compactionCount = Math.max(run.compactions[nodeId] ?? 0, snapshot.compactionCount);
|
|
1757
|
-
const result = resultFromLifecycle(
|
|
1758
|
-
status,
|
|
1759
|
-
response.id,
|
|
1760
|
-
snapshot.result,
|
|
1761
|
-
snapshot.error,
|
|
1762
|
-
compactionCount,
|
|
1763
|
-
snapshot.completedAt,
|
|
1764
|
-
snapshot.outputFile,
|
|
1765
|
-
snapshot.tokenCount,
|
|
1766
|
-
);
|
|
1767
|
-
result.attemptId = run.attemptIds[nodeId];
|
|
1768
|
-
|
|
1769
|
-
if (nodeId === SYNTHESIS_NODE_ID) {
|
|
1770
|
-
this.commitSynthesisTerminal(run, result, status);
|
|
1771
|
-
return;
|
|
1772
|
-
}
|
|
1773
|
-
if (!run.taskStatus[nodeId] || run.taskResults[nodeId] || isTerminalTask(run.taskStatus[nodeId])) return;
|
|
1774
|
-
this.commitTaskTerminal(run, nodeId, result, status);
|
|
1775
|
-
if (status === "completed" && run.status !== "stopping" && !this.settling.has(run.runId)) {
|
|
1776
|
-
this.maybePause(run);
|
|
1777
|
-
if (run.status === "running") void this.pump(run);
|
|
1778
|
-
}
|
|
1779
|
-
}
|
|
1780
|
-
|
|
1781
|
-
/**
|
|
1782
|
-
* Append the journaled results of `task.inputs` to its prompt.
|
|
1783
|
-
*
|
|
1784
|
-
* Must be a pure function of the journaled prefix: the composed prompt feeds pi-subagents'
|
|
1785
|
-
* managed fingerprint, and a replay that recomposes a different prompt is reported as a spawn
|
|
1786
|
-
* key conflict and fails the run non-retryably. So: iterate the declared `inputs` array in
|
|
1787
|
-
* order (never `Object.keys(run.taskResults)`, whose order differs between a live run and a
|
|
1788
|
-
* replay), read only journaled result fields (never `run.taskStatus`, live agent state, or the
|
|
1789
|
-
* clock), and never throw — `dispatchTask` has no error path around composition.
|
|
1790
|
-
*/
|
|
1791
|
-
private composeTaskPrompt(run: WorkflowRun, task: WorkflowTask): string {
|
|
1792
|
-
const inputs = task.inputs ?? [];
|
|
1793
|
-
if (inputs.length === 0) return task.prompt;
|
|
1794
|
-
const separator = "\n\n";
|
|
1795
|
-
// pi-subagents-protocol rejects an oversized prompt by throwing, so budget against the
|
|
1796
|
-
// remaining headroom rather than adding a fixed section and hoping it fits.
|
|
1797
|
-
const headroom = MAX_DISPATCH_PROMPT_CHARS - task.prompt.length - separator.length;
|
|
1798
|
-
const budget = Math.min(MAX_TASK_INPUT_CHARS, headroom);
|
|
1799
|
-
if (budget < MIN_TASK_INPUT_CHARS) return task.prompt;
|
|
1800
|
-
const entries: ResultEntry[] = inputs.map((id) => {
|
|
1801
|
-
const dependency = run.definition.tasks.find((candidate) => candidate.id === id);
|
|
1802
|
-
return {
|
|
1803
|
-
task: dependency ?? { id, description: id },
|
|
1804
|
-
result: run.taskResults[id],
|
|
1805
|
-
fallbackStatus: "missing",
|
|
1806
|
-
};
|
|
1807
|
-
});
|
|
1808
|
-
const section = buildBoundedResultSection([TASK_INPUT_HEADER], entries, {
|
|
1809
|
-
maxPerResult: MAX_TASK_RESULT_CHARS,
|
|
1810
|
-
maxTotal: budget,
|
|
1811
|
-
overflowMarker: "\n\n[dependency input truncated: additional results omitted]",
|
|
1812
|
-
});
|
|
1813
|
-
return cap(`${task.prompt}${separator}${section}`, MAX_DISPATCH_PROMPT_CHARS).text;
|
|
1814
|
-
}
|
|
1815
|
-
|
|
1816
|
-
private async dispatchTask(run: WorkflowRun, nodeId: string, branchEpoch: number): Promise<void> {
|
|
1817
|
-
const task = run.definition.tasks.find((candidate) => candidate.id === nodeId);
|
|
1818
|
-
if (
|
|
1819
|
-
!task ||
|
|
1820
|
-
this.branchChanging ||
|
|
1821
|
-
branchEpoch !== this.branchEpoch ||
|
|
1822
|
-
this.runs.get(run.runId) !== run ||
|
|
1823
|
-
run.status !== "running"
|
|
1824
|
-
)
|
|
1825
|
-
return;
|
|
1826
|
-
const attemptId = run.attemptIds[nodeId];
|
|
1827
|
-
const prompt = this.composeTaskPrompt(run, task);
|
|
1828
|
-
const tierOverride = task.tier === undefined ? run.definition.tier : undefined;
|
|
1829
|
-
// Keeping the `task` identity when nothing changes guarantees a byte-identical dispatch
|
|
1830
|
-
// for every workflow that does not declare `inputs`.
|
|
1831
|
-
const dispatchTaskDefinition: WorkflowTask =
|
|
1832
|
-
tierOverride === undefined && prompt === task.prompt
|
|
1833
|
-
? task
|
|
1834
|
-
: { ...task, prompt, ...(tierOverride === undefined ? {} : { tier: tierOverride }) };
|
|
1835
|
-
let response: ManagedSpawnResponse;
|
|
1836
|
-
try {
|
|
1837
|
-
response = normalizeManagedSpawnResponse(
|
|
1838
|
-
await spawnManaged(this.client, dispatchTaskDefinition, run.runId, nodeId, attemptId),
|
|
1839
|
-
);
|
|
1840
|
-
} catch (error: unknown) {
|
|
1841
|
-
if (branchEpoch !== this.branchEpoch || this.runs.get(run.runId) !== run) return;
|
|
1842
|
-
const message = boundedError(error) ?? "managed spawn failed";
|
|
1843
|
-
if ((run.status as WorkflowStatus) === "stopping") {
|
|
1844
|
-
if (!isTerminalTask(run.taskStatus[nodeId])) this.setTaskStatus(run, nodeId, "stopped", run.agentIds[nodeId]);
|
|
1845
|
-
return;
|
|
1846
|
-
}
|
|
1847
|
-
this.failTask(run, nodeId, message);
|
|
1848
|
-
return;
|
|
1849
|
-
}
|
|
1850
|
-
|
|
1851
|
-
if (
|
|
1852
|
-
this.branchChanging ||
|
|
1853
|
-
branchEpoch !== this.branchEpoch ||
|
|
1854
|
-
this.runs.get(run.runId) !== run ||
|
|
1855
|
-
run.status !== "running"
|
|
1856
|
-
) {
|
|
1857
|
-
if (
|
|
1858
|
-
!this.branchChanging &&
|
|
1859
|
-
branchEpoch === this.branchEpoch &&
|
|
1860
|
-
this.runs.get(run.runId) === run &&
|
|
1861
|
-
(run.status as WorkflowStatus) === "stopping"
|
|
1862
|
-
) {
|
|
1863
|
-
run.agentIds[nodeId] = response.id;
|
|
1864
|
-
this.rememberLateDispatchTarget(run, nodeId, response.id, attemptId);
|
|
1865
|
-
return;
|
|
1866
|
-
}
|
|
1867
|
-
this.quarantineAgent(response.id);
|
|
1868
|
-
const allowStaleBranch = this.branchChanging || branchEpoch !== this.branchEpoch;
|
|
1869
|
-
const quiesced = await this.stopAndQuiesceOwned(
|
|
1870
|
-
run,
|
|
1871
|
-
nodeId,
|
|
1872
|
-
response.id,
|
|
1873
|
-
attemptId,
|
|
1874
|
-
branchEpoch,
|
|
1875
|
-
!response.terminal,
|
|
1876
|
-
allowStaleBranch,
|
|
1877
|
-
);
|
|
1878
|
-
if (allowStaleBranch && !quiesced) this.rememberLateDispatchQuiescence(run.runId, response.id);
|
|
1879
|
-
return;
|
|
1880
|
-
}
|
|
1881
|
-
if (response.terminal) {
|
|
1882
|
-
// Keep terminal persistence failures outside the spawn-error handler. A
|
|
1883
|
-
// journal append failure must leave the current attempt untouched rather
|
|
1884
|
-
// than turning an already durable managed result into a new task failure.
|
|
1885
|
-
this.tryLifecycleMutation(run, `managed-terminal:${nodeId}:${attemptId}`, () => {
|
|
1886
|
-
this.applyManagedTerminal(run, nodeId, response);
|
|
1887
|
-
});
|
|
1888
|
-
return;
|
|
1889
|
-
}
|
|
1890
|
-
if (
|
|
1891
|
-
this.branchChanging ||
|
|
1892
|
-
branchEpoch !== this.branchEpoch ||
|
|
1893
|
-
this.runs.get(run.runId) !== run ||
|
|
1894
|
-
run.status !== "running"
|
|
1895
|
-
) {
|
|
1896
|
-
this.quarantineAgent(response.id);
|
|
1897
|
-
const allowStaleBranch = this.branchChanging || branchEpoch !== this.branchEpoch;
|
|
1898
|
-
const quiesced = await this.stopAndQuiesceOwned(
|
|
1899
|
-
run,
|
|
1900
|
-
nodeId,
|
|
1901
|
-
response.id,
|
|
1902
|
-
attemptId,
|
|
1903
|
-
branchEpoch,
|
|
1904
|
-
!response.terminal,
|
|
1905
|
-
allowStaleBranch,
|
|
1906
|
-
);
|
|
1907
|
-
if (allowStaleBranch && !quiesced) this.rememberLateDispatchQuiescence(run.runId, response.id);
|
|
1908
|
-
return;
|
|
1909
|
-
}
|
|
1910
|
-
const taskStatus = run.taskStatus[nodeId];
|
|
1911
|
-
if (response.state === "running" && (taskStatus === "dispatching" || taskStatus === "queued")) {
|
|
1912
|
-
this.setTaskStatus(run, nodeId, "running", response.id);
|
|
1913
|
-
} else if (taskStatus === "dispatching") {
|
|
1914
|
-
this.setTaskStatus(run, nodeId, "queued", response.id);
|
|
1915
|
-
}
|
|
1916
|
-
}
|
|
1917
|
-
|
|
1918
|
-
private async dispatchSynthesis(run: WorkflowRun, branchEpoch = this.branchEpoch): Promise<void> {
|
|
1919
|
-
const synthesis = run.definition.synthesis;
|
|
1920
|
-
if (
|
|
1921
|
-
!synthesis ||
|
|
1922
|
-
this.branchChanging ||
|
|
1923
|
-
branchEpoch !== this.branchEpoch ||
|
|
1924
|
-
this.runs.get(run.runId) !== run ||
|
|
1925
|
-
run.status !== "synthesizing"
|
|
1926
|
-
)
|
|
1927
|
-
return;
|
|
1928
|
-
const prompt = buildSynthesisPrompt(run.definition, run);
|
|
1929
|
-
const task: WorkflowTask = {
|
|
1930
|
-
id: SYNTHESIS_NODE_ID,
|
|
1931
|
-
subagent_type: synthesis.subagent_type,
|
|
1932
|
-
tier: synthesis.tier ?? run.definition.tier,
|
|
1933
|
-
description: "Synthesize workflow results",
|
|
1934
|
-
prompt,
|
|
1935
|
-
depends_on: [],
|
|
1936
|
-
};
|
|
1937
|
-
const attemptId = run.attemptIds[SYNTHESIS_NODE_ID];
|
|
1938
|
-
let response: ManagedSpawnResponse;
|
|
1939
|
-
try {
|
|
1940
|
-
response = normalizeManagedSpawnResponse(
|
|
1941
|
-
await spawnManaged(this.client, task, run.runId, SYNTHESIS_NODE_ID, attemptId),
|
|
1942
|
-
);
|
|
1943
|
-
} catch (error: unknown) {
|
|
1944
|
-
if (branchEpoch !== this.branchEpoch || this.runs.get(run.runId) !== run) return;
|
|
1945
|
-
const result = resultFromLifecycle("failed", run.synthesisAgentId, undefined, boundedError(error), 0);
|
|
1946
|
-
result.attemptId = run.attemptIds[SYNTHESIS_NODE_ID];
|
|
1947
|
-
this.commitSynthesisTerminal(run, result, "failed");
|
|
1948
|
-
return;
|
|
1949
|
-
}
|
|
1950
|
-
|
|
1951
|
-
if (
|
|
1952
|
-
this.branchChanging ||
|
|
1953
|
-
branchEpoch !== this.branchEpoch ||
|
|
1954
|
-
this.runs.get(run.runId) !== run ||
|
|
1955
|
-
run.status !== "synthesizing"
|
|
1956
|
-
) {
|
|
1957
|
-
if (
|
|
1958
|
-
!this.branchChanging &&
|
|
1959
|
-
branchEpoch === this.branchEpoch &&
|
|
1960
|
-
this.runs.get(run.runId) === run &&
|
|
1961
|
-
(run.status as WorkflowStatus) === "stopping"
|
|
1962
|
-
) {
|
|
1963
|
-
run.synthesisAgentId = response.id;
|
|
1964
|
-
this.rememberLateDispatchTarget(run, SYNTHESIS_NODE_ID, response.id, attemptId);
|
|
1965
|
-
return;
|
|
1966
|
-
}
|
|
1967
|
-
this.quarantineAgent(response.id);
|
|
1968
|
-
const allowStaleBranch = this.branchChanging || branchEpoch !== this.branchEpoch;
|
|
1969
|
-
const quiesced = await this.stopAndQuiesceOwned(
|
|
1970
|
-
run,
|
|
1971
|
-
SYNTHESIS_NODE_ID,
|
|
1972
|
-
response.id,
|
|
1973
|
-
attemptId,
|
|
1974
|
-
branchEpoch,
|
|
1975
|
-
!response.terminal,
|
|
1976
|
-
allowStaleBranch,
|
|
1977
|
-
);
|
|
1978
|
-
if (allowStaleBranch && !quiesced) this.rememberLateDispatchQuiescence(run.runId, response.id);
|
|
1979
|
-
return;
|
|
1980
|
-
}
|
|
1981
|
-
if (response.terminal) {
|
|
1982
|
-
// Do not route a terminal journal failure through the RPC error path; the
|
|
1983
|
-
// managed terminal fact remains retryable/recoverable as this attempt.
|
|
1984
|
-
this.tryLifecycleMutation(run, `managed-terminal:${SYNTHESIS_NODE_ID}:${attemptId}`, () => {
|
|
1985
|
-
this.applyManagedTerminal(run, SYNTHESIS_NODE_ID, response);
|
|
1986
|
-
});
|
|
1987
|
-
return;
|
|
1988
|
-
}
|
|
1989
|
-
run.synthesisAgentId ??= response.id;
|
|
1990
|
-
if (
|
|
1991
|
-
this.branchChanging ||
|
|
1992
|
-
branchEpoch !== this.branchEpoch ||
|
|
1993
|
-
this.runs.get(run.runId) !== run ||
|
|
1994
|
-
run.status !== "synthesizing"
|
|
1995
|
-
) {
|
|
1996
|
-
this.quarantineAgent(response.id);
|
|
1997
|
-
const allowStaleBranch = this.branchChanging || branchEpoch !== this.branchEpoch;
|
|
1998
|
-
const quiesced = await this.stopAndQuiesceOwned(
|
|
1999
|
-
run,
|
|
2000
|
-
SYNTHESIS_NODE_ID,
|
|
2001
|
-
response.id,
|
|
2002
|
-
attemptId,
|
|
2003
|
-
branchEpoch,
|
|
2004
|
-
!response.terminal,
|
|
2005
|
-
allowStaleBranch,
|
|
2006
|
-
);
|
|
2007
|
-
if (allowStaleBranch && !quiesced) this.rememberLateDispatchQuiescence(run.runId, response.id);
|
|
2008
|
-
return;
|
|
2009
|
-
}
|
|
2010
|
-
}
|
|
2011
|
-
/**
|
|
2012
|
-
* Gate lifecycle observations at the workflow boundary. Attempt-aware runs
|
|
2013
|
-
* must carry the exact current generation and agent identity; only a created
|
|
2014
|
-
* event may establish an agent id when dispatch has not returned one yet.
|
|
2015
|
-
* Legacy journal runs intentionally retain the pre-v2 permissive behavior.
|
|
2016
|
-
*/
|
|
2017
|
-
private acceptsLifecycleEvent(
|
|
2018
|
-
run: WorkflowRun,
|
|
2019
|
-
nodeId: string,
|
|
2020
|
-
eventName: string,
|
|
2021
|
-
ownerAttemptId: string | undefined,
|
|
2022
|
-
agentId: string | undefined,
|
|
2023
|
-
): boolean {
|
|
2024
|
-
const expectedAttemptId = run.attemptIds[nodeId];
|
|
2025
|
-
if (run.attemptTracking) {
|
|
2026
|
-
if (!expectedAttemptId || ownerAttemptId !== expectedAttemptId || !agentId) return false;
|
|
2027
|
-
}
|
|
2028
|
-
|
|
2029
|
-
const expectedAgentId = nodeId === SYNTHESIS_NODE_ID ? run.synthesisAgentId : run.agentIds[nodeId];
|
|
2030
|
-
if (agentId && expectedAgentId && agentId !== expectedAgentId) return false;
|
|
2031
|
-
if (run.attemptTracking && !expectedAgentId && eventName !== "subagents:created") return false;
|
|
2032
|
-
return true;
|
|
2033
|
-
}
|
|
2034
|
-
|
|
2035
|
-
private onLifecycle(eventName: string, data: Record<string, unknown>): void {
|
|
2036
|
-
if (this.lifecycleSuspended || this.branchChanging || this.branchEventsClosed) return;
|
|
2037
|
-
const owner = ownerFor(data);
|
|
2038
|
-
if (!owner) return;
|
|
2039
|
-
const run = this.runs.get(owner.runId);
|
|
2040
|
-
if (!run) return;
|
|
2041
|
-
const agentId = typeof data.id === "string" ? data.id : undefined;
|
|
2042
|
-
if (agentId && this.quarantinedAgentIds.has(agentId)) return;
|
|
2043
|
-
if (owner.nodeId !== SYNTHESIS_NODE_ID && !(owner.nodeId in run.taskStatus)) return;
|
|
2044
|
-
|
|
2045
|
-
const settling = run.status === "stopping" || this.settling.has(run.runId);
|
|
2046
|
-
if (isTerminalWorkflow(run.status)) return;
|
|
2047
|
-
if (!this.acceptsLifecycleEvent(run, owner.nodeId, eventName, owner.attemptId, agentId)) return;
|
|
2048
|
-
const terminalEvent = eventName === "subagents:completed" || eventName === "subagents:failed";
|
|
2049
|
-
if (run.status === "stopping" && !terminalEvent) return;
|
|
2050
|
-
if (
|
|
2051
|
-
owner.nodeId === SYNTHESIS_NODE_ID
|
|
2052
|
-
? run.synthesisResult !== undefined
|
|
2053
|
-
: run.taskResults[owner.nodeId] !== undefined || isTerminalTask(run.taskStatus[owner.nodeId])
|
|
2054
|
-
)
|
|
2055
|
-
return;
|
|
2056
|
-
|
|
2057
|
-
if (eventName === "subagents:compacted") {
|
|
2058
|
-
const previousCount = run.compactions[owner.nodeId] ?? 0;
|
|
2059
|
-
const observedCount = data.compactionCount;
|
|
2060
|
-
const count =
|
|
2061
|
-
typeof observedCount === "number" && Number.isSafeInteger(observedCount) && observedCount >= 0
|
|
2062
|
-
? Math.max(previousCount, observedCount)
|
|
2063
|
-
: previousCount + 1;
|
|
2064
|
-
const timestamp = Date.now();
|
|
2065
|
-
const event: JournalEvent = {
|
|
2066
|
-
kind: "task_compacted",
|
|
2067
|
-
schemaVersion: JOURNAL_SCHEMA_VERSION,
|
|
2068
|
-
runId: run.runId,
|
|
2069
|
-
nodeId: owner.nodeId,
|
|
2070
|
-
compactionCount: count,
|
|
2071
|
-
attemptId: run.attemptIds[owner.nodeId],
|
|
2072
|
-
owner: this.ownerForNode(run, owner.nodeId),
|
|
2073
|
-
timestamp,
|
|
2074
|
-
};
|
|
2075
|
-
this.tryLifecycleMutation(run, `compacted:${owner.nodeId}:${count}`, () => {
|
|
2076
|
-
this.write(event);
|
|
2077
|
-
run.compactions[owner.nodeId] = Math.max(run.compactions[owner.nodeId] ?? 0, count);
|
|
2078
|
-
run.updatedAt = timestamp;
|
|
2079
|
-
});
|
|
2080
|
-
return;
|
|
2081
|
-
}
|
|
2082
|
-
|
|
2083
|
-
if (owner.nodeId === SYNTHESIS_NODE_ID) {
|
|
2084
|
-
if (eventName === "subagents:created" && agentId) run.synthesisAgentId ??= agentId;
|
|
2085
|
-
if (eventName === "subagents:started" || !terminalEvent) return;
|
|
2086
|
-
const status = statusFromLifecycle(eventName, data);
|
|
2087
|
-
const timestamp = Date.now();
|
|
2088
|
-
const result = resultFromLifecycle(
|
|
2089
|
-
status,
|
|
2090
|
-
agentId ?? run.synthesisAgentId,
|
|
2091
|
-
data.result,
|
|
2092
|
-
data.error,
|
|
2093
|
-
run.compactions[SYNTHESIS_NODE_ID] ?? 0,
|
|
2094
|
-
timestamp,
|
|
2095
|
-
data.outputFile,
|
|
2096
|
-
tokenCountFrom(data.tokens),
|
|
2097
|
-
);
|
|
2098
|
-
result.attemptId = run.attemptIds[SYNTHESIS_NODE_ID];
|
|
2099
|
-
this.tryLifecycleMutation(run, `terminal:${owner.nodeId}:${owner.attemptId ?? "legacy"}`, () => {
|
|
2100
|
-
this.commitSynthesisTerminal(run, result, status);
|
|
2101
|
-
});
|
|
2102
|
-
return;
|
|
2103
|
-
}
|
|
2104
|
-
|
|
2105
|
-
if (eventName === "subagents:created") {
|
|
2106
|
-
if (run.taskStatus[owner.nodeId] === "dispatching") {
|
|
2107
|
-
this.tryLifecycleMutation(run, `created:${owner.nodeId}:${owner.attemptId ?? "legacy"}`, () => {
|
|
2108
|
-
this.setTaskStatus(run, owner.nodeId, "queued", agentId);
|
|
2109
|
-
});
|
|
2110
|
-
}
|
|
2111
|
-
return;
|
|
2112
|
-
}
|
|
2113
|
-
if (eventName === "subagents:started") {
|
|
2114
|
-
if (run.taskStatus[owner.nodeId] === "dispatching" || run.taskStatus[owner.nodeId] === "queued") {
|
|
2115
|
-
this.tryLifecycleMutation(run, `started:${owner.nodeId}:${owner.attemptId ?? "legacy"}`, () => {
|
|
2116
|
-
this.setTaskStatus(run, owner.nodeId, "running", agentId);
|
|
2117
|
-
});
|
|
2118
|
-
}
|
|
2119
|
-
return;
|
|
2120
|
-
}
|
|
2121
|
-
if (!terminalEvent) return;
|
|
2122
|
-
|
|
2123
|
-
const status = statusFromLifecycle(eventName, data);
|
|
2124
|
-
const timestamp = Date.now();
|
|
2125
|
-
const result = resultFromLifecycle(
|
|
2126
|
-
status,
|
|
2127
|
-
agentId ?? run.agentIds[owner.nodeId],
|
|
2128
|
-
data.result,
|
|
2129
|
-
data.error,
|
|
2130
|
-
run.compactions[owner.nodeId] ?? 0,
|
|
2131
|
-
timestamp,
|
|
2132
|
-
data.outputFile,
|
|
2133
|
-
tokenCountFrom(data.tokens),
|
|
2134
|
-
);
|
|
2135
|
-
result.attemptId = run.attemptIds[owner.nodeId];
|
|
2136
|
-
this.tryLifecycleMutation(run, `terminal:${owner.nodeId}:${owner.attemptId ?? "legacy"}`, () => {
|
|
2137
|
-
this.commitTaskTerminal(run, owner.nodeId, result, status);
|
|
2138
|
-
if (status === "completed" && !settling && run.status !== "stopping") {
|
|
2139
|
-
this.maybePause(run);
|
|
2140
|
-
if (run.status === "running") void this.pump(run);
|
|
2141
|
-
}
|
|
2142
|
-
});
|
|
2143
|
-
}
|
|
2144
|
-
|
|
2145
|
-
private blockDependents(run: WorkflowRun, failedNodeId: string): void {
|
|
2146
|
-
const blockedIds = this.blockedDependentIds(run, failedNodeId);
|
|
2147
|
-
if (blockedIds.length === 0) return;
|
|
2148
|
-
const timestamp = Date.now();
|
|
2149
|
-
const events: JournalEvent[] = blockedIds.map((nodeId) => ({
|
|
2150
|
-
kind: "task_transition",
|
|
2151
|
-
schemaVersion: JOURNAL_SCHEMA_VERSION,
|
|
2152
|
-
runId: run.runId,
|
|
2153
|
-
nodeId,
|
|
2154
|
-
status: "blocked",
|
|
2155
|
-
...(run.attemptIds[nodeId] ? { attemptId: run.attemptIds[nodeId] } : {}),
|
|
2156
|
-
owner: this.ownerForNode(run, nodeId),
|
|
2157
|
-
timestamp,
|
|
2158
|
-
}));
|
|
2159
|
-
this.appendEvents(events);
|
|
2160
|
-
for (const nodeId of blockedIds) run.taskStatus[nodeId] = "blocked";
|
|
2161
|
-
run.updatedAt = timestamp;
|
|
2162
|
-
}
|
|
2163
|
-
|
|
2164
|
-
private failTask(run: WorkflowRun, nodeId: string, message: string): void {
|
|
2165
|
-
if (isTerminalWorkflow(run.status) || run.terminalIntent === "stop") return;
|
|
2166
|
-
const result = resultFromLifecycle(
|
|
2167
|
-
"failed",
|
|
2168
|
-
run.agentIds[nodeId],
|
|
2169
|
-
undefined,
|
|
2170
|
-
message,
|
|
2171
|
-
run.compactions[nodeId] ?? 0,
|
|
2172
|
-
);
|
|
2173
|
-
result.attemptId = run.attemptIds[nodeId];
|
|
2174
|
-
this.commitTaskTerminal(run, nodeId, result, "failed", message);
|
|
2175
|
-
}
|
|
2176
|
-
|
|
2177
|
-
/**
|
|
2178
|
-
* Failure policy: fail-fast for scheduling, but never abandon sibling agents.
|
|
2179
|
-
* Mark the intent, stop every remaining owned agent, consume/journal terminal
|
|
2180
|
-
* callbacks, then settle the workflow as failed with dependents blocked.
|
|
2181
|
-
*/
|
|
2182
|
-
private beginFailureCleanup(run: WorkflowRun, message: string): void {
|
|
2183
|
-
if (isTerminalWorkflow(run.status) || run.terminalIntent === "stop") return;
|
|
2184
|
-
if (run.terminalIntent !== "failure" || run.status !== "stopping" || run.error !== message) {
|
|
2185
|
-
this.setWorkflowStatus(run, "stopping", message, "failure");
|
|
2186
|
-
}
|
|
2187
|
-
this.startFailureCleanup(run, message);
|
|
2188
|
-
}
|
|
2189
|
-
|
|
2190
|
-
private startFailureCleanup(run: WorkflowRun, message: string): void {
|
|
2191
|
-
if (isTerminalWorkflow(run.status) || run.terminalIntent !== "failure") return;
|
|
2192
|
-
if (this.settling.has(run.runId)) return;
|
|
2193
|
-
const cleanup = this.settleTerminalCleanup(run, message);
|
|
2194
|
-
this.settling.set(run.runId, cleanup);
|
|
2195
|
-
void cleanup.then(
|
|
2196
|
-
() => {
|
|
2197
|
-
if (this.settling.get(run.runId) === cleanup) this.settling.delete(run.runId);
|
|
2198
|
-
},
|
|
2199
|
-
() => {
|
|
2200
|
-
if (this.settling.get(run.runId) === cleanup) this.settling.delete(run.runId);
|
|
2201
|
-
},
|
|
2202
|
-
);
|
|
2203
|
-
}
|
|
2204
|
-
|
|
2205
|
-
private terminalRecoveryKey(runId: string, sourceKey: string): string {
|
|
2206
|
-
return JSON.stringify([this.recoveryBranchGeneration, runId, "terminal_recovery", sourceKey]);
|
|
2207
|
-
}
|
|
2208
|
-
|
|
2209
|
-
private applyTerminalRecoveryMemo(run: WorkflowRun, memo: TerminalRecoveryMemo): void {
|
|
2210
|
-
applyTerminalRecoveryEvent(run, memo.event);
|
|
2211
|
-
}
|
|
2212
|
-
|
|
2213
|
-
private rememberTerminalRecovery(memo: TerminalRecoveryMemo): void {
|
|
2214
|
-
const key = this.terminalRecoveryKey(memo.runId, memo.key);
|
|
2215
|
-
this.terminalRecoveryMemo.delete(key);
|
|
2216
|
-
this.terminalRecoveryMemo.set(key, memo);
|
|
2217
|
-
while (this.terminalRecoveryMemo.size > MAX_RECOVERY_MEMO_ENTRIES) {
|
|
2218
|
-
const oldest = this.terminalRecoveryMemo.keys().next().value;
|
|
2219
|
-
if (typeof oldest !== "string") break;
|
|
2220
|
-
this.terminalRecoveryMemo.delete(oldest);
|
|
2221
|
-
}
|
|
2222
|
-
}
|
|
2223
|
-
|
|
2224
|
-
/**
|
|
2225
|
-
* Complete a terminal-intent run recovered from a session branch with one
|
|
2226
|
-
* durable state transition. The event is built entirely from an immutable
|
|
2227
|
-
* source snapshot; append is the only boundary before applying it.
|
|
2228
|
-
*/
|
|
2229
|
-
private recoverTerminalIntent(run: WorkflowRun, terminalRecords: readonly DeferredTerminalRecord[]): void {
|
|
2230
|
-
const source = snapshotRun(run);
|
|
2231
|
-
const terminalRecordsForKey = [...terminalRecords]
|
|
2232
|
-
.sort((left, right) => left.nodeId.localeCompare(right.nodeId) || left.attemptId.localeCompare(right.attemptId))
|
|
2233
|
-
.map((record) => ({
|
|
2234
|
-
nodeId: record.nodeId,
|
|
2235
|
-
attemptId: record.attemptId,
|
|
2236
|
-
owner: record.owner,
|
|
2237
|
-
result: { ...record.result, updatedAt: 0 },
|
|
2238
|
-
}));
|
|
2239
|
-
const sourceKey = JSON.stringify({ source, terminalRecords: terminalRecordsForKey });
|
|
2240
|
-
const memo = this.terminalRecoveryMemo.get(this.terminalRecoveryKey(run.runId, sourceKey));
|
|
2241
|
-
if (memo) {
|
|
2242
|
-
this.applyTerminalRecoveryMemo(run, memo);
|
|
2243
|
-
return;
|
|
2244
|
-
}
|
|
2245
|
-
|
|
2246
|
-
const terminalIntent = source.terminalIntent;
|
|
2247
|
-
if (terminalIntent === undefined) return;
|
|
2248
|
-
const fallbackError =
|
|
2249
|
-
source.error ?? (terminalIntent === "failure" ? "workflow failure cleanup" : "workflow stopped");
|
|
2250
|
-
const recoveredByNode = new Map(terminalRecords.map((record) => [record.nodeId, record]));
|
|
2251
|
-
const finalStatuses: Record<string, TaskStatus> = { ...source.taskStatus };
|
|
2252
|
-
const resultForNode = (nodeId: string): WorkflowRun["taskResults"][string] | undefined => {
|
|
2253
|
-
const sourceResult = source.taskResults[nodeId];
|
|
2254
|
-
if (sourceResult) {
|
|
2255
|
-
return { ...sourceResult, attemptId: source.attemptIds[nodeId] };
|
|
2256
|
-
}
|
|
2257
|
-
const recovered = recoveredByNode.get(nodeId)?.result;
|
|
2258
|
-
if (!recovered) return undefined;
|
|
2259
|
-
return { ...recovered, attemptId: source.attemptIds[nodeId], updatedAt: source.updatedAt };
|
|
2260
|
-
};
|
|
2261
|
-
for (const task of source.definition.tasks) {
|
|
2262
|
-
const observed = resultForNode(task.id);
|
|
2263
|
-
if (observed && !isTerminalTask(source.taskStatus[task.id])) finalStatuses[task.id] = observed.status;
|
|
2264
|
-
}
|
|
2265
|
-
|
|
2266
|
-
const blockedNodeIds = new Set(
|
|
2267
|
-
source.definition.tasks.filter((task) => source.taskStatus[task.id] === "blocked").map((task) => task.id),
|
|
2268
|
-
);
|
|
2269
|
-
if (terminalIntent === "failure") {
|
|
2270
|
-
const failedOrBlocked = new Set(
|
|
2271
|
-
source.definition.tasks
|
|
2272
|
-
.filter((task) => finalStatuses[task.id] === "failed" || finalStatuses[task.id] === "blocked")
|
|
2273
|
-
.map((task) => task.id),
|
|
2274
|
-
);
|
|
2275
|
-
let changed = true;
|
|
2276
|
-
while (changed) {
|
|
2277
|
-
changed = false;
|
|
2278
|
-
for (const task of source.definition.tasks) {
|
|
2279
|
-
if (isTerminalTask(finalStatuses[task.id])) continue;
|
|
2280
|
-
if (!task.depends_on.some((dependency) => failedOrBlocked.has(dependency))) continue;
|
|
2281
|
-
finalStatuses[task.id] = "blocked";
|
|
2282
|
-
failedOrBlocked.add(task.id);
|
|
2283
|
-
blockedNodeIds.add(task.id);
|
|
2284
|
-
changed = true;
|
|
2285
|
-
}
|
|
2286
|
-
}
|
|
2287
|
-
}
|
|
2288
|
-
|
|
2289
|
-
const terminalResults: RecoveryTerminalResult[] = [];
|
|
2290
|
-
const terminalFact = (nodeId: string, result: WorkflowRun["taskResults"][string]): RecoveryTerminalResult => {
|
|
2291
|
-
const attemptId = source.attemptIds[nodeId];
|
|
2292
|
-
return {
|
|
2293
|
-
nodeId,
|
|
2294
|
-
attemptId,
|
|
2295
|
-
result: { ...result, attemptId },
|
|
2296
|
-
owner: { extension: "pi-workflows", runId: source.runId, nodeId, attemptId },
|
|
2297
|
-
};
|
|
2298
|
-
};
|
|
2299
|
-
for (const task of source.definition.tasks) {
|
|
2300
|
-
const sourceStatus = source.taskStatus[task.id];
|
|
2301
|
-
if (isTerminalTask(sourceStatus)) continue;
|
|
2302
|
-
if (blockedNodeIds.has(task.id)) continue;
|
|
2303
|
-
const result =
|
|
2304
|
-
resultForNode(task.id) ??
|
|
2305
|
-
resultFromLifecycle(
|
|
2306
|
-
"stopped",
|
|
2307
|
-
source.agentIds[task.id],
|
|
2308
|
-
undefined,
|
|
2309
|
-
fallbackError,
|
|
2310
|
-
source.compactions[task.id] ?? 0,
|
|
2311
|
-
source.updatedAt,
|
|
2312
|
-
);
|
|
2313
|
-
terminalResults.push(terminalFact(task.id, result));
|
|
2314
|
-
}
|
|
2315
|
-
|
|
2316
|
-
if (source.definition.synthesis && !source.synthesisResult) {
|
|
2317
|
-
const observed = source.synthesisResult ?? recoveredByNode.get(SYNTHESIS_NODE_ID)?.result;
|
|
2318
|
-
const result =
|
|
2319
|
-
observed ??
|
|
2320
|
-
resultFromLifecycle(
|
|
2321
|
-
"stopped",
|
|
2322
|
-
source.synthesisAgentId,
|
|
2323
|
-
undefined,
|
|
2324
|
-
fallbackError,
|
|
2325
|
-
source.compactions[SYNTHESIS_NODE_ID] ?? 0,
|
|
2326
|
-
source.updatedAt,
|
|
2327
|
-
);
|
|
2328
|
-
terminalResults.push(terminalFact(SYNTHESIS_NODE_ID, result));
|
|
2329
|
-
}
|
|
2330
|
-
|
|
2331
|
-
const recoveryBase = {
|
|
2332
|
-
kind: "terminal_recovery" as const,
|
|
2333
|
-
schemaVersion: JOURNAL_SCHEMA_VERSION as 2,
|
|
2334
|
-
runId: source.runId,
|
|
2335
|
-
status: terminalIntent === "failure" ? ("failed" as const) : ("stopped" as const),
|
|
2336
|
-
terminalIntent,
|
|
2337
|
-
branchGeneration: this.recoveryBranchGeneration,
|
|
2338
|
-
terminalResults,
|
|
2339
|
-
blockedNodeIds: [...blockedNodeIds].sort(),
|
|
2340
|
-
error: fallbackError,
|
|
2341
|
-
};
|
|
2342
|
-
const event: TerminalRecoveryEvent = {
|
|
2343
|
-
...recoveryBase,
|
|
2344
|
-
recoveryId: deriveRecoveryId(recoveryBase),
|
|
2345
|
-
timestamp: source.updatedAt,
|
|
2346
|
-
};
|
|
2347
|
-
// Do not mutate the replayed run or memoize until this single append succeeds.
|
|
2348
|
-
this.write(event);
|
|
2349
|
-
applyTerminalRecoveryEvent(run, event);
|
|
2350
|
-
this.rememberTerminalRecovery({
|
|
2351
|
-
branchGeneration: this.recoveryBranchGeneration,
|
|
2352
|
-
runId: run.runId,
|
|
2353
|
-
key: sourceKey,
|
|
2354
|
-
event,
|
|
2355
|
-
});
|
|
2356
|
-
}
|
|
2357
|
-
|
|
2358
|
-
private async settleTerminalCleanup(
|
|
2359
|
-
run: WorkflowRun,
|
|
2360
|
-
message: string,
|
|
2361
|
-
cleanupEpoch = this.branchEpoch,
|
|
2362
|
-
cleanupTargets?: TerminalCleanupTarget[],
|
|
2363
|
-
): Promise<void> {
|
|
2364
|
-
const isCurrent = (): boolean =>
|
|
2365
|
-
!this.branchChanging && cleanupEpoch === this.branchEpoch && this.runs.get(run.runId) === run;
|
|
2366
|
-
// Capture exact owners before the first yield. A pending lifecycle callback
|
|
2367
|
-
// may make a target terminal before the cleanup RPC is allowed to run.
|
|
2368
|
-
let pendingTargets = cleanupTargets ? [...cleanupTargets] : this.initialTerminalCleanupTargets(run);
|
|
2369
|
-
// Let beginFailureCleanup/stop publish the promise in `settling` before a
|
|
2370
|
-
// synchronous stop RPC can emit lifecycle callbacks.
|
|
2371
|
-
await Promise.resolve();
|
|
2372
|
-
if (!isCurrent()) return;
|
|
2373
|
-
const addTarget = (target: TerminalCleanupTarget): void => {
|
|
2374
|
-
if (
|
|
2375
|
-
pendingTargets.some(
|
|
2376
|
-
(existing) =>
|
|
2377
|
-
existing.agentId === target.agentId &&
|
|
2378
|
-
existing.owner.nodeId === target.owner.nodeId &&
|
|
2379
|
-
existing.owner.attemptId === target.owner.attemptId,
|
|
2380
|
-
)
|
|
2381
|
-
)
|
|
2382
|
-
return;
|
|
2383
|
-
pendingTargets.push(target);
|
|
2384
|
-
};
|
|
2385
|
-
for (const target of this.takeLateDispatchTargets(run)) addTarget(target);
|
|
2386
|
-
|
|
2387
|
-
let agentsSettled = pendingTargets.length === 0;
|
|
2388
|
-
if (pendingTargets.length > 0) {
|
|
2389
|
-
if (this.client.quiesceOwned) {
|
|
2390
|
-
const agentIds = pendingTargets.map(({ agentId }) => agentId);
|
|
2391
|
-
const owners = pendingTargets.map(({ owner }) => owner);
|
|
2392
|
-
const result = await this.client.quiesceOwned(run.runId, agentIds, BRANCH_QUIESCE_TIMEOUT_MS, owners).catch(
|
|
2393
|
-
(error: unknown): WorkflowQuiesceResult => ({
|
|
2394
|
-
settled: false,
|
|
2395
|
-
pending: agentIds,
|
|
2396
|
-
diagnostic: `owned quiescence failed: ${boundedError(error) ?? "unknown error"}`,
|
|
2397
|
-
}),
|
|
2398
|
-
);
|
|
2399
|
-
const pendingIds = new Set(result.pending);
|
|
2400
|
-
if (result.settled && result.pending.length === 0) {
|
|
2401
|
-
pendingTargets = [];
|
|
2402
|
-
} else {
|
|
2403
|
-
const retained = pendingTargets.filter((target) => pendingIds.has(target.agentId));
|
|
2404
|
-
// A false result without a matching pending id does not prove that
|
|
2405
|
-
// any target settled. Keep the exact targets and retry fail-closed.
|
|
2406
|
-
pendingTargets = result.pending.length === 0 || retained.length === 0 ? pendingTargets : retained;
|
|
2407
|
-
}
|
|
2408
|
-
agentsSettled = pendingTargets.length === 0;
|
|
2409
|
-
} else {
|
|
2410
|
-
// An owner-scoped stop only requests cancellation; without the v3
|
|
2411
|
-
// quiescence proof, the AgentSession may still own provider work.
|
|
2412
|
-
const stopPromises = pendingTargets
|
|
2413
|
-
.filter((target) => !target.fallbackStopRequested)
|
|
2414
|
-
.map((target) => {
|
|
2415
|
-
target.fallbackStopRequested = true;
|
|
2416
|
-
return this.stopOwned(run, target.owner.nodeId, target.agentId, target.owner.attemptId).catch(() => {});
|
|
2417
|
-
});
|
|
2418
|
-
await Promise.allSettled(stopPromises);
|
|
2419
|
-
agentsSettled = false;
|
|
2420
|
-
}
|
|
2421
|
-
}
|
|
2422
|
-
|
|
2423
|
-
const dispatchesSettled = await this.waitForDispatches(run.runId, BRANCH_QUIESCE_TIMEOUT_MS);
|
|
2424
|
-
if (!isCurrent()) return;
|
|
2425
|
-
let lateTargetsAdded = false;
|
|
2426
|
-
for (const target of this.takeLateDispatchTargets(run)) {
|
|
2427
|
-
const previousCount = pendingTargets.length;
|
|
2428
|
-
addTarget(target);
|
|
2429
|
-
lateTargetsAdded ||= pendingTargets.length > previousCount;
|
|
2430
|
-
}
|
|
2431
|
-
agentsSettled = pendingTargets.length === 0;
|
|
2432
|
-
if (lateTargetsAdded) {
|
|
2433
|
-
if (isCurrent()) await this.settleTerminalCleanup(run, message, cleanupEpoch, pendingTargets);
|
|
2434
|
-
return;
|
|
2435
|
-
}
|
|
2436
|
-
if (!agentsSettled || !dispatchesSettled) {
|
|
2437
|
-
// A stop acknowledgement only requests cancellation; the AgentSession
|
|
2438
|
-
// still owns provider work and its pool slot until its promise settles.
|
|
2439
|
-
// Stay in `stopping` and retry instead of publishing a false terminal run.
|
|
2440
|
-
await new Promise<void>((resolve) => setTimeout(resolve, 100));
|
|
2441
|
-
if (isCurrent()) await this.settleTerminalCleanup(run, message, cleanupEpoch, pendingTargets);
|
|
2442
|
-
return;
|
|
2443
|
-
}
|
|
2444
|
-
|
|
2445
|
-
if (this.hasPendingJournalWork(run.runId)) {
|
|
2446
|
-
// Do not synthesize a stopped/failed fact over a lifecycle callback whose
|
|
2447
|
-
// durable prefix is still retrying (or has become permanently blocked).
|
|
2448
|
-
// A retry may commit the observed terminal result and make the synthetic
|
|
2449
|
-
// fact unnecessary; a blocked journal must remain nonterminal fail-closed.
|
|
2450
|
-
if (this.journalBlockedRuns.has(run.runId)) return;
|
|
2451
|
-
await new Promise<void>((resolve) => setTimeout(resolve, 100));
|
|
2452
|
-
if (isCurrent()) await this.settleTerminalCleanup(run, message, cleanupEpoch, pendingTargets);
|
|
2453
|
-
return;
|
|
2454
|
-
}
|
|
2455
|
-
|
|
2456
|
-
for (const [nodeId, status] of Object.entries(run.taskStatus)) {
|
|
2457
|
-
if (isTerminalTask(status)) continue;
|
|
2458
|
-
const result =
|
|
2459
|
-
run.taskResults[nodeId] ??
|
|
2460
|
-
resultFromLifecycle(
|
|
2461
|
-
"stopped",
|
|
2462
|
-
run.agentIds[nodeId],
|
|
2463
|
-
undefined,
|
|
2464
|
-
run.terminalIntent === "failure" ? "workflow failure cleanup" : message,
|
|
2465
|
-
run.compactions[nodeId] ?? 0,
|
|
2466
|
-
);
|
|
2467
|
-
result.attemptId ??= run.attemptIds[nodeId];
|
|
2468
|
-
this.commitTaskTerminal(run, nodeId, result, "stopped", undefined, false);
|
|
2469
|
-
}
|
|
2470
|
-
|
|
2471
|
-
if (run.synthesisAgentId && !run.synthesisResult) {
|
|
2472
|
-
const result = resultFromLifecycle(
|
|
2473
|
-
"stopped",
|
|
2474
|
-
run.synthesisAgentId,
|
|
2475
|
-
undefined,
|
|
2476
|
-
run.terminalIntent === "failure" ? "workflow failure cleanup" : message,
|
|
2477
|
-
run.compactions[SYNTHESIS_NODE_ID] ?? 0,
|
|
2478
|
-
);
|
|
2479
|
-
result.attemptId = run.attemptIds[SYNTHESIS_NODE_ID];
|
|
2480
|
-
this.commitSynthesisTerminal(run, result, "stopped");
|
|
2481
|
-
}
|
|
2482
|
-
|
|
2483
|
-
if (run.status === "stopping" && isCurrent()) {
|
|
2484
|
-
this.setWorkflowStatus(run, run.terminalIntent === "failure" ? "failed" : "stopped", message);
|
|
2485
|
-
this.lateDispatchTargets.delete(run.runId);
|
|
2486
|
-
}
|
|
2487
|
-
}
|
|
2488
|
-
|
|
2489
|
-
private async failRun(run: WorkflowRun, message: string): Promise<void> {
|
|
2490
|
-
if (isTerminalWorkflow(run.status)) return;
|
|
2491
|
-
for (const task of run.definition.tasks) {
|
|
2492
|
-
if (run.taskStatus[task.id] === "failed") this.blockDependents(run, task.id);
|
|
2493
|
-
}
|
|
2494
|
-
this.beginFailureCleanup(run, message);
|
|
2495
|
-
const cleanup = this.settling.get(run.runId);
|
|
2496
|
-
if (cleanup) await cleanup;
|
|
2497
|
-
}
|
|
2498
|
-
|
|
2499
|
-
private maybePause(run: WorkflowRun): void {
|
|
2500
|
-
if (run.status !== "pausing") return;
|
|
2501
|
-
if (!Object.values(run.taskStatus).some(activeStatus) && !run.synthesisAgentId) {
|
|
2502
|
-
this.setWorkflowStatus(run, "paused");
|
|
2503
|
-
}
|
|
2504
|
-
}
|
|
2505
|
-
|
|
2506
|
-
private async stop(run: WorkflowRun): Promise<void> {
|
|
2507
|
-
if (isTerminalWorkflow(run.status)) return;
|
|
2508
|
-
if (run.status !== "stopping" || run.terminalIntent !== "stop" || run.error !== "workflow stopped") {
|
|
2509
|
-
this.setWorkflowStatus(run, "stopping", "workflow stopped", "stop");
|
|
2510
|
-
}
|
|
2511
|
-
run.nonResumable = true;
|
|
2512
|
-
const existing = this.settling.get(run.runId);
|
|
2513
|
-
if (existing) {
|
|
2514
|
-
await existing;
|
|
2515
|
-
return;
|
|
2516
|
-
}
|
|
2517
|
-
const cleanup = this.settleTerminalCleanup(run, "workflow stopped");
|
|
2518
|
-
this.settling.set(run.runId, cleanup);
|
|
2519
|
-
try {
|
|
2520
|
-
await cleanup;
|
|
2521
|
-
} finally {
|
|
2522
|
-
if (this.settling.get(run.runId) === cleanup) this.settling.delete(run.runId);
|
|
2523
|
-
}
|
|
2524
|
-
}
|
|
2525
|
-
|
|
2526
|
-
private reconcileManagedSpawnTombstone(entry: SessionEntryLike): DeferredTerminalRecord | undefined {
|
|
2527
|
-
if (entry.type !== "custom" || entry.customType !== "subagents:managed-spawn" || !isRecord(entry.data)) return;
|
|
2528
|
-
const data = entry.data;
|
|
2529
|
-
const allowedKeys = new Set([
|
|
2530
|
-
"schemaVersion",
|
|
2531
|
-
"spawnKey",
|
|
2532
|
-
"fingerprint",
|
|
2533
|
-
"id",
|
|
2534
|
-
"requestId",
|
|
2535
|
-
"type",
|
|
2536
|
-
"description",
|
|
2537
|
-
"owner",
|
|
2538
|
-
"tier",
|
|
2539
|
-
"tierSnapshot",
|
|
2540
|
-
"state",
|
|
2541
|
-
"createdAt",
|
|
2542
|
-
"updatedAt",
|
|
2543
|
-
"compactionCount",
|
|
2544
|
-
"terminal",
|
|
2545
|
-
]);
|
|
2546
|
-
if (Object.keys(data).some((key) => !allowedKeys.has(key))) return;
|
|
2547
|
-
if (data.schemaVersion !== 1 || (data.state !== "completed" && data.state !== "failed" && data.state !== "stopped"))
|
|
2548
|
-
return;
|
|
2549
|
-
const tier = data.tier;
|
|
2550
|
-
if (tier !== undefined && (typeof tier !== "string" || !WORKFLOW_TIER_VALUES.has(tier))) return;
|
|
2551
|
-
if (data.tierSnapshot !== undefined && (typeof tier !== "string" || !isValidTierSnapshot(data.tierSnapshot, tier)))
|
|
2552
|
-
return;
|
|
2553
|
-
const bounded = (value: unknown, max: number): string | undefined =>
|
|
2554
|
-
typeof value === "string" && value.length > 0 && value.length <= max && value.trim() === value
|
|
2555
|
-
? value
|
|
2556
|
-
: undefined;
|
|
2557
|
-
const spawnKey = bounded(data.spawnKey, 256);
|
|
2558
|
-
const id = bounded(data.id, 128);
|
|
2559
|
-
const fingerprint = bounded(data.fingerprint, 128);
|
|
2560
|
-
const requestId = bounded(data.requestId, 128);
|
|
2561
|
-
const type = bounded(data.type, 128);
|
|
2562
|
-
const description = bounded(data.description, 512);
|
|
2563
|
-
if (!spawnKey || !id || !fingerprint || !requestId || !type || !description || !isRecord(data.owner)) return;
|
|
2564
|
-
const ownerKeys = new Set(["extension", "runId", "nodeId", "attemptId"]);
|
|
2565
|
-
if (Object.keys(data.owner).some((key) => !ownerKeys.has(key))) return;
|
|
2566
|
-
const ownerRunId = bounded(data.owner.runId, 256);
|
|
2567
|
-
const ownerNodeId = bounded(data.owner.nodeId, 256);
|
|
2568
|
-
const ownerAttemptId = bounded(data.owner.attemptId, 256);
|
|
2569
|
-
if (data.owner.extension !== "pi-workflows" || !ownerRunId || !ownerNodeId || !ownerAttemptId) return;
|
|
2570
|
-
const run = this.runs.get(ownerRunId);
|
|
2571
|
-
if (!run || isTerminalWorkflow(run.status)) return;
|
|
2572
|
-
const isSynthesis = ownerNodeId === SYNTHESIS_NODE_ID;
|
|
2573
|
-
if (!isSynthesis && !(ownerNodeId in run.taskStatus)) return;
|
|
2574
|
-
if (isSynthesis && !run.definition.synthesis) return;
|
|
2575
|
-
const currentAttemptId = run.attemptIds[ownerNodeId];
|
|
2576
|
-
const generation = run.attempts[ownerNodeId];
|
|
2577
|
-
if (!currentAttemptId || !Number.isSafeInteger(generation) || generation < 1 || ownerAttemptId !== currentAttemptId)
|
|
2578
|
-
return;
|
|
2579
|
-
if (currentAttemptId !== `${run.runId}/${ownerNodeId}/attempt-${generation}`) return;
|
|
2580
|
-
const expectedSpawnKeys = new Set([
|
|
2581
|
-
currentAttemptId,
|
|
2582
|
-
`${run.runId}/${ownerNodeId}/${currentAttemptId}`,
|
|
2583
|
-
`${run.runId}/${ownerNodeId}/attempt-${generation}`,
|
|
2584
|
-
`${run.runId}:${ownerNodeId}:${currentAttemptId}`,
|
|
2585
|
-
]);
|
|
2586
|
-
if (!expectedSpawnKeys.has(spawnKey)) return;
|
|
2587
|
-
const currentAgentId = isSynthesis ? run.synthesisAgentId : run.agentIds[ownerNodeId];
|
|
2588
|
-
if (currentAgentId && currentAgentId !== id) return;
|
|
2589
|
-
if (isSynthesis ? run.synthesisResult : run.taskResults[ownerNodeId]) return;
|
|
2590
|
-
if (!isSynthesis && isTerminalTask(run.taskStatus[ownerNodeId])) return;
|
|
2591
|
-
|
|
2592
|
-
const terminal = data.terminal;
|
|
2593
|
-
if (!isRecord(terminal)) return;
|
|
2594
|
-
const terminalKeys = new Set([
|
|
2595
|
-
"status",
|
|
2596
|
-
"result",
|
|
2597
|
-
"error",
|
|
2598
|
-
"outputFile",
|
|
2599
|
-
"tokenCount",
|
|
2600
|
-
"compactionCount",
|
|
2601
|
-
"completedAt",
|
|
2602
|
-
]);
|
|
2603
|
-
if (Object.keys(terminal).some((key) => !terminalKeys.has(key))) return;
|
|
2604
|
-
if (terminal.status !== data.state) return;
|
|
2605
|
-
const resultText = terminal.result === undefined ? undefined : bounded(terminal.result, 8_000);
|
|
2606
|
-
const errorText = terminal.error === undefined ? undefined : bounded(terminal.error, 2_000);
|
|
2607
|
-
const outputFile = terminal.outputFile === undefined ? undefined : bounded(terminal.outputFile, 2_000);
|
|
2608
|
-
if (
|
|
2609
|
-
(terminal.result !== undefined && !resultText) ||
|
|
2610
|
-
(terminal.error !== undefined && !errorText) ||
|
|
2611
|
-
(terminal.outputFile !== undefined && !outputFile)
|
|
2612
|
-
)
|
|
2613
|
-
return;
|
|
2614
|
-
const compactionCount = terminal.compactionCount;
|
|
2615
|
-
const persistedCompactionCount = data.compactionCount;
|
|
2616
|
-
const completedAt = terminal.completedAt;
|
|
2617
|
-
const createdAt = data.createdAt;
|
|
2618
|
-
const updatedAt = data.updatedAt;
|
|
2619
|
-
if (
|
|
2620
|
-
typeof compactionCount !== "number" ||
|
|
2621
|
-
!Number.isSafeInteger(compactionCount) ||
|
|
2622
|
-
compactionCount < 0 ||
|
|
2623
|
-
typeof persistedCompactionCount !== "number" ||
|
|
2624
|
-
!Number.isSafeInteger(persistedCompactionCount) ||
|
|
2625
|
-
persistedCompactionCount !== compactionCount ||
|
|
2626
|
-
typeof completedAt !== "number" ||
|
|
2627
|
-
!Number.isFinite(completedAt) ||
|
|
2628
|
-
completedAt < 0 ||
|
|
2629
|
-
typeof createdAt !== "number" ||
|
|
2630
|
-
!Number.isFinite(createdAt) ||
|
|
2631
|
-
createdAt < 0 ||
|
|
2632
|
-
typeof updatedAt !== "number" ||
|
|
2633
|
-
!Number.isFinite(updatedAt) ||
|
|
2634
|
-
updatedAt < createdAt ||
|
|
2635
|
-
completedAt < createdAt ||
|
|
2636
|
-
completedAt > updatedAt
|
|
2637
|
-
)
|
|
2638
|
-
return;
|
|
2639
|
-
const tokenCount = terminal.tokenCount;
|
|
2640
|
-
if (
|
|
2641
|
-
tokenCount !== undefined &&
|
|
2642
|
-
(typeof tokenCount !== "number" || !Number.isSafeInteger(tokenCount) || tokenCount < 0)
|
|
2643
|
-
)
|
|
2644
|
-
return;
|
|
2645
|
-
const status = data.state === "completed" ? "completed" : data.state === "stopped" ? "stopped" : "failed";
|
|
2646
|
-
const result = resultFromLifecycle(
|
|
2647
|
-
status,
|
|
2648
|
-
id,
|
|
2649
|
-
resultText,
|
|
2650
|
-
errorText,
|
|
2651
|
-
compactionCount,
|
|
2652
|
-
completedAt,
|
|
2653
|
-
outputFile,
|
|
2654
|
-
tokenCount,
|
|
2655
|
-
);
|
|
2656
|
-
result.attemptId = currentAttemptId;
|
|
2657
|
-
return {
|
|
2658
|
-
kind: "terminal",
|
|
2659
|
-
runId: run.runId,
|
|
2660
|
-
nodeId: ownerNodeId,
|
|
2661
|
-
attemptId: currentAttemptId,
|
|
2662
|
-
result,
|
|
2663
|
-
owner: { extension: "pi-workflows", runId: run.runId, nodeId: ownerNodeId, attemptId: currentAttemptId },
|
|
2664
|
-
};
|
|
2665
|
-
}
|
|
2666
|
-
private reconcileSubagentRecord(
|
|
2667
|
-
entry: SessionEntryLike,
|
|
2668
|
-
): DeferredInterruptedRecord | DeferredTerminalRecord | undefined {
|
|
2669
|
-
if (entry.type !== "custom" || entry.customType !== "subagents:record" || !isRecord(entry.data)) return;
|
|
2670
|
-
const owner = ownerFor(entry.data);
|
|
2671
|
-
if (!owner) return;
|
|
2672
|
-
const run = this.runs.get(owner.runId);
|
|
2673
|
-
if (!run) return;
|
|
2674
|
-
if (isTerminalWorkflow(run.status)) return;
|
|
2675
|
-
if (owner.nodeId !== SYNTHESIS_NODE_ID && !(owner.nodeId in run.taskStatus)) return;
|
|
2676
|
-
|
|
2677
|
-
const currentAttemptId = run.attemptIds[owner.nodeId];
|
|
2678
|
-
const recordAgentId = typeof entry.data.id === "string" ? entry.data.id : undefined;
|
|
2679
|
-
if (!this.acceptsLifecycleEvent(run, owner.nodeId, "subagents:record", owner.attemptId, recordAgentId)) return;
|
|
2680
|
-
const currentAgentId = owner.nodeId === SYNTHESIS_NODE_ID ? run.synthesisAgentId : run.agentIds[owner.nodeId];
|
|
2681
|
-
const status = entry.data.status;
|
|
2682
|
-
if (
|
|
2683
|
-
status !== "completed" &&
|
|
2684
|
-
status !== "steered" &&
|
|
2685
|
-
status !== "error" &&
|
|
2686
|
-
status !== "stopped" &&
|
|
2687
|
-
status !== "aborted" &&
|
|
2688
|
-
status !== "interrupted"
|
|
2689
|
-
)
|
|
2690
|
-
return;
|
|
2691
|
-
|
|
2692
|
-
// An interrupted record is not a failed task result. It means the previous
|
|
2693
|
-
// AgentSession disappeared during reload. Legacy records without an
|
|
2694
|
-
// attemptId are accepted only while their agent id is still the current
|
|
2695
|
-
// journaled agent; after a recovery event clears that id, the same old
|
|
2696
|
-
// record is ignored on every later restore.
|
|
2697
|
-
if (status === "interrupted") {
|
|
2698
|
-
if (run.terminalIntent) return;
|
|
2699
|
-
if (!owner.attemptId && (!recordAgentId || !currentAgentId || recordAgentId !== currentAgentId)) return;
|
|
2700
|
-
if (owner.nodeId === SYNTHESIS_NODE_ID ? run.synthesisResult : run.taskResults[owner.nodeId]) return;
|
|
2701
|
-
return {
|
|
2702
|
-
kind: "interrupted",
|
|
2703
|
-
runId: run.runId,
|
|
2704
|
-
nodeId: owner.nodeId,
|
|
2705
|
-
oldAgentId: recordAgentId ?? currentAgentId ?? "",
|
|
2706
|
-
};
|
|
2707
|
-
}
|
|
2708
|
-
|
|
2709
|
-
const resultStatus =
|
|
2710
|
-
status === "completed" || status === "steered" ? "completed" : status === "stopped" ? "stopped" : "failed";
|
|
2711
|
-
if (owner.nodeId === SYNTHESIS_NODE_ID ? run.synthesisResult : run.taskResults[owner.nodeId]) return;
|
|
2712
|
-
if (!currentAttemptId) return;
|
|
2713
|
-
const result = resultFromLifecycle(
|
|
2714
|
-
resultStatus,
|
|
2715
|
-
recordAgentId ?? currentAgentId,
|
|
2716
|
-
entry.data.result,
|
|
2717
|
-
entry.data.error,
|
|
2718
|
-
run.compactions[owner.nodeId] ?? 0,
|
|
2719
|
-
Date.now(),
|
|
2720
|
-
entry.data.outputFile,
|
|
2721
|
-
typeof entry.data.tokens === "number" ? entry.data.tokens : undefined,
|
|
2722
|
-
);
|
|
2723
|
-
result.attemptId = currentAttemptId;
|
|
2724
|
-
return {
|
|
2725
|
-
kind: "terminal",
|
|
2726
|
-
runId: run.runId,
|
|
2727
|
-
nodeId: owner.nodeId,
|
|
2728
|
-
attemptId: currentAttemptId,
|
|
2729
|
-
result,
|
|
2730
|
-
owner: {
|
|
2731
|
-
extension: "pi-workflows",
|
|
2732
|
-
runId: run.runId,
|
|
2733
|
-
nodeId: owner.nodeId,
|
|
2734
|
-
attemptId: currentAttemptId,
|
|
2735
|
-
},
|
|
2736
|
-
};
|
|
2737
|
-
}
|
|
2738
|
-
|
|
2739
|
-
private resolveWaiters(run: WorkflowRun): void {
|
|
2740
|
-
const waiters = this.waiters.get(run.runId);
|
|
2741
|
-
if (!waiters) return;
|
|
2742
|
-
this.waiters.delete(run.runId);
|
|
2743
|
-
for (const waiter of waiters) {
|
|
2744
|
-
if (waiter.settled) continue;
|
|
2745
|
-
waiter.settled = true;
|
|
2746
|
-
waiter.cleanup();
|
|
2747
|
-
waiter.resolve(run);
|
|
2748
|
-
}
|
|
2749
|
-
}
|
|
2750
|
-
|
|
2751
|
-
private summary(run: WorkflowRun, includeDetails = false): WorkflowSummary {
|
|
2752
|
-
const statuses = Object.values(run.taskStatus);
|
|
2753
|
-
const completedTasks = statuses.filter((status) => status === "completed").length;
|
|
2754
|
-
const failedTasks = statuses.filter(
|
|
2755
|
-
(status) => status === "failed" || status === "blocked" || status === "stopped",
|
|
2756
|
-
).length;
|
|
2757
|
-
const activeTasks = statuses.filter(activeStatus).length;
|
|
2758
|
-
const phaseCompletion = run.definition.phases.map((phase) => {
|
|
2759
|
-
const phaseTasks = run.definition.tasks.filter((task) => task.phase === phase.id);
|
|
2760
|
-
return phaseTasks.length > 0 && phaseTasks.every((task) => run.taskStatus[task.id] === "completed");
|
|
2761
|
-
});
|
|
2762
|
-
const completedPhases = phaseCompletion.filter(Boolean).length;
|
|
2763
|
-
const tokenValues = [...Object.values(run.taskResults), ...(run.synthesisResult ? [run.synthesisResult] : [])]
|
|
2764
|
-
.map((result) => result.tokenCount)
|
|
2765
|
-
.filter((value): value is number => value !== undefined);
|
|
2766
|
-
const tokenCount = tokenValues.length > 0 ? tokenValues.reduce((total, value) => total + value, 0) : undefined;
|
|
2767
|
-
const taskDetails = includeDetails
|
|
2768
|
-
? run.definition.tasks.map((task) => ({
|
|
2769
|
-
id: task.id,
|
|
2770
|
-
status: run.taskStatus[task.id] ?? "pending",
|
|
2771
|
-
...(run.agentIds[task.id] ? { agentId: run.agentIds[task.id] } : {}),
|
|
2772
|
-
compactions: run.compactions[task.id] ?? 0,
|
|
2773
|
-
...resultPreview(run.taskResults[task.id]),
|
|
2774
|
-
}))
|
|
2775
|
-
: undefined;
|
|
2776
|
-
const synthesisDetails =
|
|
2777
|
-
includeDetails && run.synthesisResult
|
|
2778
|
-
? {
|
|
2779
|
-
status: run.synthesisResult.status,
|
|
2780
|
-
...(run.synthesisAgentId ? { agentId: run.synthesisAgentId } : {}),
|
|
2781
|
-
compactions: run.compactions[SYNTHESIS_NODE_ID] ?? 0,
|
|
2782
|
-
...resultPreview(run.synthesisResult),
|
|
2783
|
-
}
|
|
2784
|
-
: undefined;
|
|
2785
|
-
return {
|
|
2786
|
-
runId: run.runId,
|
|
2787
|
-
name: run.definition.name,
|
|
2788
|
-
...(run.definition.description ? { description: run.definition.description } : {}),
|
|
2789
|
-
status: run.status,
|
|
2790
|
-
startedAt: run.startedAt,
|
|
2791
|
-
updatedAt: run.updatedAt,
|
|
2792
|
-
elapsedMs: Math.max(
|
|
2793
|
-
0,
|
|
2794
|
-
(run.status === "completed" || run.status === "failed" || run.status === "stopped"
|
|
2795
|
-
? run.updatedAt
|
|
2796
|
-
: Date.now()) - run.startedAt,
|
|
2797
|
-
),
|
|
2798
|
-
taskCount: run.definition.tasks.length,
|
|
2799
|
-
phaseCount: run.definition.phases.length,
|
|
2800
|
-
completedPhases,
|
|
2801
|
-
completedTasks,
|
|
2802
|
-
failedTasks,
|
|
2803
|
-
activeTasks,
|
|
2804
|
-
...(tokenCount === undefined ? {} : { tokenCount }),
|
|
2805
|
-
...(run.synthesisResult?.text ? { synthesisPreview: run.synthesisResult.text.slice(0, 500) } : {}),
|
|
2806
|
-
...(run.error ? { error: run.error.slice(0, 2_000) } : {}),
|
|
2807
|
-
...(run.nonResumable ? { nonResumable: true } : {}),
|
|
2808
|
-
...(taskDetails ? { tasks: taskDetails } : {}),
|
|
2809
|
-
...(synthesisDetails ? { synthesis: synthesisDetails } : {}),
|
|
2810
|
-
};
|
|
1401
|
+
export class WorkflowWaitAbortedError extends Error {
|
|
1402
|
+
constructor() {
|
|
1403
|
+
super("workflow wait aborted");
|
|
1404
|
+
this.name = "WorkflowWaitAbortedError";
|
|
2811
1405
|
}
|
|
2812
1406
|
}
|
|
1407
|
+
|
|
1408
|
+
export type { ScriptRun, WorkflowOwner };
|
|
1409
|
+
export { createManagedSpawnClient, JOURNAL_ENTRY_TYPE, snapshotRun };
|