@yycholla/pi-dynamic-workflows 3.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +22 -0
- package/README.md +325 -0
- package/assets/readme/hero.png +0 -0
- package/assets/readme/hero.svg +70 -0
- package/assets/readme/package-cover.png +0 -0
- package/assets/readme/package-cover.svg +66 -0
- package/assets/readme/workflow.png +0 -0
- package/assets/readme/workflow.svg +77 -0
- package/dist/accept-workflow-guidance.d.ts +14 -0
- package/dist/accept-workflow-guidance.js +53 -0
- package/dist/adversarial-review.d.ts +30 -0
- package/dist/adversarial-review.js +107 -0
- package/dist/agent-history.d.ts +20 -0
- package/dist/agent-history.js +122 -0
- package/dist/agent-registry.d.ts +83 -0
- package/dist/agent-registry.js +190 -0
- package/dist/agent.d.ts +364 -0
- package/dist/agent.js +714 -0
- package/dist/builtin-commands.d.ts +19 -0
- package/dist/builtin-commands.js +251 -0
- package/dist/builtin-workflows.d.ts +45 -0
- package/dist/builtin-workflows.js +121 -0
- package/dist/code-review.d.ts +26 -0
- package/dist/code-review.js +181 -0
- package/dist/config.d.ts +37 -0
- package/dist/config.js +44 -0
- package/dist/deep-research.d.ts +30 -0
- package/dist/deep-research.js +124 -0
- package/dist/display.d.ts +134 -0
- package/dist/display.js +248 -0
- package/dist/effort-command.d.ts +28 -0
- package/dist/effort-command.js +68 -0
- package/dist/enums.d.ts +69 -0
- package/dist/enums.js +78 -0
- package/dist/errors.d.ts +113 -0
- package/dist/errors.js +140 -0
- package/dist/extension-reload.d.ts +37 -0
- package/dist/extension-reload.js +78 -0
- package/dist/fs-persistence.d.ts +63 -0
- package/dist/fs-persistence.js +102 -0
- package/dist/index.d.ts +57 -0
- package/dist/index.js +35 -0
- package/dist/logger.d.ts +21 -0
- package/dist/logger.js +66 -0
- package/dist/model-routing.d.ts +30 -0
- package/dist/model-routing.js +50 -0
- package/dist/model-spec.d.ts +29 -0
- package/dist/model-spec.js +252 -0
- package/dist/model-tier-config.d.ts +133 -0
- package/dist/model-tier-config.js +249 -0
- package/dist/run-persistence.d.ts +180 -0
- package/dist/run-persistence.js +294 -0
- package/dist/saved-commands.d.ts +28 -0
- package/dist/saved-commands.js +100 -0
- package/dist/shared-store.d.ts +98 -0
- package/dist/shared-store.js +212 -0
- package/dist/structured-output.d.ts +19 -0
- package/dist/structured-output.js +30 -0
- package/dist/task-panel.d.ts +61 -0
- package/dist/task-panel.js +422 -0
- package/dist/usage-limit-scheduler.d.ts +145 -0
- package/dist/usage-limit-scheduler.js +368 -0
- package/dist/web-tools.d.ts +20 -0
- package/dist/web-tools.js +120 -0
- package/dist/workflow-authoring-coverage.d.ts +70 -0
- package/dist/workflow-authoring-coverage.js +421 -0
- package/dist/workflow-authoring-reference.d.ts +20 -0
- package/dist/workflow-authoring-reference.js +156 -0
- package/dist/workflow-capability-contract.d.ts +131 -0
- package/dist/workflow-capability-contract.js +604 -0
- package/dist/workflow-commands.d.ts +18 -0
- package/dist/workflow-commands.js +260 -0
- package/dist/workflow-comprehension.d.ts +133 -0
- package/dist/workflow-comprehension.js +1321 -0
- package/dist/workflow-context-measurement.d.ts +72 -0
- package/dist/workflow-context-measurement.js +213 -0
- package/dist/workflow-control-tool.d.ts +30 -0
- package/dist/workflow-control-tool.js +176 -0
- package/dist/workflow-delivery-choice.d.ts +20 -0
- package/dist/workflow-delivery-choice.js +48 -0
- package/dist/workflow-editor.d.ts +93 -0
- package/dist/workflow-editor.js +363 -0
- package/dist/workflow-manager.d.ts +492 -0
- package/dist/workflow-manager.js +1124 -0
- package/dist/workflow-paths.d.ts +22 -0
- package/dist/workflow-paths.js +46 -0
- package/dist/workflow-release-gate.d.ts +39 -0
- package/dist/workflow-release-gate.js +309 -0
- package/dist/workflow-saved.d.ts +38 -0
- package/dist/workflow-saved.js +126 -0
- package/dist/workflow-settings.d.ts +70 -0
- package/dist/workflow-settings.js +131 -0
- package/dist/workflow-tool.d.ts +71 -0
- package/dist/workflow-tool.js +367 -0
- package/dist/workflow-ui.d.ts +182 -0
- package/dist/workflow-ui.js +1587 -0
- package/dist/workflow.d.ts +333 -0
- package/dist/workflow.js +1151 -0
- package/dist/workflows-models-command.d.ts +31 -0
- package/dist/workflows-models-command.js +156 -0
- package/dist/worktree.d.ts +25 -0
- package/dist/worktree.js +61 -0
- package/extensions/workflow.ts +151 -0
- package/package.json +104 -0
- package/skills/workflow-authoring/SKILL.md +30 -0
- package/skills/workflow-authoring/examples/adversarial-verification.js +63 -0
- package/skills/workflow-authoring/examples/bounded-semantic-retry.js +56 -0
- package/skills/workflow-authoring/examples/classify-and-act.js +67 -0
- package/skills/workflow-authoring/examples/fan-out-and-synthesize.js +47 -0
- package/skills/workflow-authoring/examples/generate-and-filter.js +68 -0
- package/skills/workflow-authoring/examples/loop-until-done.js +68 -0
- package/skills/workflow-authoring/examples/phased-budgets.js +75 -0
- package/skills/workflow-authoring/examples/saved-nested-workflows.js +44 -0
- package/skills/workflow-authoring/examples/structured-output.js +37 -0
- package/skills/workflow-authoring/examples/tournament.js +81 -0
- package/skills/workflow-authoring/examples/validated-gate.js +63 -0
- package/skills/workflow-authoring/references/capabilities.md +41 -0
- package/skills/workflow-authoring/references/capability-details.md +357 -0
- package/skills/workflow-authoring/references/common-helpers.md +4 -0
- package/skills/workflow-authoring/references/debugging.md +27 -0
- package/skills/workflow-authoring/references/focused-recipes.md +11 -0
- package/skills/workflow-authoring/references/helpers.md +9 -0
- package/skills/workflow-authoring/references/lifecycle.md +33 -0
- package/skills/workflow-authoring/references/pattern-selection.md +15 -0
- package/skills/workflow-authoring/references/quality-helpers.md +8 -0
- package/skills/workflow-authoring/references/registry-ownership.md +15 -0
- package/skills/workflow-authoring/references/retry-helper.md +5 -0
- package/skills/workflow-authoring/references/review.md +43 -0
- package/skills/workflow-authoring/references/runtime.md +27 -0
- package/skills/workflow-authoring/references/specialized-helpers.md +19 -0
- package/skills/workflow-authoring/references/versions.md +13 -0
- package/skills/workflow-patterns/SKILL.md +51 -0
- package/src/accept-workflow-guidance.ts +71 -0
- package/src/adversarial-review.ts +120 -0
- package/src/agent-history.ts +157 -0
- package/src/agent-registry.ts +221 -0
- package/src/agent.ts +929 -0
- package/src/builtin-commands.ts +286 -0
- package/src/builtin-workflows.ts +155 -0
- package/src/code-review.ts +183 -0
- package/src/config.ts +55 -0
- package/src/deep-research.ts +135 -0
- package/src/display.ts +367 -0
- package/src/effort-command.ts +87 -0
- package/src/enums.ts +77 -0
- package/src/errors.ts +199 -0
- package/src/extension-reload.ts +100 -0
- package/src/fs-persistence.ts +124 -0
- package/src/index.ts +176 -0
- package/src/logger.ts +88 -0
- package/src/model-routing.ts +73 -0
- package/src/model-spec.ts +309 -0
- package/src/model-tier-config.ts +296 -0
- package/src/run-persistence.ts +484 -0
- package/src/saved-commands.ts +115 -0
- package/src/shared-store.ts +228 -0
- package/src/structured-output.ts +47 -0
- package/src/task-panel.ts +490 -0
- package/src/usage-limit-scheduler.ts +432 -0
- package/src/web-tools.ts +124 -0
- package/src/workflow-authoring-coverage.ts +486 -0
- package/src/workflow-authoring-reference.ts +186 -0
- package/src/workflow-capability-contract.ts +806 -0
- package/src/workflow-commands.ts +287 -0
- package/src/workflow-comprehension.ts +1673 -0
- package/src/workflow-context-measurement.ts +262 -0
- package/src/workflow-control-tool.ts +238 -0
- package/src/workflow-delivery-choice.ts +69 -0
- package/src/workflow-editor.ts +444 -0
- package/src/workflow-manager.ts +1405 -0
- package/src/workflow-paths.ts +63 -0
- package/src/workflow-release-gate.ts +529 -0
- package/src/workflow-saved.ts +180 -0
- package/src/workflow-settings.ts +194 -0
- package/src/workflow-tool.ts +464 -0
- package/src/workflow-ui.ts +1789 -0
- package/src/workflow.ts +1615 -0
- package/src/workflows-models-command.ts +211 -0
- package/src/worktree.ts +76 -0
|
@@ -0,0 +1,1124 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Workflow manager for background execution, pause/resume, and run management.
|
|
3
|
+
*/
|
|
4
|
+
import { EventEmitter } from "node:events";
|
|
5
|
+
import { preview } from "./display.js";
|
|
6
|
+
import { isProviderUsageLimit, WorkflowError, WorkflowErrorCode } from "./errors.js";
|
|
7
|
+
import { createRunPersistence, generateRunId, } from "./run-persistence.js";
|
|
8
|
+
import { parseWorkflowScript, runWorkflow } from "./workflow.js";
|
|
9
|
+
/**
|
|
10
|
+
* Statuses in which a run's execution has genuinely settled — no promise is
|
|
11
|
+
* still pending, no lease is still held, nothing will asynchronously mutate
|
|
12
|
+
* this ManagedRun again. "paused" is deliberately excluded: both a manual
|
|
13
|
+
* pause() and a usage-limit checkpoint leave the run resumable and, from the
|
|
14
|
+
* in-memory-retention question's point of view, still "the run the user is
|
|
15
|
+
* looking at" — only completed/failed/aborted runs are eviction candidates.
|
|
16
|
+
* See the `runs` field doc comment for the full eviction lifecycle contract.
|
|
17
|
+
*/
|
|
18
|
+
const IN_MEMORY_TERMINAL_STATUSES = new Set(["completed", "failed", "aborted"]);
|
|
19
|
+
/**
|
|
20
|
+
* How many terminal (completed/failed/aborted) runs' full in-memory state
|
|
21
|
+
* (agents array, journal, snapshot, agentTimestamps) to retain in `runs`
|
|
22
|
+
* before the oldest is evicted. Kept small: a terminal run's data is fully
|
|
23
|
+
* on disk (run-persistence.ts) by the time it's eviction-eligible, so the
|
|
24
|
+
* in-memory copy exists only to serve a `getRun()`/`getSnapshot()` caller
|
|
25
|
+
* that wants the LIVE object (vs. listRuns()'s persisted view) for a run
|
|
26
|
+
* that *just* finished — a handful is enough for that; unbounded retention
|
|
27
|
+
* is exactly the leak this bounds (run-level analog of the subagent
|
|
28
|
+
* memory-retention mitigation in agent.ts).
|
|
29
|
+
*/
|
|
30
|
+
const DEFAULT_MAX_TERMINAL_RUNS_IN_MEMORY = 20;
|
|
31
|
+
export class WorkflowManager extends EventEmitter {
|
|
32
|
+
/**
|
|
33
|
+
* Lifecycle contract for `runs`:
|
|
34
|
+
*
|
|
35
|
+
* - An entry is added when a run starts (startInBackground/runSync) or is
|
|
36
|
+
* resumed (resume()), always with a live AbortController and (usually)
|
|
37
|
+
* an active RunLease.
|
|
38
|
+
* - While status is "running" or "paused", the entry is NEVER evicted —
|
|
39
|
+
* its execution could still settle (a pending executeRun() promise) or
|
|
40
|
+
* it is mid-usage-limit-checkpoint/manually-paused and still considered
|
|
41
|
+
* "the current state of this run" by callers. Eviction only ever
|
|
42
|
+
* considers an entry AFTER executeRun() has fully settled it to
|
|
43
|
+
* "completed" | "failed" | "aborted" (see IN_MEMORY_TERMINAL_STATUSES)
|
|
44
|
+
* and persisted + released its lease — i.e. strictly after the same
|
|
45
|
+
* isCurrent()-gated persistRun()/releaseRunLease() calls in
|
|
46
|
+
* executeRun()'s success/catch tails.
|
|
47
|
+
* - Once terminal, an entry becomes eviction-ELIGIBLE (recordTerminalRun())
|
|
48
|
+
* but is not necessarily evicted immediately: up to
|
|
49
|
+
* maxTerminalRunsInMemory terminal entries are kept, oldest evicted
|
|
50
|
+
* first, so a `getRun()` call immediately after completion (e.g. the
|
|
51
|
+
* "complete" event's own synchronous listeners — task-panel's result
|
|
52
|
+
* delivery, `/workflows watch`) still sees the live object. Once
|
|
53
|
+
* evicted, the entry is simply removed from `runs`; nothing else reads
|
|
54
|
+
* or writes it again.
|
|
55
|
+
* - Every caller of getRun()/getSnapshot() must treat "undefined"/null as
|
|
56
|
+
* "no live in-memory copy right now" and fall back to listRuns() (backed
|
|
57
|
+
* by run-persistence.ts, which is what's authoritative for a run once
|
|
58
|
+
* the in-memory copy is gone) — this mirrors how those callers already
|
|
59
|
+
* treat any run this process never had in memory (e.g. one started by a
|
|
60
|
+
* different process and only ever seen via listRuns()). resume() never
|
|
61
|
+
* depends on `runs` for a run's state either: it always reloads from
|
|
62
|
+
* persistence, so an evicted runId resumes exactly like one from a
|
|
63
|
+
* prior process.
|
|
64
|
+
* - isCurrent(managed) composes with eviction the same way it composes
|
|
65
|
+
* with resume()/deleteRun() replacing or removing an entry: eviction
|
|
66
|
+
* removes the map entry outright, so a stale execution's later settle
|
|
67
|
+
* (isCurrent() check) sees `this.runs.get(runId) !== managed` (in fact
|
|
68
|
+
* undefined) and correctly no-ops, exactly as it would after
|
|
69
|
+
* resume()/deleteRun().
|
|
70
|
+
*/
|
|
71
|
+
runs = new Map();
|
|
72
|
+
/**
|
|
73
|
+
* FIFO of runIds that reached IN_MEMORY_TERMINAL_STATUSES, oldest first —
|
|
74
|
+
* the eviction order for `runs` (see its doc comment). A runId can appear
|
|
75
|
+
* more than once (e.g. resumed after eviction, then terminates again);
|
|
76
|
+
* evicting is idempotent (recordTerminalRun() re-checks the CURRENT status
|
|
77
|
+
* of the current map entry for that id before deleting), so duplicates
|
|
78
|
+
* are harmless.
|
|
79
|
+
*/
|
|
80
|
+
terminalRunQueue = [];
|
|
81
|
+
maxTerminalRunsInMemory;
|
|
82
|
+
persistence;
|
|
83
|
+
cwd;
|
|
84
|
+
concurrency;
|
|
85
|
+
loadSavedWorkflow;
|
|
86
|
+
agent;
|
|
87
|
+
/** The session's main model (provider/id), for auto-tiering explore agents. */
|
|
88
|
+
mainModel;
|
|
89
|
+
/** The host Pi session's model registry, shared with subagents. */
|
|
90
|
+
modelRegistry;
|
|
91
|
+
/** The current pi session id; runs are stamped with it and listRuns() filters by it. */
|
|
92
|
+
sessionId;
|
|
93
|
+
defaultAgentTimeoutMs;
|
|
94
|
+
defaultAgentRetries;
|
|
95
|
+
defaultTokenBudget;
|
|
96
|
+
toolsets;
|
|
97
|
+
excludeSubagentTools;
|
|
98
|
+
persistAgentSessions;
|
|
99
|
+
constructor(options = {}) {
|
|
100
|
+
super();
|
|
101
|
+
this.cwd = options.cwd ?? process.cwd();
|
|
102
|
+
this.concurrency = options.concurrency ?? 8;
|
|
103
|
+
this.loadSavedWorkflow = options.loadSavedWorkflow;
|
|
104
|
+
this.agent = options.agent;
|
|
105
|
+
this.mainModel = options.mainModel;
|
|
106
|
+
this.modelRegistry = options.modelRegistry;
|
|
107
|
+
this.sessionId = options.sessionId;
|
|
108
|
+
this.defaultAgentTimeoutMs = options.defaultAgentTimeoutMs ?? null;
|
|
109
|
+
this.defaultAgentRetries = options.defaultAgentRetries ?? 0;
|
|
110
|
+
this.defaultTokenBudget = options.defaultTokenBudget ?? null;
|
|
111
|
+
this.toolsets = options.toolsets;
|
|
112
|
+
this.excludeSubagentTools = options.excludeSubagentTools;
|
|
113
|
+
this.persistAgentSessions = options.persistAgentSessions ?? false;
|
|
114
|
+
this.maxTerminalRunsInMemory = options.maxTerminalRunsInMemory ?? DEFAULT_MAX_TERMINAL_RUNS_IN_MEMORY;
|
|
115
|
+
this.persistence = createRunPersistence(this.cwd);
|
|
116
|
+
this.recoverStaleRuns();
|
|
117
|
+
}
|
|
118
|
+
/** Bind the manager to the current pi session, so new runs are tagged with it and
|
|
119
|
+
* the navigator/task-panel show only this session's runs (set on session_start). */
|
|
120
|
+
setSessionId(id) {
|
|
121
|
+
this.sessionId = id;
|
|
122
|
+
}
|
|
123
|
+
/**
|
|
124
|
+
* On startup, any persisted run still marked "running" belongs to a process
|
|
125
|
+
* that died mid-run (this fresh manager has it nowhere in memory). Reconcile it
|
|
126
|
+
* to "paused" — never "failed" — so its journal is preserved and resume() can
|
|
127
|
+
* replay the completed prefix and finish the rest.
|
|
128
|
+
*/
|
|
129
|
+
recoverStaleRuns() {
|
|
130
|
+
try {
|
|
131
|
+
for (const p of this.listAllRuns()) {
|
|
132
|
+
if (p.status === "running" && !this.runs.has(p.runId)) {
|
|
133
|
+
const lease = this.persistence.acquireRunLease(p.runId);
|
|
134
|
+
if (!lease)
|
|
135
|
+
continue;
|
|
136
|
+
try {
|
|
137
|
+
this.persistence.save({ ...p, status: "paused" });
|
|
138
|
+
}
|
|
139
|
+
finally {
|
|
140
|
+
this.persistence.releaseRunLease(lease);
|
|
141
|
+
}
|
|
142
|
+
}
|
|
143
|
+
}
|
|
144
|
+
}
|
|
145
|
+
catch {
|
|
146
|
+
// Recovery is best-effort; never let it block manager construction.
|
|
147
|
+
}
|
|
148
|
+
}
|
|
149
|
+
/**
|
|
150
|
+
* Refresh host configuration after Pi reloads the extension while retaining
|
|
151
|
+
* this manager's live runs, controllers, leases, and event listeners.
|
|
152
|
+
* Existing executions keep the options they captured at start; subsequent
|
|
153
|
+
* runs and resumes use these refreshed defaults.
|
|
154
|
+
*/
|
|
155
|
+
reconfigureAfterReload(options) {
|
|
156
|
+
this.concurrency = options.concurrency ?? 8;
|
|
157
|
+
this.loadSavedWorkflow = options.loadSavedWorkflow;
|
|
158
|
+
this.defaultAgentTimeoutMs = options.defaultAgentTimeoutMs ?? null;
|
|
159
|
+
this.defaultAgentRetries = options.defaultAgentRetries ?? 0;
|
|
160
|
+
this.defaultTokenBudget = options.defaultTokenBudget ?? null;
|
|
161
|
+
this.toolsets = options.toolsets;
|
|
162
|
+
this.excludeSubagentTools = options.excludeSubagentTools;
|
|
163
|
+
this.persistAgentSessions = options.persistAgentSessions ?? false;
|
|
164
|
+
}
|
|
165
|
+
/** Set the session's main model (provider/id). Used to auto-tier explore agents. */
|
|
166
|
+
setMainModel(spec) {
|
|
167
|
+
this.mainModel = spec;
|
|
168
|
+
}
|
|
169
|
+
/** Set the host session's model registry so subagents resolve models consistently. */
|
|
170
|
+
setModelRegistry(registry) {
|
|
171
|
+
this.modelRegistry = registry;
|
|
172
|
+
}
|
|
173
|
+
/**
|
|
174
|
+
* Expose the host session's model registry to integrations sharing this
|
|
175
|
+
* manager. Workflow execution reads the same registry internally.
|
|
176
|
+
*/
|
|
177
|
+
getModelRegistry() {
|
|
178
|
+
return this.modelRegistry;
|
|
179
|
+
}
|
|
180
|
+
/**
|
|
181
|
+
* Start a workflow in the background.
|
|
182
|
+
* Returns immediately with a run ID; the workflow executes asynchronously.
|
|
183
|
+
*/
|
|
184
|
+
startInBackground(script, args, exec = {}) {
|
|
185
|
+
const parsed = parseWorkflowScript(script);
|
|
186
|
+
const slug = parsed.meta.name
|
|
187
|
+
? parsed.meta.name
|
|
188
|
+
.toLowerCase()
|
|
189
|
+
.replace(/[^a-z0-9]+/g, "-")
|
|
190
|
+
.replace(/^-+|-+$/g, "")
|
|
191
|
+
.slice(0, 40) || "workflow"
|
|
192
|
+
: "";
|
|
193
|
+
const runId = slug ? `${slug}-${generateRunId()}` : generateRunId();
|
|
194
|
+
const controller = new AbortController();
|
|
195
|
+
const lease = this.persistence.acquireRunLease(runId);
|
|
196
|
+
if (!lease)
|
|
197
|
+
throw new Error(`Could not acquire workflow run lease for ${runId}`);
|
|
198
|
+
const managed = {
|
|
199
|
+
runId,
|
|
200
|
+
status: "running",
|
|
201
|
+
snapshot: {
|
|
202
|
+
name: parsed.meta.name,
|
|
203
|
+
description: parsed.meta.description,
|
|
204
|
+
phases: parsed.meta.phases?.map((p) => p.title) ?? [],
|
|
205
|
+
logs: [],
|
|
206
|
+
agents: [],
|
|
207
|
+
agentCount: 0,
|
|
208
|
+
runningCount: 0,
|
|
209
|
+
doneCount: 0,
|
|
210
|
+
errorCount: 0,
|
|
211
|
+
},
|
|
212
|
+
controller,
|
|
213
|
+
startedAt: new Date(),
|
|
214
|
+
script,
|
|
215
|
+
args,
|
|
216
|
+
journal: [],
|
|
217
|
+
background: true,
|
|
218
|
+
lease,
|
|
219
|
+
autoResume: exec.autoResume,
|
|
220
|
+
// Resolve the budget once at start and freeze it on the run (see
|
|
221
|
+
// ManagedRun.tokenBudget) so resume keeps start-time semantics.
|
|
222
|
+
tokenBudget: exec.tokenBudget !== undefined ? exec.tokenBudget : this.defaultTokenBudget,
|
|
223
|
+
toolset: exec.toolset,
|
|
224
|
+
// Same freeze-at-start pattern as tokenBudget, for the same reason: a
|
|
225
|
+
// resumed run must keep these values, not re-resolve against the
|
|
226
|
+
// manager's current defaults (see ManagedRun doc comments).
|
|
227
|
+
maxAgents: exec.maxAgents,
|
|
228
|
+
agentTimeoutMs: exec.agentTimeoutMs !== undefined ? exec.agentTimeoutMs : this.defaultAgentTimeoutMs,
|
|
229
|
+
concurrency: exec.concurrency !== undefined ? exec.concurrency : this.concurrency,
|
|
230
|
+
agentRetries: exec.agentRetries !== undefined ? exec.agentRetries : this.defaultAgentRetries,
|
|
231
|
+
agentTimestamps: new Map(),
|
|
232
|
+
agentsById: new Map(),
|
|
233
|
+
};
|
|
234
|
+
this.runs.set(runId, managed);
|
|
235
|
+
try {
|
|
236
|
+
// Persist initial state
|
|
237
|
+
this.persistence.save({
|
|
238
|
+
runId,
|
|
239
|
+
workflowName: parsed.meta.name,
|
|
240
|
+
script,
|
|
241
|
+
args,
|
|
242
|
+
sessionId: this.sessionId,
|
|
243
|
+
status: "running",
|
|
244
|
+
phases: managed.snapshot.phases,
|
|
245
|
+
agents: [],
|
|
246
|
+
logs: [],
|
|
247
|
+
startedAt: managed.startedAt.toISOString(),
|
|
248
|
+
updatedAt: managed.startedAt.toISOString(),
|
|
249
|
+
autoResume: managed.autoResume,
|
|
250
|
+
tokenBudget: managed.tokenBudget,
|
|
251
|
+
toolset: managed.toolset,
|
|
252
|
+
maxAgents: managed.maxAgents,
|
|
253
|
+
agentTimeoutMs: managed.agentTimeoutMs,
|
|
254
|
+
concurrency: managed.concurrency,
|
|
255
|
+
agentRetries: managed.agentRetries,
|
|
256
|
+
});
|
|
257
|
+
}
|
|
258
|
+
catch (err) {
|
|
259
|
+
this.releaseRunLease(managed);
|
|
260
|
+
this.runs.delete(runId);
|
|
261
|
+
throw err;
|
|
262
|
+
}
|
|
263
|
+
// Run workflow asynchronously.
|
|
264
|
+
// Attach a side-channel catch to prevent Node.js unhandled-rejection crashes
|
|
265
|
+
// when a workflow is aborted/paused/stopped — executeRun()'s catch block
|
|
266
|
+
// already records status/event/persist, but the promise still rejects.
|
|
267
|
+
// The original promise is returned so callers can await it in try/catch.
|
|
268
|
+
const promise = this.executeRun(managed, script, args, exec);
|
|
269
|
+
promise.catch(() => { });
|
|
270
|
+
return { runId, promise };
|
|
271
|
+
}
|
|
272
|
+
/**
|
|
273
|
+
* Execute a workflow synchronously (blocking) while still tracking it like a
|
|
274
|
+
* background run, so the `/workflows` navigator and the live task panel see it.
|
|
275
|
+
* `onProgress` fires on every progress event with the current snapshot, letting
|
|
276
|
+
* a caller (e.g. the workflow tool) drive its own inline display.
|
|
277
|
+
*/
|
|
278
|
+
async runSync(script, args, exec = {}) {
|
|
279
|
+
const managed = this.createManaged(script, args);
|
|
280
|
+
const lease = this.persistence.acquireRunLease(managed.runId);
|
|
281
|
+
if (!lease)
|
|
282
|
+
throw new Error(`Could not acquire workflow run lease for ${managed.runId}`);
|
|
283
|
+
managed.lease = lease;
|
|
284
|
+
managed.autoResume = exec.autoResume;
|
|
285
|
+
managed.tokenBudget = exec.tokenBudget !== undefined ? exec.tokenBudget : this.defaultTokenBudget;
|
|
286
|
+
managed.toolset = exec.toolset;
|
|
287
|
+
// Same freeze-at-start pattern as tokenBudget (see startInBackground/ManagedRun).
|
|
288
|
+
managed.maxAgents = exec.maxAgents;
|
|
289
|
+
managed.agentTimeoutMs = exec.agentTimeoutMs !== undefined ? exec.agentTimeoutMs : this.defaultAgentTimeoutMs;
|
|
290
|
+
managed.concurrency = exec.concurrency !== undefined ? exec.concurrency : this.concurrency;
|
|
291
|
+
managed.agentRetries = exec.agentRetries !== undefined ? exec.agentRetries : this.defaultAgentRetries;
|
|
292
|
+
this.runs.set(managed.runId, managed);
|
|
293
|
+
// Persist the initial state immediately so listRuns()/the task panel can see
|
|
294
|
+
// the run the moment it starts, not only after the first agent journals.
|
|
295
|
+
this.persistRun(managed);
|
|
296
|
+
return this.executeRun(managed, script, args, exec);
|
|
297
|
+
}
|
|
298
|
+
/** Build a fresh managed run with an empty snapshot. */
|
|
299
|
+
createManaged(script, args) {
|
|
300
|
+
const parsed = parseWorkflowScript(script);
|
|
301
|
+
const slug = parsed.meta.name
|
|
302
|
+
? parsed.meta.name
|
|
303
|
+
.toLowerCase()
|
|
304
|
+
.replace(/[^a-z0-9]+/g, "-")
|
|
305
|
+
.replace(/^-+|-+$/g, "")
|
|
306
|
+
.slice(0, 40) || "workflow"
|
|
307
|
+
: "";
|
|
308
|
+
const runId = slug ? `${slug}-${generateRunId()}` : generateRunId();
|
|
309
|
+
return {
|
|
310
|
+
runId,
|
|
311
|
+
status: "running",
|
|
312
|
+
snapshot: {
|
|
313
|
+
name: parsed.meta.name,
|
|
314
|
+
description: parsed.meta.description,
|
|
315
|
+
phases: parsed.meta.phases?.map((p) => p.title) ?? [],
|
|
316
|
+
logs: [],
|
|
317
|
+
agents: [],
|
|
318
|
+
agentCount: 0,
|
|
319
|
+
runningCount: 0,
|
|
320
|
+
doneCount: 0,
|
|
321
|
+
errorCount: 0,
|
|
322
|
+
},
|
|
323
|
+
controller: new AbortController(),
|
|
324
|
+
startedAt: new Date(),
|
|
325
|
+
script,
|
|
326
|
+
args,
|
|
327
|
+
journal: [],
|
|
328
|
+
background: false,
|
|
329
|
+
agentTimestamps: new Map(),
|
|
330
|
+
agentsById: new Map(),
|
|
331
|
+
};
|
|
332
|
+
}
|
|
333
|
+
async executeRun(managed, script, args, exec = {}) {
|
|
334
|
+
const { resumeJournal, maxAgents, agentTimeoutMs, externalSignal, onProgress, tokenBudget, concurrency, agentRetries, confirm, tools, initialTokenUsage, } = exec;
|
|
335
|
+
// maxAgents/agentTimeoutMs/concurrency/agentRetries were resolved (per-run
|
|
336
|
+
// value, else the manager default at the time) and frozen on the managed
|
|
337
|
+
// run at start/resume (see ManagedRun doc comments) — read them from there
|
|
338
|
+
// first, exactly like resolvedTokenBudget below, so a resumed run keeps the
|
|
339
|
+
// values it started with instead of re-resolving against the manager's
|
|
340
|
+
// CURRENT defaults. The exec.* fallbacks are a safety net for direct
|
|
341
|
+
// executeRun callers that skipped the start paths (same rationale as
|
|
342
|
+
// resolvedTokenBudget's tokenBudget fallback).
|
|
343
|
+
const resolvedMaxAgents = managed.maxAgents !== undefined ? managed.maxAgents : maxAgents;
|
|
344
|
+
const resolvedAgentTimeoutMs = managed.agentTimeoutMs !== undefined
|
|
345
|
+
? managed.agentTimeoutMs
|
|
346
|
+
: agentTimeoutMs !== undefined
|
|
347
|
+
? agentTimeoutMs
|
|
348
|
+
: this.defaultAgentTimeoutMs;
|
|
349
|
+
const resolvedConcurrency = managed.concurrency !== undefined ? managed.concurrency : (concurrency ?? this.concurrency);
|
|
350
|
+
const resolvedAgentRetries = managed.agentRetries !== undefined ? managed.agentRetries : (agentRetries ?? this.defaultAgentRetries);
|
|
351
|
+
// The budget was resolved (per-run value, else defaultTokenBudget) and frozen
|
|
352
|
+
// on the managed run at start/resume — read it from there so a resumed run
|
|
353
|
+
// keeps the budget it started with. exec.tokenBudget is a safety net for
|
|
354
|
+
// direct executeRun callers that skipped the start paths.
|
|
355
|
+
const resolvedTokenBudget = managed.tokenBudget !== undefined ? managed.tokenBudget : (tokenBudget ?? null);
|
|
356
|
+
// Explicit tools win for this execution; else re-resolve the run's persisted
|
|
357
|
+
// toolset tag (how a resumed /deep-research keeps its web tools); else the
|
|
358
|
+
// agent layer's default coding tools.
|
|
359
|
+
const resolvedTools = tools ?? (managed.toolset ? this.toolsets?.[managed.toolset]?.() : undefined);
|
|
360
|
+
// Gated the same way as this.emitLive() below (see isCurrent()) — a stale
|
|
361
|
+
// execution's progress callback would otherwise keep driving live UI
|
|
362
|
+
// (task panel, etc.) for a run that's been superseded or deleted.
|
|
363
|
+
const progress = () => {
|
|
364
|
+
if (this.isCurrent(managed))
|
|
365
|
+
onProgress?.(managed.snapshot);
|
|
366
|
+
};
|
|
367
|
+
// Let a host abort (e.g. Esc during a blocking tool call) cancel this run.
|
|
368
|
+
if (externalSignal) {
|
|
369
|
+
if (externalSignal.aborted)
|
|
370
|
+
managed.controller.abort();
|
|
371
|
+
else
|
|
372
|
+
externalSignal.addEventListener("abort", () => managed.controller.abort(), { once: true });
|
|
373
|
+
}
|
|
374
|
+
try {
|
|
375
|
+
const result = await runWorkflow(script, {
|
|
376
|
+
cwd: this.cwd,
|
|
377
|
+
args,
|
|
378
|
+
// Use the managed run's persisted id as the workflow runId so the value
|
|
379
|
+
// returned in result.runId matches the id that listRuns()/resume() use.
|
|
380
|
+
// Otherwise runWorkflow mints an ephemeral `run-<ts>` id and the sync
|
|
381
|
+
// path would surface a non-resumable id to the model.
|
|
382
|
+
runId: managed.runId,
|
|
383
|
+
agent: this.agent,
|
|
384
|
+
mainModel: this.mainModel,
|
|
385
|
+
modelRegistry: this.modelRegistry,
|
|
386
|
+
persistAgentSessions: this.persistAgentSessions,
|
|
387
|
+
signal: managed.controller.signal,
|
|
388
|
+
concurrency: resolvedConcurrency,
|
|
389
|
+
agentRetries: resolvedAgentRetries,
|
|
390
|
+
maxAgents: resolvedMaxAgents,
|
|
391
|
+
agentTimeoutMs: resolvedAgentTimeoutMs,
|
|
392
|
+
tokenBudget: resolvedTokenBudget,
|
|
393
|
+
tools: resolvedTools,
|
|
394
|
+
excludeTools: this.excludeSubagentTools,
|
|
395
|
+
confirm,
|
|
396
|
+
loadSavedWorkflow: this.loadSavedWorkflow,
|
|
397
|
+
resumeJournal,
|
|
398
|
+
resumeFromRunId: resumeJournal ? managed.runId : undefined,
|
|
399
|
+
// Seed the fresh SharedRuntime's spend counter from the persisted total
|
|
400
|
+
// (resume()) so the hard tokenBudget cap holds cumulatively across a
|
|
401
|
+
// pause/resume cycle instead of resetting to zero each time (see A2 —
|
|
402
|
+
// runWorkflow only applies this on the fresh-SharedRuntime branch, never
|
|
403
|
+
// overriding an inherited options.sharedRuntime from a nested workflow()).
|
|
404
|
+
initialTokenUsage,
|
|
405
|
+
// Retried-attempt spend (see WorkflowRunOptions.onRetrySpend and A2):
|
|
406
|
+
// recordTokens() in workflow.ts already folded this into
|
|
407
|
+
// shared.spent/tokenUsage, but onAgentEnd never sees a retried
|
|
408
|
+
// (non-final) attempt — fold it into the same persisted aggregate here
|
|
409
|
+
// so a run paused after a retry doesn't under-count against the budget.
|
|
410
|
+
onRetrySpend: (tokens) => {
|
|
411
|
+
this.accumulateTokenUsage(managed, tokens);
|
|
412
|
+
},
|
|
413
|
+
onAgentJournal: (entry) => {
|
|
414
|
+
// Append (crash-safe-ish): keep the latest entry per (runId, index)
|
|
415
|
+
// pair, then persist. Matching on index ALONE would let a nested
|
|
416
|
+
// workflow()'s callIndex-0 entry evict the parent's own
|
|
417
|
+
// callIndex-0 entry (and vice versa) — they're only distinguished
|
|
418
|
+
// by runId (see JournalEntry.runId). This is the high-frequency
|
|
419
|
+
// progress persist (fires once per completed agent, can burst
|
|
420
|
+
// under concurrency) — throttled (trailing edge). Every
|
|
421
|
+
// lifecycle-critical persist below (status transitions, run end,
|
|
422
|
+
// pause/resume/stop) still calls persistRun() directly and flushes this.
|
|
423
|
+
managed.journal = managed.journal.filter((e) => !(e.index === entry.index && e.runId === entry.runId));
|
|
424
|
+
managed.journal.push(entry);
|
|
425
|
+
this.schedulePersist(managed);
|
|
426
|
+
},
|
|
427
|
+
onLog: (message) => {
|
|
428
|
+
managed.snapshot.logs.push(message);
|
|
429
|
+
this.emitLive(managed, "log", { runId: managed.runId, message });
|
|
430
|
+
progress();
|
|
431
|
+
},
|
|
432
|
+
onPhase: (title) => {
|
|
433
|
+
managed.snapshot.currentPhase = title;
|
|
434
|
+
if (!managed.snapshot.phases.includes(title)) {
|
|
435
|
+
managed.snapshot.phases.push(title);
|
|
436
|
+
}
|
|
437
|
+
this.emitLive(managed, "phase", { runId: managed.runId, title });
|
|
438
|
+
progress();
|
|
439
|
+
},
|
|
440
|
+
onAgentStart: (event) => {
|
|
441
|
+
const id = managed.snapshot.agents.length + 1;
|
|
442
|
+
const agentSnapshot = {
|
|
443
|
+
id,
|
|
444
|
+
callId: event.id,
|
|
445
|
+
label: event.label,
|
|
446
|
+
phase: event.phase,
|
|
447
|
+
prompt: event.prompt,
|
|
448
|
+
status: "running",
|
|
449
|
+
model: event.model,
|
|
450
|
+
};
|
|
451
|
+
managed.snapshot.agents.push(agentSnapshot);
|
|
452
|
+
// Index by the call's unique id (never label — see agentsById's doc
|
|
453
|
+
// comment) so onAgentEnd/onAgentHistory can resolve back to exactly
|
|
454
|
+
// THIS entry even when a concurrent sibling shares its label.
|
|
455
|
+
managed.agentsById.set(event.id, agentSnapshot);
|
|
456
|
+
// Real per-agent start time, captured the moment the agent actually
|
|
457
|
+
// starts (not the run's startedAt) — see agentTimestamps.
|
|
458
|
+
managed.agentTimestamps.set(id, { startedAt: new Date().toISOString() });
|
|
459
|
+
this.emitLive(managed, "agentStart", { runId: managed.runId, ...event });
|
|
460
|
+
progress();
|
|
461
|
+
},
|
|
462
|
+
onAgentEnd: (event) => {
|
|
463
|
+
const agent = managed.agentsById.get(event.id);
|
|
464
|
+
if (agent) {
|
|
465
|
+
agent.status = event.result === null ? "error" : "done";
|
|
466
|
+
// Keep the full value for the interactive pager; compact surfaces
|
|
467
|
+
// continue to use resultPreview.
|
|
468
|
+
agent.result = event.result;
|
|
469
|
+
agent.resultPreview = preview(event.result);
|
|
470
|
+
agent.error = event.error;
|
|
471
|
+
agent.errorCode = event.errorCode;
|
|
472
|
+
agent.recoverable = event.recoverable;
|
|
473
|
+
agent.tokens = event.tokens;
|
|
474
|
+
if (event.tokenUsage)
|
|
475
|
+
agent.tokenUsage = event.tokenUsage;
|
|
476
|
+
if (event.model)
|
|
477
|
+
agent.model = event.model;
|
|
478
|
+
// Real per-agent end time — only terminal agents get one; a still-
|
|
479
|
+
// running agent's entry keeps endedAt undefined.
|
|
480
|
+
const ts = managed.agentTimestamps.get(agent.id);
|
|
481
|
+
if (ts)
|
|
482
|
+
ts.endedAt = new Date().toISOString();
|
|
483
|
+
}
|
|
484
|
+
// Progressive run-wide token aggregate (A2): workflow.ts's onTokenUsage
|
|
485
|
+
// callback below fires exactly once, only when the whole script finishes
|
|
486
|
+
// successfully (a deliberate, tested contract — see
|
|
487
|
+
// "agent() accumulates usage across multiple agents" in agent.test.ts,
|
|
488
|
+
// which asserts one final event, not one per agent). A run that
|
|
489
|
+
// pauses/aborts/fails mid-flight never reaches it, so without tracking
|
|
490
|
+
// it here too, a paused run's persisted tokenUsage would stay whatever
|
|
491
|
+
// it was (usually unset) — starving resume()'s spend-seeding of the
|
|
492
|
+
// very data it needs. Accumulate additively from every onAgentEnd
|
|
493
|
+
// instead: a cache-hit replay reports tokens: 0 (see agent()'s replay
|
|
494
|
+
// branch in workflow.ts), so replaying the unchanged prefix on resume
|
|
495
|
+
// is a no-op add here, matching the "already historically spent, don't
|
|
496
|
+
// double-count" semantics of journal replay.
|
|
497
|
+
this.accumulateTokenUsage(managed, event.tokens ?? 0, event.tokenUsage);
|
|
498
|
+
this.emitLive(managed, "agentEnd", { runId: managed.runId, ...event });
|
|
499
|
+
progress();
|
|
500
|
+
},
|
|
501
|
+
onAgentHistory: (event) => {
|
|
502
|
+
const agent = managed.agentsById.get(event.id);
|
|
503
|
+
if (agent) {
|
|
504
|
+
agent.history = event.history;
|
|
505
|
+
}
|
|
506
|
+
this.emitLive(managed, "agentHistory", { runId: managed.runId, agentId: agent?.id, ...event });
|
|
507
|
+
progress();
|
|
508
|
+
},
|
|
509
|
+
onTokenUsage: (usage) => {
|
|
510
|
+
managed.snapshot.tokenUsage = usage;
|
|
511
|
+
this.emitLive(managed, "tokenUsage", { runId: managed.runId, usage });
|
|
512
|
+
progress();
|
|
513
|
+
},
|
|
514
|
+
});
|
|
515
|
+
managed.status = "completed";
|
|
516
|
+
managed.result = result;
|
|
517
|
+
// Gated the same way as disk/lease below (see emitLive()): a stale
|
|
518
|
+
// execution's "complete" would otherwise still deliver a result for a
|
|
519
|
+
// run that's been superseded or deleted (e.g. background result
|
|
520
|
+
// delivery into the conversation) even though it's no longer current.
|
|
521
|
+
this.emitLive(managed, "complete", { runId: managed.runId, result });
|
|
522
|
+
// Persist final state. persistRun()/writeRunToDisk() already no-op if
|
|
523
|
+
// `managed` has been superseded (resume()/deleteRun() took over this
|
|
524
|
+
// runId) — see isCurrent(). Guard the lease release the same way: a
|
|
525
|
+
// stale execution settling after resume() has already acquired a NEW
|
|
526
|
+
// lease for this runId must not touch that newer lease's bookkeeping.
|
|
527
|
+
this.persistRun(managed);
|
|
528
|
+
if (this.isCurrent(managed)) {
|
|
529
|
+
this.releaseRunLease(managed);
|
|
530
|
+
// Now (and only now — after the run's data is safely on disk and its
|
|
531
|
+
// lease released) does this run become eviction-eligible; see the
|
|
532
|
+
// `runs` field doc comment.
|
|
533
|
+
this.recordTerminalRun(managed.runId);
|
|
534
|
+
}
|
|
535
|
+
return result;
|
|
536
|
+
}
|
|
537
|
+
catch (error) {
|
|
538
|
+
const workflowError = error instanceof WorkflowError
|
|
539
|
+
? error
|
|
540
|
+
: new WorkflowError(error instanceof Error ? error.message : String(error), WorkflowErrorCode.WORKFLOW_ABORTED, { recoverable: true });
|
|
541
|
+
const usageLimitPaused = !managed.controller.signal.aborted && isProviderUsageLimit(workflowError);
|
|
542
|
+
if (managed.controller.signal.aborted) {
|
|
543
|
+
// Intentional abort (pause/stop/Esc) — preserve status set by pause()/stop()
|
|
544
|
+
if (managed.status === "running") {
|
|
545
|
+
managed.status = "aborted";
|
|
546
|
+
}
|
|
547
|
+
}
|
|
548
|
+
else if (usageLimitPaused) {
|
|
549
|
+
// Provider quota/usage limit: NOT a failure. Checkpoint the run as paused so
|
|
550
|
+
// the persisted journal (completed agent results) is replayed by resume()
|
|
551
|
+
// once the budget refills — instead of the user starting from scratch.
|
|
552
|
+
managed.status = "paused";
|
|
553
|
+
}
|
|
554
|
+
else {
|
|
555
|
+
managed.status = "failed";
|
|
556
|
+
}
|
|
557
|
+
managed.error = workflowError;
|
|
558
|
+
// Both branches gated via emitLive() (see its doc comment) — a stale
|
|
559
|
+
// execution's "paused"/"error" is equally misleading once superseded.
|
|
560
|
+
if (usageLimitPaused) {
|
|
561
|
+
this.emitLive(managed, "paused", {
|
|
562
|
+
runId: managed.runId,
|
|
563
|
+
reason: "usage_limit",
|
|
564
|
+
error: workflowError,
|
|
565
|
+
resetHint: workflowError.resetHint,
|
|
566
|
+
});
|
|
567
|
+
}
|
|
568
|
+
else if (this.listenerCount("error") > 0) {
|
|
569
|
+
// Guarded: EventEmitter throws on an unlistened "error" emit, which
|
|
570
|
+
// would abort this catch block mid-way — skipping the final persist,
|
|
571
|
+
// the lease release, and the real error rethrow below.
|
|
572
|
+
this.emitLive(managed, "error", { runId: managed.runId, error: workflowError });
|
|
573
|
+
}
|
|
574
|
+
// Persist final state (see the success-path comment above for the
|
|
575
|
+
// isCurrent() rationale — same guard, same reason).
|
|
576
|
+
this.persistRun(managed);
|
|
577
|
+
if (this.isCurrent(managed)) {
|
|
578
|
+
this.releaseRunLease(managed);
|
|
579
|
+
// "paused" (manual pause() or a usage-limit checkpoint) is
|
|
580
|
+
// deliberately NOT eviction-eligible — only a genuinely settled
|
|
581
|
+
// terminal status is (see IN_MEMORY_TERMINAL_STATUSES / the `runs`
|
|
582
|
+
// field doc comment). recordTerminalRun() itself re-checks this too,
|
|
583
|
+
// but skip the call entirely here so a paused run never even enters
|
|
584
|
+
// the eviction queue.
|
|
585
|
+
if (IN_MEMORY_TERMINAL_STATUSES.has(managed.status))
|
|
586
|
+
this.recordTerminalRun(managed.runId);
|
|
587
|
+
}
|
|
588
|
+
throw workflowError;
|
|
589
|
+
}
|
|
590
|
+
}
|
|
591
|
+
/**
|
|
592
|
+
* True when `managed` is still the live, current entry for its runId in
|
|
593
|
+
* `this.runs` — false once resume() has replaced it with a new ManagedRun
|
|
594
|
+
* object for the same runId, or deleteRun() has removed it entirely. A
|
|
595
|
+
* superseded ManagedRun's async completion (executeRun's promise settling
|
|
596
|
+
* well after something else already took over or tore down that runId)
|
|
597
|
+
* must not write to disk or touch lease state on the newer execution's
|
|
598
|
+
* behalf — see writeRunToDisk() and executeRun()'s post-await persist calls.
|
|
599
|
+
*/
|
|
600
|
+
isCurrent(managed) {
|
|
601
|
+
return this.runs.get(managed.runId) === managed;
|
|
602
|
+
}
|
|
603
|
+
/**
|
|
604
|
+
* Emit an event on behalf of `managed`, but only while it's still the
|
|
605
|
+
* current entry for its runId (see isCurrent()) — mirrors the disk/lease
|
|
606
|
+
* guard for the observer-facing side of the same problem. A superseded
|
|
607
|
+
* execution's progress/terminal events (log, phase, agentStart/End,
|
|
608
|
+
* tokenUsage, complete, error, paused) are not just stale-but-harmless:
|
|
609
|
+
* "complete" in particular can drive background result delivery into the
|
|
610
|
+
* conversation, so letting a deleted/superseded run's stale settle still
|
|
611
|
+
* fire it would deliver a result for a run that, from the caller's POV, no
|
|
612
|
+
* longer exists (or has since been superseded by a newer execution whose
|
|
613
|
+
* own events already tell the true story). No event in this set has a
|
|
614
|
+
* legitimate reason to still reach listeners once superseded — unlike
|
|
615
|
+
* disk writes there's no "expected race, harmless no-op" nuance here, it's
|
|
616
|
+
* simply wrong to notify twice (or for a run that's gone). Events emitted
|
|
617
|
+
* directly by pause()/stop()/resume()/deleteRun() themselves are NOT routed
|
|
618
|
+
* through this helper — those methods own the transition and ARE current
|
|
619
|
+
* at the moment they fire, same precedent as their persist/lease calls.
|
|
620
|
+
*/
|
|
621
|
+
emitLive(managed, event, payload) {
|
|
622
|
+
if (this.isCurrent(managed))
|
|
623
|
+
this.emit(event, payload);
|
|
624
|
+
}
|
|
625
|
+
/**
|
|
626
|
+
* Mark `runId` as eviction-eligible now that its execution has genuinely
|
|
627
|
+
* settled to a terminal status (completed/failed/aborted — see
|
|
628
|
+
* IN_MEMORY_TERMINAL_STATUSES), and evict the oldest eligible entries
|
|
629
|
+
* beyond maxTerminalRunsInMemory. Callers must only invoke this after the
|
|
630
|
+
* same isCurrent()-gated persistRun()/releaseRunLease() sequence executeRun()
|
|
631
|
+
* already uses (see the `runs` field doc comment for the full contract) —
|
|
632
|
+
* this method itself re-validates the CURRENT entry's status before
|
|
633
|
+
* deleting anything, so it never evicts a run that isn't (or is no longer)
|
|
634
|
+
* genuinely terminal, including one resumed back to "running" after being
|
|
635
|
+
* queued here but before its turn to be evicted came up.
|
|
636
|
+
*/
|
|
637
|
+
recordTerminalRun(runId) {
|
|
638
|
+
this.terminalRunQueue.push(runId);
|
|
639
|
+
while (this.terminalRunQueue.length > this.maxTerminalRunsInMemory) {
|
|
640
|
+
const oldest = this.terminalRunQueue.shift();
|
|
641
|
+
if (oldest === undefined)
|
|
642
|
+
break;
|
|
643
|
+
const current = this.runs.get(oldest);
|
|
644
|
+
// Re-check the CURRENT entry for this id (not the ManagedRun object
|
|
645
|
+
// that was terminal when queued) — resume() may have since replaced
|
|
646
|
+
// it with a fresh, live execution, which must never be evicted here.
|
|
647
|
+
if (current && IN_MEMORY_TERMINAL_STATUSES.has(current.status)) {
|
|
648
|
+
this.runs.delete(oldest);
|
|
649
|
+
}
|
|
650
|
+
}
|
|
651
|
+
}
|
|
652
|
+
/**
|
|
653
|
+
* Additively fold one agent-call's token cost into the run-wide persisted
|
|
654
|
+
* aggregate (managed.snapshot.tokenUsage), seeded (on resume) from the
|
|
655
|
+
* persisted total-at-pause — see A2. Shared by onAgentEnd (a completed or
|
|
656
|
+
* finally-failed agent call) and onRetrySpend (a failed attempt that WILL
|
|
657
|
+
* be retried, whose cost recordTokens() already folded into
|
|
658
|
+
* shared.spent/tokenUsage in workflow.ts, but which onAgentEnd never sees —
|
|
659
|
+
* see WorkflowRunOptions.onRetrySpend for why that needs its own channel).
|
|
660
|
+
*/
|
|
661
|
+
accumulateTokenUsage(managed, tokens, tokenUsage) {
|
|
662
|
+
const prior = managed.snapshot.tokenUsage;
|
|
663
|
+
const usage = {
|
|
664
|
+
input: prior?.input ?? 0,
|
|
665
|
+
output: prior?.output ?? 0,
|
|
666
|
+
total: prior?.total ?? 0,
|
|
667
|
+
cost: prior?.cost ?? 0,
|
|
668
|
+
cacheRead: prior?.cacheRead ?? 0,
|
|
669
|
+
cacheWrite: prior?.cacheWrite ?? 0,
|
|
670
|
+
};
|
|
671
|
+
usage.total += tokens;
|
|
672
|
+
if (tokenUsage) {
|
|
673
|
+
usage.input += tokenUsage.input;
|
|
674
|
+
usage.output += tokenUsage.output;
|
|
675
|
+
usage.cost += tokenUsage.cost;
|
|
676
|
+
usage.cacheRead += tokenUsage.cacheRead;
|
|
677
|
+
usage.cacheWrite += tokenUsage.cacheWrite;
|
|
678
|
+
}
|
|
679
|
+
managed.snapshot.tokenUsage = usage;
|
|
680
|
+
}
|
|
681
|
+
releaseRunLease(managed) {
|
|
682
|
+
if (!managed.lease)
|
|
683
|
+
return;
|
|
684
|
+
this.persistence.releaseRunLease(managed.lease);
|
|
685
|
+
managed.lease = undefined;
|
|
686
|
+
}
|
|
687
|
+
/** Trailing-edge throttle window for high-frequency progress persists (see schedulePersist). */
|
|
688
|
+
static PERSIST_THROTTLE_MS = 400;
|
|
689
|
+
/** Pending trailing-edge persist timers for high-frequency progress events, keyed by runId. */
|
|
690
|
+
persistTimers = new Map();
|
|
691
|
+
/**
|
|
692
|
+
* Coalesce rapid progress persists (currently: onAgentJournal, which fires
|
|
693
|
+
* once per completed agent and can burst under concurrency) to at most one
|
|
694
|
+
* disk write per PERSIST_THROTTLE_MS (trailing edge) instead of one write
|
|
695
|
+
* per tick — persistRun() does a full JSON.stringify of the run plus up to
|
|
696
|
+
* 3 sync writes, so firing it once per agent in a long run is O(N^2).
|
|
697
|
+
*
|
|
698
|
+
* Lifecycle-critical writes (status transitions, run end, pause/resume/stop)
|
|
699
|
+
* must NOT use this — call persistRun() directly, which flushes (and cancels)
|
|
700
|
+
* any pending timer first so a stale trailing write can never fire after, and
|
|
701
|
+
* resurrect, a terminal state.
|
|
702
|
+
*/
|
|
703
|
+
schedulePersist(managed) {
|
|
704
|
+
if (this.persistTimers.has(managed.runId))
|
|
705
|
+
return; // already scheduled; the trailing write reads live state
|
|
706
|
+
const timer = setTimeout(() => {
|
|
707
|
+
this.persistTimers.delete(managed.runId);
|
|
708
|
+
this.writeRunToDisk(managed);
|
|
709
|
+
}, WorkflowManager.PERSIST_THROTTLE_MS);
|
|
710
|
+
// A pending progress persist should never keep the process alive on its own.
|
|
711
|
+
timer.unref?.();
|
|
712
|
+
this.persistTimers.set(managed.runId, timer);
|
|
713
|
+
}
|
|
714
|
+
/**
|
|
715
|
+
* Persist immediately and synchronously. Cancels any pending throttled write
|
|
716
|
+
* for this run first, so the write that lands is always the caller's current
|
|
717
|
+
* (final) state — never superseded by a stale deferred write. Use this for
|
|
718
|
+
* every lifecycle-critical persist: run start, status transitions, run end,
|
|
719
|
+
* pause()/resume()/stop().
|
|
720
|
+
*/
|
|
721
|
+
persistRun(managed) {
|
|
722
|
+
// A superseded execution's persist call must not touch the CURRENT
|
|
723
|
+
// execution's pending-timer bookkeeping for this runId (see isCurrent()).
|
|
724
|
+
// writeRunToDisk() below re-checks this too (it's the sole choke point
|
|
725
|
+
// schedulePersist()'s deferred timer also funnels through), so this is a
|
|
726
|
+
// belt-and-suspenders early-out specifically for the timer-clearing side
|
|
727
|
+
// effect, which writeRunToDisk() alone wouldn't prevent.
|
|
728
|
+
if (!this.isCurrent(managed))
|
|
729
|
+
return;
|
|
730
|
+
const timer = this.persistTimers.get(managed.runId);
|
|
731
|
+
if (timer) {
|
|
732
|
+
clearTimeout(timer);
|
|
733
|
+
this.persistTimers.delete(managed.runId);
|
|
734
|
+
}
|
|
735
|
+
this.writeRunToDisk(managed);
|
|
736
|
+
}
|
|
737
|
+
writeRunToDisk(managed) {
|
|
738
|
+
// The sole choke point for every disk write (both persistRun()'s direct
|
|
739
|
+
// calls and schedulePersist()'s deferred timer funnel through here) — skip
|
|
740
|
+
// silently when `managed` is no longer the current entry for its runId
|
|
741
|
+
// (see isCurrent()). This is an expected race outcome (resume() replaced
|
|
742
|
+
// it, or deleteRun() removed it), not an error: writing anyway would
|
|
743
|
+
// resurrect a torn-down run's file, or clobber a newer execution's
|
|
744
|
+
// in-progress/completed state with this stale one's.
|
|
745
|
+
//
|
|
746
|
+
// This check is redundant with persistRun()'s own early-return for every
|
|
747
|
+
// CURRENT call site — it earns its keep solely for schedulePersist()'s
|
|
748
|
+
// deferred setTimeout callback, the one path into this method that skips
|
|
749
|
+
// persistRun() entirely. That callback only fires from onAgentJournal, and
|
|
750
|
+
// onAgentJournal only fires for a call that got PAST agent()'s
|
|
751
|
+
// throwIfAborted() check (see workflow.ts) — which, since run-fatal abort
|
|
752
|
+
// (SharedRuntime.runFatalController) now seals every top-level run's
|
|
753
|
+
// shared runtime the instant any error escapes it uncaught, means a
|
|
754
|
+
// genuinely superseded-but-never-aborted execution (the only kind that
|
|
755
|
+
// could previously still journal a stray call after resume() replaced it)
|
|
756
|
+
// is structurally impossible to construct anymore — see the "unreachable
|
|
757
|
+
// defense-in-depth (#2)" test in workflow-manager.test.ts for the worked
|
|
758
|
+
// example and its own note. This check is KEPT anyway: it costs nothing,
|
|
759
|
+
// and removing it would silently reopen a stale-write path the moment any
|
|
760
|
+
// future change (e.g. a new way to journal without throwIfAborted()'s
|
|
761
|
+
// gate) reintroduces a producer for it.
|
|
762
|
+
if (!this.isCurrent(managed))
|
|
763
|
+
return;
|
|
764
|
+
try {
|
|
765
|
+
// Resumable states need their journal; completed/aborted states need rich
|
|
766
|
+
// agent details. Persist exactly one full copy of each agent result instead
|
|
767
|
+
// of writing it to both agents[].result and journal[].result.
|
|
768
|
+
const keepsResumeJournal = managed.status !== "completed" && managed.status !== "aborted";
|
|
769
|
+
this.persistence.save({
|
|
770
|
+
runId: managed.runId,
|
|
771
|
+
workflowName: managed.snapshot.name,
|
|
772
|
+
// Persist the real script + journal so the run can be resumed. Runs live
|
|
773
|
+
// in workflow run storage — protect via directory permissions, not blanking.
|
|
774
|
+
script: managed.script,
|
|
775
|
+
args: managed.args,
|
|
776
|
+
sessionId: this.sessionId,
|
|
777
|
+
journal: keepsResumeJournal ? managed.journal : undefined,
|
|
778
|
+
status: managed.status,
|
|
779
|
+
// Persisted every write (not just at pause) so a stale read during the
|
|
780
|
+
// "paused" event race (see UsageLimitScheduler) is still correct — this
|
|
781
|
+
// is fixed at run-start and doesn't change over the run's lifetime.
|
|
782
|
+
autoResume: managed.autoResume,
|
|
783
|
+
// Start-time execution context, re-read by resume() (see ManagedRun).
|
|
784
|
+
tokenBudget: managed.tokenBudget,
|
|
785
|
+
toolset: managed.toolset,
|
|
786
|
+
maxAgents: managed.maxAgents,
|
|
787
|
+
agentTimeoutMs: managed.agentTimeoutMs,
|
|
788
|
+
concurrency: managed.concurrency,
|
|
789
|
+
agentRetries: managed.agentRetries,
|
|
790
|
+
// Why a usage-limit pause happened, so the navigator / a future cold start
|
|
791
|
+
// can show it and (eventually) re-arm resume after the budget refills.
|
|
792
|
+
pauseReason: managed.status === "paused" && isProviderUsageLimit(managed.error) ? "usage_limit" : undefined,
|
|
793
|
+
resetHint: managed.status === "paused" && isProviderUsageLimit(managed.error) ? managed.error.resetHint : undefined,
|
|
794
|
+
phases: managed.snapshot.phases,
|
|
795
|
+
currentPhase: managed.snapshot.currentPhase,
|
|
796
|
+
// Real per-agent timestamps only (see agentTimestamps) — never the run's
|
|
797
|
+
// own startedAt or "now" stamped onto every agent on every write. A
|
|
798
|
+
// still-running agent is persisted with no endedAt.
|
|
799
|
+
agents: managed.snapshot.agents.map((a) => {
|
|
800
|
+
const { result, ...summary } = a;
|
|
801
|
+
const ts = managed.agentTimestamps.get(a.id);
|
|
802
|
+
return {
|
|
803
|
+
...summary,
|
|
804
|
+
// Live runs keep the rich value in memory. Cold resumable runs use
|
|
805
|
+
// the journal and retain resultPreview until replay reconstructs it.
|
|
806
|
+
...(keepsResumeJournal || result === undefined ? {} : { result }),
|
|
807
|
+
startedAt: ts?.startedAt,
|
|
808
|
+
endedAt: ts?.endedAt,
|
|
809
|
+
};
|
|
810
|
+
}),
|
|
811
|
+
logs: managed.snapshot.logs,
|
|
812
|
+
result: managed.result?.result,
|
|
813
|
+
tokenUsage: managed.snapshot.tokenUsage
|
|
814
|
+
? {
|
|
815
|
+
input: managed.snapshot.tokenUsage.input,
|
|
816
|
+
output: managed.snapshot.tokenUsage.output,
|
|
817
|
+
total: managed.snapshot.tokenUsage.total,
|
|
818
|
+
cost: managed.snapshot.tokenUsage.cost,
|
|
819
|
+
cacheRead: managed.snapshot.tokenUsage.cacheRead,
|
|
820
|
+
cacheWrite: managed.snapshot.tokenUsage.cacheWrite,
|
|
821
|
+
}
|
|
822
|
+
: undefined,
|
|
823
|
+
startedAt: managed.startedAt.toISOString(),
|
|
824
|
+
updatedAt: new Date().toISOString(),
|
|
825
|
+
completedAt: managed.status === "completed" ? new Date().toISOString() : undefined,
|
|
826
|
+
durationMs: managed.result?.durationMs,
|
|
827
|
+
});
|
|
828
|
+
}
|
|
829
|
+
catch (err) {
|
|
830
|
+
// Persistence is best-effort: the run is still healthy in memory.
|
|
831
|
+
// Log so an operator debugging state-loss has a lead, but never crash
|
|
832
|
+
// the workflow over a disk-full situation.
|
|
833
|
+
console.warn("[workflow-manager] Persist run failed:", err);
|
|
834
|
+
}
|
|
835
|
+
}
|
|
836
|
+
/**
|
|
837
|
+
* Pause a running workflow.
|
|
838
|
+
*/
|
|
839
|
+
pause(runId) {
|
|
840
|
+
const managed = this.runs.get(runId);
|
|
841
|
+
if (managed?.status !== "running")
|
|
842
|
+
return false;
|
|
843
|
+
managed.controller.abort();
|
|
844
|
+
managed.status = "paused";
|
|
845
|
+
this.emit("paused", { runId });
|
|
846
|
+
this.persistRun(managed);
|
|
847
|
+
this.releaseRunLease(managed);
|
|
848
|
+
return true;
|
|
849
|
+
}
|
|
850
|
+
/**
|
|
851
|
+
* Resume an interrupted run: replay journaled results for the unchanged prefix
|
|
852
|
+
* and run the rest live. Returns false if there is nothing resumable.
|
|
853
|
+
*
|
|
854
|
+
* `opts.script` lets the orchestrating model resume with an EDITED script
|
|
855
|
+
* (cached-prefix reuse / iteration): unchanged agent() calls whose content
|
|
856
|
+
* hash still matches the journal entry at their positional callIndex replay
|
|
857
|
+
* from cache, while the first changed or newly inserted call — and everything
|
|
858
|
+
* after it — re-runs live. When `opts.script` is omitted, resume behaves
|
|
859
|
+
* exactly as before and uses the persisted script (auto-resume, TUI resume);
|
|
860
|
+
* this keeps the existing single-arg `resume(runId)` callers (e.g. the
|
|
861
|
+
* UsageLimitScheduler) unchanged. `opts.args` overrides the persisted args
|
|
862
|
+
* only when provided; otherwise the persisted args are kept.
|
|
863
|
+
*/
|
|
864
|
+
async resume(runId, opts) {
|
|
865
|
+
// Guard: refuse to resume a run that is already running, or one that was
|
|
866
|
+
// intentionally aborted (pause/stop/Esc). Paused and failed runs can restart.
|
|
867
|
+
const active = this.runs.get(runId);
|
|
868
|
+
if (active?.status === "running")
|
|
869
|
+
return false;
|
|
870
|
+
if (active?.status === "aborted")
|
|
871
|
+
return false;
|
|
872
|
+
const persisted = this.persistence.load(runId);
|
|
873
|
+
if (!persisted?.script || persisted.status === "completed" || persisted.status === "aborted")
|
|
874
|
+
return false;
|
|
875
|
+
const lease = this.persistence.acquireRunLease(runId);
|
|
876
|
+
if (!lease)
|
|
877
|
+
return false;
|
|
878
|
+
// Use the edited script when supplied, else the persisted one (backward-compat).
|
|
879
|
+
const script = opts?.script ?? persisted.script;
|
|
880
|
+
const args = opts?.args !== undefined ? opts.args : persisted.args;
|
|
881
|
+
// Normalize the persisted total-at-pause once: PersistedRunState.tokenUsage
|
|
882
|
+
// has optional cost/cacheRead/cacheWrite (legacy runs may lack them), but
|
|
883
|
+
// both the seeded snapshot and initialTokenUsage need concrete numbers.
|
|
884
|
+
const priorTokenUsage = persisted.tokenUsage
|
|
885
|
+
? {
|
|
886
|
+
input: persisted.tokenUsage.input,
|
|
887
|
+
output: persisted.tokenUsage.output,
|
|
888
|
+
total: persisted.tokenUsage.total,
|
|
889
|
+
cost: persisted.tokenUsage.cost ?? 0,
|
|
890
|
+
cacheRead: persisted.tokenUsage.cacheRead ?? 0,
|
|
891
|
+
cacheWrite: persisted.tokenUsage.cacheWrite ?? 0,
|
|
892
|
+
}
|
|
893
|
+
: undefined;
|
|
894
|
+
const controller = new AbortController();
|
|
895
|
+
const managed = {
|
|
896
|
+
runId,
|
|
897
|
+
status: "running",
|
|
898
|
+
snapshot: {
|
|
899
|
+
name: persisted.workflowName,
|
|
900
|
+
phases: persisted.phases ?? [],
|
|
901
|
+
logs: persisted.logs ?? [],
|
|
902
|
+
agents: [],
|
|
903
|
+
agentCount: 0,
|
|
904
|
+
runningCount: 0,
|
|
905
|
+
doneCount: 0,
|
|
906
|
+
errorCount: 0,
|
|
907
|
+
// Seed the live snapshot's aggregate from the persisted total-at-pause
|
|
908
|
+
// (see A2) so a pause that lands before this resume's first agent
|
|
909
|
+
// completes doesn't lose the prior spend — onAgentEnd accumulates on
|
|
910
|
+
// top of this rather than starting from scratch.
|
|
911
|
+
tokenUsage: priorTokenUsage,
|
|
912
|
+
},
|
|
913
|
+
controller,
|
|
914
|
+
startedAt: new Date(),
|
|
915
|
+
// The (possibly edited) script + args become the run's own — persistRun()
|
|
916
|
+
// writes them below, so a later resume of this run sees the edited script.
|
|
917
|
+
script,
|
|
918
|
+
args,
|
|
919
|
+
journal: persisted.journal ?? [],
|
|
920
|
+
background: true,
|
|
921
|
+
lease,
|
|
922
|
+
// Carry the original opt-out forward across resumes; it's fixed at
|
|
923
|
+
// run-start and persistRun() re-persists it on every subsequent write.
|
|
924
|
+
autoResume: persisted.autoResume,
|
|
925
|
+
// Restore start-time execution context: the budget the run started with
|
|
926
|
+
// (legacy runs without one resume unbudgeted — never re-apply the current
|
|
927
|
+
// default to a run that predates it) and the toolset tag executeRun
|
|
928
|
+
// re-resolves so e.g. a resumed /deep-research keeps its web tools.
|
|
929
|
+
tokenBudget: persisted.tokenBudget !== undefined ? persisted.tokenBudget : null,
|
|
930
|
+
toolset: persisted.toolset,
|
|
931
|
+
// Restore the same start-time execution context for the other four
|
|
932
|
+
// per-run knobs (see ManagedRun doc comments) — same rationale as
|
|
933
|
+
// tokenBudget: never re-resolve against the manager's CURRENT defaults.
|
|
934
|
+
// maxAgents: legacy/never-set runs resume with no cap carried forward
|
|
935
|
+
// (runWorkflow's own MAX_AGENTS_PER_RUN default applies), exactly as if
|
|
936
|
+
// maxAgents had never been passed at all.
|
|
937
|
+
maxAgents: persisted.maxAgents,
|
|
938
|
+
// agentTimeoutMs: unlike tokenBudget, a legacy run's real timeout at
|
|
939
|
+
// start was never "no timeout" by omission — it was always
|
|
940
|
+
// this.defaultAgentTimeoutMs, because pre-A1 resume() never threaded
|
|
941
|
+
// agentTimeoutMs through at all and unconditionally fell back to the
|
|
942
|
+
// manager default (see executeRun's resolvedAgentTimeoutMs fallback
|
|
943
|
+
// chain). Falling back to null here would change what a legacy run's
|
|
944
|
+
// resume actually does versus both its original start AND pre-fix
|
|
945
|
+
// resume behavior. So — deliberately unlike tokenBudget's null
|
|
946
|
+
// fallback — legacy runs resume with the manager's CURRENT default,
|
|
947
|
+
// matching the only semantics such a run ever had.
|
|
948
|
+
agentTimeoutMs: persisted.agentTimeoutMs !== undefined ? persisted.agentTimeoutMs : this.defaultAgentTimeoutMs,
|
|
949
|
+
// concurrency/agentRetries have no "explicit opt-out sentinel" the way
|
|
950
|
+
// tokenBudget's null does — a legacy run without a persisted value falls
|
|
951
|
+
// back to the manager's current values, matching how this execution
|
|
952
|
+
// resolved unset concurrency/agentRetries before this fix ever existed.
|
|
953
|
+
concurrency: persisted.concurrency !== undefined ? persisted.concurrency : this.concurrency,
|
|
954
|
+
agentRetries: persisted.agentRetries !== undefined ? persisted.agentRetries : this.defaultAgentRetries,
|
|
955
|
+
// Fresh per-resume: agents (and any prior timing) are rebuilt live as
|
|
956
|
+
// onAgentStart/onAgentEnd fire again for this attempt (see `agents: []`
|
|
957
|
+
// above); the journal, not this map, is what makes replayed agents cheap.
|
|
958
|
+
agentTimestamps: new Map(),
|
|
959
|
+
agentsById: new Map(),
|
|
960
|
+
};
|
|
961
|
+
this.runs.set(runId, managed);
|
|
962
|
+
// Persist before notifying renderers: listRuns() is their source of truth for
|
|
963
|
+
// lifecycle status, while getRun() supplies the live in-memory snapshot.
|
|
964
|
+
this.persistRun(managed);
|
|
965
|
+
// Namespace by (runId, index) exactly like the live onAgentJournal dedup
|
|
966
|
+
// above and like SharedStore's deltaKey — see JournalEntry.runId. A
|
|
967
|
+
// legacy entry persisted before namespacing existed has no `runId`; it is
|
|
968
|
+
// assumed to belong to this run's own top-level runId (the only frame
|
|
969
|
+
// that existed before nested workflow() journaling was namespaced), so it
|
|
970
|
+
// still resume-hits for a top-level call and safely cache-misses (re-runs
|
|
971
|
+
// live, does not misapply) for what was actually a nested-run entry.
|
|
972
|
+
const resumeJournal = new Map((persisted.journal ?? []).map((e) => [`${e.runId ?? runId}:${e.index}`, e]));
|
|
973
|
+
this.emit("resumed", { runId });
|
|
974
|
+
// Run in the background; executeRun records status/errors on the managed run.
|
|
975
|
+
// initialTokenUsage seeds the resumed execution's fresh SharedRuntime.spent
|
|
976
|
+
// (A2) from the persisted total-at-pause, so the tokenBudget cap holds
|
|
977
|
+
// cumulatively instead of resetting to zero. Note: shared.agentCount is
|
|
978
|
+
// deliberately NOT seeded the same way — it doesn't need to be. Unlike
|
|
979
|
+
// token spend (whose cache-hit replay branch skips recordTokens() to avoid
|
|
980
|
+
// double-counting already-spent tokens), agent()'s shared.agentCount++
|
|
981
|
+
// fires unconditionally for EVERY call, cache-hit or live, before the
|
|
982
|
+
// replay check runs (see workflow.ts). Because resume() always replays the
|
|
983
|
+
// whole script from callIndex 0, that replay alone reconstructs the
|
|
984
|
+
// correct cumulative count inside this fresh SharedRuntime by the time any
|
|
985
|
+
// new live agent runs — so maxAgents (via A1) is already a genuine
|
|
986
|
+
// cumulative cap across resume with no extra seeding required.
|
|
987
|
+
void this.executeRun(managed, script, args, { resumeJournal, initialTokenUsage: priorTokenUsage }).catch(() => { });
|
|
988
|
+
return true;
|
|
989
|
+
}
|
|
990
|
+
/**
|
|
991
|
+
* Stop a running workflow.
|
|
992
|
+
*
|
|
993
|
+
* Fast path: the run is live in this process (`this.runs`) — abort its
|
|
994
|
+
* controller and persist "aborted" as before. Fallback: the run is not in
|
|
995
|
+
* memory but is persisted as "running" or "paused" — e.g. it belongs to a
|
|
996
|
+
* prior pi session that this process's recoverStaleRuns() flipped to
|
|
997
|
+
* "paused" on disk without repopulating this.runs (see workflow-control-tool's
|
|
998
|
+
* findRun(), which resolves candidates from disk via listRuns()). There is no
|
|
999
|
+
* live controller to abort in that case — the run simply isn't executing in
|
|
1000
|
+
* this process — so mark it aborted on disk directly, mirroring resume()'s
|
|
1001
|
+
* persisted-fallback lease handling.
|
|
1002
|
+
*/
|
|
1003
|
+
stop(runId) {
|
|
1004
|
+
const managed = this.runs.get(runId);
|
|
1005
|
+
if (managed) {
|
|
1006
|
+
if (managed.status !== "running" && managed.status !== "paused")
|
|
1007
|
+
return false;
|
|
1008
|
+
// Whether this run's OWN executeRun() promise has already fully settled
|
|
1009
|
+
// matters for whether stop() itself must be the one to call
|
|
1010
|
+
// recordTerminalRun(): a usage-limit checkpoint runs executeRun()'s
|
|
1011
|
+
// catch tail to completion before "paused" is ever observable (it
|
|
1012
|
+
// deliberately skipped recordTerminalRun() then, since "paused" isn't
|
|
1013
|
+
// terminal) — so there is no FUTURE tail left that will ever call it
|
|
1014
|
+
// for this managed object. A manual pause() sets "paused" while its
|
|
1015
|
+
// cooperative abort may still be settling; in that narrow window the
|
|
1016
|
+
// tail later settles this object to "aborted" (terminal) and records a
|
|
1017
|
+
// SECOND time — a tolerated duplicate: recordTerminalRun() is
|
|
1018
|
+
// idempotent-safe under duplicates (re-validates the current entry),
|
|
1019
|
+
// the lease was already cleared here, and the worst case is the
|
|
1020
|
+
// stopped run leaving memory earlier than FIFO order (persistence
|
|
1021
|
+
// fallback covers every consumer). A "running" run, by contrast,
|
|
1022
|
+
// always still has that tail pending;
|
|
1023
|
+
// it (not stop()) is what calls recordTerminalRun() once it actually
|
|
1024
|
+
// settles to "aborted" — see the `runs` field doc comment's rule that
|
|
1025
|
+
// eviction eligibility must wait for the real settle, not a request to
|
|
1026
|
+
// abort. Without this, stopping an already-paused run left it in
|
|
1027
|
+
// `runs` forever (no future tail to mark it eviction-eligible) — a
|
|
1028
|
+
// small leak in exactly the class this manager otherwise bounds.
|
|
1029
|
+
const hadNoPendingSettle = managed.status === "paused";
|
|
1030
|
+
managed.controller.abort();
|
|
1031
|
+
managed.status = "aborted";
|
|
1032
|
+
this.emit("stopped", { runId });
|
|
1033
|
+
this.persistRun(managed);
|
|
1034
|
+
this.releaseRunLease(managed);
|
|
1035
|
+
if (hadNoPendingSettle)
|
|
1036
|
+
this.recordTerminalRun(runId);
|
|
1037
|
+
return true;
|
|
1038
|
+
}
|
|
1039
|
+
const persisted = this.persistence.load(runId);
|
|
1040
|
+
if (!persisted || (persisted.status !== "running" && persisted.status !== "paused"))
|
|
1041
|
+
return false;
|
|
1042
|
+
const lease = this.persistence.acquireRunLease(runId);
|
|
1043
|
+
if (!lease)
|
|
1044
|
+
return false;
|
|
1045
|
+
try {
|
|
1046
|
+
this.persistence.save({ ...persisted, status: "aborted", updatedAt: new Date().toISOString() });
|
|
1047
|
+
}
|
|
1048
|
+
finally {
|
|
1049
|
+
this.persistence.releaseRunLease(lease);
|
|
1050
|
+
}
|
|
1051
|
+
this.emit("stopped", { runId });
|
|
1052
|
+
return true;
|
|
1053
|
+
}
|
|
1054
|
+
/**
|
|
1055
|
+
* Get status of a specific run.
|
|
1056
|
+
*/
|
|
1057
|
+
getRun(runId) {
|
|
1058
|
+
return this.runs.get(runId);
|
|
1059
|
+
}
|
|
1060
|
+
/**
|
|
1061
|
+
* List all runs (active + persisted).
|
|
1062
|
+
*/
|
|
1063
|
+
/**
|
|
1064
|
+
* Runs for the navigator/task panel. Once bound to a session (setSessionId), only
|
|
1065
|
+
* that session's runs are returned — runs from other sessions stay on disk and
|
|
1066
|
+
* reappear when you switch back. Unbound (tests/legacy) returns everything.
|
|
1067
|
+
*/
|
|
1068
|
+
listRuns() {
|
|
1069
|
+
const all = this.persistence.list();
|
|
1070
|
+
return this.sessionId ? all.filter((r) => r.sessionId === this.sessionId) : all;
|
|
1071
|
+
}
|
|
1072
|
+
/** All persisted runs regardless of session (used by cross-session recovery). */
|
|
1073
|
+
listAllRuns() {
|
|
1074
|
+
return this.persistence.list();
|
|
1075
|
+
}
|
|
1076
|
+
/**
|
|
1077
|
+
* Get snapshot of a run.
|
|
1078
|
+
*/
|
|
1079
|
+
getSnapshot(runId) {
|
|
1080
|
+
return this.runs.get(runId)?.snapshot ?? null;
|
|
1081
|
+
}
|
|
1082
|
+
/**
|
|
1083
|
+
* Delete a persisted run.
|
|
1084
|
+
*
|
|
1085
|
+
* If `runId` is still live in this process (running or paused-in-memory),
|
|
1086
|
+
* abort its controller FIRST, before any teardown below — a live run left
|
|
1087
|
+
* un-aborted would otherwise keep executing in the background indefinitely
|
|
1088
|
+
* (burning API calls/tokens/holding a worktree) after its record is gone.
|
|
1089
|
+
* Aborting first, while `managed` is still `this.runs.get(runId)`, costs
|
|
1090
|
+
* nothing extra: the abort signal is fire-and-forget (cooperative — the
|
|
1091
|
+
* execution winds down on its own schedule), so the exact instant we flip
|
|
1092
|
+
* `this.runs`/release the lease/delete files relative to it doesn't matter
|
|
1093
|
+
* for correctness. What DOES matter is that once this method returns, the
|
|
1094
|
+
* aborted execution's eventual settle (executeRun's success/catch path,
|
|
1095
|
+
* asynchronously, possibly much later) must be a harmless no-op rather than
|
|
1096
|
+
* a resurrection — that's what isCurrent() guarantees: `this.runs.delete()`
|
|
1097
|
+
* below means executeRun's later persistRun()/releaseRunLease() calls on
|
|
1098
|
+
* this same `managed` object find `this.runs.get(runId) !== managed` (in
|
|
1099
|
+
* fact `undefined`, since the entry is gone) and skip writing/releasing.
|
|
1100
|
+
*/
|
|
1101
|
+
deleteRun(runId) {
|
|
1102
|
+
const managed = this.runs.get(runId);
|
|
1103
|
+
if (managed) {
|
|
1104
|
+
if (!managed.controller.signal.aborted)
|
|
1105
|
+
managed.controller.abort();
|
|
1106
|
+
this.releaseRunLease(managed);
|
|
1107
|
+
}
|
|
1108
|
+
this.runs.delete(runId);
|
|
1109
|
+
// Cancel any pending throttled write so a deferred persist can't fire after
|
|
1110
|
+
// deletion and resurrect the run's file on disk.
|
|
1111
|
+
const timer = this.persistTimers.get(runId);
|
|
1112
|
+
if (timer) {
|
|
1113
|
+
clearTimeout(timer);
|
|
1114
|
+
this.persistTimers.delete(runId);
|
|
1115
|
+
}
|
|
1116
|
+
return this.persistence.delete(runId);
|
|
1117
|
+
}
|
|
1118
|
+
/**
|
|
1119
|
+
* Get the persistence layer (for saving workflows).
|
|
1120
|
+
*/
|
|
1121
|
+
getPersistence() {
|
|
1122
|
+
return this.persistence;
|
|
1123
|
+
}
|
|
1124
|
+
}
|