@fradser/pi-agent-teams 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.memory/MEMORY.md +3 -0
- package/.memory/openai_teammate_api_reference.md +118 -0
- package/.memory/run-centric-design.md +22 -0
- package/README.md +111 -0
- package/agents/observer.md +10 -0
- package/agents/reviewer.md +11 -0
- package/agents/specialist.md +10 -0
- package/agents/worker.md +11 -0
- package/package.json +40 -0
- package/src/agents.ts +138 -0
- package/src/console-viewport.ts +30 -0
- package/src/guidance.ts +77 -0
- package/src/index.ts +79 -0
- package/src/run-machine.ts +580 -0
- package/src/spawner.ts +549 -0
- package/src/state.ts +684 -0
- package/src/statefile.ts +140 -0
- package/src/terminal.ts +34 -0
- package/src/tools.ts +154 -0
- package/src/types.ts +231 -0
- package/src/ui.ts +411 -0
- package/src/worker.ts +56 -0
- package/src/worktree.ts +115 -0
package/src/state.ts
ADDED
|
@@ -0,0 +1,684 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Run-centric state management for the current session only.
|
|
3
|
+
*
|
|
4
|
+
* A run is a dispatched dependency-aware task graph; each node is one bounded
|
|
5
|
+
* child-process worker with a stable identity of `${runId}:${nodeId}`.
|
|
6
|
+
*/
|
|
7
|
+
|
|
8
|
+
import type { MailboxMessage, Node, NodeStatus, Run, RunStatus, SpawnInfo, TeammateState, WorkerMessageEvent } from "./types";
|
|
9
|
+
|
|
10
|
+
function emptyState(): TeammateState {
|
|
11
|
+
return {
|
|
12
|
+
runs: {},
|
|
13
|
+
leaderMailbox: [],
|
|
14
|
+
messageCounter: 0,
|
|
15
|
+
runCounter: 0,
|
|
16
|
+
workerEventOffsets: {},
|
|
17
|
+
workerEventIds: {},
|
|
18
|
+
};
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
let state = emptyState();
|
|
22
|
+
let stateDirty = false;
|
|
23
|
+
|
|
24
|
+
export function markStateDirty(): void {
|
|
25
|
+
stateDirty = true;
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
/** Return whether state changed since the last persisted snapshot. */
|
|
29
|
+
export function isStateDirty(): boolean {
|
|
30
|
+
return stateDirty;
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
export function clearStateDirty(): void {
|
|
34
|
+
stateDirty = false;
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
/** Return whether state changed since the last persisted snapshot, then clear it. */
|
|
38
|
+
export function consumeStateDirty(): boolean {
|
|
39
|
+
const dirty = stateDirty;
|
|
40
|
+
clearStateDirty();
|
|
41
|
+
return dirty;
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
function nextMessageId(): string {
|
|
45
|
+
return `msg_${++state.messageCounter}`;
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
function nextRunId(): string {
|
|
49
|
+
return `run_${++state.runCounter}`;
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
export function resetState(): void {
|
|
53
|
+
state = emptyState();
|
|
54
|
+
markStateDirty();
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
// ── Path normalization ────────────────────────────────────────────
|
|
58
|
+
|
|
59
|
+
export function pathsOverlap(left: string, right: string): boolean {
|
|
60
|
+
return left === "." || right === "." || left === right || left.startsWith(`${right}/`) || right.startsWith(`${left}/`);
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
export function normalizeNodePaths(paths: string[]): { ok: true; paths: string[] } | { ok: false; error: string } {
|
|
64
|
+
if (paths.length === 0) return { ok: false, error: "Provide at least one node path." };
|
|
65
|
+
const normalized: string[] = [];
|
|
66
|
+
for (const rawPath of paths) {
|
|
67
|
+
const candidate = rawPath.replaceAll("\\", "/").replace(/^\.\//, "").replace(/\/+$/, "");
|
|
68
|
+
const valid = candidate === "." || (
|
|
69
|
+
candidate
|
|
70
|
+
&& !/^[A-Za-z]:\//.test(candidate)
|
|
71
|
+
&& !candidate.startsWith("/")
|
|
72
|
+
&& !/[*?[\]{}]/.test(candidate)
|
|
73
|
+
&& candidate.split("/").every((part) => part !== "" && part !== "." && part !== "..")
|
|
74
|
+
);
|
|
75
|
+
if (!valid) return { ok: false, error: `Invalid node path: "${rawPath}".` };
|
|
76
|
+
if (normalized.includes(candidate)) return { ok: false, error: `Duplicate node path: "${rawPath}".` };
|
|
77
|
+
if (normalized.some((path) => pathsOverlap(path, candidate))) {
|
|
78
|
+
return { ok: false, error: `Overlapping node paths: "${rawPath}".` };
|
|
79
|
+
}
|
|
80
|
+
normalized.push(candidate);
|
|
81
|
+
}
|
|
82
|
+
return { ok: true, paths: normalized };
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
// ── Run creation ──────────────────────────────────────────────────
|
|
86
|
+
|
|
87
|
+
export interface RunNodeInput {
|
|
88
|
+
id: string;
|
|
89
|
+
agent: string;
|
|
90
|
+
prompt: string;
|
|
91
|
+
paths: string[];
|
|
92
|
+
access: Node["access"];
|
|
93
|
+
model?: string;
|
|
94
|
+
timeoutMs?: number;
|
|
95
|
+
dependsOn: string[];
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
/** Reserved node id for the optional synthesized summary. */
|
|
99
|
+
export const SUMMARY_NODE_ID = "__summary";
|
|
100
|
+
|
|
101
|
+
/** System-generated task text for the summary node. */
|
|
102
|
+
function summaryNodePrompt(): string {
|
|
103
|
+
return [
|
|
104
|
+
"You are the final summary step of this run. Use the upstream handoffs from",
|
|
105
|
+
"every completed leaf node, then write ONE concise final summary covering:",
|
|
106
|
+
"overall outcome, what changed or was produced,",
|
|
107
|
+
"verification status, confirmed risks or failures, and recommended follow-ups.",
|
|
108
|
+
"Synthesize — do not repeat node results verbatim and do not report your own",
|
|
109
|
+
"plan or process. The leader sees this summary as the run's headline result.",
|
|
110
|
+
].join(" ");
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
/**
|
|
114
|
+
* Create a run from validated task inputs. Rejects duplicate node ids,
|
|
115
|
+
* unknown dependsOn references, dependency cycles, and invalid paths before
|
|
116
|
+
* any worker starts. When summarize=true, appends a reserved summary node that
|
|
117
|
+
* depends on every leaf node.
|
|
118
|
+
*/
|
|
119
|
+
export function createRun(
|
|
120
|
+
input: {
|
|
121
|
+
cwd: string;
|
|
122
|
+
concurrency: number;
|
|
123
|
+
worktree: boolean;
|
|
124
|
+
background?: boolean;
|
|
125
|
+
timeoutMs?: number;
|
|
126
|
+
summarize?: boolean;
|
|
127
|
+
summaryAgent?: string;
|
|
128
|
+
nodes: RunNodeInput[];
|
|
129
|
+
},
|
|
130
|
+
): { ok: true; run: Run } | { ok: false; error: string } {
|
|
131
|
+
if (input.nodes.length === 0) return { ok: false, error: "Provide at least one task." };
|
|
132
|
+
const summarize = input.summarize ?? input.nodes.length > 1;
|
|
133
|
+
if (summarize && input.nodes.some((node) => node.id === SUMMARY_NODE_ID)) {
|
|
134
|
+
return { ok: false, error: `Node id "${SUMMARY_NODE_ID}" is reserved for the run summary.` };
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
const ids = new Set<string>();
|
|
138
|
+
const normalized: Node[] = [];
|
|
139
|
+
for (const node of input.nodes) {
|
|
140
|
+
if (ids.has(node.id)) return { ok: false, error: `Duplicate node id: "${node.id}".` };
|
|
141
|
+
ids.add(node.id);
|
|
142
|
+
const normalizedPaths = normalizeNodePaths(node.paths);
|
|
143
|
+
if (!normalizedPaths.ok) return normalizedPaths;
|
|
144
|
+
for (const dep of node.dependsOn) {
|
|
145
|
+
if (!ids.has(dep) && !input.nodes.some((candidate) => candidate.id === dep)) {
|
|
146
|
+
return { ok: false, error: `Node "${node.id}" depends on unknown node "${dep}".` };
|
|
147
|
+
}
|
|
148
|
+
}
|
|
149
|
+
normalized.push({
|
|
150
|
+
id: node.id,
|
|
151
|
+
workerKey: "", // filled after the run id is assigned
|
|
152
|
+
agent: node.agent,
|
|
153
|
+
prompt: node.prompt,
|
|
154
|
+
paths: normalizedPaths.paths,
|
|
155
|
+
access: node.access,
|
|
156
|
+
model: node.model,
|
|
157
|
+
timeoutMs: node.timeoutMs,
|
|
158
|
+
dependsOn: [...new Set(node.dependsOn)],
|
|
159
|
+
status: "pending",
|
|
160
|
+
createdAt: Date.now(),
|
|
161
|
+
updatedAt: Date.now(),
|
|
162
|
+
});
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
// Cycle detection (DFS over the dependency edges).
|
|
166
|
+
const visiting = new Set<string>();
|
|
167
|
+
const visited = new Set<string>();
|
|
168
|
+
const visit = (id: string): boolean => {
|
|
169
|
+
if (visiting.has(id)) return true;
|
|
170
|
+
if (visited.has(id)) return false;
|
|
171
|
+
visiting.add(id);
|
|
172
|
+
const node = normalized.find((candidate) => candidate.id === id)!;
|
|
173
|
+
for (const dep of node.dependsOn) {
|
|
174
|
+
if (visit(dep)) return true;
|
|
175
|
+
}
|
|
176
|
+
visiting.delete(id);
|
|
177
|
+
visited.add(id);
|
|
178
|
+
return false;
|
|
179
|
+
};
|
|
180
|
+
for (const node of normalized) {
|
|
181
|
+
if (visit(node.id)) return { ok: false, error: "The task graph contains a dependency cycle." };
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
// Synthesized summary: default on for multi-node runs. Depends on every leaf.
|
|
185
|
+
if (summarize) {
|
|
186
|
+
const leaves = normalized.filter((node) => !normalized.some((other) => other.dependsOn.includes(node.id)));
|
|
187
|
+
if (leaves.length > 0) {
|
|
188
|
+
normalized.push({
|
|
189
|
+
id: SUMMARY_NODE_ID,
|
|
190
|
+
workerKey: "",
|
|
191
|
+
agent: input.summaryAgent ?? "observer",
|
|
192
|
+
prompt: summaryNodePrompt(),
|
|
193
|
+
paths: ["."],
|
|
194
|
+
access: "read",
|
|
195
|
+
dependsOn: leaves.map((node) => node.id),
|
|
196
|
+
status: "pending",
|
|
197
|
+
createdAt: Date.now(),
|
|
198
|
+
updatedAt: Date.now(),
|
|
199
|
+
});
|
|
200
|
+
}
|
|
201
|
+
}
|
|
202
|
+
|
|
203
|
+
const runId = nextRunId();
|
|
204
|
+
const run: Run = {
|
|
205
|
+
id: runId,
|
|
206
|
+
cwd: input.cwd,
|
|
207
|
+
status: "running",
|
|
208
|
+
concurrency: input.concurrency,
|
|
209
|
+
worktree: input.worktree,
|
|
210
|
+
background: input.background ?? true,
|
|
211
|
+
timeoutMs: input.timeoutMs,
|
|
212
|
+
deadlineAt: input.timeoutMs ? Date.now() + input.timeoutMs : undefined,
|
|
213
|
+
nodes: {},
|
|
214
|
+
createdAt: Date.now(),
|
|
215
|
+
updatedAt: Date.now(),
|
|
216
|
+
};
|
|
217
|
+
for (const node of normalized) {
|
|
218
|
+
node.workerKey = `${runId}:${node.id}`;
|
|
219
|
+
run.nodes[node.id] = node;
|
|
220
|
+
}
|
|
221
|
+
state.runs[runId] = run;
|
|
222
|
+
markStateDirty();
|
|
223
|
+
return { ok: true, run };
|
|
224
|
+
}
|
|
225
|
+
|
|
226
|
+
// ── Run / node queries ────────────────────────────────────────────
|
|
227
|
+
|
|
228
|
+
export function getRun(runId: string): Run | undefined {
|
|
229
|
+
return state.runs[runId];
|
|
230
|
+
}
|
|
231
|
+
|
|
232
|
+
export function getNode(runId: string, nodeId: string): Node | undefined {
|
|
233
|
+
return state.runs[runId]?.nodes[nodeId];
|
|
234
|
+
}
|
|
235
|
+
|
|
236
|
+
export function getNodeByWorkerKey(workerKey: string): { run: Run; node: Node } | undefined {
|
|
237
|
+
for (const run of Object.values(state.runs)) {
|
|
238
|
+
const node = Object.values(run.nodes).find((candidate) => candidate.workerKey === workerKey);
|
|
239
|
+
if (node) return { run, node };
|
|
240
|
+
}
|
|
241
|
+
return undefined;
|
|
242
|
+
}
|
|
243
|
+
|
|
244
|
+
export function listRuns(): Run[] {
|
|
245
|
+
return Object.values(state.runs).sort((a, b) => b.updatedAt - a.updatedAt);
|
|
246
|
+
}
|
|
247
|
+
|
|
248
|
+
export function listActiveRuns(): Run[] {
|
|
249
|
+
return listRuns().filter((run) => run.status === "running");
|
|
250
|
+
}
|
|
251
|
+
|
|
252
|
+
export function runningNodeCount(runId: string): number {
|
|
253
|
+
const run = state.runs[runId];
|
|
254
|
+
if (!run) return 0;
|
|
255
|
+
return Object.values(run.nodes).filter((node) => node.status === "running").length;
|
|
256
|
+
}
|
|
257
|
+
|
|
258
|
+
export function isRunTerminal(run: Run): boolean {
|
|
259
|
+
return run.status === "completed" || run.status === "failed" || run.status === "cancelled";
|
|
260
|
+
}
|
|
261
|
+
|
|
262
|
+
/** Claim a run's completion delivery so the follow-up is suppressed (foreground gather). */
|
|
263
|
+
export function markRunCompletionDelivered(runId: string): void {
|
|
264
|
+
const run = state.runs[runId];
|
|
265
|
+
if (run) {
|
|
266
|
+
run.completionNotified = true;
|
|
267
|
+
markStateDirty();
|
|
268
|
+
}
|
|
269
|
+
}
|
|
270
|
+
|
|
271
|
+
/** Every node across every run (for console rows and liveness polls). */
|
|
272
|
+
export function listNodes(): Node[] {
|
|
273
|
+
return Object.values(state.runs).flatMap((run) => Object.values(run.nodes));
|
|
274
|
+
}
|
|
275
|
+
|
|
276
|
+
// ── Node lifecycle ────────────────────────────────────────────────
|
|
277
|
+
|
|
278
|
+
export function markNodeRunning(runId: string, nodeId: string): { ok: boolean; error?: string } {
|
|
279
|
+
const node = getNode(runId, nodeId);
|
|
280
|
+
if (!node) return { ok: false, error: `Node "${nodeId}" not found.` };
|
|
281
|
+
node.status = "running";
|
|
282
|
+
node.updatedAt = Date.now();
|
|
283
|
+
state.runs[runId].updatedAt = Date.now();
|
|
284
|
+
markStateDirty();
|
|
285
|
+
return { ok: true };
|
|
286
|
+
}
|
|
287
|
+
|
|
288
|
+
export function updateNodeStatus(
|
|
289
|
+
runId: string,
|
|
290
|
+
nodeId: string,
|
|
291
|
+
status: NodeStatus,
|
|
292
|
+
result?: string,
|
|
293
|
+
errorMessage?: string,
|
|
294
|
+
): { ok: boolean; node?: Node; error?: string } {
|
|
295
|
+
const node = getNode(runId, nodeId);
|
|
296
|
+
if (!node) return { ok: false, error: `Node "${nodeId}" not found.` };
|
|
297
|
+
node.status = status;
|
|
298
|
+
node.updatedAt = Date.now();
|
|
299
|
+
if (result !== undefined) node.result = result;
|
|
300
|
+
if (errorMessage !== undefined) node.errorMessage = errorMessage;
|
|
301
|
+
if (status === "completed" || status === "failed" || status === "cancelled") {
|
|
302
|
+
node.completedAt = Date.now();
|
|
303
|
+
}
|
|
304
|
+
state.runs[runId].updatedAt = Date.now();
|
|
305
|
+
markStateDirty();
|
|
306
|
+
return { ok: true, node };
|
|
307
|
+
}
|
|
308
|
+
|
|
309
|
+
/**
|
|
310
|
+
* Attach child-process execution info to a node and derive its status from
|
|
311
|
+
* the spawn outcome (unless the node was cancelled).
|
|
312
|
+
*/
|
|
313
|
+
export function setNodeSpawnInfo(runId: string, nodeId: string, info: SpawnInfo): { ok: boolean; node?: Node; error?: string } {
|
|
314
|
+
const node = getNode(runId, nodeId);
|
|
315
|
+
if (!node) return { ok: false, error: `Node "${nodeId}" not found.` };
|
|
316
|
+
node.spawn = info;
|
|
317
|
+
node.updatedAt = Date.now();
|
|
318
|
+
state.runs[runId].updatedAt = Date.now();
|
|
319
|
+
|
|
320
|
+
if (info.status === "running") {
|
|
321
|
+
// A live spawn implies the node is running (idempotent with markNodeRunning).
|
|
322
|
+
if (node.status === "pending") node.status = "running";
|
|
323
|
+
} else if (info.status === "completed" && node.status !== "cancelled") {
|
|
324
|
+
node.status = "completed";
|
|
325
|
+
node.completedAt = Date.now();
|
|
326
|
+
if (info.stdout && node.result === undefined) node.result = info.stdout;
|
|
327
|
+
} else if (info.status === "failed" && node.status !== "cancelled") {
|
|
328
|
+
node.status = "failed";
|
|
329
|
+
node.completedAt = Date.now();
|
|
330
|
+
node.errorMessage = info.error ?? info.stderr ?? `Child process exited with code ${info.exitCode ?? "unknown"}.`;
|
|
331
|
+
}
|
|
332
|
+
markStateDirty();
|
|
333
|
+
return { ok: true, node };
|
|
334
|
+
}
|
|
335
|
+
|
|
336
|
+
/** Merge a streaming child-process update into its matching running node. */
|
|
337
|
+
export function updateNodeSpawnProgress(
|
|
338
|
+
runId: string,
|
|
339
|
+
nodeId: string,
|
|
340
|
+
spawnId: string,
|
|
341
|
+
progress: Pick<SpawnInfo, "liveText" | "activeTool" | "liveThinking" | "turns" | "finalResponse">,
|
|
342
|
+
): { ok: boolean; node?: Node; error?: string } {
|
|
343
|
+
const node = getNode(runId, nodeId);
|
|
344
|
+
if (!node) return { ok: false, error: `Node "${nodeId}" not found.` };
|
|
345
|
+
if (node.spawn?.spawnId !== spawnId || node.spawn.status !== "running") {
|
|
346
|
+
return { ok: false, error: `Node "${nodeId}" is not running the expected worker.` };
|
|
347
|
+
}
|
|
348
|
+
node.spawn.liveText = progress.liveText;
|
|
349
|
+
node.spawn.activeTool = progress.activeTool;
|
|
350
|
+
node.spawn.liveThinking = progress.liveThinking;
|
|
351
|
+
node.spawn.turns = progress.turns;
|
|
352
|
+
node.spawn.finalResponse = progress.finalResponse;
|
|
353
|
+
node.updatedAt = Date.now();
|
|
354
|
+
state.runs[runId].updatedAt = Date.now();
|
|
355
|
+
markStateDirty();
|
|
356
|
+
return { ok: true, node };
|
|
357
|
+
}
|
|
358
|
+
|
|
359
|
+
// ── Dependency readiness ──────────────────────────────────────────
|
|
360
|
+
|
|
361
|
+
/** A node is ready when every dependency has completed. */
|
|
362
|
+
export function nodeIsReady(run: Run, node: Node): boolean {
|
|
363
|
+
return node.dependsOn.every((dep) => run.nodes[dep]?.status === "completed");
|
|
364
|
+
}
|
|
365
|
+
|
|
366
|
+
/** Nodes that are pending, have no unmet dependencies, and are not yet spawned. */
|
|
367
|
+
export function readyPendingNodes(run: Run): Node[] {
|
|
368
|
+
return Object.values(run.nodes)
|
|
369
|
+
.filter((node) => node.status === "pending" && nodeIsReady(run, node))
|
|
370
|
+
.sort((a, b) => a.id.localeCompare(b.id));
|
|
371
|
+
}
|
|
372
|
+
|
|
373
|
+
/**
|
|
374
|
+
* The running shared-workspace write node whose paths overlap this node's,
|
|
375
|
+
* if any. Searched across EVERY run in the session: a write node deferred in
|
|
376
|
+
* one run must also wait for an overlapping writer dispatched by another run.
|
|
377
|
+
* Worktree-isolated nodes never conflict. This is advisory scheduling
|
|
378
|
+
* coordination — it defers node starts; it does not isolate file access.
|
|
379
|
+
*/
|
|
380
|
+
export function findSharedWorkspaceWriteConflict(runId: string, nodeId: string): Node | undefined {
|
|
381
|
+
const run = state.runs[runId];
|
|
382
|
+
const node = run?.nodes[nodeId];
|
|
383
|
+
if (!run || !node || node.access !== "write") return undefined;
|
|
384
|
+
for (const candidateRun of Object.values(state.runs)) {
|
|
385
|
+
for (const other of Object.values(candidateRun.nodes)) {
|
|
386
|
+
if (candidateRun.id === runId && other.id === node.id) continue;
|
|
387
|
+
if (
|
|
388
|
+
other.status === "running"
|
|
389
|
+
&& other.spawn?.status === "running"
|
|
390
|
+
&& other.spawn.isolation !== "worktree"
|
|
391
|
+
&& other.access === "write"
|
|
392
|
+
&& node.paths.some((path) => other.paths.some((otherPath) => pathsOverlap(path, otherPath)))
|
|
393
|
+
) {
|
|
394
|
+
return other;
|
|
395
|
+
}
|
|
396
|
+
}
|
|
397
|
+
}
|
|
398
|
+
return undefined;
|
|
399
|
+
}
|
|
400
|
+
|
|
401
|
+
/** Cancel pending nodes that transitively depend on the given node. */
|
|
402
|
+
export function cancelBlockedDependents(runId: string, nodeId: string): number {
|
|
403
|
+
const run = state.runs[runId];
|
|
404
|
+
if (!run) return 0;
|
|
405
|
+
const cancelled = new Set<string>([nodeId]);
|
|
406
|
+
let changed = true;
|
|
407
|
+
while (changed) {
|
|
408
|
+
changed = false;
|
|
409
|
+
for (const node of Object.values(run.nodes)) {
|
|
410
|
+
if (node.status !== "pending" || cancelled.has(node.id)) continue;
|
|
411
|
+
if (node.dependsOn.some((dep) => cancelled.has(dep))) {
|
|
412
|
+
node.status = "cancelled";
|
|
413
|
+
node.completedAt = Date.now();
|
|
414
|
+
node.updatedAt = Date.now();
|
|
415
|
+
cancelled.add(node.id);
|
|
416
|
+
changed = true;
|
|
417
|
+
}
|
|
418
|
+
}
|
|
419
|
+
}
|
|
420
|
+
run.updatedAt = Date.now();
|
|
421
|
+
if (cancelled.size > 1) markStateDirty();
|
|
422
|
+
return cancelled.size - 1;
|
|
423
|
+
}
|
|
424
|
+
|
|
425
|
+
// ── Run settlement ────────────────────────────────────────────────
|
|
426
|
+
|
|
427
|
+
/**
|
|
428
|
+
* Recompute a running run's status from its nodes. Settles only when no node
|
|
429
|
+
* is running or pending: any failed node fails the run, an all-cancelled run
|
|
430
|
+
* is cancelled, otherwise the run completed.
|
|
431
|
+
*/
|
|
432
|
+
export function settleRun(runId: string): RunStatus | undefined {
|
|
433
|
+
const run = state.runs[runId];
|
|
434
|
+
if (!run) return undefined;
|
|
435
|
+
if (run.status !== "running") return run.status;
|
|
436
|
+
const nodes = Object.values(run.nodes);
|
|
437
|
+
const anyActive = nodes.some((node) => node.status === "running" || node.status === "pending");
|
|
438
|
+
if (anyActive) return "running";
|
|
439
|
+
if (nodes.some((node) => node.status === "failed")) {
|
|
440
|
+
run.status = "failed";
|
|
441
|
+
} else if (nodes.every((node) => node.status === "cancelled")) {
|
|
442
|
+
run.status = "cancelled";
|
|
443
|
+
} else {
|
|
444
|
+
run.status = "completed";
|
|
445
|
+
}
|
|
446
|
+
run.finishedAt = Date.now();
|
|
447
|
+
run.updatedAt = Date.now();
|
|
448
|
+
markStateDirty();
|
|
449
|
+
return run.status;
|
|
450
|
+
}
|
|
451
|
+
|
|
452
|
+
// ── Cancel / cleanup ──────────────────────────────────────────────
|
|
453
|
+
|
|
454
|
+
/**
|
|
455
|
+
* Mark a running run's pending nodes cancelled. Returns the ids of nodes with
|
|
456
|
+
* live workers that the caller must terminate.
|
|
457
|
+
*/
|
|
458
|
+
export function cancelRun(runId: string): { ok: boolean; runningNodeIds: string[]; error?: string } {
|
|
459
|
+
const run = state.runs[runId];
|
|
460
|
+
if (!run) return { ok: false, runningNodeIds: [], error: `Run "${runId}" not found.` };
|
|
461
|
+
if (isRunTerminal(run)) return { ok: false, runningNodeIds: [], error: `Run "${runId}" is already ${run.status}.` };
|
|
462
|
+
const runningNodeIds: string[] = [];
|
|
463
|
+
for (const node of Object.values(run.nodes)) {
|
|
464
|
+
if (node.status === "running") {
|
|
465
|
+
runningNodeIds.push(node.id);
|
|
466
|
+
} else if (node.status === "pending") {
|
|
467
|
+
node.status = "cancelled";
|
|
468
|
+
node.completedAt = Date.now();
|
|
469
|
+
node.updatedAt = Date.now();
|
|
470
|
+
}
|
|
471
|
+
}
|
|
472
|
+
run.updatedAt = Date.now();
|
|
473
|
+
markStateDirty();
|
|
474
|
+
return { ok: true, runningNodeIds };
|
|
475
|
+
}
|
|
476
|
+
|
|
477
|
+
/** Cancel a single node (and its not-yet-started transitive dependents).
|
|
478
|
+
* Returns the ids of nodes with live workers the caller must terminate,
|
|
479
|
+
* including running dependents whose cancelled prerequisites invalidate them. */
|
|
480
|
+
export function cancelNode(runId: string, nodeId: string): { ok: boolean; runningNodeIds: string[]; error?: string } {
|
|
481
|
+
const run = state.runs[runId];
|
|
482
|
+
const node = run?.nodes[nodeId];
|
|
483
|
+
if (!run || !node) return { ok: false, runningNodeIds: [], error: `Node "${nodeId}" not found in run "${runId}".` };
|
|
484
|
+
if (node.status === "completed" || node.status === "failed" || node.status === "cancelled") {
|
|
485
|
+
return { ok: false, runningNodeIds: [], error: `Node "${nodeId}" is already ${node.status}.` };
|
|
486
|
+
}
|
|
487
|
+
|
|
488
|
+
// Cancel pending transitive dependents of the target.
|
|
489
|
+
cancelBlockedDependents(runId, nodeId);
|
|
490
|
+
|
|
491
|
+
// Propagate to RUNNING dependents whose (now cancelled) prerequisites make
|
|
492
|
+
// their remaining work meaningless; the caller terminates their workers and
|
|
493
|
+
// finalizeNode records them cancelled.
|
|
494
|
+
const runningDependents: string[] = [];
|
|
495
|
+
const invalidated = new Set<string>([nodeId]);
|
|
496
|
+
let changed = true;
|
|
497
|
+
while (changed) {
|
|
498
|
+
changed = false;
|
|
499
|
+
for (const candidate of Object.values(run.nodes)) {
|
|
500
|
+
if (candidate.status === "cancelled") {
|
|
501
|
+
invalidated.add(candidate.id);
|
|
502
|
+
continue;
|
|
503
|
+
}
|
|
504
|
+
if (candidate.status === "running" && !invalidated.has(candidate.id)
|
|
505
|
+
&& candidate.dependsOn.some((dep) => invalidated.has(dep))) {
|
|
506
|
+
invalidated.add(candidate.id);
|
|
507
|
+
runningDependents.push(candidate.id);
|
|
508
|
+
changed = true;
|
|
509
|
+
}
|
|
510
|
+
}
|
|
511
|
+
}
|
|
512
|
+
|
|
513
|
+
const runningNodeIds: string[] = [];
|
|
514
|
+
if (node.status === "running") {
|
|
515
|
+
runningNodeIds.push(node.id);
|
|
516
|
+
} else if (node.status === "pending") {
|
|
517
|
+
node.status = "cancelled";
|
|
518
|
+
node.completedAt = Date.now();
|
|
519
|
+
node.updatedAt = Date.now();
|
|
520
|
+
}
|
|
521
|
+
for (const id of runningDependents) {
|
|
522
|
+
if (!runningNodeIds.includes(id)) runningNodeIds.push(id);
|
|
523
|
+
}
|
|
524
|
+
run.updatedAt = Date.now();
|
|
525
|
+
markStateDirty();
|
|
526
|
+
return { ok: true, runningNodeIds };
|
|
527
|
+
}
|
|
528
|
+
|
|
529
|
+
/**
|
|
530
|
+
* Fail a run for exceeding its hard wall-clock cap: pending nodes are
|
|
531
|
+
* cancelled and the run is marked failed; live workers must be terminated by
|
|
532
|
+
* the caller.
|
|
533
|
+
*/
|
|
534
|
+
export function failRunTimeout(runId: string, errorMessage: string): { ok: boolean; runningNodeIds: string[]; error?: string } {
|
|
535
|
+
const run = state.runs[runId];
|
|
536
|
+
if (!run) return { ok: false, runningNodeIds: [], error: `Run "${runId}" not found.` };
|
|
537
|
+
if (run.status !== "running") return { ok: false, runningNodeIds: [], error: `Run "${runId}" is already ${run.status}.` };
|
|
538
|
+
const runningNodeIds: string[] = [];
|
|
539
|
+
for (const node of Object.values(run.nodes)) {
|
|
540
|
+
if (node.status === "running") {
|
|
541
|
+
// The node's worker is terminated by the caller; clearing the spawn means
|
|
542
|
+
// its late close event cannot pollute a retried (reset) node.
|
|
543
|
+
node.status = "failed";
|
|
544
|
+
node.errorMessage = errorMessage;
|
|
545
|
+
node.completedAt = Date.now();
|
|
546
|
+
node.spawn = undefined;
|
|
547
|
+
node.updatedAt = Date.now();
|
|
548
|
+
runningNodeIds.push(node.id);
|
|
549
|
+
} else if (node.status === "pending") {
|
|
550
|
+
node.status = "cancelled";
|
|
551
|
+
node.completedAt = Date.now();
|
|
552
|
+
node.updatedAt = Date.now();
|
|
553
|
+
}
|
|
554
|
+
}
|
|
555
|
+
run.status = "failed";
|
|
556
|
+
run.errorMessage = errorMessage;
|
|
557
|
+
run.finishedAt = Date.now();
|
|
558
|
+
run.updatedAt = Date.now();
|
|
559
|
+
markStateDirty();
|
|
560
|
+
return { ok: true, runningNodeIds };
|
|
561
|
+
}
|
|
562
|
+
|
|
563
|
+
/**
|
|
564
|
+
* Reset failed and cancelled nodes of a settled run back to pending so the
|
|
565
|
+
* run can be re-dispatched. Completed nodes are retained. The run returns to
|
|
566
|
+
* running; callers must scheduleRun afterwards.
|
|
567
|
+
*/
|
|
568
|
+
export function retryRun(runId: string, nodeIds?: string[]): { ok: boolean; reset: string[]; error?: string } {
|
|
569
|
+
const run = state.runs[runId];
|
|
570
|
+
if (!run) return { ok: false, reset: [], error: `Run "${runId}" not found.` };
|
|
571
|
+
if (run.status === "running") return { ok: false, reset: [], error: `Run "${runId}" is still running; wait for it to settle first.` };
|
|
572
|
+
const targets = nodeIds && nodeIds.length > 0 ? nodeIds : [];
|
|
573
|
+
for (const id of targets) {
|
|
574
|
+
if (!run.nodes[id]) return { ok: false, reset: [], error: `Node "${id}" not found in run "${runId}".` };
|
|
575
|
+
}
|
|
576
|
+
|
|
577
|
+
// Explicit targets may only reset failed or cancelled nodes.
|
|
578
|
+
const resetSet = new Set<string>();
|
|
579
|
+
for (const node of Object.values(run.nodes)) {
|
|
580
|
+
if (targets.length > 0 && !targets.includes(node.id)) continue;
|
|
581
|
+
if (node.status !== "failed" && node.status !== "cancelled") {
|
|
582
|
+
if (targets.includes(node.id)) {
|
|
583
|
+
return { ok: false, reset: [], error: `Node "${node.id}" is ${node.status}, not failed or cancelled.` };
|
|
584
|
+
}
|
|
585
|
+
continue;
|
|
586
|
+
}
|
|
587
|
+
resetSet.add(node.id);
|
|
588
|
+
}
|
|
589
|
+
|
|
590
|
+
// Propagate to cancelled dependents whose (now reset) prerequisites failed:
|
|
591
|
+
// a targeted retry of node A must also re-run B, which was cancelled because
|
|
592
|
+
// it depended on A.
|
|
593
|
+
let changed = true;
|
|
594
|
+
while (changed) {
|
|
595
|
+
changed = false;
|
|
596
|
+
for (const node of Object.values(run.nodes)) {
|
|
597
|
+
if (resetSet.has(node.id) || node.status !== "cancelled") continue;
|
|
598
|
+
if (node.dependsOn.some((dep) => resetSet.has(dep))) {
|
|
599
|
+
resetSet.add(node.id);
|
|
600
|
+
changed = true;
|
|
601
|
+
}
|
|
602
|
+
}
|
|
603
|
+
}
|
|
604
|
+
|
|
605
|
+
if (resetSet.size === 0) {
|
|
606
|
+
return { ok: false, reset: [], error: "No failed or cancelled nodes to retry." };
|
|
607
|
+
}
|
|
608
|
+
for (const node of Object.values(run.nodes)) {
|
|
609
|
+
if (!resetSet.has(node.id)) continue;
|
|
610
|
+
node.status = "pending";
|
|
611
|
+
node.result = undefined;
|
|
612
|
+
node.errorMessage = undefined;
|
|
613
|
+
node.completedAt = undefined;
|
|
614
|
+
node.spawn = undefined;
|
|
615
|
+
node.updatedAt = Date.now();
|
|
616
|
+
}
|
|
617
|
+
const reset = [...resetSet];
|
|
618
|
+
run.status = "running";
|
|
619
|
+
run.errorMessage = undefined;
|
|
620
|
+
run.finishedAt = undefined;
|
|
621
|
+
run.completionNotified = false;
|
|
622
|
+
run.settledMessageSent = false;
|
|
623
|
+
// Re-arm the run-level cap: a stale deadline would re-fail the retried run
|
|
624
|
+
// on the very next poll.
|
|
625
|
+
run.deadlineAt = run.timeoutMs ? Date.now() + run.timeoutMs : undefined;
|
|
626
|
+
run.updatedAt = Date.now();
|
|
627
|
+
markStateDirty();
|
|
628
|
+
return { ok: true, reset };
|
|
629
|
+
}
|
|
630
|
+
|
|
631
|
+
/** Drop per-spawn replay metadata after its final state snapshot was persisted. */
|
|
632
|
+
export function clearWorkerRunEvents(workerKey: string, spawnId: string): void {
|
|
633
|
+
const outboxKey = `${workerKey}:${spawnId}`;
|
|
634
|
+
delete state.workerEventOffsets[outboxKey];
|
|
635
|
+
for (const id of Object.keys(state.workerEventIds)) {
|
|
636
|
+
if (id.startsWith(`${spawnId}:`)) delete state.workerEventIds[id];
|
|
637
|
+
}
|
|
638
|
+
markStateDirty();
|
|
639
|
+
}
|
|
640
|
+
|
|
641
|
+
// ── Message storage ───────────────────────────────────────────────
|
|
642
|
+
|
|
643
|
+
/** Deliver a message to the single leader inbox. */
|
|
644
|
+
export function deliverToLeader(msg: Omit<MailboxMessage, "id" | "timestamp">): MailboxMessage {
|
|
645
|
+
const full: MailboxMessage = {
|
|
646
|
+
...msg,
|
|
647
|
+
id: nextMessageId(),
|
|
648
|
+
timestamp: Date.now(),
|
|
649
|
+
};
|
|
650
|
+
state.leaderMailbox.push(full);
|
|
651
|
+
markStateDirty();
|
|
652
|
+
return full;
|
|
653
|
+
}
|
|
654
|
+
|
|
655
|
+
/** Apply a validated worker event exactly once by event id. */
|
|
656
|
+
export function receiveWorkerMessage(event: WorkerMessageEvent): boolean {
|
|
657
|
+
const sender = getNodeByWorkerKey(event.worker);
|
|
658
|
+
if (!sender || sender.node.spawn?.spawnId !== event.spawnId) return false;
|
|
659
|
+
if (state.leaderMailbox.some((message) => message.id === event.id)) return false;
|
|
660
|
+
state.leaderMailbox.push({
|
|
661
|
+
id: event.id,
|
|
662
|
+
from: event.worker,
|
|
663
|
+
subject: event.subject,
|
|
664
|
+
body: event.body,
|
|
665
|
+
runId: sender.run.id,
|
|
666
|
+
timestamp: Date.now(),
|
|
667
|
+
});
|
|
668
|
+
markStateDirty();
|
|
669
|
+
return true;
|
|
670
|
+
}
|
|
671
|
+
|
|
672
|
+
// ── State inspection ──────────────────────────────────────────────
|
|
673
|
+
|
|
674
|
+
export function getState(): TeammateState {
|
|
675
|
+
return state;
|
|
676
|
+
}
|
|
677
|
+
|
|
678
|
+
export function getSummary(): string | undefined {
|
|
679
|
+
const runCount = Object.keys(state.runs).length;
|
|
680
|
+
if (runCount === 0) return undefined;
|
|
681
|
+
const messageCount = state.leaderMailbox.length;
|
|
682
|
+
const activeNodes = listNodes().filter((node) => node.status === "running" || node.status === "pending").length;
|
|
683
|
+
return `${runCount} run(s) | ${messageCount} message(s) | ${activeNodes} active node(s)`;
|
|
684
|
+
}
|