agent-horde 0.1.5 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +42 -10
- package/docs/REFERENCE.md +141 -27
- package/package.json +1 -1
- package/src/args.ts +125 -50
- package/src/build-info.ts +65 -0
- package/src/collaboration-report.ts +23 -9
- package/src/dashboard-data.ts +480 -49
- package/src/dashboard-html.ts +390 -95
- package/src/dashboard.ts +205 -96
- package/src/ledger/collaboration.ts +36 -6
- package/src/ledger/facts.ts +52 -5
- package/src/ledger/files.ts +6 -1
- package/src/ledger/findings.ts +91 -1
- package/src/ledger/run-signals.ts +17 -2
- package/src/member.ts +62 -3
- package/src/prompt.ts +102 -21
- package/src/run/command.ts +68 -37
- package/src/run/completions.ts +32 -15
- package/src/run/controls.ts +74 -15
- package/src/run/finalize.ts +30 -21
- package/src/run/launch.ts +179 -34
- package/src/run/repair.ts +15 -7
- package/src/run/schedule.ts +43 -13
- package/src/run/setup.ts +36 -25
- package/src/run/solved.ts +125 -15
- package/src/run/supervise.ts +77 -10
- package/src/run/swarm-finalize.ts +121 -0
- package/src/run/swarm-report.ts +152 -0
- package/src/run/swarm.ts +156 -0
- package/src/run/synthesis.ts +141 -20
- package/src/run/types.ts +70 -5
- package/src/run/ui.ts +73 -28
- package/src/run-graph.ts +16 -9
- package/src/swarm-graph.ts +127 -0
- package/src/tool/actions.ts +206 -41
- package/src/tool/collaboration-actions.ts +75 -25
- package/src/tool/params.ts +10 -7
- package/src/tool/register.ts +13 -6
- package/src/widget.ts +80 -8
package/src/run/solved.ts
CHANGED
|
@@ -3,9 +3,10 @@ import { oneLine } from "../format.js";
|
|
|
3
3
|
import { loadCollaborationReport } from "../collaboration-report.js";
|
|
4
4
|
import { openMessages } from "../ledger/collaboration.js";
|
|
5
5
|
import { postBoard } from "../ledger/board.js";
|
|
6
|
-
import {
|
|
6
|
+
import { unsettledFactCitations } from "../ledger/facts.js";
|
|
7
|
+
import { listFindingsForRun, staleAnswerCitations, unverifiedFindings } from "../ledger/findings.js";
|
|
7
8
|
import { listIdeas } from "../ledger/ideas.js";
|
|
8
|
-
import { archiveSolved, clearCleanStop, clearSolved, readSolvedState, resolveSolvedState } from "../ledger/run-signals.js";
|
|
9
|
+
import { archiveSolved, clearCleanStop, clearSolved, readSolvedState, reopenSolved, resolveSolvedState, writeAnswer } from "../ledger/run-signals.js";
|
|
9
10
|
import type { SolvedSignal } from "../ledger/run-signals.js";
|
|
10
11
|
import { requestCleanStop } from "./controls.js";
|
|
11
12
|
import { recordLifecycle } from "./lifecycle.js";
|
|
@@ -46,14 +47,12 @@ function adoptProposal(run: HordeRun, proposal: SolvedSignal): void {
|
|
|
46
47
|
refreshRun(run);
|
|
47
48
|
}
|
|
48
49
|
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
`SOLVED proposal contested by ${contest.member}: ${contest.text}. Work continues.`,
|
|
56
|
-
);
|
|
50
|
+
/** Idea ids a contest names as what is still missing; the proposal reopens once they close. */
|
|
51
|
+
export function contestBlockers(text: string): number[] {
|
|
52
|
+
return [...new Set([...text.matchAll(/(?:^|[\s(,;])#(\d{1,6})\b/g)].map((m) => Number(m[1])))];
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
function clearProposal(run: HordeRun): void {
|
|
57
56
|
archiveSolved(run.runDir);
|
|
58
57
|
clearSolved(run.runDir);
|
|
59
58
|
if (!run.stopped) clearCleanStop(run.runDir);
|
|
@@ -62,10 +61,76 @@ function dropContested(run: HordeRun, contest: SolvedSignal): void {
|
|
|
62
61
|
run.solvedCommitted = false;
|
|
63
62
|
run.solvedCoverageNotice = undefined;
|
|
64
63
|
run.solvedVerificationNotice = undefined;
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
function dropContested(run: HordeRun, proposal: SolvedSignal, contest: SolvedSignal): void {
|
|
67
|
+
const blockers = contestBlockers(contest.text);
|
|
68
|
+
safeNotify(run, `horde ${run.id}: solved proposal contested by ${contest.member}`, "warning");
|
|
69
|
+
void recordLifecycle(run, "solved-contested", { member: contest.member, text: oneLine(contest.text, 200), blockers });
|
|
70
|
+
const reopens = blockers.length ? ` It reopens for validation once ${blockers.map((id) => `#${id}`).join(", ")} are done or ruled out.` : "";
|
|
71
|
+
void postBoard(
|
|
72
|
+
run.hordeDir,
|
|
73
|
+
"parent",
|
|
74
|
+
`SOLVED proposal contested by ${contest.member}: ${contest.text}. Work continues.${reopens}`,
|
|
75
|
+
);
|
|
76
|
+
clearProposal(run);
|
|
65
77
|
run.solvedContested = true;
|
|
78
|
+
run.solvedContest = blockers.length ? { proposal, blockers } : undefined;
|
|
66
79
|
refreshRun(run);
|
|
67
80
|
}
|
|
68
81
|
|
|
82
|
+
/**
|
|
83
|
+
* A proposal that cites a finding superseded or amended after it was made would ship the stale
|
|
84
|
+
* reading as the answer. It is withdrawn for re-proposal rather than validated; the board names
|
|
85
|
+
* the successor or the amendment.
|
|
86
|
+
*/
|
|
87
|
+
function withdrawStale(run: HordeRun, proposal: SolvedSignal, stale: string[]): void {
|
|
88
|
+
const reason = `cites ${stale.join(", ")}`;
|
|
89
|
+
safeNotify(run, `horde ${run.id}: solved proposal withdrawn; it ${reason}`, "warning");
|
|
90
|
+
void recordLifecycle(run, "solved-withdrawn", { member: proposal.member, reason });
|
|
91
|
+
void postBoard(
|
|
92
|
+
run.hordeDir,
|
|
93
|
+
"parent",
|
|
94
|
+
`SOLVED proposal by ${proposal.member} withdrawn: it ${reason}. ${proposal.member}: propose again citing the current findings.`,
|
|
95
|
+
);
|
|
96
|
+
clearProposal(run);
|
|
97
|
+
refreshRun(run);
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
/**
|
|
101
|
+
* A contest that named open ideas as what was missing is answered by their closing: nothing else
|
|
102
|
+
* reopens the vote, so the run ended `solved-contested` with the closed negatives unapplied.
|
|
103
|
+
*/
|
|
104
|
+
function reopenContested(run: HordeRun): void {
|
|
105
|
+
const contest = run.solvedContest;
|
|
106
|
+
if (!contest || run.stopped) return;
|
|
107
|
+
const closed = new Map(listIdeas(run.hordeDir).map((idea) => [idea.id, idea.mark === "x" || idea.mark === "!"]));
|
|
108
|
+
if (!contest.blockers.every((id) => closed.get(id))) return;
|
|
109
|
+
const named = contest.blockers.map((id) => `#${id}`).join(", ");
|
|
110
|
+
const reason = `contest blockers ${named} closed`;
|
|
111
|
+
// The proposal returns with the citations it pinned when it was made; what went stale or
|
|
112
|
+
// unsettled meanwhile makes it a re-proposal for its author, not a reopening.
|
|
113
|
+
const stale = [
|
|
114
|
+
...staleAnswerCitations(listFindingsForRun(run.hordeDir, run.id), contest.proposal.text, contest.proposal.cites),
|
|
115
|
+
...unsettledFactCitations(run.hordeDir, contest.proposal.text).map((citation) => `provisional ${citation}`),
|
|
116
|
+
];
|
|
117
|
+
run.solvedContest = undefined;
|
|
118
|
+
if (stale.length) {
|
|
119
|
+
const why = `cites ${stale.join(", ")}`;
|
|
120
|
+
void recordLifecycle(run, "solved-withdrawn", { member: contest.proposal.member, reason: why });
|
|
121
|
+
void postBoard(run.hordeDir, "parent", `SOLVED proposal by ${contest.proposal.member} is not re-opened after ${reason}: it ${why}. ${contest.proposal.member}: propose again citing the current findings.`);
|
|
122
|
+
return;
|
|
123
|
+
}
|
|
124
|
+
const signal = reopenSolved(run.runDir, run.id, contest.proposal, `re-opened: ${reason}; awaiting validation`, contest.proposal.cites);
|
|
125
|
+
run.solvedContested = false;
|
|
126
|
+
void recordLifecycle(run, "solved-reopened", { member: signal.member, reason, revision: signal.revision });
|
|
127
|
+
void postBoard(
|
|
128
|
+
run.hordeDir,
|
|
129
|
+
"parent",
|
|
130
|
+
`SOLVED proposal by ${signal.member} re-opened: ${reason}. Another member must validate it against the closed ideas with 'horde confirm-solved' or contest it again.`,
|
|
131
|
+
);
|
|
132
|
+
}
|
|
133
|
+
|
|
69
134
|
/**
|
|
70
135
|
* Coverage is structural: a run that created ideas must close them before a
|
|
71
136
|
* solve winds it down, regardless of the prompt's wording. A run that created
|
|
@@ -124,11 +189,43 @@ function verificationPending(run: HordeRun, othersRunning: boolean): boolean {
|
|
|
124
189
|
return true;
|
|
125
190
|
}
|
|
126
191
|
|
|
192
|
+
/**
|
|
193
|
+
* `--until-stop`: bank the answer as the run's current one and reopen the proposal slot, so a
|
|
194
|
+
* materially better answer can be proposed later. Nobody is stopped; the supervisor gives every
|
|
195
|
+
* member that finishes a fresh session.
|
|
196
|
+
*/
|
|
197
|
+
function bankSolved(run: HordeRun, proposal: SolvedSignal, validator: string | undefined): void {
|
|
198
|
+
run.solveRounds = (run.solveRounds ?? 0) + 1;
|
|
199
|
+
run.lastRound = { proposal, validator };
|
|
200
|
+
const status = validator ? `validated by ${validator}` : "committed without a separate validator";
|
|
201
|
+
try {
|
|
202
|
+
writeAnswer(run.runDir, run.id, proposal, `${status} (banked as round ${run.solveRounds}; the run continues until stopped)`);
|
|
203
|
+
} catch (error) {
|
|
204
|
+
safeNotify(run, `horde ${run.id}: could not write ANSWER.md (${oneLine(String(error), 200)})`, "error");
|
|
205
|
+
}
|
|
206
|
+
clearProposal(run);
|
|
207
|
+
run.solvedContest = undefined;
|
|
208
|
+
void recordLifecycle(run, "solved-banked", { round: run.solveRounds, member: proposal.member, validator });
|
|
209
|
+
safeNotify(run, `horde ${run.id}: solve banked as round ${run.solveRounds}; the run continues until stopped`, "info");
|
|
210
|
+
void postBoard(
|
|
211
|
+
run.hordeDir,
|
|
212
|
+
"parent",
|
|
213
|
+
`SOLVED ${status}, banked as round ${run.solveRounds}: ${oneLine(proposal.text, 200)}. ` +
|
|
214
|
+
"This run continues until the user stops it. Keep working: extend, stress-test, or deepen the work, " +
|
|
215
|
+
"add ideas for what is still unexplored, and propose SOLVED again only when a materially better answer exists.",
|
|
216
|
+
);
|
|
217
|
+
refreshRun(run);
|
|
218
|
+
}
|
|
219
|
+
|
|
127
220
|
function commitSolved(
|
|
128
221
|
run: HordeRun,
|
|
129
222
|
ctx: ExtensionCommandContext,
|
|
130
223
|
validator: string | undefined,
|
|
131
224
|
): void {
|
|
225
|
+
if (run.untilStop && run.solved && !run.stopped) {
|
|
226
|
+
bankSolved(run, run.solved, validator);
|
|
227
|
+
return;
|
|
228
|
+
}
|
|
132
229
|
run.solvedValidatedBy = validator;
|
|
133
230
|
run.solvedCommitted = true;
|
|
134
231
|
void recordLifecycle(run, "solved-validated", { validator });
|
|
@@ -150,13 +247,24 @@ function commitSolved(
|
|
|
150
247
|
}
|
|
151
248
|
|
|
152
249
|
/** One poll of the two-phase solved protocol. */
|
|
153
|
-
function checkSolved(run: HordeRun, ctx: ExtensionCommandContext): void {
|
|
250
|
+
export function checkSolved(run: HordeRun, ctx: ExtensionCommandContext): void {
|
|
154
251
|
const state = resolveSolvedState(readSolvedState(run.runDir));
|
|
155
|
-
if (!run.solved && state.proposal)
|
|
252
|
+
if (!run.solved && state.proposal) {
|
|
253
|
+
run.solvedContest = undefined;
|
|
254
|
+
adoptProposal(run, state.proposal);
|
|
255
|
+
}
|
|
156
256
|
const solved = run.solved;
|
|
157
|
-
if (!solved)
|
|
257
|
+
if (!solved) {
|
|
258
|
+
reopenContested(run);
|
|
259
|
+
return;
|
|
260
|
+
}
|
|
158
261
|
if (state.contest) {
|
|
159
|
-
dropContested(run, state.contest);
|
|
262
|
+
dropContested(run, solved, state.contest);
|
|
263
|
+
return;
|
|
264
|
+
}
|
|
265
|
+
const stale = staleAnswerCitations(listFindingsForRun(run.hordeDir, run.id), solved.text, solved.cites);
|
|
266
|
+
if (stale.length && !run.solvedCommitted) {
|
|
267
|
+
withdrawStale(run, solved, stale);
|
|
160
268
|
return;
|
|
161
269
|
}
|
|
162
270
|
const validator =
|
|
@@ -167,7 +275,9 @@ function checkSolved(run: HordeRun, ctx: ExtensionCommandContext): void {
|
|
|
167
275
|
if (coveragePending(run, validator)) return;
|
|
168
276
|
const collaboration = loadCollaborationReport(run.hordeDir, run.id);
|
|
169
277
|
if (collaboration.issues.length || openMessages(collaboration.state).length) return;
|
|
170
|
-
|
|
278
|
+
// Members of an `--until-stop` run are always coming back, so a lull between sessions is not
|
|
279
|
+
// the absence of a validator; the proposal waits for one or for the timeout.
|
|
280
|
+
const othersRunning = run.untilStop === true || run.members.some(
|
|
171
281
|
(member) =>
|
|
172
282
|
(member.status === "running" || member.status === "starting") &&
|
|
173
283
|
member.name !== solved.member,
|
package/src/run/supervise.ts
CHANGED
|
@@ -4,13 +4,14 @@ import * as path from "node:path";
|
|
|
4
4
|
import type { ExtensionCommandContext } from "@earendil-works/pi-coding-agent";
|
|
5
5
|
import { envNumber, errorMessage } from "../format.js";
|
|
6
6
|
import { postBoard } from "../ledger/board.js";
|
|
7
|
+
import { pendingReviewsForMember } from "../ledger/findings.js";
|
|
7
8
|
import { reconcileOwnedIdeas } from "../ledger/ideas.js";
|
|
8
9
|
import { cleanStopRequestedFor, clearHeartbeat, readHeartbeat } from "../ledger/run-signals.js";
|
|
9
10
|
import { resetMemberState, spawnMember } from "../member.js";
|
|
10
11
|
import type { MemberState } from "../member.js";
|
|
11
12
|
import { hardStopMembers } from "./controls.js";
|
|
12
13
|
import { recordLifecycle } from "./lifecycle.js";
|
|
13
|
-
import type { HordeRun, HordeState } from "./types.js";
|
|
14
|
+
import type { HordeRun, HordeState, Swarm } from "./types.js";
|
|
14
15
|
import { refreshRun, safeNotify } from "./ui.js";
|
|
15
16
|
|
|
16
17
|
/** A running member with no ledger write for this long is reset; 0 disables. */
|
|
@@ -22,9 +23,22 @@ const RESET_CAP = envNumber("PI_HORDE_RESET_CAP", 2);
|
|
|
22
23
|
/** Member exits that a fresh session may recover from; a user stop is never one. */
|
|
23
24
|
const RESETTABLE_CAUSES = new Set(["model_error", "no_output", "exit_nonzero"]);
|
|
24
25
|
|
|
26
|
+
/**
|
|
27
|
+
* An `--until-stop` session that finished sooner than this is held back until it has been this
|
|
28
|
+
* long since it started, so a member that keeps finding nothing to do cannot spin through
|
|
29
|
+
* sessions, each paying the full prompt for nothing.
|
|
30
|
+
*/
|
|
31
|
+
const CONTINUE_COOLDOWN_MS = envNumber("PI_HORDE_CONTINUE_COOLDOWN_MS", 60_000);
|
|
32
|
+
|
|
33
|
+
const CONTINUE_REASON = "until-stop: earlier session finished";
|
|
34
|
+
|
|
35
|
+
const sleep = (ms: number): Promise<void> => new Promise((resolve) => setTimeout(resolve, ms).unref?.());
|
|
36
|
+
|
|
25
37
|
/** What spawning and supervising a run's members needs beyond the run itself. */
|
|
26
38
|
export interface RunLaunch {
|
|
27
39
|
state: HordeState;
|
|
40
|
+
/** The swarm the run belongs to; a plain run is a swarm of one. */
|
|
41
|
+
swarm: Swarm;
|
|
28
42
|
run: HordeRun;
|
|
29
43
|
ctx: ExtensionCommandContext;
|
|
30
44
|
/** Per-member role instruction, indexed by member index - 1. */
|
|
@@ -80,16 +94,34 @@ export async function spawnOne(
|
|
|
80
94
|
* stop all mean the run is ending; the cap keeps a broken provider from being
|
|
81
95
|
* retried forever.
|
|
82
96
|
*/
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
97
|
+
/**
|
|
98
|
+
* A verifier that closes its session with high-importance findings by other members unreviewed
|
|
99
|
+
* has not done its job, whatever its summary says. Its exit is answered with a fresh session
|
|
100
|
+
* that is told which F# it owes, within the same reset cap as a crash.
|
|
101
|
+
*/
|
|
102
|
+
export function verificationBacklog(run: HordeRun, member: MemberState): string | undefined {
|
|
103
|
+
if (member.name !== run.verifier || member.endCause !== "completed") return undefined;
|
|
104
|
+
try {
|
|
105
|
+
const owed = pendingReviewsForMember(run.hordeDir, run.id, member.name)
|
|
106
|
+
.filter((finding) => ["critical", "high"].includes((finding.importance ?? "").toLowerCase()));
|
|
107
|
+
if (!owed.length) return undefined;
|
|
108
|
+
return `verification backlog: ${owed.map((finding) => `F${finding.id}`).join(", ")} unreviewed at close`;
|
|
109
|
+
} catch {
|
|
86
110
|
return undefined;
|
|
87
111
|
}
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
function resetReasonForExit(launch: RunLaunch, member: MemberState): string | undefined {
|
|
115
|
+
const { state, run } = launch;
|
|
116
|
+
if (state.disposed || run.stopped || run.userStop || run.canary !== "ok") return undefined;
|
|
117
|
+
// A pending proposal means the run is about to end, unless it runs until stopped.
|
|
118
|
+
if (run.solved && !run.untilStop) return undefined;
|
|
88
119
|
if (cleanStopRequestedFor(run.hordeDir, run.id, member.name)) return undefined;
|
|
89
120
|
const reason =
|
|
90
121
|
member.resetPending ??
|
|
91
|
-
(RESETTABLE_CAUSES.has(member.endCause ?? "") ? member.endCause : undefined)
|
|
92
|
-
|
|
122
|
+
(RESETTABLE_CAUSES.has(member.endCause ?? "") ? member.endCause : undefined) ??
|
|
123
|
+
verificationBacklog(run, member);
|
|
124
|
+
if (!reason) return run.untilStop && member.endCause === "completed" ? CONTINUE_REASON : undefined;
|
|
93
125
|
if ((member.resets ?? 0) >= RESET_CAP) {
|
|
94
126
|
fs.appendFileSync(
|
|
95
127
|
path.join(run.runDir, `${member.name}.log`),
|
|
@@ -115,10 +147,13 @@ async function releaseForReset(
|
|
|
115
147
|
const releasedText = released.map((id) => `#${id}`).join(", ") || "none";
|
|
116
148
|
const closedText =
|
|
117
149
|
result.closed.map((item) => `#${item.id} (F${item.findingId})`).join(", ") || "none";
|
|
150
|
+
const head = reason === CONTINUE_REASON
|
|
151
|
+
? `${member.name} finished and continues in a fresh session (${run.id} runs until stopped)`
|
|
152
|
+
: `reset ${member.name} (${reason})`;
|
|
118
153
|
await postBoard(
|
|
119
154
|
run.hordeDir,
|
|
120
155
|
"parent",
|
|
121
|
-
|
|
156
|
+
`${head}: released ${releasedText}; closed ${closedText}. ` +
|
|
122
157
|
`A fresh session of ${member.name} starts now; earlier claims by ${member.name} ` +
|
|
123
158
|
"are void.",
|
|
124
159
|
);
|
|
@@ -157,13 +192,41 @@ async function performReset(run: HordeRun, member: MemberState, reason: string):
|
|
|
157
192
|
await recordLifecycle(run, "member-reset", { member: member.name, reason, released, closed, resets: member.resets });
|
|
158
193
|
member.resetNote =
|
|
159
194
|
`You replace an earlier session of ${member.name} that was reset for ${reason}; ` +
|
|
160
|
-
"its ideas were released. Read BOARD.md and FINDINGS.md before claiming."
|
|
195
|
+
"its ideas were released. Read BOARD.md and FINDINGS.md before claiming." +
|
|
196
|
+
(reason.startsWith("verification backlog") ? " Review the findings named above, with any verdict and an honest unchecked scope, before anything else." : "");
|
|
161
197
|
clearHeartbeat(run.runDir, member.name);
|
|
162
198
|
resetMemberState(member);
|
|
163
199
|
safeNotify(run, `horde ${run.id}: reset ${member.name} (${reason})`, "warning");
|
|
164
200
|
refreshRun(run);
|
|
165
201
|
}
|
|
166
202
|
|
|
203
|
+
/**
|
|
204
|
+
* `--until-stop`: a member that finished its session gets a fresh one. Not a recovery, so it
|
|
205
|
+
* counts against no cap and is not a reset in the report; the cooldown keeps a member that
|
|
206
|
+
* finds nothing to do from spinning.
|
|
207
|
+
*/
|
|
208
|
+
async function performContinue(run: HordeRun, member: MemberState): Promise<boolean> {
|
|
209
|
+
const elapsed = Date.now() - member.startedAt;
|
|
210
|
+
if (elapsed < CONTINUE_COOLDOWN_MS) await sleep(CONTINUE_COOLDOWN_MS - elapsed);
|
|
211
|
+
// A stop during the cooldown leaves the member as it ended: finished, not killed.
|
|
212
|
+
if (run.stopped) return false;
|
|
213
|
+
member.continuations = (member.continuations ?? 0) + 1;
|
|
214
|
+
fs.appendFileSync(
|
|
215
|
+
path.join(run.runDir, `${member.name}.log`),
|
|
216
|
+
`[horde] continue ${member.continuations}: ${CONTINUE_REASON}\n`,
|
|
217
|
+
);
|
|
218
|
+
const { released, closed } = await releaseForReset(run, member, CONTINUE_REASON);
|
|
219
|
+
await recordLifecycle(run, "member-continue", { member: member.name, released, closed, continuations: member.continuations });
|
|
220
|
+
member.resetNote =
|
|
221
|
+
`You are a fresh session of ${member.name}; its earlier session finished and its claims were released. ` +
|
|
222
|
+
"Read BOARD.md, FINDINGS.md and ANSWER.md if present, then add ideas that extend, stress-test, or deepen the work, claim them, and pursue them.";
|
|
223
|
+
clearHeartbeat(run.runDir, member.name);
|
|
224
|
+
resetMemberState(member);
|
|
225
|
+
safeNotify(run, `horde ${run.id}: ${member.name} finished; continuing in a fresh session`, "info");
|
|
226
|
+
refreshRun(run);
|
|
227
|
+
return true;
|
|
228
|
+
}
|
|
229
|
+
|
|
167
230
|
/**
|
|
168
231
|
* Resolves only when the member is down for good, so the fan-out's
|
|
169
232
|
* Promise.all cannot finalize the run while a replacement is running.
|
|
@@ -179,7 +242,11 @@ export async function superviseMember(
|
|
|
179
242
|
await exited;
|
|
180
243
|
const reason = resetReasonForExit(launch, member);
|
|
181
244
|
if (!reason || state.disposed) return;
|
|
182
|
-
|
|
245
|
+
if (reason === CONTINUE_REASON) {
|
|
246
|
+
if (!(await performContinue(run, member))) return;
|
|
247
|
+
} else {
|
|
248
|
+
await performReset(run, member, reason);
|
|
249
|
+
}
|
|
183
250
|
if (run.stopped || state.disposed) {
|
|
184
251
|
// A stop landed during the reset: leave the member down, not "starting".
|
|
185
252
|
member.status = "stopped";
|
|
@@ -196,7 +263,7 @@ export async function superviseMember(
|
|
|
196
263
|
* hung. Mark it and kill it; the supervise loop turns the kill into a reset.
|
|
197
264
|
*/
|
|
198
265
|
function checkStalls(run: HordeRun): void {
|
|
199
|
-
if (run.stopped || run.solved) return;
|
|
266
|
+
if (run.stopped || (run.solved && !run.untilStop)) return;
|
|
200
267
|
const now = Date.now();
|
|
201
268
|
for (const member of run.members) {
|
|
202
269
|
if (member.status !== "running" || member.paused || member.resetPending) continue;
|
|
@@ -0,0 +1,121 @@
|
|
|
1
|
+
import * as fs from "node:fs";
|
|
2
|
+
import * as path from "node:path";
|
|
3
|
+
import { errorMessage } from "../format.js";
|
|
4
|
+
import { SWARM_REPORT_TYPE, swarmHordeRows, swarmReportMarkdown, swarmReportText } from "./swarm-report.js";
|
|
5
|
+
import type { SwarmReportDetails } from "./swarm-report.js";
|
|
6
|
+
import { runSwarmDeliverable } from "./synthesis.js";
|
|
7
|
+
import type { HordeRun, HordeState, Swarm } from "./types.js";
|
|
8
|
+
import { refreshSwarm, safeNotify, unmountWidget } from "./ui.js";
|
|
9
|
+
|
|
10
|
+
/** `SWARM.json`: the swarm's timeline for the dashboard once nothing is in memory. */
|
|
11
|
+
export interface SwarmRecord {
|
|
12
|
+
id: string;
|
|
13
|
+
prompt: string;
|
|
14
|
+
hordes: number;
|
|
15
|
+
count: number;
|
|
16
|
+
startedAt: number;
|
|
17
|
+
untilStop: boolean;
|
|
18
|
+
partition: string[];
|
|
19
|
+
runs: Array<{ label: string; runId: string; runDir: string; hordeDir: string; startedAt: number; endedAt?: number | undefined; outcome?: string | undefined }>;
|
|
20
|
+
/** The merge step: when the swarm deliverable helper ran and whether it wrote anything. */
|
|
21
|
+
merge?: { startedAt: number; endedAt?: number | undefined; written?: boolean | undefined } | undefined;
|
|
22
|
+
endedAt?: number | undefined;
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
export function swarmRecord(swarm: Swarm): SwarmRecord {
|
|
26
|
+
return {
|
|
27
|
+
id: swarm.id,
|
|
28
|
+
prompt: swarm.prompt,
|
|
29
|
+
hordes: swarm.hordes,
|
|
30
|
+
count: swarm.count,
|
|
31
|
+
startedAt: swarm.startedAt,
|
|
32
|
+
untilStop: swarm.untilStop === true,
|
|
33
|
+
partition: swarm.partition ?? [],
|
|
34
|
+
runs: swarm.runs.map((run) => ({
|
|
35
|
+
label: `h${run.horde}`,
|
|
36
|
+
runId: run.id,
|
|
37
|
+
runDir: run.runDir,
|
|
38
|
+
hordeDir: run.hordeDir,
|
|
39
|
+
startedAt: run.startedAt,
|
|
40
|
+
endedAt: run.finalized ? Math.max(run.startedAt, ...run.members.map((m) => m.endedAt ?? 0)) : undefined,
|
|
41
|
+
outcome: run.outcome,
|
|
42
|
+
})),
|
|
43
|
+
};
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
/** Write a swarm file, reporting (not throwing) a failure so finalize keeps going. */
|
|
47
|
+
function writeSwarmFile(swarm: Swarm, name: string, content: string): void {
|
|
48
|
+
if (!swarm.dir) return;
|
|
49
|
+
try {
|
|
50
|
+
fs.writeFileSync(path.join(swarm.dir, name), content, "utf-8");
|
|
51
|
+
} catch (error) {
|
|
52
|
+
safeNotify(swarm, `swarm ${swarm.id}: could not write ${name} (${errorMessage(error)})`, "error");
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
export function writeSwarmRecord(swarm: Swarm, extra: Partial<SwarmRecord> = {}): void {
|
|
57
|
+
writeSwarmFile(swarm, "SWARM.json", JSON.stringify({ ...swarmRecord(swarm), ...extra }, null, 2));
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
/**
|
|
61
|
+
* A horde's members exited and its REPORT.md is written. Once every horde is there the swarm
|
|
62
|
+
* slot frees, so a new launch can start while the (possibly slow) synthesis steps run.
|
|
63
|
+
*/
|
|
64
|
+
export function noteRunReported(state: HordeState, swarm: Swarm, run: HordeRun): void {
|
|
65
|
+
run.reported = true;
|
|
66
|
+
if (!swarm.runs.every((other) => other.reported === true)) return;
|
|
67
|
+
if (state.active === swarm) state.active = undefined;
|
|
68
|
+
state.finalizing = swarm;
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
/** The merged deliverable, REPORT.md, and the injected message of a swarm of several hordes. */
|
|
72
|
+
async function mergeSwarm(state: HordeState, swarm: Swarm): Promise<void> {
|
|
73
|
+
swarm.finalizing = ["merge"];
|
|
74
|
+
refreshSwarm(swarm);
|
|
75
|
+
const merge: NonNullable<SwarmRecord["merge"]> = { startedAt: Date.now() };
|
|
76
|
+
writeSwarmRecord(swarm, { merge });
|
|
77
|
+
swarm.deliverable = process.env.PI_HORDE_DELIVERABLE === "off" ? undefined : await runSwarmDeliverable(state, swarm);
|
|
78
|
+
if (swarm.deliverable) writeSwarmFile(swarm, "DELIVERABLE.md", swarm.deliverable);
|
|
79
|
+
const now = Date.now();
|
|
80
|
+
merge.endedAt = now;
|
|
81
|
+
merge.written = Boolean(swarm.deliverable);
|
|
82
|
+
writeSwarmFile(swarm, "REPORT.md", swarmReportMarkdown(swarm, now));
|
|
83
|
+
writeSwarmRecord(swarm, { merge, endedAt: now });
|
|
84
|
+
if (state.disposed) return;
|
|
85
|
+
const details: SwarmReportDetails = {
|
|
86
|
+
swarmId: swarm.id,
|
|
87
|
+
elapsedMs: now - swarm.startedAt,
|
|
88
|
+
hordes: swarmHordeRows(swarm, now),
|
|
89
|
+
deliverable: swarm.deliverable,
|
|
90
|
+
};
|
|
91
|
+
const content = [swarmReportText(swarm, now), ...(swarm.deliverable ? ["## Deliverable", "", swarm.deliverable] : [])].join("\n\n");
|
|
92
|
+
// A hard stop of every horde keeps the parent idle, as it does for one horde.
|
|
93
|
+
const triggerTurn = swarm.runs.some((run) => run.cleanStopNames.length > 0 || !run.stopped);
|
|
94
|
+
state.pi.sendMessage(
|
|
95
|
+
{ customType: SWARM_REPORT_TYPE, content, display: true, details },
|
|
96
|
+
{ deliverAs: "followUp", triggerTurn },
|
|
97
|
+
);
|
|
98
|
+
safeNotify(swarm, `swarm ${swarm.id} finished`, "info");
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
/**
|
|
102
|
+
* A horde's finalize completed, synthesis included. The swarm step runs once every horde is
|
|
103
|
+
* done: a swarm of one has nothing to merge and only releases the widget.
|
|
104
|
+
*/
|
|
105
|
+
export async function noteRunFinalized(state: HordeState, swarm: Swarm, run: HordeRun): Promise<void> {
|
|
106
|
+
run.finalized = true;
|
|
107
|
+
if (!swarm.runs.every((other) => other.finalized === true)) {
|
|
108
|
+
refreshSwarm(swarm);
|
|
109
|
+
return;
|
|
110
|
+
}
|
|
111
|
+
try {
|
|
112
|
+
if (swarm.dir && !state.disposed) await mergeSwarm(state, swarm);
|
|
113
|
+
} catch (error) {
|
|
114
|
+
safeNotify(swarm, `swarm ${swarm.id} failed to finalize: ${errorMessage(error)}`, "error");
|
|
115
|
+
} finally {
|
|
116
|
+
swarm.finalizing = undefined;
|
|
117
|
+
if (state.active === swarm) state.active = undefined;
|
|
118
|
+
if (state.finalizing === swarm) state.finalizing = undefined;
|
|
119
|
+
unmountWidget(swarm);
|
|
120
|
+
}
|
|
121
|
+
}
|
|
@@ -0,0 +1,152 @@
|
|
|
1
|
+
import * as path from "node:path";
|
|
2
|
+
import { formatDateTime, formatElapsed, formatTokens, oneLine } from "../format.js";
|
|
3
|
+
import { listFindingsForRun } from "../ledger/findings.js";
|
|
4
|
+
import { statusCounts, totalUsage } from "../prompt.js";
|
|
5
|
+
import { hordeLabel } from "./types.js";
|
|
6
|
+
import type { HordeRun, Swarm } from "./types.js";
|
|
7
|
+
|
|
8
|
+
/** One horde's totals for the swarm report and the injected message. */
|
|
9
|
+
export interface SwarmHordeRow {
|
|
10
|
+
label: string;
|
|
11
|
+
runId: string;
|
|
12
|
+
runDir: string;
|
|
13
|
+
outcome: string;
|
|
14
|
+
members: number;
|
|
15
|
+
done: number;
|
|
16
|
+
failed: number;
|
|
17
|
+
stopped: number;
|
|
18
|
+
turns: number;
|
|
19
|
+
tokens: number;
|
|
20
|
+
cost: number;
|
|
21
|
+
findings: number;
|
|
22
|
+
elapsedMs: number;
|
|
23
|
+
deliverable: boolean;
|
|
24
|
+
postmortem: boolean;
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
/** What the swarm report message carries for its renderer. */
|
|
28
|
+
export interface SwarmReportDetails {
|
|
29
|
+
swarmId: string;
|
|
30
|
+
elapsedMs: number;
|
|
31
|
+
hordes: SwarmHordeRow[];
|
|
32
|
+
deliverable?: string | undefined;
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
export const SWARM_REPORT_TYPE = "agent-horde-swarm-report";
|
|
36
|
+
|
|
37
|
+
function findingCount(run: HordeRun): number {
|
|
38
|
+
try {
|
|
39
|
+
return listFindingsForRun(run.hordeDir, run.id).length;
|
|
40
|
+
} catch {
|
|
41
|
+
return 0;
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
function hordeRow(run: HordeRun, now: number): SwarmHordeRow {
|
|
46
|
+
const counts = statusCounts(run.members);
|
|
47
|
+
const usage = totalUsage(run.members);
|
|
48
|
+
const ended = Math.max(...run.members.map((member) => member.endedAt ?? now), run.startedAt);
|
|
49
|
+
return {
|
|
50
|
+
label: hordeLabel(run.horde),
|
|
51
|
+
runId: run.id,
|
|
52
|
+
runDir: run.runDir,
|
|
53
|
+
outcome: run.outcome ?? (run.finalized ? "finished" : "running"),
|
|
54
|
+
members: run.members.length,
|
|
55
|
+
done: counts.done,
|
|
56
|
+
failed: counts.failed,
|
|
57
|
+
stopped: counts.stopped,
|
|
58
|
+
turns: usage.turns,
|
|
59
|
+
tokens: usage.input + usage.output,
|
|
60
|
+
cost: usage.cost,
|
|
61
|
+
findings: findingCount(run),
|
|
62
|
+
elapsedMs: ended - run.startedAt,
|
|
63
|
+
deliverable: Boolean(run.deliverable),
|
|
64
|
+
postmortem: Boolean(run.postmortem),
|
|
65
|
+
};
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
export function swarmHordeRows(swarm: Swarm, now: number = Date.now()): SwarmHordeRow[] {
|
|
69
|
+
return swarm.runs.map((run) => hordeRow(run, now));
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
function sum(rows: SwarmHordeRow[], pick: (row: SwarmHordeRow) => number): number {
|
|
73
|
+
return rows.reduce((total, row) => total + pick(row), 0);
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
function hordeTable(rows: SwarmHordeRow[]): string[] {
|
|
77
|
+
return [
|
|
78
|
+
"| horde | run | outcome | members | done | failed | stopped | turns | tokens | cost | findings | elapsed |",
|
|
79
|
+
"|---|---|---|---|---|---|---|---|---|---|---|---|",
|
|
80
|
+
...rows.map((row) =>
|
|
81
|
+
`| ${row.label} | ${row.runId} | ${row.outcome} | ${row.members} | ${row.done} | ${row.failed} | ` +
|
|
82
|
+
`${row.stopped} | ${row.turns} | ${formatTokens(row.tokens)} | $${row.cost.toFixed(4)} | ${row.findings} | ${formatElapsed(row.elapsedMs)} |`),
|
|
83
|
+
`| total | | | ${sum(rows, (r) => r.members)} | ${sum(rows, (r) => r.done)} | ${sum(rows, (r) => r.failed)} | ` +
|
|
84
|
+
`${sum(rows, (r) => r.stopped)} | ${sum(rows, (r) => r.turns)} | ${formatTokens(sum(rows, (r) => r.tokens))} | ` +
|
|
85
|
+
`$${sum(rows, (r) => r.cost).toFixed(4)} | ${sum(rows, (r) => r.findings)} | |`,
|
|
86
|
+
];
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
/** REPORT.md of a swarm: the partition, per-horde totals, where each horde's files are, the merged deliverable. */
|
|
90
|
+
export function swarmReportMarkdown(swarm: Swarm, now: number = Date.now()): string {
|
|
91
|
+
const rows = swarmHordeRows(swarm, now);
|
|
92
|
+
return [
|
|
93
|
+
`# Swarm ${swarm.id}`,
|
|
94
|
+
"",
|
|
95
|
+
`- started: ${formatDateTime(new Date(swarm.startedAt))}`,
|
|
96
|
+
`- elapsed: ${formatElapsed(now - swarm.startedAt)}`,
|
|
97
|
+
`- hordes: ${swarm.hordes} x ${swarm.count} members`,
|
|
98
|
+
`- until stop: ${swarm.untilStop ? "yes" : "no"}`,
|
|
99
|
+
`- outcomes: ${rows.map((row) => `${row.label} ${row.outcome}`).join(", ")}`,
|
|
100
|
+
`- merged deliverable: ${swarm.deliverable ? "DELIVERABLE.md" : "not written"}`,
|
|
101
|
+
"",
|
|
102
|
+
"## Prompt",
|
|
103
|
+
"",
|
|
104
|
+
swarm.prompt,
|
|
105
|
+
"",
|
|
106
|
+
"## Partition",
|
|
107
|
+
"",
|
|
108
|
+
...(swarm.partition ?? []).map((slice, index) => `- ${hordeLabel(index + 1)}: ${slice}`),
|
|
109
|
+
"",
|
|
110
|
+
"## Hordes",
|
|
111
|
+
"",
|
|
112
|
+
...hordeTable(rows),
|
|
113
|
+
"",
|
|
114
|
+
...rows.flatMap((row) => [
|
|
115
|
+
`### ${row.label}: ${row.runId}`,
|
|
116
|
+
"",
|
|
117
|
+
`- ${path.join(row.runDir, "REPORT.md")}`,
|
|
118
|
+
...(row.deliverable ? [`- ${path.join(row.runDir, "DELIVERABLE.md")}`] : []),
|
|
119
|
+
...(row.postmortem ? [`- ${path.join(row.runDir, "POSTMORTEM.md")}`] : []),
|
|
120
|
+
"",
|
|
121
|
+
]),
|
|
122
|
+
...(swarm.deliverable ? ["## Deliverable", "", swarm.deliverable, ""] : []),
|
|
123
|
+
].join("\n");
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
/** The message injected into the parent conversation when a swarm finishes. */
|
|
127
|
+
export function swarmReportText(swarm: Swarm, now: number = Date.now()): string {
|
|
128
|
+
const rows = swarmHordeRows(swarm, now);
|
|
129
|
+
const dir = swarm.dir ?? "";
|
|
130
|
+
return [
|
|
131
|
+
`Swarm ${swarm.id} finished after ${formatElapsed(now - swarm.startedAt)}: ${swarm.hordes} hordes of ` +
|
|
132
|
+
`${swarm.count} members on one task, partitioned as ${rows.map((row) => `${row.label} (${row.outcome})`).join(", ")}.`,
|
|
133
|
+
`${sum(rows, (r) => r.turns)} turns, ${formatTokens(sum(rows, (r) => r.tokens))} tokens, ` +
|
|
134
|
+
`$${sum(rows, (r) => r.cost).toFixed(2)}, ${sum(rows, (r) => r.findings)} findings across the hordes.`,
|
|
135
|
+
"",
|
|
136
|
+
"Partition:",
|
|
137
|
+
...(swarm.partition ?? []).map((slice, index) => `- ${hordeLabel(index + 1)}: ${oneLine(slice, 200)}`),
|
|
138
|
+
"",
|
|
139
|
+
"Files:",
|
|
140
|
+
`- ${path.join(dir, "REPORT.md")} per-horde totals and where each horde's files are`,
|
|
141
|
+
...(swarm.deliverable ? [`- ${path.join(dir, "DELIVERABLE.md")} merged deliverable, each claim attributed to its horde`] : []),
|
|
142
|
+
`- ${path.join(dir, "SWARM.md")} the prompt and the partition`,
|
|
143
|
+
...rows.map((row) => `- ${row.runDir}/ ${row.label}: REPORT.md, DELIVERABLE.md, POSTMORTEM.md, SUMMARY.md`),
|
|
144
|
+
"",
|
|
145
|
+
...(swarm.deliverable
|
|
146
|
+
? [
|
|
147
|
+
"The merged deliverable is included below. No horde reviewed another's work, so verify claims",
|
|
148
|
+
"against the owning horde's FINDINGS.md and ACCEPTANCE.md before presenting them to the user.",
|
|
149
|
+
]
|
|
150
|
+
: ["No merged deliverable was written; read each horde's REPORT.md and DELIVERABLE.md."]),
|
|
151
|
+
].join("\n");
|
|
152
|
+
}
|