agent-horde 0.2.0 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/prompt.ts CHANGED
@@ -1,4 +1,4 @@
1
- import { formatClock, formatDateTime, formatElapsed, formatTokens, oneLine, taskHash } from "./format.js";
1
+ import { formatClock, formatDateTime, formatElapsed, formatTokens, formatUsd, oneLine, taskHash } from "./format.js";
2
2
  import type { BoardEntry } from "./ledger/board.js";
3
3
  import type { ClaimEntry } from "./ledger/claims.js";
4
4
  import type { FindingCluster } from "./ledger/clusters.js";
@@ -26,6 +26,8 @@ export interface RunSummaryInput {
26
26
  userStop?: boolean | undefined;
27
27
  /** Human-readable stop reason, e.g. "stopped by user" or "stopped at deadline". */
28
28
  stopLabel?: string | undefined;
29
+ /** The `--budget` in dollars the run (or its swarm) clean-stops at. */
30
+ budget?: number | undefined;
29
31
  /** Set when a member reported the goal solved; the end sentence states whether the run wound down on it. */
30
32
  solved?: SolvedSignal | undefined;
31
33
  /** The other member that validated the proposal, if any. */
@@ -34,6 +36,8 @@ export interface RunSummaryInput {
34
36
  solvedCommitted?: boolean | undefined;
35
37
  /** Members whose queued repair work was skipped for budget. */
36
38
  repairSkipped?: string[] | undefined;
39
+ /** Members held back from later rounds because their attempt left every thread unchanged. */
40
+ repairStalled?: string[] | undefined;
37
41
  /** The REPORT.md outcome, so SUMMARY.md states the same end cause. */
38
42
  outcome?: string | undefined;
39
43
  /** One-line run diagnosis shown in the report message. */
@@ -126,6 +130,8 @@ Collaboration loop:
126
130
  a finding on that idea, explicitly set responseTarget=F#. Superseded F# requests follow the correction.
127
131
  Requester: attach/review evidence on the current response target, then resolve target=Q# refs=[E#]
128
132
  revision=current-hash text=decision, or defer with a reason. Disclosed finding limitations remain debt.
133
+ A blocking request about your own work resolves on the recipient's full supporting review of the
134
+ current revision, since you cannot review it yourself.
129
135
  Do not repeat your existing check: inspect collaboration first and test a specific remaining gap.
130
136
  A deferred blocking issue stays visible as acceptance debt; its recipient can still respond to it and
131
137
  the requester then resolves it. Never silently ignore a peer's failed test.
@@ -273,7 +279,8 @@ Protocol:
273
279
  proposal reopens for validation once every idea it named is done or ruled out. Pass the revision
274
280
  from ANSWER.md or 'horde inspect' target=answer to either action, so stale reviews cannot approve
275
281
  a newer proposal. An answer that cites a superseded F# or quotes a provisional T# is refused, and a
276
- pending proposal is withdrawn for re-proposal when a finding it cites is superseded. A request to
282
+ pending proposal is withdrawn for re-proposal when a finding it cites is superseded or amended,
283
+ or an artifact it cites is republished. A request to
277
284
  the proposer or a validator about the pending answer (target=answer) is served with the blockers.
278
285
  The parent enforces coverage before wind-down.
279
286
 
@@ -294,10 +301,16 @@ export function taskMarkdown(input: {
294
301
  startedAt: number;
295
302
  verifier?: string | undefined;
296
303
  untilStop?: boolean | undefined;
304
+ /** `--budget` in dollars; a swarm's budget covers every horde. */
305
+ budget?: number | undefined;
297
306
  /** Set for a horde of a larger swarm: which horde this is and its workstream. */
298
307
  swarm?: { id: string; horde: number; hordes: number; slice?: string | undefined } | undefined;
299
308
  }): string {
300
309
  const swarm = input.swarm;
310
+ const budget =
311
+ input.budget === undefined
312
+ ? "none"
313
+ : `${formatUsd(input.budget)} (the run clean-stops when member spend${swarm ? " across the swarm" : ""} reaches it)`;
301
314
  return [
302
315
  `# Horde run ${input.id}`,
303
316
  "",
@@ -307,6 +320,7 @@ export function taskMarkdown(input: {
307
320
  `Thinking: ${input.thinkingLevel ?? "parent default"}`,
308
321
  `Verifier: ${input.verifier ?? "none"}`,
309
322
  `Until stop: ${input.untilStop ? "yes (the run continues until stop-clean or stop-hard)" : "no"}`,
323
+ `Budget: ${budget}`,
310
324
  ...(swarm ? [`Swarm: ${swarm.id} (horde h${swarm.horde} of ${swarm.hordes})`] : []),
311
325
  ...(swarm?.slice ? [`Workstream: ${swarm.slice}`] : []),
312
326
  `Task-SHA256: ${taskHash(input.prompt)}`,
@@ -326,6 +340,8 @@ interface TaskInfo {
326
340
  model?: string | undefined;
327
341
  thinking?: string | undefined;
328
342
  hash?: string | undefined;
343
+ /** The `--budget` in dollars, when the run was started with one. */
344
+ budget?: number | undefined;
329
345
  prompt: string;
330
346
  }
331
347
 
@@ -337,13 +353,17 @@ export function parseTaskMarkdown(content: string): TaskInfo {
337
353
  for (const line of lines.slice(0, promptAt === -1 ? lines.length : promptAt)) {
338
354
  const title = /^# Horde run (\S+)$/.exec(line);
339
355
  if (title) info.id = title[1];
340
- const field = /^(Started|Members|Model|Thinking|Task-SHA256): (.*)$/.exec(line);
356
+ const field = /^(Started|Members|Model|Thinking|Budget|Task-SHA256): (.*)$/.exec(line);
341
357
  if (!field) continue;
342
358
  const value = field[2] ?? "";
343
359
  if (field[1] === "Started") info.started = value;
344
360
  else if (field[1] === "Members") info.count = Number(/^\d+/.exec(value)?.[0] ?? NaN) || undefined;
345
361
  else if (field[1] === "Model") info.model = value;
346
362
  else if (field[1] === "Thinking") info.thinking = value;
363
+ else if (field[1] === "Budget") {
364
+ const amount = Number(/^\$(\d+(?:\.\d+)?)/.exec(value)?.[1] ?? NaN);
365
+ if (amount > 0) info.budget = amount;
366
+ }
347
367
  else if (field[1] === "Task-SHA256") info.hash = value;
348
368
  }
349
369
  return info;
@@ -369,13 +389,14 @@ export interface UsageTotals {
369
389
  cost: number;
370
390
  }
371
391
 
392
+ /** Run totals over every session each member has had; turns count the current sessions only. */
372
393
  export function totalUsage(members: MemberState[]): UsageTotals {
373
394
  return members.reduce(
374
395
  (acc, m) => ({
375
396
  turns: acc.turns + m.turns,
376
- input: acc.input + m.usage.input,
377
- output: acc.output + m.usage.output,
378
- cost: acc.cost + m.usage.cost,
397
+ input: acc.input + m.lifetime.input,
398
+ output: acc.output + m.lifetime.output,
399
+ cost: acc.cost + m.lifetime.cost,
379
400
  }),
380
401
  { turns: 0, input: 0, output: 0, cost: 0 },
381
402
  );
@@ -462,10 +483,12 @@ export function runEndSentence(run: RunSummaryInput): string {
462
483
  const attempts = run.members.reduce((n, m) => n + (m.repairs ?? 0), 0);
463
484
  const killed = run.members.filter((m) => ["tokens", "turns", "time"].includes(m.repairStopReason ?? ""));
464
485
  const skipped = run.repairSkipped ?? [];
486
+ const stalled = run.repairStalled ?? [];
465
487
  if (attempts > 0 || skipped.length > 0) {
466
488
  const detail = [
467
489
  killed.length ? `${killed.length} killed on budget (${killed.map((m) => `${m.name}: ${m.repairStopReason}`).join(", ")})` : "",
468
490
  skipped.length ? `${skipped.length} skipped for budget (${skipped.join(", ")})` : "",
491
+ stalled.length ? `${stalled.length} held after no progress (${stalled.join(", ")})` : "",
469
492
  ].filter(Boolean).join(", ");
470
493
  parts.push(`Repair: ${attempts} attempt(s)${detail ? `, ${detail}` : ""}.`);
471
494
  }
@@ -579,6 +602,9 @@ export function formatStatusReport(run: RunSummaryInput, now: number = Date.now(
579
602
  const left = Math.max(0, Math.ceil((run.cleanStopUntil - now) / 1000));
580
603
  lines.push(`clean stop requested: members wrap up now, hard stop in ${left}s`);
581
604
  }
605
+ if (run.budget !== undefined) {
606
+ lines.push(`budget: ${formatUsd(totalUsage(run.members).cost)} of ${formatUsd(run.budget)} spent`);
607
+ }
582
608
  for (const m of run.members) {
583
609
  lines.push(
584
610
  `${m.name.padEnd(10)} ${m.status.padEnd(8)} ${memberActivity(m)} ` +
@@ -646,7 +672,8 @@ function reportHeader(input: RunReportInput, now: number): string[] {
646
672
  if (input.model) lines.push(`Model: ${input.model}`);
647
673
  lines.push(
648
674
  `Usage: ${usage.turns} turns, ${formatTokens(usage.input)} in / ` +
649
- `${formatTokens(usage.output)} out, $${usage.cost.toFixed(4)} cost`,
675
+ `${formatTokens(usage.output)} out, $${usage.cost.toFixed(4)} cost` +
676
+ (input.budget === undefined ? "" : ` (budget ${formatUsd(input.budget)})`),
650
677
  );
651
678
  return lines;
652
679
  }
@@ -0,0 +1,30 @@
1
+ import type { ExtensionCommandContext } from "@earendil-works/pi-coding-agent";
2
+ import { formatUsd } from "../format.js";
3
+ import { totalUsage } from "../prompt.js";
4
+ import { requestCleanStop } from "./controls.js";
5
+ import type { HordeState, Swarm } from "./types.js";
6
+
7
+ /** Dollars the swarm's members have spent over every session, replaced ones included. */
8
+ export function swarmSpend(swarm: Pick<Swarm, "runs">): number {
9
+ return swarm.runs.reduce((sum, run) => sum + totalUsage(run.members).cost, 0);
10
+ }
11
+
12
+ /**
13
+ * `--budget`: once the members' combined spend reaches the threshold, clean-stop every horde
14
+ * still working. Called on every child event, so it fires on the usage update that crosses the
15
+ * line. The flag is set before the first await so a burst of events requests the stop once.
16
+ * Hordes that already reported are skipped: a clean stop on one would re-arm a timer that
17
+ * `stopRunTimers` cleared and record a stop-hard on a finished run.
18
+ */
19
+ export function checkBudget(launch: { state: HordeState; swarm: Swarm; ctx: ExtensionCommandContext }): void {
20
+ const { state, swarm, ctx } = launch;
21
+ if (swarm.budget === undefined || swarm.budgetHit || state.disposed) return;
22
+ const spent = swarmSpend(swarm);
23
+ if (spent < swarm.budget) return;
24
+ swarm.budgetHit = true;
25
+ ctx.ui.notify(`horde ${swarm.id}: budget reached (${formatUsd(spent)} of ${formatUsd(swarm.budget)}); stopping`, "warning");
26
+ for (const run of swarm.runs) {
27
+ if (run.stopped || run.reported || run.finalized) continue;
28
+ void requestCleanStop(run, { kind: "all" }, ctx, { postmortem: false, label: "budget" });
29
+ }
30
+ }
@@ -31,7 +31,12 @@ const UNTIL_STOP_COMPLETION = {
31
31
  description: "keep the whole horde working until stop-clean or stop-hard; validated solves are banked",
32
32
  };
33
33
 
34
- const LAUNCH_FLAG_COMPLETIONS = [DASHBOARD_COMPLETION, NO_VERIFIER_COMPLETION, UNTIL_STOP_COMPLETION];
34
+ const BUDGET_COMPLETION = {
35
+ value: "--budget=5 ",
36
+ description: "clean-stop the run once the members' combined spend reaches this many dollars; edit the number",
37
+ };
38
+
39
+ const LAUNCH_FLAG_COMPLETIONS = [DASHBOARD_COMPLETION, NO_VERIFIER_COMPLETION, UNTIL_STOP_COMPLETION, BUDGET_COMPLETION];
35
40
 
36
41
  /** Argument completions that are not member targets. */
37
42
  const EXTRA_COMPLETIONS: Record<string, Array<{ value: string; description: string }>> = {
@@ -19,7 +19,7 @@ const CLEAN_STOP_ACTIVE_GRACE_MS = Number(process.env.PI_HORDE_ACTIVE_GRACE_MS)
19
19
  const SOLVED_STOP_MS = Number(process.env.PI_HORDE_SOLVED_STOP_MS) || 30_000;
20
20
 
21
21
  /** Why a clean stop was requested; selects the report's stop label. */
22
- type CleanStopLabel = "stop-clean" | "deadline" | "solved";
22
+ type CleanStopLabel = "stop-clean" | "deadline" | "solved" | "budget";
23
23
 
24
24
  interface CleanStopOptions {
25
25
  graceMs?: number | undefined;
@@ -168,6 +168,7 @@ export function setPaused(
168
168
 
169
169
  function stopLabelFor(label: CleanStopLabel): string {
170
170
  if (label === "deadline") return "stopped at deadline";
171
+ if (label === "budget") return "stopped at budget";
171
172
  if (label === "solved") return "stopped during solved wind-down";
172
173
  return "stopped by user";
173
174
  }
@@ -20,7 +20,7 @@ import { clusterFindings } from "../ledger/clusters.js";
20
20
  import { listFacts } from "../ledger/facts.js";
21
21
  import type { FactEntry } from "../ledger/facts.js";
22
22
  import { readHordeFile } from "../ledger/files.js";
23
- import { listFindingsForRun, parseFindings, supersededCitations } from "../ledger/findings.js";
23
+ import { listFindingsForRun, parseFindings, staleAnswerFor } from "../ledger/findings.js";
24
24
  import type { FindingEntry } from "../ledger/findings.js";
25
25
  import { countIdeas, parseIdeas, reconcileOwnedIdeas } from "../ledger/ideas.js";
26
26
  import type { Idea } from "../ledger/ideas.js";
@@ -29,7 +29,7 @@ import type { PendingCleanup } from "../ledger/resources.js";
29
29
  import { archiveSolved, clearCleanStop, clearSolved, readSolvedState, resolveSolvedState, writeAnswer } from "../ledger/run-signals.js";
30
30
  import type { SolvedSignal } from "../ledger/run-signals.js";
31
31
  import { childEnv, execPi } from "../member.js";
32
- import { reportText, runReportMarkdown, summaryMarkdown } from "../prompt.js";
32
+ import { reportText, runReportMarkdown, summaryMarkdown, totalUsage } from "../prompt.js";
33
33
  import { buildRunGraph } from "../run-graph.js";
34
34
  import { runGraphMarkdown } from "../run-graph-mermaid.js";
35
35
  import type { CleanupStatus, ReportResource, RunReportInput } from "../prompt.js";
@@ -137,19 +137,18 @@ interface SettledAnswer {
137
137
  status: string;
138
138
  }
139
139
 
140
- /** Findings the proposal names that a later finding corrected; labeled, not re-versioned. */
141
140
  /**
142
141
  * The proposal body is the member's verbatim text and can predate later challenges and corrections.
143
- * The status names the blockers still open at freeze and the findings a later finding superseded,
144
- * and, once synthesis wrote it, points the operator at DELIVERABLE.md as the reconciled output.
142
+ * The status names the blockers still open at freeze and the citations that changed after the
143
+ * proposal pinned them (superseded or amended findings, republished artifacts), and, once
144
+ * synthesis wrote it, points the operator at DELIVERABLE.md as the reconciled output.
145
145
  */
146
- function annotateAnswer(run: HordeRun, answer: SettledAnswer, acceptance: { hash: string; issues: string[] }, findings: FindingEntry[]): void {
146
+ function annotateAnswer(run: HordeRun, answer: SettledAnswer, acceptance: { hash: string; issues: string[] }, stale: string[]): void {
147
147
  const blockers = acceptance.issues.filter((issue) => /^Q\d+ .*: \w+ blocking /.test(issue)).map((issue) => issue.replace(/:/, ""));
148
148
  const parts = [answer.status];
149
149
  if (acceptance.issues.length) parts.push("acceptance incomplete");
150
150
  if (blockers.length) parts.push(`open blocking at freeze: ${blockers.join(", ")}`);
151
- const superseded = supersededCitations(answer.proposal.text, findings);
152
- if (superseded.length) parts.push(`cites superseded ${superseded.join(", ")}`);
151
+ if (stale.length) parts.push(`cites ${stale.join(", ")}`);
153
152
  parts.push(`see ACCEPTANCE.md (snapshot ${acceptance.hash})`);
154
153
  if (run.deliverable) parts.push("reconciled operator output: DELIVERABLE.md; the proposal text below is verbatim and may be superseded there");
155
154
  try {
@@ -559,18 +558,21 @@ export async function finalizeRun(state: HordeState, swarm: Swarm, run: HordeRun
559
558
  run.acceptance = acceptance.status;
560
559
  run.snapshotHash = acceptance.hash;
561
560
  await recordLifecycle(run, "acceptance", { status: acceptance.status, hash: acceptance.hash, issues: acceptance.issues.length });
562
- const annotate = answer !== undefined &&
563
- (acceptance.issues.length > 0 || supersededCitations(answer.proposal.text, ledgers.findings).length > 0);
564
- if (annotate) annotateAnswer(run, answer, acceptance, ledgers.findings);
561
+ const stale = answer ? staleAnswerFor(run.hordeDir, run.id, answer.proposal.text, answer.proposal.cites) : [];
562
+ const annotate = answer !== undefined && (acceptance.issues.length > 0 || stale.length > 0);
563
+ if (annotate) annotateAnswer(run, answer, acceptance, stale);
565
564
  // Workers and bounded repairs have exited; only now may their labs be torn down.
566
565
  const cleanups = await runCleanups(run);
567
566
  const reportInput = writeRunReports(run, ledgers, releasedIdeas, cleanups, now);
568
567
  // Free the slot before the (possibly slow) synthesis so a new run can start.
569
568
  noteRunReported(state, swarm, run);
570
569
  await synthesize(state, run, ledgers, reportInput, now);
571
- if (annotate && run.deliverable) annotateAnswer(run, answer, acceptance, ledgers.findings);
570
+ if (annotate && run.deliverable) annotateAnswer(run, answer, acceptance, stale);
572
571
  sendReport(state, swarm, run, ledgers, now);
573
- await recordLifecycle(run, "run-end", { outcome: runOutcome(run), elapsedMs: Date.now() - run.startedAt, stopLabel: run.stopLabel });
572
+ await recordLifecycle(run, "run-end", {
573
+ outcome: runOutcome(run), elapsedMs: Date.now() - run.startedAt, stopLabel: run.stopLabel,
574
+ spent: totalUsage(run.members).cost, budget: run.budget,
575
+ });
574
576
  writeRunGraph(run);
575
577
  } catch (error) {
576
578
  safeNotify(run, `horde ${run.id} failed to finalize: ${errorMessage(error)}`, "error");
package/src/run/launch.ts CHANGED
@@ -277,6 +277,7 @@ function buildSwarm(
277
277
  plan: parsed.plan,
278
278
  partition: partition.slices,
279
279
  untilStop: parsed.untilStop === true ? true : undefined,
280
+ budget: parsed.budget,
280
281
  dashboardUrl,
281
282
  };
282
283
  return { swarm, launches };
@@ -287,7 +288,7 @@ async function seedHorde(swarm: Swarm, run: HordeRun, roles: string[]): Promise<
287
288
  await seedRun(run);
288
289
  await recordLifecycle(run, "run-start", {
289
290
  count: run.count, model: run.model, verifier: run.verifier, plan: run.plan, untilStop: run.untilStop,
290
- roles: roles.map((role) => oneLine(role, 120)), dashboardUrl: run.dashboardUrl,
291
+ budget: run.budget, roles: roles.map((role) => oneLine(role, 120)), dashboardUrl: run.dashboardUrl,
291
292
  ...(swarm.hordes > 1 ? { swarm: swarm.id, horde: run.horde, hordes: swarm.hordes, slice: run.slice } : {}),
292
293
  });
293
294
  await recordPhase(run, "discovery", "start");
package/src/run/repair.ts CHANGED
@@ -4,9 +4,10 @@ import { randomUUID } from "node:crypto";
4
4
  import { inboxFor } from "../tool/inbox.js";
5
5
  import type { PeerMessage } from "../ledger/collaboration.js";
6
6
  import { recordEvent } from "../ledger/events.js";
7
+ import type { EventContext } from "../ledger/events.js";
7
8
  import { recordPhase } from "./lifecycle.js";
8
9
  import { envNumber } from "../format.js";
9
- import { resetMemberState } from "../member.js";
10
+ import { resetMemberState, USAGE_KEYS } from "../member.js";
10
11
  import type { MemberState } from "../member.js";
11
12
  import { hardStopMembers } from "./controls.js";
12
13
  import { scheduleCollaboration, stopReason } from "./schedule.js";
@@ -24,11 +25,94 @@ function repairPriority(message: PeerMessage): number {
24
25
  return 3;
25
26
  }
26
27
 
27
- function queueFor(launch: RunLaunch, member: MemberState): PeerMessage[] {
28
- return inboxFor({ hordeDir: launch.run.hordeDir, runId: launch.run.id, actor: member.name })
28
+ /** The repair phase's scheduling state, shared by every round. */
29
+ interface RepairPhase {
30
+ launch: RunLaunch;
31
+ floors: RepairFloors;
32
+ /**
33
+ * Threads a member's completed attempts left exactly as they found them, by member then
34
+ * message id: the signature the thread was stuck at and how many consecutive attempts left it
35
+ * there. The thread is held back from that member once the count reaches `UNMOVED_ATTEMPTS`,
36
+ * until its signature changes, which only another member's action or a new response can cause.
37
+ */
38
+ stalled: Map<string, Map<string, UnmovedThread>>;
39
+ }
40
+
41
+ interface UnmovedThread {
42
+ signature: string;
43
+ attempts: number;
44
+ }
45
+
46
+ /**
47
+ * Completed attempts that may leave a thread untouched before it is held. One miss is routine: a
48
+ * member that answered the rest of its queue, or re-answered the task, often clears the thread on
49
+ * its next attempt within seconds. Two misses on the same signature are the loop this guards.
50
+ */
51
+ const UNMOVED_ATTEMPTS = 2;
52
+
53
+ function memberContext(phase: RepairPhase, member: MemberState): EventContext {
54
+ return { hordeDir: phase.launch.run.hordeDir, runId: phase.launch.run.id, actor: member.name };
55
+ }
56
+
57
+ /**
58
+ * What an attempt can change on a thread: its state and the revision it was answered with.
59
+ * Delivery receipts and a re-worded response on the same revision are not progress, so neither
60
+ * the inbox read of a fresh attempt nor a recipient repeating itself re-qualifies a thread.
61
+ */
62
+ function threadSignature(m: PeerMessage): string {
63
+ return `${m.state}:${m.responseRevision ?? ""}:${m.responseTarget ?? ""}`;
64
+ }
65
+
66
+ function isStalled(phase: RepairPhase, member: MemberState, m: PeerMessage): boolean {
67
+ const unmoved = phase.stalled.get(member.name)?.get(m.id);
68
+ return unmoved !== undefined && unmoved.signature === threadSignature(m) && unmoved.attempts >= UNMOVED_ATTEMPTS;
69
+ }
70
+
71
+ /** The threads currently held back from the member, in inbox order. */
72
+ function heldThreads(phase: RepairPhase, member: MemberState): string[] {
73
+ return inboxFor(memberContext(phase, member)).filter((m) => isStalled(phase, member, m)).map((m) => m.id);
74
+ }
75
+
76
+ /** The threads an attempt for the member would be handed: its inbox minus the threads held back. */
77
+ function queueFor(phase: RepairPhase, member: MemberState): PeerMessage[] {
78
+ return inboxFor(memberContext(phase, member)).filter((m) => !isStalled(phase, member, m))
29
79
  .sort((a, b) => repairPriority(a) - repairPriority(b) || a.createdAt.localeCompare(b.createdAt));
30
80
  }
31
81
 
82
+ /**
83
+ * Counts, per thread, the consecutive completed attempts that left it as they found it; a thread
84
+ * the attempt moved starts over. A killed attempt counts nothing: it may have been about to act.
85
+ * Returns the ids this attempt left unchanged.
86
+ */
87
+ function noteUnchanged(phase: RepairPhase, member: MemberState, queue: PeerMessage[]): string[] {
88
+ if (member.endCause !== "completed") return [];
89
+ const after = new Map(inboxFor(memberContext(phase, member)).map((m) => [m.id, threadSignature(m)]));
90
+ const unmoved = phase.stalled.get(member.name) ?? new Map<string, UnmovedThread>();
91
+ const unchanged: string[] = [];
92
+ for (const m of queue) {
93
+ const signature = threadSignature(m);
94
+ if (after.get(m.id) !== signature) { unmoved.delete(m.id); continue; }
95
+ const previous = unmoved.get(m.id);
96
+ unmoved.set(m.id, { signature, attempts: previous?.signature === signature ? previous.attempts + 1 : 1 });
97
+ unchanged.push(m.id);
98
+ }
99
+ phase.stalled.set(member.name, unmoved);
100
+ return unchanged;
101
+ }
102
+
103
+ /**
104
+ * Once every thread in the member's inbox is one its own completed attempt could not move, later
105
+ * rounds skip the member and its threads drop out of the demand that sizes the shares, until
106
+ * another member changes one. Recorded so the post-mortem sees why the member stopped appearing.
107
+ */
108
+ async function recordStalled(phase: RepairPhase, member: MemberState): Promise<void> {
109
+ const held = heldThreads(phase, member);
110
+ if (!held.length || queueFor(phase, member).length) return;
111
+ const { run } = phase.launch;
112
+ run.repairStalled = [...new Set([...(run.repairStalled ?? []), member.name])];
113
+ await recordEvent({ ...memberContext(phase, member), actor: "parent", attempt: "scheduler" }, "repair-skipped", { payload: { reason: "no-progress", members: [{ member: member.name, messages: held }] } });
114
+ }
115
+
32
116
  /**
33
117
  * What a queue costs in floors: one cold session covers the administrative threads in it, and
34
118
  * each review request or blocking challenge is an inspect/attach/review cycle of about a floor
@@ -114,10 +198,10 @@ export function repairBudget(demands: number[], floors: RepairFloors, rounds: nu
114
198
 
115
199
  type SkipReason = "insufficient-budget" | "insufficient-time";
116
200
 
117
- async function recordSkipped(launch: RunLaunch, pending: MemberState[], reason: SkipReason, detail: Record<string, number>): Promise<void> {
118
- const { run } = launch;
201
+ async function recordSkipped(phase: RepairPhase, pending: MemberState[], reason: SkipReason, detail: Record<string, number>): Promise<void> {
202
+ const { run } = phase.launch;
119
203
  const ctx = { hordeDir: run.hordeDir, runId: run.id, actor: "parent", attempt: "scheduler" };
120
- const members = pending.map((member) => ({ member: member.name, messages: queueFor(launch, member).map((m) => m.id) }))
204
+ const members = pending.map((member) => ({ member: member.name, messages: queueFor(phase, member).map((m) => m.id) }))
121
205
  .filter((entry) => entry.messages.length);
122
206
  if (!members.length) return;
123
207
  run.repairSkipped = [...new Set([...(run.repairSkipped ?? []), ...members.map((entry) => entry.member)])];
@@ -143,10 +227,11 @@ interface FirstTurnCost {
143
227
  * share is the allowance recorded and the deadline is derived from it once, so the record cannot
144
228
  * read a millisecond under the floor the share was raised to.
145
229
  */
146
- async function repairMember(launch: RunLaunch, member: MemberState, ms: number, limits: RepairLimits, floors: RepairFloors): Promise<void> {
230
+ async function repairMember(phase: RepairPhase, member: MemberState, ms: number, limits: RepairLimits): Promise<void> {
231
+ const { launch, floors } = phase;
147
232
  const { run } = launch;
148
- const ctx = { hordeDir: run.hordeDir, runId: run.id, actor: member.name };
149
- const queue = queueFor(launch, member);
233
+ const ctx = memberContext(phase, member);
234
+ const queue = queueFor(phase, member);
150
235
  if (!queue.length || stopReason(launch)) return;
151
236
  const prior = { turns: member.turns, toolCalls: member.toolCalls, usage: { ...member.usage }, startedAt: member.startedAt };
152
237
  resetMemberState(member);
@@ -181,7 +266,8 @@ async function repairMember(launch: RunLaunch, member: MemberState, ms: number,
181
266
  const timeout = setTimeout(() => halt("time"), Math.max(1, deadline - Date.now()));
182
267
  const watchdog = setInterval(check, 100);
183
268
  try {
184
- await recordEvent({ ...ctx, attempt: member.attempt }, "repair-start", { payload: { messages: queue.map((m) => m.id), demand: repairDemand(queue), prior, allowance, floors } });
269
+ const held = heldThreads(phase, member);
270
+ await recordEvent({ ...ctx, attempt: member.attempt }, "repair-start", { payload: { messages: queue.map((m) => m.id), demand: repairDemand(queue), prior, allowance, floors, ...(held.length ? { held } : {}) } });
185
271
  if (stopReason(launch) || Date.now() >= deadline) {
186
272
  halt(stopReason(launch) ?? "time");
187
273
  member.status = "stopped";
@@ -191,17 +277,19 @@ async function repairMember(launch: RunLaunch, member: MemberState, ms: number,
191
277
  // Enforce a share on every usage update, rather than waiting for the next poll.
192
278
  await spawnOne(launch, member, undefined, check);
193
279
  }
280
+ const unchanged = noteUnchanged(phase, member, queue);
194
281
  await recordEvent({ ...ctx, attempt: member.attempt }, "repair-end", { payload: {
195
282
  cause: member.endCause, stopReason: member.repairStopReason, allowance, usage: member.usage, turns: member.turns, firstTurn,
196
- messages: queue.map((m) => m.id),
283
+ messages: queue.map((m) => m.id), unchanged,
197
284
  } });
285
+ if (unchanged.length) await recordStalled(phase, member);
198
286
  } finally {
199
287
  clearTimeout(timeout);
200
288
  clearInterval(watchdog);
201
289
  run.repairTokens = (run.repairTokens ?? 0) + spent();
202
290
  member.turns += prior.turns;
203
291
  member.toolCalls += prior.toolCalls;
204
- for (const key of ["input", "output", "cacheRead", "cacheWrite", "cost"] as const) member.usage[key] += prior.usage[key];
292
+ for (const key of USAGE_KEYS) member.usage[key] += prior.usage[key];
205
293
  member.startedAt = prior.startedAt;
206
294
  }
207
295
  }
@@ -223,19 +311,42 @@ export async function drainRepairs(launch: RunLaunch): Promise<void> {
223
311
  }
224
312
  }
225
313
 
226
- function eligibleMembers(launch: RunLaunch): Array<{ member: MemberState; queue: PeerMessage[] }> {
227
- return launch.run.members.map((member) => ({ member, queue: queueFor(launch, member) })).filter((entry) => entry.queue.length)
314
+ function eligibleMembers(phase: RepairPhase): Array<{ member: MemberState; queue: PeerMessage[] }> {
315
+ return phase.launch.run.members.map((member) => ({ member, queue: queueFor(phase, member) })).filter((entry) => entry.queue.length)
228
316
  .sort((a, b) => repairPriority(a.queue[0]!) - repairPriority(b.queue[0]!) ||
229
317
  (a.member.repairs ?? 0) - (b.member.repairs ?? 0) || a.queue[0]!.createdAt.localeCompare(b.queue[0]!.createdAt));
230
318
  }
231
319
 
320
+ /** The queue a member was handed this round, by message id, at the signature it was handed. */
321
+ type HandedQueues = Map<string, Map<string, string>>;
322
+
323
+ /**
324
+ * True when every blocking thread left belongs to a member already served this round, and one of
325
+ * them is new or moved since that member was handed its queue. A correction answered mid-round
326
+ * leaves its requester to recheck it; finishing the round on administrative rechecks first spent
327
+ * the clock that recheck needed, so when a later round exists it starts now. A blocker the member
328
+ * left as it was handed is no reason to serve it again early; the last round serves everyone.
329
+ */
330
+ export function blockedOnServed(eligible: Array<{ member: MemberState; queue: PeerMessage[] }>, handed: HandedQueues): boolean {
331
+ let moved = false;
332
+ for (const { member, queue } of eligible) {
333
+ const seen = handed.get(member.name);
334
+ for (const m of queue) {
335
+ if (repairPriority(m) !== 0) continue;
336
+ if (!seen) return false;
337
+ if (seen.get(m.id) !== threadSignature(m)) moved = true;
338
+ }
339
+ }
340
+ return moved;
341
+ }
342
+
232
343
  async function runRepairRounds(launch: RunLaunch, begin: (budget: RepairBudget) => Promise<void>): Promise<void> {
233
344
  const { run } = launch;
234
345
  const rounds = Math.max(0, Math.min(6, envNumber("PI_HORDE_REPAIR_ROUNDS", 4)));
235
346
  if (rounds === 0 || stopReason(launch)) return;
236
- const floors = repairFloors();
347
+ const phase: RepairPhase = { launch, floors: repairFloors(), stalled: new Map() };
237
348
  await scheduleCollaboration(launch, { settleMs: 0, liveOnly: false });
238
- const budget = repairBudget(eligibleMembers(launch).map((entry) => repairDemand(entry.queue)), floors, rounds);
349
+ const budget = repairBudget(eligibleMembers(phase).map((entry) => repairDemand(entry.queue)), phase.floors, rounds);
239
350
  if (budget.queued === 0) return;
240
351
  const deadline = Date.now() + budget.ms;
241
352
  const exhausted = () => stopReason(launch) || Date.now() >= deadline || (run.repairTokens ?? 0) >= budget.tokens;
@@ -245,34 +356,35 @@ async function runRepairRounds(launch: RunLaunch, begin: (budget: RepairBudget)
245
356
  // One attempt per member per round, picked in priority order re-read before each attempt:
246
357
  // a thread that became blocking during an attempt moves its recipient ahead of the
247
358
  // administrative queues that were ordered first at round start.
248
- const served = new Set<string>();
359
+ const handed: HandedQueues = new Map();
249
360
  for (;;) {
250
361
  if (exhausted()) return;
251
- const pending = eligibleMembers(launch).filter((entry) => !served.has(entry.member.name));
252
- if (!pending.length) break;
362
+ const eligible = eligibleMembers(phase);
363
+ const pending = eligible.filter((entry) => !handed.has(entry.member.name));
364
+ if (!pending.length || (round < rounds - 1 && blockedOnServed(eligible, handed))) break;
253
365
  const next = pending[0]!;
254
366
  const pendingUnits = pending.reduce((sum, entry) => sum + repairDemand(entry.queue), 0);
255
367
  const units = repairDemand(next.queue);
256
368
  const unserved = pending.map((entry) => entry.member);
257
369
  const remaining = budget.tokens - (run.repairTokens ?? 0);
258
370
  const laterRounds = rounds - round - 1;
259
- const tokens = repairAllowance(repairShare(remaining, pendingUnits, laterRounds, units), remaining, floors.tokens);
371
+ const tokens = repairAllowance(repairShare(remaining, pendingUnits, laterRounds, units), remaining, phase.floors.tokens);
260
372
  if (tokens === undefined) {
261
373
  await begin(budget);
262
- await recordSkipped(launch, unserved, "insufficient-budget", { remaining, floor: floors.tokens });
374
+ await recordSkipped(phase, unserved, "insufficient-budget", { remaining, floor: phase.floors.tokens });
263
375
  return;
264
376
  }
265
377
  const remainingMs = Math.max(0, deadline - Date.now());
266
- const fundable = Math.floor(remaining / Math.max(1, floors.tokens));
267
- const ms = repairAllowance(repairTimeShare(remainingMs, pendingUnits, laterRounds, fundable, units), remainingMs, floors.ms);
378
+ const fundable = Math.floor(remaining / Math.max(1, phase.floors.tokens));
379
+ const ms = repairAllowance(repairTimeShare(remainingMs, pendingUnits, laterRounds, fundable, units), remainingMs, phase.floors.ms);
268
380
  if (ms === undefined || ms < 1) {
269
381
  await begin(budget);
270
- await recordSkipped(launch, unserved, "insufficient-time", { remainingMs, floorMs: floors.ms });
382
+ await recordSkipped(phase, unserved, "insufficient-time", { remainingMs, floorMs: phase.floors.ms });
271
383
  return;
272
384
  }
273
385
  await begin(budget);
274
- served.add(next.member.name);
275
- await repairMember(launch, next.member, ms, { tokens, turns: budget.turns }, floors);
386
+ handed.set(next.member.name, new Map(next.queue.map((m) => [m.id, threadSignature(m)])));
387
+ await repairMember(phase, next.member, ms, { tokens, turns: budget.turns });
276
388
  }
277
389
  }
278
390
  }
package/src/run/setup.ts CHANGED
@@ -93,7 +93,8 @@ export function createRun(
93
93
  fs.writeFileSync(
94
94
  path.join(runDir, "TASK.md"),
95
95
  taskMarkdown({
96
- id, prompt, count, model, thinkingLevel: ctx.thinkingLevel, startedAt, verifier, untilStop: parsed.untilStop, swarm,
96
+ id, prompt, count, model, thinkingLevel: ctx.thinkingLevel, startedAt, verifier, untilStop: parsed.untilStop,
97
+ budget: parsed.budget, swarm,
97
98
  }),
98
99
  "utf-8",
99
100
  );
@@ -123,6 +124,7 @@ export function createRun(
123
124
  plan,
124
125
  verifier,
125
126
  untilStop: parsed.untilStop === true ? true : undefined,
127
+ budget: parsed.budget,
126
128
  resets: [],
127
129
  dashboardUrl,
128
130
  };
package/src/run/solved.ts CHANGED
@@ -4,7 +4,7 @@ import { loadCollaborationReport } from "../collaboration-report.js";
4
4
  import { openMessages } from "../ledger/collaboration.js";
5
5
  import { postBoard } from "../ledger/board.js";
6
6
  import { unsettledFactCitations } from "../ledger/facts.js";
7
- import { listFindingsForRun, staleAnswerCitations, unverifiedFindings } from "../ledger/findings.js";
7
+ import { staleAnswerFor, unverifiedFindings } from "../ledger/findings.js";
8
8
  import { listIdeas } from "../ledger/ideas.js";
9
9
  import { archiveSolved, clearCleanStop, clearSolved, readSolvedState, reopenSolved, resolveSolvedState, writeAnswer } from "../ledger/run-signals.js";
10
10
  import type { SolvedSignal } from "../ledger/run-signals.js";
@@ -80,14 +80,27 @@ function dropContested(run: HordeRun, proposal: SolvedSignal, contest: SolvedSig
80
80
  }
81
81
 
82
82
  /**
83
- * A proposal that cites a finding superseded or amended after it was made would ship the stale
84
- * reading as the answer. It is withdrawn for re-proposal rather than validated; the board names
85
- * the successor or the amendment.
83
+ * ANSWER.md keeps the withdrawn text for the record, but its status must stop naming the
84
+ * validation it held, or the run reads as validated after the proposal is gone.
85
+ */
86
+ function recordWithdrawn(run: HordeRun, proposal: SolvedSignal, reason: string): void {
87
+ void recordLifecycle(run, "solved-withdrawn", { member: proposal.member, reason });
88
+ try {
89
+ writeAnswer(run.runDir, run.id, proposal, `withdrawn: it ${reason}; no validation carries over and no proposal is pending`);
90
+ } catch (error) {
91
+ safeNotify(run, `horde ${run.id}: could not write ANSWER.md (${oneLine(String(error), 200)})`, "error");
92
+ }
93
+ }
94
+
95
+ /**
96
+ * A proposal that cites a finding superseded or amended, or an artifact republished, after it was
97
+ * made would ship the stale reading as the answer. It is withdrawn for re-proposal rather than
98
+ * validated; the board names the successor, the amendment, or the republish.
86
99
  */
87
100
  function withdrawStale(run: HordeRun, proposal: SolvedSignal, stale: string[]): void {
88
101
  const reason = `cites ${stale.join(", ")}`;
89
102
  safeNotify(run, `horde ${run.id}: solved proposal withdrawn; it ${reason}`, "warning");
90
- void recordLifecycle(run, "solved-withdrawn", { member: proposal.member, reason });
103
+ recordWithdrawn(run, proposal, reason);
91
104
  void postBoard(
92
105
  run.hordeDir,
93
106
  "parent",
@@ -111,13 +124,13 @@ function reopenContested(run: HordeRun): void {
111
124
  // The proposal returns with the citations it pinned when it was made; what went stale or
112
125
  // unsettled meanwhile makes it a re-proposal for its author, not a reopening.
113
126
  const stale = [
114
- ...staleAnswerCitations(listFindingsForRun(run.hordeDir, run.id), contest.proposal.text, contest.proposal.cites),
127
+ ...staleAnswerFor(run.hordeDir, run.id, contest.proposal.text, contest.proposal.cites),
115
128
  ...unsettledFactCitations(run.hordeDir, contest.proposal.text).map((citation) => `provisional ${citation}`),
116
129
  ];
117
130
  run.solvedContest = undefined;
118
131
  if (stale.length) {
119
132
  const why = `cites ${stale.join(", ")}`;
120
- void recordLifecycle(run, "solved-withdrawn", { member: contest.proposal.member, reason: why });
133
+ recordWithdrawn(run, contest.proposal, why);
121
134
  void postBoard(run.hordeDir, "parent", `SOLVED proposal by ${contest.proposal.member} is not re-opened after ${reason}: it ${why}. ${contest.proposal.member}: propose again citing the current findings.`);
122
135
  return;
123
136
  }
@@ -262,7 +275,7 @@ export function checkSolved(run: HordeRun, ctx: ExtensionCommandContext): void {
262
275
  dropContested(run, solved, state.contest);
263
276
  return;
264
277
  }
265
- const stale = staleAnswerCitations(listFindingsForRun(run.hordeDir, run.id), solved.text, solved.cites);
278
+ const stale = staleAnswerFor(run.hordeDir, run.id, solved.text, solved.cites);
266
279
  if (stale.length && !run.solvedCommitted) {
267
280
  withdrawStale(run, solved, stale);
268
281
  return;