agent-horde 0.2.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +13 -2
- package/docs/REFERENCE.md +36 -9
- package/package.json +2 -2
- package/src/args.ts +30 -4
- package/src/citations.ts +16 -5
- package/src/dashboard-data.ts +31 -4
- package/src/dashboard-html.ts +4 -0
- package/src/format.ts +5 -0
- package/src/ledger/findings.ts +25 -6
- package/src/member.ts +22 -7
- package/src/prompt.ts +34 -7
- package/src/run/budget.ts +30 -0
- package/src/run/completions.ts +6 -1
- package/src/run/controls.ts +2 -1
- package/src/run/finalize.ts +15 -13
- package/src/run/launch.ts +2 -1
- package/src/run/repair.ts +138 -26
- package/src/run/setup.ts +3 -1
- package/src/run/solved.ts +21 -8
- package/src/run/supervise.ts +3 -2
- package/src/run/swarm-finalize.ts +3 -0
- package/src/run/swarm-report.ts +2 -1
- package/src/run/swarm.ts +2 -1
- package/src/run/types.ts +8 -0
- package/src/tool/actions.ts +7 -7
- package/src/tool/collaboration-actions.ts +31 -6
- package/src/tool/params.ts +14 -2
- package/src/widget.ts +12 -2
package/src/prompt.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { formatClock, formatDateTime, formatElapsed, formatTokens, oneLine, taskHash } from "./format.js";
|
|
1
|
+
import { formatClock, formatDateTime, formatElapsed, formatTokens, formatUsd, oneLine, taskHash } from "./format.js";
|
|
2
2
|
import type { BoardEntry } from "./ledger/board.js";
|
|
3
3
|
import type { ClaimEntry } from "./ledger/claims.js";
|
|
4
4
|
import type { FindingCluster } from "./ledger/clusters.js";
|
|
@@ -26,6 +26,8 @@ export interface RunSummaryInput {
|
|
|
26
26
|
userStop?: boolean | undefined;
|
|
27
27
|
/** Human-readable stop reason, e.g. "stopped by user" or "stopped at deadline". */
|
|
28
28
|
stopLabel?: string | undefined;
|
|
29
|
+
/** The `--budget` in dollars the run (or its swarm) clean-stops at. */
|
|
30
|
+
budget?: number | undefined;
|
|
29
31
|
/** Set when a member reported the goal solved; the end sentence states whether the run wound down on it. */
|
|
30
32
|
solved?: SolvedSignal | undefined;
|
|
31
33
|
/** The other member that validated the proposal, if any. */
|
|
@@ -34,6 +36,8 @@ export interface RunSummaryInput {
|
|
|
34
36
|
solvedCommitted?: boolean | undefined;
|
|
35
37
|
/** Members whose queued repair work was skipped for budget. */
|
|
36
38
|
repairSkipped?: string[] | undefined;
|
|
39
|
+
/** Members held back from later rounds because their attempt left every thread unchanged. */
|
|
40
|
+
repairStalled?: string[] | undefined;
|
|
37
41
|
/** The REPORT.md outcome, so SUMMARY.md states the same end cause. */
|
|
38
42
|
outcome?: string | undefined;
|
|
39
43
|
/** One-line run diagnosis shown in the report message. */
|
|
@@ -126,6 +130,8 @@ Collaboration loop:
|
|
|
126
130
|
a finding on that idea, explicitly set responseTarget=F#. Superseded F# requests follow the correction.
|
|
127
131
|
Requester: attach/review evidence on the current response target, then resolve target=Q# refs=[E#]
|
|
128
132
|
revision=current-hash text=decision, or defer with a reason. Disclosed finding limitations remain debt.
|
|
133
|
+
A blocking request about your own work resolves on the recipient's full supporting review of the
|
|
134
|
+
current revision, since you cannot review it yourself.
|
|
129
135
|
Do not repeat your existing check: inspect collaboration first and test a specific remaining gap.
|
|
130
136
|
A deferred blocking issue stays visible as acceptance debt; its recipient can still respond to it and
|
|
131
137
|
the requester then resolves it. Never silently ignore a peer's failed test.
|
|
@@ -273,7 +279,8 @@ Protocol:
|
|
|
273
279
|
proposal reopens for validation once every idea it named is done or ruled out. Pass the revision
|
|
274
280
|
from ANSWER.md or 'horde inspect' target=answer to either action, so stale reviews cannot approve
|
|
275
281
|
a newer proposal. An answer that cites a superseded F# or quotes a provisional T# is refused, and a
|
|
276
|
-
pending proposal is withdrawn for re-proposal when a finding it cites is superseded
|
|
282
|
+
pending proposal is withdrawn for re-proposal when a finding it cites is superseded or amended,
|
|
283
|
+
or an artifact it cites is republished. A request to
|
|
277
284
|
the proposer or a validator about the pending answer (target=answer) is served with the blockers.
|
|
278
285
|
The parent enforces coverage before wind-down.
|
|
279
286
|
|
|
@@ -294,10 +301,16 @@ export function taskMarkdown(input: {
|
|
|
294
301
|
startedAt: number;
|
|
295
302
|
verifier?: string | undefined;
|
|
296
303
|
untilStop?: boolean | undefined;
|
|
304
|
+
/** `--budget` in dollars; a swarm's budget covers every horde. */
|
|
305
|
+
budget?: number | undefined;
|
|
297
306
|
/** Set for a horde of a larger swarm: which horde this is and its workstream. */
|
|
298
307
|
swarm?: { id: string; horde: number; hordes: number; slice?: string | undefined } | undefined;
|
|
299
308
|
}): string {
|
|
300
309
|
const swarm = input.swarm;
|
|
310
|
+
const budget =
|
|
311
|
+
input.budget === undefined
|
|
312
|
+
? "none"
|
|
313
|
+
: `${formatUsd(input.budget)} (the run clean-stops when member spend${swarm ? " across the swarm" : ""} reaches it)`;
|
|
301
314
|
return [
|
|
302
315
|
`# Horde run ${input.id}`,
|
|
303
316
|
"",
|
|
@@ -307,6 +320,7 @@ export function taskMarkdown(input: {
|
|
|
307
320
|
`Thinking: ${input.thinkingLevel ?? "parent default"}`,
|
|
308
321
|
`Verifier: ${input.verifier ?? "none"}`,
|
|
309
322
|
`Until stop: ${input.untilStop ? "yes (the run continues until stop-clean or stop-hard)" : "no"}`,
|
|
323
|
+
`Budget: ${budget}`,
|
|
310
324
|
...(swarm ? [`Swarm: ${swarm.id} (horde h${swarm.horde} of ${swarm.hordes})`] : []),
|
|
311
325
|
...(swarm?.slice ? [`Workstream: ${swarm.slice}`] : []),
|
|
312
326
|
`Task-SHA256: ${taskHash(input.prompt)}`,
|
|
@@ -326,6 +340,8 @@ interface TaskInfo {
|
|
|
326
340
|
model?: string | undefined;
|
|
327
341
|
thinking?: string | undefined;
|
|
328
342
|
hash?: string | undefined;
|
|
343
|
+
/** The `--budget` in dollars, when the run was started with one. */
|
|
344
|
+
budget?: number | undefined;
|
|
329
345
|
prompt: string;
|
|
330
346
|
}
|
|
331
347
|
|
|
@@ -337,13 +353,17 @@ export function parseTaskMarkdown(content: string): TaskInfo {
|
|
|
337
353
|
for (const line of lines.slice(0, promptAt === -1 ? lines.length : promptAt)) {
|
|
338
354
|
const title = /^# Horde run (\S+)$/.exec(line);
|
|
339
355
|
if (title) info.id = title[1];
|
|
340
|
-
const field = /^(Started|Members|Model|Thinking|Task-SHA256): (.*)$/.exec(line);
|
|
356
|
+
const field = /^(Started|Members|Model|Thinking|Budget|Task-SHA256): (.*)$/.exec(line);
|
|
341
357
|
if (!field) continue;
|
|
342
358
|
const value = field[2] ?? "";
|
|
343
359
|
if (field[1] === "Started") info.started = value;
|
|
344
360
|
else if (field[1] === "Members") info.count = Number(/^\d+/.exec(value)?.[0] ?? NaN) || undefined;
|
|
345
361
|
else if (field[1] === "Model") info.model = value;
|
|
346
362
|
else if (field[1] === "Thinking") info.thinking = value;
|
|
363
|
+
else if (field[1] === "Budget") {
|
|
364
|
+
const amount = Number(/^\$(\d+(?:\.\d+)?)/.exec(value)?.[1] ?? NaN);
|
|
365
|
+
if (amount > 0) info.budget = amount;
|
|
366
|
+
}
|
|
347
367
|
else if (field[1] === "Task-SHA256") info.hash = value;
|
|
348
368
|
}
|
|
349
369
|
return info;
|
|
@@ -369,13 +389,14 @@ export interface UsageTotals {
|
|
|
369
389
|
cost: number;
|
|
370
390
|
}
|
|
371
391
|
|
|
392
|
+
/** Run totals over every session each member has had; turns count the current sessions only. */
|
|
372
393
|
export function totalUsage(members: MemberState[]): UsageTotals {
|
|
373
394
|
return members.reduce(
|
|
374
395
|
(acc, m) => ({
|
|
375
396
|
turns: acc.turns + m.turns,
|
|
376
|
-
input: acc.input + m.
|
|
377
|
-
output: acc.output + m.
|
|
378
|
-
cost: acc.cost + m.
|
|
397
|
+
input: acc.input + m.lifetime.input,
|
|
398
|
+
output: acc.output + m.lifetime.output,
|
|
399
|
+
cost: acc.cost + m.lifetime.cost,
|
|
379
400
|
}),
|
|
380
401
|
{ turns: 0, input: 0, output: 0, cost: 0 },
|
|
381
402
|
);
|
|
@@ -462,10 +483,12 @@ export function runEndSentence(run: RunSummaryInput): string {
|
|
|
462
483
|
const attempts = run.members.reduce((n, m) => n + (m.repairs ?? 0), 0);
|
|
463
484
|
const killed = run.members.filter((m) => ["tokens", "turns", "time"].includes(m.repairStopReason ?? ""));
|
|
464
485
|
const skipped = run.repairSkipped ?? [];
|
|
486
|
+
const stalled = run.repairStalled ?? [];
|
|
465
487
|
if (attempts > 0 || skipped.length > 0) {
|
|
466
488
|
const detail = [
|
|
467
489
|
killed.length ? `${killed.length} killed on budget (${killed.map((m) => `${m.name}: ${m.repairStopReason}`).join(", ")})` : "",
|
|
468
490
|
skipped.length ? `${skipped.length} skipped for budget (${skipped.join(", ")})` : "",
|
|
491
|
+
stalled.length ? `${stalled.length} held after no progress (${stalled.join(", ")})` : "",
|
|
469
492
|
].filter(Boolean).join(", ");
|
|
470
493
|
parts.push(`Repair: ${attempts} attempt(s)${detail ? `, ${detail}` : ""}.`);
|
|
471
494
|
}
|
|
@@ -579,6 +602,9 @@ export function formatStatusReport(run: RunSummaryInput, now: number = Date.now(
|
|
|
579
602
|
const left = Math.max(0, Math.ceil((run.cleanStopUntil - now) / 1000));
|
|
580
603
|
lines.push(`clean stop requested: members wrap up now, hard stop in ${left}s`);
|
|
581
604
|
}
|
|
605
|
+
if (run.budget !== undefined) {
|
|
606
|
+
lines.push(`budget: ${formatUsd(totalUsage(run.members).cost)} of ${formatUsd(run.budget)} spent`);
|
|
607
|
+
}
|
|
582
608
|
for (const m of run.members) {
|
|
583
609
|
lines.push(
|
|
584
610
|
`${m.name.padEnd(10)} ${m.status.padEnd(8)} ${memberActivity(m)} ` +
|
|
@@ -646,7 +672,8 @@ function reportHeader(input: RunReportInput, now: number): string[] {
|
|
|
646
672
|
if (input.model) lines.push(`Model: ${input.model}`);
|
|
647
673
|
lines.push(
|
|
648
674
|
`Usage: ${usage.turns} turns, ${formatTokens(usage.input)} in / ` +
|
|
649
|
-
`${formatTokens(usage.output)} out, $${usage.cost.toFixed(4)} cost
|
|
675
|
+
`${formatTokens(usage.output)} out, $${usage.cost.toFixed(4)} cost` +
|
|
676
|
+
(input.budget === undefined ? "" : ` (budget ${formatUsd(input.budget)})`),
|
|
650
677
|
);
|
|
651
678
|
return lines;
|
|
652
679
|
}
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
import type { ExtensionCommandContext } from "@earendil-works/pi-coding-agent";
|
|
2
|
+
import { formatUsd } from "../format.js";
|
|
3
|
+
import { totalUsage } from "../prompt.js";
|
|
4
|
+
import { requestCleanStop } from "./controls.js";
|
|
5
|
+
import type { HordeState, Swarm } from "./types.js";
|
|
6
|
+
|
|
7
|
+
/** Dollars the swarm's members have spent over every session, replaced ones included. */
|
|
8
|
+
export function swarmSpend(swarm: Pick<Swarm, "runs">): number {
|
|
9
|
+
return swarm.runs.reduce((sum, run) => sum + totalUsage(run.members).cost, 0);
|
|
10
|
+
}
|
|
11
|
+
|
|
12
|
+
/**
|
|
13
|
+
* `--budget`: once the members' combined spend reaches the threshold, clean-stop every horde
|
|
14
|
+
* still working. Called on every child event, so it fires on the usage update that crosses the
|
|
15
|
+
* line. The flag is set before the first await so a burst of events requests the stop once.
|
|
16
|
+
* Hordes that already reported are skipped: a clean stop on one would re-arm a timer that
|
|
17
|
+
* `stopRunTimers` cleared and record a stop-hard on a finished run.
|
|
18
|
+
*/
|
|
19
|
+
export function checkBudget(launch: { state: HordeState; swarm: Swarm; ctx: ExtensionCommandContext }): void {
|
|
20
|
+
const { state, swarm, ctx } = launch;
|
|
21
|
+
if (swarm.budget === undefined || swarm.budgetHit || state.disposed) return;
|
|
22
|
+
const spent = swarmSpend(swarm);
|
|
23
|
+
if (spent < swarm.budget) return;
|
|
24
|
+
swarm.budgetHit = true;
|
|
25
|
+
ctx.ui.notify(`horde ${swarm.id}: budget reached (${formatUsd(spent)} of ${formatUsd(swarm.budget)}); stopping`, "warning");
|
|
26
|
+
for (const run of swarm.runs) {
|
|
27
|
+
if (run.stopped || run.reported || run.finalized) continue;
|
|
28
|
+
void requestCleanStop(run, { kind: "all" }, ctx, { postmortem: false, label: "budget" });
|
|
29
|
+
}
|
|
30
|
+
}
|
package/src/run/completions.ts
CHANGED
|
@@ -31,7 +31,12 @@ const UNTIL_STOP_COMPLETION = {
|
|
|
31
31
|
description: "keep the whole horde working until stop-clean or stop-hard; validated solves are banked",
|
|
32
32
|
};
|
|
33
33
|
|
|
34
|
-
const
|
|
34
|
+
const BUDGET_COMPLETION = {
|
|
35
|
+
value: "--budget=5 ",
|
|
36
|
+
description: "clean-stop the run once the members' combined spend reaches this many dollars; edit the number",
|
|
37
|
+
};
|
|
38
|
+
|
|
39
|
+
const LAUNCH_FLAG_COMPLETIONS = [DASHBOARD_COMPLETION, NO_VERIFIER_COMPLETION, UNTIL_STOP_COMPLETION, BUDGET_COMPLETION];
|
|
35
40
|
|
|
36
41
|
/** Argument completions that are not member targets. */
|
|
37
42
|
const EXTRA_COMPLETIONS: Record<string, Array<{ value: string; description: string }>> = {
|
package/src/run/controls.ts
CHANGED
|
@@ -19,7 +19,7 @@ const CLEAN_STOP_ACTIVE_GRACE_MS = Number(process.env.PI_HORDE_ACTIVE_GRACE_MS)
|
|
|
19
19
|
const SOLVED_STOP_MS = Number(process.env.PI_HORDE_SOLVED_STOP_MS) || 30_000;
|
|
20
20
|
|
|
21
21
|
/** Why a clean stop was requested; selects the report's stop label. */
|
|
22
|
-
type CleanStopLabel = "stop-clean" | "deadline" | "solved";
|
|
22
|
+
type CleanStopLabel = "stop-clean" | "deadline" | "solved" | "budget";
|
|
23
23
|
|
|
24
24
|
interface CleanStopOptions {
|
|
25
25
|
graceMs?: number | undefined;
|
|
@@ -168,6 +168,7 @@ export function setPaused(
|
|
|
168
168
|
|
|
169
169
|
function stopLabelFor(label: CleanStopLabel): string {
|
|
170
170
|
if (label === "deadline") return "stopped at deadline";
|
|
171
|
+
if (label === "budget") return "stopped at budget";
|
|
171
172
|
if (label === "solved") return "stopped during solved wind-down";
|
|
172
173
|
return "stopped by user";
|
|
173
174
|
}
|
package/src/run/finalize.ts
CHANGED
|
@@ -20,7 +20,7 @@ import { clusterFindings } from "../ledger/clusters.js";
|
|
|
20
20
|
import { listFacts } from "../ledger/facts.js";
|
|
21
21
|
import type { FactEntry } from "../ledger/facts.js";
|
|
22
22
|
import { readHordeFile } from "../ledger/files.js";
|
|
23
|
-
import { listFindingsForRun, parseFindings,
|
|
23
|
+
import { listFindingsForRun, parseFindings, staleAnswerFor } from "../ledger/findings.js";
|
|
24
24
|
import type { FindingEntry } from "../ledger/findings.js";
|
|
25
25
|
import { countIdeas, parseIdeas, reconcileOwnedIdeas } from "../ledger/ideas.js";
|
|
26
26
|
import type { Idea } from "../ledger/ideas.js";
|
|
@@ -29,7 +29,7 @@ import type { PendingCleanup } from "../ledger/resources.js";
|
|
|
29
29
|
import { archiveSolved, clearCleanStop, clearSolved, readSolvedState, resolveSolvedState, writeAnswer } from "../ledger/run-signals.js";
|
|
30
30
|
import type { SolvedSignal } from "../ledger/run-signals.js";
|
|
31
31
|
import { childEnv, execPi } from "../member.js";
|
|
32
|
-
import { reportText, runReportMarkdown, summaryMarkdown } from "../prompt.js";
|
|
32
|
+
import { reportText, runReportMarkdown, summaryMarkdown, totalUsage } from "../prompt.js";
|
|
33
33
|
import { buildRunGraph } from "../run-graph.js";
|
|
34
34
|
import { runGraphMarkdown } from "../run-graph-mermaid.js";
|
|
35
35
|
import type { CleanupStatus, ReportResource, RunReportInput } from "../prompt.js";
|
|
@@ -137,19 +137,18 @@ interface SettledAnswer {
|
|
|
137
137
|
status: string;
|
|
138
138
|
}
|
|
139
139
|
|
|
140
|
-
/** Findings the proposal names that a later finding corrected; labeled, not re-versioned. */
|
|
141
140
|
/**
|
|
142
141
|
* The proposal body is the member's verbatim text and can predate later challenges and corrections.
|
|
143
|
-
* The status names the blockers still open at freeze and the
|
|
144
|
-
*
|
|
142
|
+
* The status names the blockers still open at freeze and the citations that changed after the
|
|
143
|
+
* proposal pinned them (superseded or amended findings, republished artifacts), and, once
|
|
144
|
+
* synthesis wrote it, points the operator at DELIVERABLE.md as the reconciled output.
|
|
145
145
|
*/
|
|
146
|
-
function annotateAnswer(run: HordeRun, answer: SettledAnswer, acceptance: { hash: string; issues: string[] },
|
|
146
|
+
function annotateAnswer(run: HordeRun, answer: SettledAnswer, acceptance: { hash: string; issues: string[] }, stale: string[]): void {
|
|
147
147
|
const blockers = acceptance.issues.filter((issue) => /^Q\d+ .*: \w+ blocking /.test(issue)).map((issue) => issue.replace(/:/, ""));
|
|
148
148
|
const parts = [answer.status];
|
|
149
149
|
if (acceptance.issues.length) parts.push("acceptance incomplete");
|
|
150
150
|
if (blockers.length) parts.push(`open blocking at freeze: ${blockers.join(", ")}`);
|
|
151
|
-
|
|
152
|
-
if (superseded.length) parts.push(`cites superseded ${superseded.join(", ")}`);
|
|
151
|
+
if (stale.length) parts.push(`cites ${stale.join(", ")}`);
|
|
153
152
|
parts.push(`see ACCEPTANCE.md (snapshot ${acceptance.hash})`);
|
|
154
153
|
if (run.deliverable) parts.push("reconciled operator output: DELIVERABLE.md; the proposal text below is verbatim and may be superseded there");
|
|
155
154
|
try {
|
|
@@ -559,18 +558,21 @@ export async function finalizeRun(state: HordeState, swarm: Swarm, run: HordeRun
|
|
|
559
558
|
run.acceptance = acceptance.status;
|
|
560
559
|
run.snapshotHash = acceptance.hash;
|
|
561
560
|
await recordLifecycle(run, "acceptance", { status: acceptance.status, hash: acceptance.hash, issues: acceptance.issues.length });
|
|
562
|
-
const
|
|
563
|
-
|
|
564
|
-
if (annotate) annotateAnswer(run, answer, acceptance,
|
|
561
|
+
const stale = answer ? staleAnswerFor(run.hordeDir, run.id, answer.proposal.text, answer.proposal.cites) : [];
|
|
562
|
+
const annotate = answer !== undefined && (acceptance.issues.length > 0 || stale.length > 0);
|
|
563
|
+
if (annotate) annotateAnswer(run, answer, acceptance, stale);
|
|
565
564
|
// Workers and bounded repairs have exited; only now may their labs be torn down.
|
|
566
565
|
const cleanups = await runCleanups(run);
|
|
567
566
|
const reportInput = writeRunReports(run, ledgers, releasedIdeas, cleanups, now);
|
|
568
567
|
// Free the slot before the (possibly slow) synthesis so a new run can start.
|
|
569
568
|
noteRunReported(state, swarm, run);
|
|
570
569
|
await synthesize(state, run, ledgers, reportInput, now);
|
|
571
|
-
if (annotate && run.deliverable) annotateAnswer(run, answer, acceptance,
|
|
570
|
+
if (annotate && run.deliverable) annotateAnswer(run, answer, acceptance, stale);
|
|
572
571
|
sendReport(state, swarm, run, ledgers, now);
|
|
573
|
-
await recordLifecycle(run, "run-end", {
|
|
572
|
+
await recordLifecycle(run, "run-end", {
|
|
573
|
+
outcome: runOutcome(run), elapsedMs: Date.now() - run.startedAt, stopLabel: run.stopLabel,
|
|
574
|
+
spent: totalUsage(run.members).cost, budget: run.budget,
|
|
575
|
+
});
|
|
574
576
|
writeRunGraph(run);
|
|
575
577
|
} catch (error) {
|
|
576
578
|
safeNotify(run, `horde ${run.id} failed to finalize: ${errorMessage(error)}`, "error");
|
package/src/run/launch.ts
CHANGED
|
@@ -277,6 +277,7 @@ function buildSwarm(
|
|
|
277
277
|
plan: parsed.plan,
|
|
278
278
|
partition: partition.slices,
|
|
279
279
|
untilStop: parsed.untilStop === true ? true : undefined,
|
|
280
|
+
budget: parsed.budget,
|
|
280
281
|
dashboardUrl,
|
|
281
282
|
};
|
|
282
283
|
return { swarm, launches };
|
|
@@ -287,7 +288,7 @@ async function seedHorde(swarm: Swarm, run: HordeRun, roles: string[]): Promise<
|
|
|
287
288
|
await seedRun(run);
|
|
288
289
|
await recordLifecycle(run, "run-start", {
|
|
289
290
|
count: run.count, model: run.model, verifier: run.verifier, plan: run.plan, untilStop: run.untilStop,
|
|
290
|
-
roles: roles.map((role) => oneLine(role, 120)), dashboardUrl: run.dashboardUrl,
|
|
291
|
+
budget: run.budget, roles: roles.map((role) => oneLine(role, 120)), dashboardUrl: run.dashboardUrl,
|
|
291
292
|
...(swarm.hordes > 1 ? { swarm: swarm.id, horde: run.horde, hordes: swarm.hordes, slice: run.slice } : {}),
|
|
292
293
|
});
|
|
293
294
|
await recordPhase(run, "discovery", "start");
|
package/src/run/repair.ts
CHANGED
|
@@ -4,9 +4,10 @@ import { randomUUID } from "node:crypto";
|
|
|
4
4
|
import { inboxFor } from "../tool/inbox.js";
|
|
5
5
|
import type { PeerMessage } from "../ledger/collaboration.js";
|
|
6
6
|
import { recordEvent } from "../ledger/events.js";
|
|
7
|
+
import type { EventContext } from "../ledger/events.js";
|
|
7
8
|
import { recordPhase } from "./lifecycle.js";
|
|
8
9
|
import { envNumber } from "../format.js";
|
|
9
|
-
import { resetMemberState } from "../member.js";
|
|
10
|
+
import { resetMemberState, USAGE_KEYS } from "../member.js";
|
|
10
11
|
import type { MemberState } from "../member.js";
|
|
11
12
|
import { hardStopMembers } from "./controls.js";
|
|
12
13
|
import { scheduleCollaboration, stopReason } from "./schedule.js";
|
|
@@ -24,11 +25,94 @@ function repairPriority(message: PeerMessage): number {
|
|
|
24
25
|
return 3;
|
|
25
26
|
}
|
|
26
27
|
|
|
27
|
-
|
|
28
|
-
|
|
28
|
+
/** The repair phase's scheduling state, shared by every round. */
|
|
29
|
+
interface RepairPhase {
|
|
30
|
+
launch: RunLaunch;
|
|
31
|
+
floors: RepairFloors;
|
|
32
|
+
/**
|
|
33
|
+
* Threads a member's completed attempts left exactly as they found them, by member then
|
|
34
|
+
* message id: the signature the thread was stuck at and how many consecutive attempts left it
|
|
35
|
+
* there. The thread is held back from that member once the count reaches `UNMOVED_ATTEMPTS`,
|
|
36
|
+
* until its signature changes, which only another member's action or a new response can cause.
|
|
37
|
+
*/
|
|
38
|
+
stalled: Map<string, Map<string, UnmovedThread>>;
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
interface UnmovedThread {
|
|
42
|
+
signature: string;
|
|
43
|
+
attempts: number;
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
/**
|
|
47
|
+
* Completed attempts that may leave a thread untouched before it is held. One miss is routine: a
|
|
48
|
+
* member that answered the rest of its queue, or re-answered the task, often clears the thread on
|
|
49
|
+
* its next attempt within seconds. Two misses on the same signature are the loop this guards.
|
|
50
|
+
*/
|
|
51
|
+
const UNMOVED_ATTEMPTS = 2;
|
|
52
|
+
|
|
53
|
+
function memberContext(phase: RepairPhase, member: MemberState): EventContext {
|
|
54
|
+
return { hordeDir: phase.launch.run.hordeDir, runId: phase.launch.run.id, actor: member.name };
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
/**
|
|
58
|
+
* What an attempt can change on a thread: its state and the revision it was answered with.
|
|
59
|
+
* Delivery receipts and a re-worded response on the same revision are not progress, so neither
|
|
60
|
+
* the inbox read of a fresh attempt nor a recipient repeating itself re-qualifies a thread.
|
|
61
|
+
*/
|
|
62
|
+
function threadSignature(m: PeerMessage): string {
|
|
63
|
+
return `${m.state}:${m.responseRevision ?? ""}:${m.responseTarget ?? ""}`;
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
function isStalled(phase: RepairPhase, member: MemberState, m: PeerMessage): boolean {
|
|
67
|
+
const unmoved = phase.stalled.get(member.name)?.get(m.id);
|
|
68
|
+
return unmoved !== undefined && unmoved.signature === threadSignature(m) && unmoved.attempts >= UNMOVED_ATTEMPTS;
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
/** The threads currently held back from the member, in inbox order. */
|
|
72
|
+
function heldThreads(phase: RepairPhase, member: MemberState): string[] {
|
|
73
|
+
return inboxFor(memberContext(phase, member)).filter((m) => isStalled(phase, member, m)).map((m) => m.id);
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
/** The threads an attempt for the member would be handed: its inbox minus the threads held back. */
|
|
77
|
+
function queueFor(phase: RepairPhase, member: MemberState): PeerMessage[] {
|
|
78
|
+
return inboxFor(memberContext(phase, member)).filter((m) => !isStalled(phase, member, m))
|
|
29
79
|
.sort((a, b) => repairPriority(a) - repairPriority(b) || a.createdAt.localeCompare(b.createdAt));
|
|
30
80
|
}
|
|
31
81
|
|
|
82
|
+
/**
|
|
83
|
+
* Counts, per thread, the consecutive completed attempts that left it as they found it; a thread
|
|
84
|
+
* the attempt moved starts over. A killed attempt counts nothing: it may have been about to act.
|
|
85
|
+
* Returns the ids this attempt left unchanged.
|
|
86
|
+
*/
|
|
87
|
+
function noteUnchanged(phase: RepairPhase, member: MemberState, queue: PeerMessage[]): string[] {
|
|
88
|
+
if (member.endCause !== "completed") return [];
|
|
89
|
+
const after = new Map(inboxFor(memberContext(phase, member)).map((m) => [m.id, threadSignature(m)]));
|
|
90
|
+
const unmoved = phase.stalled.get(member.name) ?? new Map<string, UnmovedThread>();
|
|
91
|
+
const unchanged: string[] = [];
|
|
92
|
+
for (const m of queue) {
|
|
93
|
+
const signature = threadSignature(m);
|
|
94
|
+
if (after.get(m.id) !== signature) { unmoved.delete(m.id); continue; }
|
|
95
|
+
const previous = unmoved.get(m.id);
|
|
96
|
+
unmoved.set(m.id, { signature, attempts: previous?.signature === signature ? previous.attempts + 1 : 1 });
|
|
97
|
+
unchanged.push(m.id);
|
|
98
|
+
}
|
|
99
|
+
phase.stalled.set(member.name, unmoved);
|
|
100
|
+
return unchanged;
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
/**
|
|
104
|
+
* Once every thread in the member's inbox is one its own completed attempt could not move, later
|
|
105
|
+
* rounds skip the member and its threads drop out of the demand that sizes the shares, until
|
|
106
|
+
* another member changes one. Recorded so the post-mortem sees why the member stopped appearing.
|
|
107
|
+
*/
|
|
108
|
+
async function recordStalled(phase: RepairPhase, member: MemberState): Promise<void> {
|
|
109
|
+
const held = heldThreads(phase, member);
|
|
110
|
+
if (!held.length || queueFor(phase, member).length) return;
|
|
111
|
+
const { run } = phase.launch;
|
|
112
|
+
run.repairStalled = [...new Set([...(run.repairStalled ?? []), member.name])];
|
|
113
|
+
await recordEvent({ ...memberContext(phase, member), actor: "parent", attempt: "scheduler" }, "repair-skipped", { payload: { reason: "no-progress", members: [{ member: member.name, messages: held }] } });
|
|
114
|
+
}
|
|
115
|
+
|
|
32
116
|
/**
|
|
33
117
|
* What a queue costs in floors: one cold session covers the administrative threads in it, and
|
|
34
118
|
* each review request or blocking challenge is an inspect/attach/review cycle of about a floor
|
|
@@ -114,10 +198,10 @@ export function repairBudget(demands: number[], floors: RepairFloors, rounds: nu
|
|
|
114
198
|
|
|
115
199
|
type SkipReason = "insufficient-budget" | "insufficient-time";
|
|
116
200
|
|
|
117
|
-
async function recordSkipped(
|
|
118
|
-
const { run } = launch;
|
|
201
|
+
async function recordSkipped(phase: RepairPhase, pending: MemberState[], reason: SkipReason, detail: Record<string, number>): Promise<void> {
|
|
202
|
+
const { run } = phase.launch;
|
|
119
203
|
const ctx = { hordeDir: run.hordeDir, runId: run.id, actor: "parent", attempt: "scheduler" };
|
|
120
|
-
const members = pending.map((member) => ({ member: member.name, messages: queueFor(
|
|
204
|
+
const members = pending.map((member) => ({ member: member.name, messages: queueFor(phase, member).map((m) => m.id) }))
|
|
121
205
|
.filter((entry) => entry.messages.length);
|
|
122
206
|
if (!members.length) return;
|
|
123
207
|
run.repairSkipped = [...new Set([...(run.repairSkipped ?? []), ...members.map((entry) => entry.member)])];
|
|
@@ -143,10 +227,11 @@ interface FirstTurnCost {
|
|
|
143
227
|
* share is the allowance recorded and the deadline is derived from it once, so the record cannot
|
|
144
228
|
* read a millisecond under the floor the share was raised to.
|
|
145
229
|
*/
|
|
146
|
-
async function repairMember(
|
|
230
|
+
async function repairMember(phase: RepairPhase, member: MemberState, ms: number, limits: RepairLimits): Promise<void> {
|
|
231
|
+
const { launch, floors } = phase;
|
|
147
232
|
const { run } = launch;
|
|
148
|
-
const ctx =
|
|
149
|
-
const queue = queueFor(
|
|
233
|
+
const ctx = memberContext(phase, member);
|
|
234
|
+
const queue = queueFor(phase, member);
|
|
150
235
|
if (!queue.length || stopReason(launch)) return;
|
|
151
236
|
const prior = { turns: member.turns, toolCalls: member.toolCalls, usage: { ...member.usage }, startedAt: member.startedAt };
|
|
152
237
|
resetMemberState(member);
|
|
@@ -181,7 +266,8 @@ async function repairMember(launch: RunLaunch, member: MemberState, ms: number,
|
|
|
181
266
|
const timeout = setTimeout(() => halt("time"), Math.max(1, deadline - Date.now()));
|
|
182
267
|
const watchdog = setInterval(check, 100);
|
|
183
268
|
try {
|
|
184
|
-
|
|
269
|
+
const held = heldThreads(phase, member);
|
|
270
|
+
await recordEvent({ ...ctx, attempt: member.attempt }, "repair-start", { payload: { messages: queue.map((m) => m.id), demand: repairDemand(queue), prior, allowance, floors, ...(held.length ? { held } : {}) } });
|
|
185
271
|
if (stopReason(launch) || Date.now() >= deadline) {
|
|
186
272
|
halt(stopReason(launch) ?? "time");
|
|
187
273
|
member.status = "stopped";
|
|
@@ -191,17 +277,19 @@ async function repairMember(launch: RunLaunch, member: MemberState, ms: number,
|
|
|
191
277
|
// Enforce a share on every usage update, rather than waiting for the next poll.
|
|
192
278
|
await spawnOne(launch, member, undefined, check);
|
|
193
279
|
}
|
|
280
|
+
const unchanged = noteUnchanged(phase, member, queue);
|
|
194
281
|
await recordEvent({ ...ctx, attempt: member.attempt }, "repair-end", { payload: {
|
|
195
282
|
cause: member.endCause, stopReason: member.repairStopReason, allowance, usage: member.usage, turns: member.turns, firstTurn,
|
|
196
|
-
messages: queue.map((m) => m.id),
|
|
283
|
+
messages: queue.map((m) => m.id), unchanged,
|
|
197
284
|
} });
|
|
285
|
+
if (unchanged.length) await recordStalled(phase, member);
|
|
198
286
|
} finally {
|
|
199
287
|
clearTimeout(timeout);
|
|
200
288
|
clearInterval(watchdog);
|
|
201
289
|
run.repairTokens = (run.repairTokens ?? 0) + spent();
|
|
202
290
|
member.turns += prior.turns;
|
|
203
291
|
member.toolCalls += prior.toolCalls;
|
|
204
|
-
for (const key of
|
|
292
|
+
for (const key of USAGE_KEYS) member.usage[key] += prior.usage[key];
|
|
205
293
|
member.startedAt = prior.startedAt;
|
|
206
294
|
}
|
|
207
295
|
}
|
|
@@ -223,19 +311,42 @@ export async function drainRepairs(launch: RunLaunch): Promise<void> {
|
|
|
223
311
|
}
|
|
224
312
|
}
|
|
225
313
|
|
|
226
|
-
function eligibleMembers(
|
|
227
|
-
return launch.run.members.map((member) => ({ member, queue: queueFor(
|
|
314
|
+
function eligibleMembers(phase: RepairPhase): Array<{ member: MemberState; queue: PeerMessage[] }> {
|
|
315
|
+
return phase.launch.run.members.map((member) => ({ member, queue: queueFor(phase, member) })).filter((entry) => entry.queue.length)
|
|
228
316
|
.sort((a, b) => repairPriority(a.queue[0]!) - repairPriority(b.queue[0]!) ||
|
|
229
317
|
(a.member.repairs ?? 0) - (b.member.repairs ?? 0) || a.queue[0]!.createdAt.localeCompare(b.queue[0]!.createdAt));
|
|
230
318
|
}
|
|
231
319
|
|
|
320
|
+
/** The queue a member was handed this round, by message id, at the signature it was handed. */
|
|
321
|
+
type HandedQueues = Map<string, Map<string, string>>;
|
|
322
|
+
|
|
323
|
+
/**
|
|
324
|
+
* True when every blocking thread left belongs to a member already served this round, and one of
|
|
325
|
+
* them is new or moved since that member was handed its queue. A correction answered mid-round
|
|
326
|
+
* leaves its requester to recheck it; finishing the round on administrative rechecks first spent
|
|
327
|
+
* the clock that recheck needed, so when a later round exists it starts now. A blocker the member
|
|
328
|
+
* left as it was handed is no reason to serve it again early; the last round serves everyone.
|
|
329
|
+
*/
|
|
330
|
+
export function blockedOnServed(eligible: Array<{ member: MemberState; queue: PeerMessage[] }>, handed: HandedQueues): boolean {
|
|
331
|
+
let moved = false;
|
|
332
|
+
for (const { member, queue } of eligible) {
|
|
333
|
+
const seen = handed.get(member.name);
|
|
334
|
+
for (const m of queue) {
|
|
335
|
+
if (repairPriority(m) !== 0) continue;
|
|
336
|
+
if (!seen) return false;
|
|
337
|
+
if (seen.get(m.id) !== threadSignature(m)) moved = true;
|
|
338
|
+
}
|
|
339
|
+
}
|
|
340
|
+
return moved;
|
|
341
|
+
}
|
|
342
|
+
|
|
232
343
|
async function runRepairRounds(launch: RunLaunch, begin: (budget: RepairBudget) => Promise<void>): Promise<void> {
|
|
233
344
|
const { run } = launch;
|
|
234
345
|
const rounds = Math.max(0, Math.min(6, envNumber("PI_HORDE_REPAIR_ROUNDS", 4)));
|
|
235
346
|
if (rounds === 0 || stopReason(launch)) return;
|
|
236
|
-
const
|
|
347
|
+
const phase: RepairPhase = { launch, floors: repairFloors(), stalled: new Map() };
|
|
237
348
|
await scheduleCollaboration(launch, { settleMs: 0, liveOnly: false });
|
|
238
|
-
const budget = repairBudget(eligibleMembers(
|
|
349
|
+
const budget = repairBudget(eligibleMembers(phase).map((entry) => repairDemand(entry.queue)), phase.floors, rounds);
|
|
239
350
|
if (budget.queued === 0) return;
|
|
240
351
|
const deadline = Date.now() + budget.ms;
|
|
241
352
|
const exhausted = () => stopReason(launch) || Date.now() >= deadline || (run.repairTokens ?? 0) >= budget.tokens;
|
|
@@ -245,34 +356,35 @@ async function runRepairRounds(launch: RunLaunch, begin: (budget: RepairBudget)
|
|
|
245
356
|
// One attempt per member per round, picked in priority order re-read before each attempt:
|
|
246
357
|
// a thread that became blocking during an attempt moves its recipient ahead of the
|
|
247
358
|
// administrative queues that were ordered first at round start.
|
|
248
|
-
const
|
|
359
|
+
const handed: HandedQueues = new Map();
|
|
249
360
|
for (;;) {
|
|
250
361
|
if (exhausted()) return;
|
|
251
|
-
const
|
|
252
|
-
|
|
362
|
+
const eligible = eligibleMembers(phase);
|
|
363
|
+
const pending = eligible.filter((entry) => !handed.has(entry.member.name));
|
|
364
|
+
if (!pending.length || (round < rounds - 1 && blockedOnServed(eligible, handed))) break;
|
|
253
365
|
const next = pending[0]!;
|
|
254
366
|
const pendingUnits = pending.reduce((sum, entry) => sum + repairDemand(entry.queue), 0);
|
|
255
367
|
const units = repairDemand(next.queue);
|
|
256
368
|
const unserved = pending.map((entry) => entry.member);
|
|
257
369
|
const remaining = budget.tokens - (run.repairTokens ?? 0);
|
|
258
370
|
const laterRounds = rounds - round - 1;
|
|
259
|
-
const tokens = repairAllowance(repairShare(remaining, pendingUnits, laterRounds, units), remaining, floors.tokens);
|
|
371
|
+
const tokens = repairAllowance(repairShare(remaining, pendingUnits, laterRounds, units), remaining, phase.floors.tokens);
|
|
260
372
|
if (tokens === undefined) {
|
|
261
373
|
await begin(budget);
|
|
262
|
-
await recordSkipped(
|
|
374
|
+
await recordSkipped(phase, unserved, "insufficient-budget", { remaining, floor: phase.floors.tokens });
|
|
263
375
|
return;
|
|
264
376
|
}
|
|
265
377
|
const remainingMs = Math.max(0, deadline - Date.now());
|
|
266
|
-
const fundable = Math.floor(remaining / Math.max(1, floors.tokens));
|
|
267
|
-
const ms = repairAllowance(repairTimeShare(remainingMs, pendingUnits, laterRounds, fundable, units), remainingMs, floors.ms);
|
|
378
|
+
const fundable = Math.floor(remaining / Math.max(1, phase.floors.tokens));
|
|
379
|
+
const ms = repairAllowance(repairTimeShare(remainingMs, pendingUnits, laterRounds, fundable, units), remainingMs, phase.floors.ms);
|
|
268
380
|
if (ms === undefined || ms < 1) {
|
|
269
381
|
await begin(budget);
|
|
270
|
-
await recordSkipped(
|
|
382
|
+
await recordSkipped(phase, unserved, "insufficient-time", { remainingMs, floorMs: phase.floors.ms });
|
|
271
383
|
return;
|
|
272
384
|
}
|
|
273
385
|
await begin(budget);
|
|
274
|
-
|
|
275
|
-
await repairMember(
|
|
386
|
+
handed.set(next.member.name, new Map(next.queue.map((m) => [m.id, threadSignature(m)])));
|
|
387
|
+
await repairMember(phase, next.member, ms, { tokens, turns: budget.turns });
|
|
276
388
|
}
|
|
277
389
|
}
|
|
278
390
|
}
|
package/src/run/setup.ts
CHANGED
|
@@ -93,7 +93,8 @@ export function createRun(
|
|
|
93
93
|
fs.writeFileSync(
|
|
94
94
|
path.join(runDir, "TASK.md"),
|
|
95
95
|
taskMarkdown({
|
|
96
|
-
id, prompt, count, model, thinkingLevel: ctx.thinkingLevel, startedAt, verifier, untilStop: parsed.untilStop,
|
|
96
|
+
id, prompt, count, model, thinkingLevel: ctx.thinkingLevel, startedAt, verifier, untilStop: parsed.untilStop,
|
|
97
|
+
budget: parsed.budget, swarm,
|
|
97
98
|
}),
|
|
98
99
|
"utf-8",
|
|
99
100
|
);
|
|
@@ -123,6 +124,7 @@ export function createRun(
|
|
|
123
124
|
plan,
|
|
124
125
|
verifier,
|
|
125
126
|
untilStop: parsed.untilStop === true ? true : undefined,
|
|
127
|
+
budget: parsed.budget,
|
|
126
128
|
resets: [],
|
|
127
129
|
dashboardUrl,
|
|
128
130
|
};
|
package/src/run/solved.ts
CHANGED
|
@@ -4,7 +4,7 @@ import { loadCollaborationReport } from "../collaboration-report.js";
|
|
|
4
4
|
import { openMessages } from "../ledger/collaboration.js";
|
|
5
5
|
import { postBoard } from "../ledger/board.js";
|
|
6
6
|
import { unsettledFactCitations } from "../ledger/facts.js";
|
|
7
|
-
import {
|
|
7
|
+
import { staleAnswerFor, unverifiedFindings } from "../ledger/findings.js";
|
|
8
8
|
import { listIdeas } from "../ledger/ideas.js";
|
|
9
9
|
import { archiveSolved, clearCleanStop, clearSolved, readSolvedState, reopenSolved, resolveSolvedState, writeAnswer } from "../ledger/run-signals.js";
|
|
10
10
|
import type { SolvedSignal } from "../ledger/run-signals.js";
|
|
@@ -80,14 +80,27 @@ function dropContested(run: HordeRun, proposal: SolvedSignal, contest: SolvedSig
|
|
|
80
80
|
}
|
|
81
81
|
|
|
82
82
|
/**
|
|
83
|
-
*
|
|
84
|
-
*
|
|
85
|
-
|
|
83
|
+
* ANSWER.md keeps the withdrawn text for the record, but its status must stop naming the
|
|
84
|
+
* validation it held, or the run reads as validated after the proposal is gone.
|
|
85
|
+
*/
|
|
86
|
+
function recordWithdrawn(run: HordeRun, proposal: SolvedSignal, reason: string): void {
|
|
87
|
+
void recordLifecycle(run, "solved-withdrawn", { member: proposal.member, reason });
|
|
88
|
+
try {
|
|
89
|
+
writeAnswer(run.runDir, run.id, proposal, `withdrawn: it ${reason}; no validation carries over and no proposal is pending`);
|
|
90
|
+
} catch (error) {
|
|
91
|
+
safeNotify(run, `horde ${run.id}: could not write ANSWER.md (${oneLine(String(error), 200)})`, "error");
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
/**
|
|
96
|
+
* A proposal that cites a finding superseded or amended, or an artifact republished, after it was
|
|
97
|
+
* made would ship the stale reading as the answer. It is withdrawn for re-proposal rather than
|
|
98
|
+
* validated; the board names the successor, the amendment, or the republish.
|
|
86
99
|
*/
|
|
87
100
|
function withdrawStale(run: HordeRun, proposal: SolvedSignal, stale: string[]): void {
|
|
88
101
|
const reason = `cites ${stale.join(", ")}`;
|
|
89
102
|
safeNotify(run, `horde ${run.id}: solved proposal withdrawn; it ${reason}`, "warning");
|
|
90
|
-
|
|
103
|
+
recordWithdrawn(run, proposal, reason);
|
|
91
104
|
void postBoard(
|
|
92
105
|
run.hordeDir,
|
|
93
106
|
"parent",
|
|
@@ -111,13 +124,13 @@ function reopenContested(run: HordeRun): void {
|
|
|
111
124
|
// The proposal returns with the citations it pinned when it was made; what went stale or
|
|
112
125
|
// unsettled meanwhile makes it a re-proposal for its author, not a reopening.
|
|
113
126
|
const stale = [
|
|
114
|
-
...
|
|
127
|
+
...staleAnswerFor(run.hordeDir, run.id, contest.proposal.text, contest.proposal.cites),
|
|
115
128
|
...unsettledFactCitations(run.hordeDir, contest.proposal.text).map((citation) => `provisional ${citation}`),
|
|
116
129
|
];
|
|
117
130
|
run.solvedContest = undefined;
|
|
118
131
|
if (stale.length) {
|
|
119
132
|
const why = `cites ${stale.join(", ")}`;
|
|
120
|
-
|
|
133
|
+
recordWithdrawn(run, contest.proposal, why);
|
|
121
134
|
void postBoard(run.hordeDir, "parent", `SOLVED proposal by ${contest.proposal.member} is not re-opened after ${reason}: it ${why}. ${contest.proposal.member}: propose again citing the current findings.`);
|
|
122
135
|
return;
|
|
123
136
|
}
|
|
@@ -262,7 +275,7 @@ export function checkSolved(run: HordeRun, ctx: ExtensionCommandContext): void {
|
|
|
262
275
|
dropContested(run, solved, state.contest);
|
|
263
276
|
return;
|
|
264
277
|
}
|
|
265
|
-
const stale =
|
|
278
|
+
const stale = staleAnswerFor(run.hordeDir, run.id, solved.text, solved.cites);
|
|
266
279
|
if (stale.length && !run.solvedCommitted) {
|
|
267
280
|
withdrawStale(run, solved, stale);
|
|
268
281
|
return;
|