omp-conductor 0.18.2 → 0.19.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +105 -40
- package/REFERENCE.md +865 -30
- package/package.json +1 -1
- package/schema/config.schema.json +26 -0
- package/src/admission.ts +212 -26
- package/src/ask.ts +288 -1
- package/src/briefs/orchestrator.md +6 -5
- package/src/cli.ts +5 -1
- package/src/command-help.ts +9 -1
- package/src/command-manifest.ts +36 -3
- package/src/commands/arm.ts +5 -1
- package/src/commands/context.ts +2 -0
- package/src/commands/message.ts +26 -2
- package/src/commands/reconcile-units.ts +104 -0
- package/src/commands/release-composition.ts +232 -0
- package/src/commands/resume.ts +2 -27
- package/src/commands/setup.ts +101 -16
- package/src/commands/stats.ts +11 -30
- package/src/commands/tail.ts +31 -1
- package/src/commands/upgrade.ts +20 -3
- package/src/commands/verb.ts +2 -1
- package/src/config-schema.ts +19 -0
- package/src/config.ts +80 -0
- package/src/credential-class.ts +366 -0
- package/src/daemon.ts +1218 -288
- package/src/dashboard/app.js +504 -2
- package/src/dashboard/controls.ts +336 -0
- package/src/dashboard/index.html +30 -0
- package/src/dashboard/server.ts +271 -30
- package/src/dashboard/style.css +116 -0
- package/src/dashboard/transcript.ts +173 -0
- package/src/doctor.ts +377 -20
- package/src/failure-class.ts +59 -0
- package/src/fleet.ts +497 -15
- package/src/host.ts +6 -130
- package/src/omp.ts +29 -0
- package/src/orchestrator-tick.ts +343 -88
- package/src/pause.ts +233 -0
- package/src/settlement.ts +159 -2
- package/src/setup-answers.ts +97 -0
- package/src/setup-host.ts +321 -1155
- package/src/setup-install.ts +204 -27
- package/src/setup-wizard.ts +111 -50
- package/src/setup.ts +33 -0
- package/src/spend-telemetry.ts +117 -0
- package/src/stats.ts +35 -0
- package/src/status-render.ts +348 -19
- package/src/store.ts +1229 -55
- package/src/telegram-freshness.ts +269 -0
- package/src/to-spec.ts +27 -0
- package/src/types.ts +697 -4
- package/src/unblock.ts +22 -0
- package/src/unit-reconcile.ts +303 -0
- package/src/upgrade-verify.ts +8 -1
- package/src/upgrade.ts +299 -12
- package/src/verbs/actions.ts +124 -10
- package/src/verbs/protocol.ts +70 -2
- package/src/verbs/server.ts +447 -8
- package/src/wake.ts +48 -0
- package/src/worker.ts +403 -3
package/src/daemon.ts
CHANGED
|
@@ -39,11 +39,35 @@ import { graphHint } from "./graph.ts";
|
|
|
39
39
|
import { hostConstraintsNotice } from "./host.ts";
|
|
40
40
|
import { acquireOnceLease, healthCheck, livingDaemon, probeUnit, SYSTEMD_UNIT } from "./lifecycle.ts";
|
|
41
41
|
import { requestImmediateTick, resolveTickConfigCwd, STALL_MARKER_FILE } from "./orchestrator-tick.ts";
|
|
42
|
-
import { runDoctor } from "./doctor.ts";
|
|
42
|
+
import { SPEND_SAMPLE_ROWS, SPEND_SAMPLE_RUNS, runDoctor } from "./doctor.ts";
|
|
43
|
+
import { judgeSpendTelemetry, type SpendTelemetryVerdict } from "./spend-telemetry.ts";
|
|
44
|
+
import {
|
|
45
|
+
clearPauseIfUnchanged,
|
|
46
|
+
isPaused,
|
|
47
|
+
pausedAt,
|
|
48
|
+
pausedPath,
|
|
49
|
+
pauseInstance,
|
|
50
|
+
pauseInstanceAt,
|
|
51
|
+
pauseProvenance,
|
|
52
|
+
pauseSourceToken,
|
|
53
|
+
setPaused,
|
|
54
|
+
} from "./pause.ts";
|
|
43
55
|
import { appendJournal, launchTransientUnit, readUpgradeJournal } from "./upgrade-journal.ts";
|
|
44
56
|
import { runCommand, verifyPendingUpgrade, type UpgradeVerifyDeps } from "./upgrade-verify.ts";
|
|
57
|
+
import { inspectSurfaces, type InstalledSurfaces } from "./upgrade.ts";
|
|
58
|
+
import { checkTelegramFreshness, type TelegramFreshness } from "./telegram-freshness.ts";
|
|
45
59
|
import { processStartTimeMs } from "./upgrade-verify.ts";
|
|
46
|
-
import {
|
|
60
|
+
import {
|
|
61
|
+
fleetLayers,
|
|
62
|
+
herdrPaneOmpStarts,
|
|
63
|
+
openWorkerPane,
|
|
64
|
+
reconcileWorkerPanes,
|
|
65
|
+
releaseOrphanedWorkerPane,
|
|
66
|
+
releaseWorkerPane,
|
|
67
|
+
reportWorkerPaneState,
|
|
68
|
+
resolveHerdrSession,
|
|
69
|
+
type WorkerPaneOutcome,
|
|
70
|
+
} from "./fleet.ts";
|
|
47
71
|
import { startOrchestrator } from "./orchestrator.ts";
|
|
48
72
|
import type { OrchestratorHandle } from "./orchestrator.ts";
|
|
49
73
|
import {
|
|
@@ -71,11 +95,18 @@ import {
|
|
|
71
95
|
effectiveModel,
|
|
72
96
|
hasContinuationBudget,
|
|
73
97
|
hasFailedAttemptBudget,
|
|
98
|
+
runSpendAllowanceUsd,
|
|
99
|
+
startOfToday,
|
|
74
100
|
laneEcho,
|
|
75
101
|
} from "./admission.ts";
|
|
76
102
|
import type { Admission, AdmissionHold } from "./admission.ts";
|
|
77
103
|
import type { EffectiveModel, FileLane } from "./types.ts";
|
|
78
104
|
import { log, errText, safeEscalate } from "./log.ts";
|
|
105
|
+
import {
|
|
106
|
+
oauthFenceVerdict,
|
|
107
|
+
probeCredentialClass,
|
|
108
|
+
type CredentialClassProbeResult,
|
|
109
|
+
} from "./credential-class.ts";
|
|
79
110
|
import {
|
|
80
111
|
adoptSalvagedPrs,
|
|
81
112
|
classifyAndRecover,
|
|
@@ -85,6 +116,7 @@ import {
|
|
|
85
116
|
reactToProviderCredit,
|
|
86
117
|
readSessionError,
|
|
87
118
|
reconcileOrphanedRuns,
|
|
119
|
+
reconcileGroomingClosures,
|
|
88
120
|
reconcileStaleLabels,
|
|
89
121
|
recordOperatorStop,
|
|
90
122
|
settlePushedGreen,
|
|
@@ -93,7 +125,13 @@ import {
|
|
|
93
125
|
} from "./settlement.ts";
|
|
94
126
|
import { materializeOmpSettings } from "./omp-settings.ts";
|
|
95
127
|
import { evaluateDecisionConditions, probeNpmVersion, probeRateLimitReset } from "./decisions.ts";
|
|
96
|
-
import {
|
|
128
|
+
import {
|
|
129
|
+
infraLogSignature,
|
|
130
|
+
infraSignatureVersion,
|
|
131
|
+
providerCreditRefusal,
|
|
132
|
+
providerTransientFault,
|
|
133
|
+
reviewRoundNeverWorked,
|
|
134
|
+
} from "./failure-class.ts";
|
|
97
135
|
import {
|
|
98
136
|
DEFAULT_MODEL_FALLBACK_THRESHOLD,
|
|
99
137
|
fallbackClause,
|
|
@@ -112,10 +150,11 @@ import {
|
|
|
112
150
|
utcDay,
|
|
113
151
|
} from "./store.ts";
|
|
114
152
|
import { GhPrMissingError, GraphqlBreaker, makeTracker, type RateLimitStatus } from "./tracker/github.ts";
|
|
115
|
-
import { DEFAULT_REPORT_POLICY, RELEASE_SHAPES } from "./types.ts";
|
|
153
|
+
import { DEFAULT_REPORT_POLICY, RELEASE_SHAPES, REVIEW_ROUND_INFRA_MAX_RETRIES } from "./types.ts";
|
|
116
154
|
import type {
|
|
117
155
|
BaseFreeze,
|
|
118
156
|
BaseHealth,
|
|
157
|
+
InstallSurfaceObservation,
|
|
119
158
|
AdmissionHoldReason,
|
|
120
159
|
Caps,
|
|
121
160
|
ConductorConfig,
|
|
@@ -125,6 +164,8 @@ import type {
|
|
|
125
164
|
InterruptCategory,
|
|
126
165
|
IssueComment,
|
|
127
166
|
IssueSnapshot,
|
|
167
|
+
PrDiff,
|
|
168
|
+
ReviewAdjudicationRecord,
|
|
128
169
|
MergedPrInfo,
|
|
129
170
|
OpenCloser,
|
|
130
171
|
ReleaseShape,
|
|
@@ -141,6 +182,7 @@ import type {
|
|
|
141
182
|
FailureClass,
|
|
142
183
|
HostConstraints,
|
|
143
184
|
RecoveryAction,
|
|
185
|
+
ReviewHeadBlocker,
|
|
144
186
|
ReviewPolicy,
|
|
145
187
|
ReviewRevisionOutcome,
|
|
146
188
|
ReviewRevisionRecord,
|
|
@@ -154,14 +196,19 @@ import type {
|
|
|
154
196
|
WorkflowRun,
|
|
155
197
|
} from "./types.ts";
|
|
156
198
|
import {
|
|
199
|
+
type AdjudicationRound,
|
|
200
|
+
type AdjudicatorOpts,
|
|
201
|
+
type AdjudicationResult,
|
|
157
202
|
type KilledBy,
|
|
158
203
|
type WorkerPauseControl,
|
|
159
204
|
type WorkerPausePhase,
|
|
160
205
|
type WorkerResult,
|
|
161
206
|
type RunWorkerDeps,
|
|
162
207
|
ORPHAN_RESUME_PROMPT,
|
|
208
|
+
renderAdjudicationBrief,
|
|
163
209
|
renderBrief,
|
|
164
210
|
renderReviewRevisionPrompt,
|
|
211
|
+
runAdjudicator,
|
|
165
212
|
runWorker,
|
|
166
213
|
} from "./worker.ts";
|
|
167
214
|
import {
|
|
@@ -332,6 +379,17 @@ interface Deps {
|
|
|
332
379
|
* probe the way `criticalBase` does.
|
|
333
380
|
*/
|
|
334
381
|
probeWorktreeLane?: RunLaneProbe;
|
|
382
|
+
/**
|
|
383
|
+
* Reads one provider's current credential class from the harness, out of
|
|
384
|
+
* process (#852). Wired by `runDaemon` to {@link probeCredentialClass}; a test
|
|
385
|
+
* injects a fake.
|
|
386
|
+
*
|
|
387
|
+
* Absent, both fences refuse a project that declares `requireOauthProviders` —
|
|
388
|
+
* this one fails CLOSED, unlike `probeWorktreeLane`, because an unverified
|
|
389
|
+
* credential class costs exactly what a wrong one costs. A project declaring
|
|
390
|
+
* nothing never calls it, so an unwired test dispatches as it always has.
|
|
391
|
+
*/
|
|
392
|
+
probeCredentialClass?: (provider: string) => Promise<CredentialClassProbeResult>;
|
|
335
393
|
/**
|
|
336
394
|
* Reads one issue's tracker state in a repository the admission tracker is
|
|
337
395
|
* not bound to — the cross-repo Depends-on interlock (#420). Wired by
|
|
@@ -346,6 +404,24 @@ interface Deps {
|
|
|
346
404
|
* body fails that branch closed rather than synthesising a cycle.
|
|
347
405
|
*/
|
|
348
406
|
probeBodyIn?: (repo: string, issue: number) => Promise<string | undefined>;
|
|
407
|
+
/**
|
|
408
|
+
* Reads the three install identities this host carries (#919). Wired by
|
|
409
|
+
* `runDaemon` to `inspectSurfaces`, the same seam `doctor` and `upgrade`
|
|
410
|
+
* read, so nothing re-implements the probe. A test injects its own; absent,
|
|
411
|
+
* the pass records nothing and every cheap surface honestly says "not
|
|
412
|
+
* observed yet" rather than claiming agreement.
|
|
413
|
+
*/
|
|
414
|
+
probeInstallSurfaces?: () => Promise<InstalledSurfaces>;
|
|
415
|
+
/** The `omp-telegram` install/daemon/published triple, read on the same
|
|
416
|
+
* periodic pass as the surfaces above (#961). */
|
|
417
|
+
probeTelegramFreshness?: () => Promise<TelegramFreshness>;
|
|
418
|
+
/**
|
|
419
|
+
* Runs one review-ceiling adjudication (#932). Production is
|
|
420
|
+
* {@link runAdjudicator}; a test injects a fake so the assertion can be the
|
|
421
|
+
* launch arguments and the assembled brief — which is what this pass actually
|
|
422
|
+
* produces — rather than a stored flag.
|
|
423
|
+
*/
|
|
424
|
+
runAdjudicatorImpl?: (opts: AdjudicatorOpts) => Promise<AdjudicationResult>;
|
|
349
425
|
}
|
|
350
426
|
|
|
351
427
|
/**
|
|
@@ -375,6 +451,7 @@ export function verbDeps(d: Pick<Deps, "project" | "store" | "tracker" | "verbAc
|
|
|
375
451
|
log,
|
|
376
452
|
now: () => Date.now(),
|
|
377
453
|
chain: { readBaseChain },
|
|
454
|
+
lane: { probeRunLane },
|
|
378
455
|
};
|
|
379
456
|
}
|
|
380
457
|
|
|
@@ -516,180 +593,21 @@ export async function watchOrchestrator(d: Deps, now = Date.now()): Promise<void
|
|
|
516
593
|
markPaged(d.stall, delivered, now);
|
|
517
594
|
}
|
|
518
595
|
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
export function isPaused(project?: string): boolean {
|
|
535
|
-
return activePausePaths(project).length !== 0;
|
|
536
|
-
}
|
|
537
|
-
|
|
538
|
-
/**
|
|
539
|
-
* The epoch-ms timestamp at which the current pause began. A project pause also
|
|
540
|
-
* observes the legacy bare sentinel, which pauses every project. If any active
|
|
541
|
-
* sentinel is unreadable or unparseable, the timestamp is unknown so callers
|
|
542
|
-
* continue to fail closed.
|
|
543
|
-
*/
|
|
544
|
-
export function pausedAt(project?: string): number | undefined {
|
|
545
|
-
const paths = activePausePaths(project);
|
|
546
|
-
if (paths.length === 0) return undefined;
|
|
547
|
-
const times: number[] = [];
|
|
548
|
-
for (const path of paths) {
|
|
549
|
-
try {
|
|
550
|
-
const first = readFileSync(path, "utf8").split("\n")[0]?.trim();
|
|
551
|
-
if (first === undefined || first === "") return undefined;
|
|
552
|
-
const time = Date.parse(first);
|
|
553
|
-
if (Number.isNaN(time)) return undefined;
|
|
554
|
-
times.push(time);
|
|
555
|
-
} catch {
|
|
556
|
-
return undefined;
|
|
557
|
-
}
|
|
558
|
-
}
|
|
559
|
-
return Math.min(...times);
|
|
560
|
-
}
|
|
561
|
-
|
|
562
|
-
/**
|
|
563
|
-
* Who paused the project and why. Per-project provenance wins when both its
|
|
564
|
-
* sentinel and the legacy all-project sentinel are active.
|
|
565
|
-
*/
|
|
566
|
-
export function pauseProvenance(
|
|
567
|
-
project?: string,
|
|
568
|
-
): { source: string; reason?: string } | undefined {
|
|
569
|
-
const paths =
|
|
570
|
-
project === undefined
|
|
571
|
-
? [pausedPath()]
|
|
572
|
-
: [pausedPath(project), pausedPath()];
|
|
573
|
-
const path = paths.find((candidate) => existsSync(candidate));
|
|
574
|
-
if (path === undefined) return undefined;
|
|
575
|
-
try {
|
|
576
|
-
const second = readFileSync(path, "utf8").split("\n")[1];
|
|
577
|
-
if (second === undefined) return undefined;
|
|
578
|
-
const match = /^source=(\S+)(?: reason="(.*)")?$/.exec(second.trim());
|
|
579
|
-
if (match === null) return undefined;
|
|
580
|
-
const source = match[1]!;
|
|
581
|
-
const reason = match[2];
|
|
582
|
-
return { source, ...(reason === undefined ? {} : { reason }) };
|
|
583
|
-
} catch {
|
|
584
|
-
return undefined;
|
|
585
|
-
}
|
|
586
|
-
}
|
|
587
|
-
|
|
588
|
-
/**
|
|
589
|
-
* One pause sentinel FILE read as an instance identity: who set it, why, and
|
|
590
|
-
* the creation instant, all from that exact file. An unreadable or malformed
|
|
591
|
-
* file is undefined — the caller may treat it as absence.
|
|
592
|
-
*/
|
|
593
|
-
function pauseInstanceAt(
|
|
594
|
-
path: string,
|
|
595
|
-
): { source: string; reason?: string; since: number } | undefined {
|
|
596
|
-
try {
|
|
597
|
-
const [line1, line2] = readFileSync(path, "utf8").split("\n");
|
|
598
|
-
const since = Date.parse(line1?.trim() ?? "");
|
|
599
|
-
if (!Number.isFinite(since)) return undefined;
|
|
600
|
-
if (line2 === undefined) return undefined;
|
|
601
|
-
const match = /^source=(\S+)(?: reason="(.*)")?$/.exec(line2.trim());
|
|
602
|
-
if (match === null) return undefined;
|
|
603
|
-
const source = match[1]!;
|
|
604
|
-
const reason = match[2];
|
|
605
|
-
return { source, since, ...(reason === undefined ? {} : { reason }) };
|
|
606
|
-
} catch {
|
|
607
|
-
return undefined;
|
|
608
|
-
}
|
|
609
|
-
}
|
|
610
|
-
|
|
611
|
-
/**
|
|
612
|
-
* One pause sentinel read as a single identity: who set it, why, and the
|
|
613
|
-
* creation instant, all from the SAME file that was selected. Unlike pairing
|
|
614
|
-
* {@link pauseProvenance} with {@link pausedAt} — which can describe
|
|
615
|
-
* different files when a project pause coexists with the legacy global
|
|
616
|
-
* sentinel, letting a stale global timestamp mask a recreated project pause —
|
|
617
|
-
* this reads provenance and timestamp from one sentinel, so a caller can prove
|
|
618
|
-
* "the pause I set still exists" instead of "some pause with the same labels
|
|
619
|
-
* still exists" (#377). Per-project sentinel wins, like {@link pauseProvenance}.
|
|
620
|
-
*/
|
|
621
|
-
export function pauseInstance(
|
|
622
|
-
project?: string,
|
|
623
|
-
): { source: string; reason?: string; since: number } | undefined {
|
|
624
|
-
const paths =
|
|
625
|
-
project === undefined
|
|
626
|
-
? [pausedPath()]
|
|
627
|
-
: [pausedPath(project), pausedPath()];
|
|
628
|
-
const path = paths.find((candidate) => existsSync(candidate));
|
|
629
|
-
if (path === undefined) return undefined;
|
|
630
|
-
return pauseInstanceAt(path);
|
|
631
|
-
}
|
|
632
|
-
|
|
633
|
-
/**
|
|
634
|
-
* Compare-and-clear one pause sentinel (#780 review): remove `path` only while
|
|
635
|
-
* it still holds exactly the `expected` instance — same source, same reason,
|
|
636
|
-
* same creation instant — as read by {@link pauseInstance}. A hold or pause
|
|
637
|
-
* that replaced or recreated the sentinel between the read and the clear is a
|
|
638
|
-
* newer instance (writes always re-stamp `since`), so it is never destroyed:
|
|
639
|
-
* returning false keeps the newer fence in force. Scoped strictly to one
|
|
640
|
-
* caller-provided path, so auto-expiry can clear the per-project spend-cap
|
|
641
|
-
* sentinel without ever touching the legacy global sentinel.
|
|
642
|
-
*/
|
|
643
|
-
export function clearPauseIfUnchanged(
|
|
644
|
-
path: string,
|
|
645
|
-
expected: { source: string; reason?: string; since: number },
|
|
646
|
-
): boolean {
|
|
647
|
-
const current = pauseInstanceAt(path);
|
|
648
|
-
if (current === undefined) return false;
|
|
649
|
-
if (
|
|
650
|
-
current.source !== expected.source ||
|
|
651
|
-
current.since !== expected.since ||
|
|
652
|
-
current.reason !== expected.reason
|
|
653
|
-
) {
|
|
654
|
-
return false;
|
|
655
|
-
}
|
|
656
|
-
rmSync(path, { force: true });
|
|
657
|
-
return true;
|
|
658
|
-
}
|
|
659
|
-
|
|
660
|
-
/**
|
|
661
|
-
* The pause sentinel's `source=` token, proven from a verb. The sentinel's
|
|
662
|
-
* source line is read back as a single `\S+` token (see {@link pauseInstance}
|
|
663
|
-
* and {@link pauseProvenance}), so a verb that contains a space (`setup host`)
|
|
664
|
-
* is unrepresentable verbatim and must be encoded before it reaches disk —
|
|
665
|
-
* otherwise the fence cannot prove its own pause and refuses forever (#552).
|
|
666
|
-
* Spaces become `-`; the human-readable verb is preserved in the sentinel's
|
|
667
|
-
* `reason=` instead.
|
|
668
|
-
*/
|
|
669
|
-
export function pauseSourceToken(verb: string): string {
|
|
670
|
-
return verb.trim().replace(/\s+/g, "-");
|
|
671
|
-
}
|
|
672
|
-
|
|
673
|
-
export function setPaused(
|
|
674
|
-
v: boolean,
|
|
675
|
-
why?: { source: string; reason?: string },
|
|
676
|
-
project?: string,
|
|
677
|
-
): void {
|
|
678
|
-
const path = pausedPath(project);
|
|
679
|
-
if (v) {
|
|
680
|
-
mkdirSync(dirname(path), { recursive: true });
|
|
681
|
-
const line1 = `${new Date().toISOString()}\n`;
|
|
682
|
-
if (why === undefined) {
|
|
683
|
-
writeFileSync(path, line1);
|
|
684
|
-
} else {
|
|
685
|
-
const reason = why.reason === undefined ? "" : ` reason="${why.reason.replaceAll('"', "")}"`;
|
|
686
|
-
writeFileSync(path, `${line1}source=${why.source}${reason}\n`);
|
|
687
|
-
}
|
|
688
|
-
} else {
|
|
689
|
-
rmSync(path, { force: true });
|
|
690
|
-
if (project !== undefined) rmSync(pausedPath(), { force: true });
|
|
691
|
-
}
|
|
692
|
-
}
|
|
596
|
+
// ------------------------------------------------------------------- pause
|
|
597
|
+
// The sentinel itself lives in `pause.ts` (#938): this module imports
|
|
598
|
+
// `doctor.ts`, so a health check that needs to read a fence cannot import back
|
|
599
|
+
// here. Re-exported so every existing `from "./daemon.ts"` import still
|
|
600
|
+
// resolves — the move is structural, not a change of surface.
|
|
601
|
+
export {
|
|
602
|
+
clearPauseIfUnchanged,
|
|
603
|
+
isPaused,
|
|
604
|
+
pausedAt,
|
|
605
|
+
pausedPath,
|
|
606
|
+
pauseInstance,
|
|
607
|
+
pauseProvenance,
|
|
608
|
+
pauseSourceToken,
|
|
609
|
+
setPaused,
|
|
610
|
+
};
|
|
693
611
|
|
|
694
612
|
// ------------------------------------------------------------------- drain
|
|
695
613
|
// (#484 slice 1) A project drain is a durable, self-expiring admission fence:
|
|
@@ -1090,19 +1008,6 @@ export function markPaged(
|
|
|
1090
1008
|
|
|
1091
1009
|
// ---------------------------------------------------------------------- helpers
|
|
1092
1010
|
|
|
1093
|
-
/**
|
|
1094
|
-
* Local midnight, matching how a human reads "today".
|
|
1095
|
-
*
|
|
1096
|
-
* ponytail: a rolling 24h window would be fairer to a run that started at
|
|
1097
|
-
* 23:50, but midnight is what someone checking a morning spend report expects.
|
|
1098
|
-
* Upgrade path is a `capWindow: "day" | "rolling24h"` config key.
|
|
1099
|
-
*/
|
|
1100
|
-
function startOfToday(): number {
|
|
1101
|
-
const d = new Date();
|
|
1102
|
-
d.setHours(0, 0, 0, 0);
|
|
1103
|
-
return d.getTime();
|
|
1104
|
-
}
|
|
1105
|
-
|
|
1106
1011
|
/**
|
|
1107
1012
|
* `owner/repo` for `gh`, derived from the clone URL.
|
|
1108
1013
|
*
|
|
@@ -1300,6 +1205,7 @@ function swapLabel(store: Store, projectName: string, issue: number, from: strin
|
|
|
1300
1205
|
function endedBy(killedBy: KilledBy | undefined): string {
|
|
1301
1206
|
if (killedBy === "turns") return "killed by the turns cap";
|
|
1302
1207
|
if (killedBy === "wallclock") return "killed by the wall-clock cap";
|
|
1208
|
+
if (killedBy === "spend") return "killed by the per-run spend cap";
|
|
1303
1209
|
return "killed by a failed run";
|
|
1304
1210
|
}
|
|
1305
1211
|
|
|
@@ -1505,7 +1411,19 @@ export interface WorkerControlSlot {
|
|
|
1505
1411
|
}
|
|
1506
1412
|
|
|
1507
1413
|
export interface WorkerControlRegistry {
|
|
1508
|
-
|
|
1414
|
+
/**
|
|
1415
|
+
* `onPhase` is called after a pause/resume/stop that actually changed this
|
|
1416
|
+
* worker's phase (#842) — the authoritative transition, from the same control
|
|
1417
|
+
* that performed it. It exists so a surface outside this registry (the run's
|
|
1418
|
+
* Herdr representation) can follow the phase without polling and without
|
|
1419
|
+
* inventing a second notion of "paused".
|
|
1420
|
+
*/
|
|
1421
|
+
open(
|
|
1422
|
+
project: string,
|
|
1423
|
+
issue: number,
|
|
1424
|
+
runId: string,
|
|
1425
|
+
onPhase?: (phase: WorkerPausePhase) => void,
|
|
1426
|
+
): WorkerControlSlot;
|
|
1509
1427
|
pause(project: string, issue: number): Promise<WorkerControlResult>;
|
|
1510
1428
|
resume(project: string, issue: number): WorkerControlResult;
|
|
1511
1429
|
stop(project: string, issue: number, reason: string): Promise<WorkerControlResult>;
|
|
@@ -1523,12 +1441,13 @@ export function createWorkerControlRegistry(): WorkerControlRegistry {
|
|
|
1523
1441
|
stopReason?: string;
|
|
1524
1442
|
stopError?: string;
|
|
1525
1443
|
finished: PromiseWithResolvers<void>;
|
|
1444
|
+
onPhase?: (phase: WorkerPausePhase) => void;
|
|
1526
1445
|
}
|
|
1527
1446
|
|
|
1528
1447
|
const active = new Map<string, Entry>();
|
|
1529
1448
|
const key = (project: string, issue: number): string => `${project}\0${issue}`;
|
|
1530
1449
|
return {
|
|
1531
|
-
open(project, issue, runId) {
|
|
1450
|
+
open(project, issue, runId, onPhase) {
|
|
1532
1451
|
const k = key(project, issue);
|
|
1533
1452
|
if (active.has(k)) throw new Error(`#${issue} already has a live worker controller`);
|
|
1534
1453
|
const entry: Entry = {
|
|
@@ -1536,6 +1455,7 @@ export function createWorkerControlRegistry(): WorkerControlRegistry {
|
|
|
1536
1455
|
issue,
|
|
1537
1456
|
runId,
|
|
1538
1457
|
finished: Promise.withResolvers<void>(),
|
|
1458
|
+
...(onPhase === undefined ? {} : { onPhase }),
|
|
1539
1459
|
};
|
|
1540
1460
|
active.set(k, entry);
|
|
1541
1461
|
return {
|
|
@@ -1561,7 +1481,9 @@ export function createWorkerControlRegistry(): WorkerControlRegistry {
|
|
|
1561
1481
|
if (entry?.control === undefined) return { kind: "not-active" };
|
|
1562
1482
|
try {
|
|
1563
1483
|
await entry.control.pause();
|
|
1564
|
-
|
|
1484
|
+
const phase = entry.control.phase();
|
|
1485
|
+
entry.onPhase?.(phase);
|
|
1486
|
+
return { kind: "ok", runId: entry.runId, phase };
|
|
1565
1487
|
} catch (err) {
|
|
1566
1488
|
return {
|
|
1567
1489
|
kind: "refused",
|
|
@@ -1575,7 +1497,9 @@ export function createWorkerControlRegistry(): WorkerControlRegistry {
|
|
|
1575
1497
|
if (entry?.control === undefined) return { kind: "not-active" };
|
|
1576
1498
|
try {
|
|
1577
1499
|
entry.control.resume();
|
|
1578
|
-
|
|
1500
|
+
const phase = entry.control.phase();
|
|
1501
|
+
entry.onPhase?.(phase);
|
|
1502
|
+
return { kind: "ok", runId: entry.runId, phase };
|
|
1579
1503
|
} catch (err) {
|
|
1580
1504
|
return {
|
|
1581
1505
|
kind: "refused",
|
|
@@ -1936,6 +1860,32 @@ export async function handleIssue(
|
|
|
1936
1860
|
let turnLimit: TurnLimitController | undefined;
|
|
1937
1861
|
let workerControl: WorkerControlSlot | undefined;
|
|
1938
1862
|
let workerSessionInstalled = false;
|
|
1863
|
+
// The run's Herdr representation (#840), hoisted for the same reason the verb
|
|
1864
|
+
// listener is: whoever tears the run down has to hand the pane's lifecycle
|
|
1865
|
+
// authority back, and a representation that outlives its child is a worker the
|
|
1866
|
+
// workspace still shows as live.
|
|
1867
|
+
let workerPane: Extract<WorkerPaneOutcome, { kind: "tracked" }> | undefined;
|
|
1868
|
+
// Herdr orders lifecycle reports by `seq`, so this run owns one counter and
|
|
1869
|
+
// every report takes the next value (#842). A shared or restarted counter would
|
|
1870
|
+
// let a late report overwrite a newer state — the pane would then show `working`
|
|
1871
|
+
// for a worker that has already blocked.
|
|
1872
|
+
let paneSeq = 0;
|
|
1873
|
+
const nextPaneSeq = (): number => (paneSeq += 1);
|
|
1874
|
+
/**
|
|
1875
|
+
* Project one authoritative transition onto the pane (#842).
|
|
1876
|
+
*
|
|
1877
|
+
* The input is always a durable run transition or a typed session event —
|
|
1878
|
+
* never the pane's own output, which is display and nothing more. A failure is
|
|
1879
|
+
* logged, not raised: the workspace lagging is not worth failing a run over.
|
|
1880
|
+
*/
|
|
1881
|
+
const projectPaneState = (state: "working" | "idle" | "blocked" | "unknown", message?: string): void => {
|
|
1882
|
+
if (workerPane === undefined) return;
|
|
1883
|
+
const reported = reportWorkerPaneState(workerPane.paneId, workerPane.label, state, {
|
|
1884
|
+
seq: nextPaneSeq(),
|
|
1885
|
+
...(message === undefined ? {} : { message }),
|
|
1886
|
+
});
|
|
1887
|
+
if (!reported.ok) log(`#${issue} herdr pane state ${state} not reported: ${reported.reason}`);
|
|
1888
|
+
};
|
|
1939
1889
|
// The run's own repository. Hoisted for the same reason `worktreePath` is —
|
|
1940
1890
|
// the catch and finally paths have to publish the branch.
|
|
1941
1891
|
let runRepo: RunRepoRef | undefined;
|
|
@@ -2052,6 +2002,73 @@ export async function handleIssue(
|
|
|
2052
2002
|
return true;
|
|
2053
2003
|
};
|
|
2054
2004
|
|
|
2005
|
+
/**
|
|
2006
|
+
* The launch half of the credential-class fence (#852).
|
|
2007
|
+
*
|
|
2008
|
+
* Admission already held every candidate while a required provider billed to
|
|
2009
|
+
* the wrong credential, but a grant can be disabled in the window *between*
|
|
2010
|
+
* that check and this spawn — a refresh failing, an operator revoking, a token
|
|
2011
|
+
* expiring — and that window is exactly what the incident's acceptance
|
|
2012
|
+
* criterion names. So the same question is asked again here, as the last thing
|
|
2013
|
+
* before the session exists, and the run is closed rather than launched.
|
|
2014
|
+
*
|
|
2015
|
+
* Settled the same way the stop and drain fences settle: the claim closes, the
|
|
2016
|
+
* tree is salvaged, nothing is launched. It is deliberately NOT a failed
|
|
2017
|
+
* attempt against the issue's budget — the issue is not what is wrong, the
|
|
2018
|
+
* host's credentials are, and charging an attempt for it would exhaust an
|
|
2019
|
+
* issue while the operator re-authenticates.
|
|
2020
|
+
*/
|
|
2021
|
+
const settleCredentialClassBeforeSession = async (): Promise<boolean> => {
|
|
2022
|
+
const required = project.requireOauthProviders ?? [];
|
|
2023
|
+
if (required.length === 0 || run === undefined || workerSessionInstalled) return false;
|
|
2024
|
+
const fence = await oauthFenceVerdict(
|
|
2025
|
+
required,
|
|
2026
|
+
d.probeCredentialClass ??
|
|
2027
|
+
(async (provider) => ({
|
|
2028
|
+
ok: false,
|
|
2029
|
+
reason: `no credential probe is wired for ${provider}`,
|
|
2030
|
+
})),
|
|
2031
|
+
);
|
|
2032
|
+
if (fence.ok) return false;
|
|
2033
|
+
turnLimit?.close();
|
|
2034
|
+
turnLimit = undefined;
|
|
2035
|
+
const settlement =
|
|
2036
|
+
worktreePath === undefined
|
|
2037
|
+
? undefined
|
|
2038
|
+
: await settleWorktree({
|
|
2039
|
+
issue,
|
|
2040
|
+
attempt,
|
|
2041
|
+
ending: "not launched: a required subscription credential is unavailable",
|
|
2042
|
+
worktree: worktreePath,
|
|
2043
|
+
branch,
|
|
2044
|
+
publish,
|
|
2045
|
+
tree: "remove",
|
|
2046
|
+
mirrorPath,
|
|
2047
|
+
});
|
|
2048
|
+
recordOperatorStop(store, {
|
|
2049
|
+
project: project.name,
|
|
2050
|
+
issue,
|
|
2051
|
+
runId: run.id,
|
|
2052
|
+
inProgress,
|
|
2053
|
+
reason: `required subscription credential unavailable: ${fence.reason}`,
|
|
2054
|
+
patch: {
|
|
2055
|
+
endedAt: Date.now(),
|
|
2056
|
+
turns: run.turns,
|
|
2057
|
+
spendUsd: run.spendUsd,
|
|
2058
|
+
worktree: worktreePath ?? run.worktree,
|
|
2059
|
+
report: [
|
|
2060
|
+
"This run was closed instead of launched: a provider this project requires to bill",
|
|
2061
|
+
"to its subscription would have billed to something else.",
|
|
2062
|
+
fence.reason,
|
|
2063
|
+
...(settlement?.lines ?? []),
|
|
2064
|
+
].join("\n"),
|
|
2065
|
+
...settlement?.patch,
|
|
2066
|
+
},
|
|
2067
|
+
});
|
|
2068
|
+
log(`#${issue} not launched: ${fence.reason}`);
|
|
2069
|
+
return true;
|
|
2070
|
+
};
|
|
2071
|
+
|
|
2055
2072
|
try {
|
|
2056
2073
|
// The claim-side of the stop fence (#374): the run row is the boundary the
|
|
2057
2074
|
// shutdown drain waits on, so the claim itself refuses once the daemon is
|
|
@@ -2134,13 +2151,27 @@ export async function handleIssue(
|
|
|
2134
2151
|
// (#286) keeps its semantics in both cases — it is the same resolution,
|
|
2135
2152
|
// one different primary.
|
|
2136
2153
|
const declaredModel = admittedModel?.model;
|
|
2154
|
+
// #807: a chain that already bought its one model escalation dispatches on
|
|
2155
|
+
// that stronger selector, outranking both the declaration and the project
|
|
2156
|
+
// default — the settlement that wrote the marker did so precisely because
|
|
2157
|
+
// the previous tier spun to a cap with nothing to show. Read from the
|
|
2158
|
+
// store, so it is sticky to this issue's chain and a fresh issue (no
|
|
2159
|
+
// marker) resolves exactly as it always has. The provider chain still
|
|
2160
|
+
// layers on top: an escalated run whose provider then aborts fails over
|
|
2161
|
+
// normally, because escalation only ever changes the primary.
|
|
2162
|
+
const escalation = store.modelEscalation(project.name, issue);
|
|
2163
|
+
const primaryModel = escalation?.model ?? declaredModel ?? project.workerModel;
|
|
2137
2164
|
const choice = resolveDispatchModel({
|
|
2138
|
-
workerModel:
|
|
2165
|
+
workerModel: primaryModel,
|
|
2139
2166
|
modelFallbacks: project.modelFallbacks,
|
|
2140
2167
|
threshold: project.modelFallbackThreshold ?? DEFAULT_MODEL_FALLBACK_THRESHOLD,
|
|
2141
2168
|
streak: chainFacts.streak,
|
|
2142
2169
|
});
|
|
2143
|
-
const clause =
|
|
2170
|
+
const clause =
|
|
2171
|
+
fallbackClause(choice, chainFacts, primaryModel) ??
|
|
2172
|
+
(escalation === undefined
|
|
2173
|
+
? undefined
|
|
2174
|
+
: `on ${escalation.model} — this chain's one model escalation, after ${escalation.failureClass}`);
|
|
2144
2175
|
|
|
2145
2176
|
// #536: an orphan-clean requeue resumes the interrupted session instead of
|
|
2146
2177
|
// re-reading the repo from zero. Decided here, before this attempt's row
|
|
@@ -2193,6 +2224,12 @@ export async function handleIssue(
|
|
|
2193
2224
|
// enforced and the brief rendered, never a re-parse. Absent for a run
|
|
2194
2225
|
// with no declaration (fail open), exactly as it was admitted.
|
|
2195
2226
|
lane: admittedLane,
|
|
2227
|
+
// #851: the allowance admission reserved for this run, recorded on the
|
|
2228
|
+
// row before anything launches. Derived from the same caps the gate
|
|
2229
|
+
// read, so the reservation the pass enforced and the one the row holds
|
|
2230
|
+
// are one value; `undefined` (NULL) when the fleet has no spend cap at
|
|
2231
|
+
// all, which is the truth rather than "reserved nothing".
|
|
2232
|
+
spendReservedUsd: runSpendAllowanceUsd(caps) ?? undefined,
|
|
2196
2233
|
});
|
|
2197
2234
|
// #434: carry a terminal predecessor's open PR onto the continuation row.
|
|
2198
2235
|
// The claim itself stays synchronous — `handleIssue` claims at the top of
|
|
@@ -2227,9 +2264,23 @@ export async function handleIssue(
|
|
|
2227
2264
|
store.enqueueLabelOps(project.name, [{ issue, op: "add", label: inProgress }]);
|
|
2228
2265
|
claimed = true;
|
|
2229
2266
|
turnLimit = d.turnLimits.open(project.name, issue, runId, maxTurns);
|
|
2230
|
-
|
|
2267
|
+
// The pane follows the phase the registry actually reached (#842) — never a
|
|
2268
|
+
// guess made at the call site, and never the pane's own output read back.
|
|
2269
|
+
workerControl = d.workerControls.open(project.name, issue, runId, (phase) => {
|
|
2270
|
+
projectPaneState(
|
|
2271
|
+
phase === "running" ? "working" : "idle",
|
|
2272
|
+
phase === "running" ? "resumed" : `${phase} by the operator`,
|
|
2273
|
+
);
|
|
2274
|
+
});
|
|
2231
2275
|
if (await settleStopBeforeSession()) return;
|
|
2232
2276
|
if (await settleDrainBeforeSession()) return;
|
|
2277
|
+
// The credential fence, twice, for the same reason the stop fence is
|
|
2278
|
+
// consulted more than once (#852). Here it is the cheap one: a run whose
|
|
2279
|
+
// required subscription credential is already unavailable is closed before
|
|
2280
|
+
// anything clones a mirror or cuts a worktree, because provisioning for a
|
|
2281
|
+
// launch that cannot happen is pure waste. The second call, immediately
|
|
2282
|
+
// before the spawn, is the one that closes the admission-to-session window.
|
|
2283
|
+
if (await settleCredentialClassBeforeSession()) return;
|
|
2233
2284
|
|
|
2234
2285
|
|
|
2235
2286
|
// A run's tree is <workspaceRoot>/<issue> and addRunRepo refuses to reuse
|
|
@@ -2328,7 +2379,10 @@ export async function handleIssue(
|
|
|
2328
2379
|
// Recorded before the launch, so even a run killed mid-flight leaves the
|
|
2329
2380
|
// model it chose on its row. Only a chain-configured project writes the
|
|
2330
2381
|
// column: absent `modelFallbacks` must preserve today's rows byte for byte.
|
|
2331
|
-
|
|
2382
|
+
// An escalated chain (#807) writes it too — the selector it dispatched on
|
|
2383
|
+
// is the whole provenance of that recovery, and its harness-resolved model
|
|
2384
|
+
// lands beside it in `resolvedModel` when the run settles.
|
|
2385
|
+
if ((chainConfigured || escalation !== undefined) && choice.model !== undefined) {
|
|
2332
2386
|
store.updateRun(runId, { model: choice.model });
|
|
2333
2387
|
}
|
|
2334
2388
|
|
|
@@ -2377,18 +2431,27 @@ export async function handleIssue(
|
|
|
2377
2431
|
}
|
|
2378
2432
|
if (await settleStopBeforeSession()) return;
|
|
2379
2433
|
if (await settleDrainBeforeSession()) return;
|
|
2434
|
+
// Last, so it is the newest fact anything has about the credentials — a
|
|
2435
|
+
// grant disabled since admission is caught here rather than paid for.
|
|
2436
|
+
if (await settleCredentialClassBeforeSession()) return;
|
|
2380
2437
|
|
|
2381
2438
|
const repoSlug = githubRepo(r.repo.cloneUrl);
|
|
2382
2439
|
|
|
2383
2440
|
|
|
2384
2441
|
let result: WorkerResult;
|
|
2385
2442
|
try {
|
|
2443
|
+
const runAllowanceUsd = runSpendAllowanceUsd(caps);
|
|
2386
2444
|
result = await runWorker({
|
|
2387
2445
|
brief,
|
|
2388
2446
|
cwd: worktreePath,
|
|
2389
2447
|
caps,
|
|
2390
2448
|
...(repoSlug === undefined ? {} : { repoSlug }),
|
|
2391
2449
|
maxTurns: () => turnLimit?.maxTurns() ?? maxTurns,
|
|
2450
|
+
// #851: the allowance this run reserved at admission is also its live
|
|
2451
|
+
// ceiling. Without it the reservation would only bound how many runs
|
|
2452
|
+
// start, not what one of them spends, and the day's total would be
|
|
2453
|
+
// unbounded again.
|
|
2454
|
+
...(runAllowanceUsd === null ? {} : { maxSpendUsd: runAllowanceUsd }),
|
|
2392
2455
|
onPauseControl: (control) => {
|
|
2393
2456
|
workerSessionInstalled = true;
|
|
2394
2457
|
workerControl?.install(control);
|
|
@@ -2407,6 +2470,69 @@ export async function handleIssue(
|
|
|
2407
2470
|
onSpawn: (pid) => {
|
|
2408
2471
|
verbListener?.bindPid(pid);
|
|
2409
2472
|
},
|
|
2473
|
+
// The workspace representation of this exact child (#840). Best effort by
|
|
2474
|
+
// construction: a worker that cannot be *shown* is still a worker, so a
|
|
2475
|
+
// failure is logged with its reason and the run proceeds. What a failure
|
|
2476
|
+
// should mean for the launch — fail closed, or a named degraded state —
|
|
2477
|
+
// is #841's policy and is deliberately not decided here. Nothing is
|
|
2478
|
+
// claimed silently: either the pane id is logged, or the reason is.
|
|
2479
|
+
pane: {
|
|
2480
|
+
open: (pid) => {
|
|
2481
|
+
// Adopt before creating (#842): a run that already carries a pane
|
|
2482
|
+
// identity gets that pane reported against, never a second one. The
|
|
2483
|
+
// durable row is what makes this survive the process that opened it,
|
|
2484
|
+
// and duplicate-prevention is exactly what it buys — a re-entered
|
|
2485
|
+
// launch for one run must not leave two panes claiming it.
|
|
2486
|
+
const recorded = store.getRun(runId);
|
|
2487
|
+
if (recorded?.paneId !== undefined && recorded.paneLabel !== undefined) {
|
|
2488
|
+
workerPane = { kind: "tracked", paneId: recorded.paneId, label: recorded.paneLabel, pid };
|
|
2489
|
+
store.updateRun(runId, { workerPid: pid });
|
|
2490
|
+
const state = reportWorkerPaneState(recorded.paneId, recorded.paneLabel, "working", {
|
|
2491
|
+
seq: nextPaneSeq(),
|
|
2492
|
+
});
|
|
2493
|
+
log(
|
|
2494
|
+
state.ok
|
|
2495
|
+
? `#${issue} herdr pane ${recorded.paneId} re-reported for pid ${pid}`
|
|
2496
|
+
: `#${issue} herdr pane ${recorded.paneId} could not be re-reported: ${state.reason}`,
|
|
2497
|
+
);
|
|
2498
|
+
return;
|
|
2499
|
+
}
|
|
2500
|
+
const outcome = openWorkerPane({
|
|
2501
|
+
project: project.name,
|
|
2502
|
+
issue,
|
|
2503
|
+
attempt,
|
|
2504
|
+
runId,
|
|
2505
|
+
pid,
|
|
2506
|
+
...(recorded?.sessionFile === undefined ? {} : { sessionFile: recorded.sessionFile }),
|
|
2507
|
+
});
|
|
2508
|
+
if (outcome.kind === "tracked") {
|
|
2509
|
+
workerPane = outcome;
|
|
2510
|
+
// Durable before it is announced: a pane the store does not know
|
|
2511
|
+
// about is a pane a restart cannot reconcile.
|
|
2512
|
+
store.updateRun(runId, {
|
|
2513
|
+
workerPid: pid,
|
|
2514
|
+
paneId: outcome.paneId,
|
|
2515
|
+
paneLabel: outcome.label,
|
|
2516
|
+
// Cleared, because there is now a pane: a leftover reason beside a
|
|
2517
|
+
// tracked run would keep `status` saying degraded forever (#841).
|
|
2518
|
+
paneUnavailable: null,
|
|
2519
|
+
});
|
|
2520
|
+
log(`#${issue} herdr pane ${outcome.paneId} (${outcome.label}) tracks pid ${pid}`);
|
|
2521
|
+
} else {
|
|
2522
|
+
// Named, never silent (#841): the run proceeds untracked, and the
|
|
2523
|
+
// reason is durable so `status` can say the fleet is running blind
|
|
2524
|
+
// rather than looking identical to a fleet with no workers.
|
|
2525
|
+
store.updateRun(runId, { workerPid: pid, paneUnavailable: outcome.reason });
|
|
2526
|
+
log(`#${issue} no herdr pane: ${outcome.reason}`);
|
|
2527
|
+
}
|
|
2528
|
+
},
|
|
2529
|
+
release: () => {
|
|
2530
|
+
if (workerPane === undefined) return;
|
|
2531
|
+
const released = releaseWorkerPane(workerPane.paneId, workerPane.label, { seq: nextPaneSeq() });
|
|
2532
|
+
if (!released.ok) log(`#${issue} herdr pane release failed: ${released.reason}`);
|
|
2533
|
+
workerPane = undefined;
|
|
2534
|
+
},
|
|
2535
|
+
},
|
|
2410
2536
|
onChildLog: (line) => {
|
|
2411
2537
|
log(`#${issue} ${line}`);
|
|
2412
2538
|
},
|
|
@@ -2421,6 +2547,11 @@ export async function handleIssue(
|
|
|
2421
2547
|
onReleaseBlocked: workerReleaseBlockRecorder(project.name, issue, runId),
|
|
2422
2548
|
onTurn: (n) => store.updateRun(runId, { turns: n }),
|
|
2423
2549
|
onSpend: (usd) => store.updateRun(runId, { spendUsd: usd }),
|
|
2550
|
+
// #518: recorded live, not at settlement — the moment this explains is
|
|
2551
|
+
// forty minutes before the wall-clock cap fires, so a number only a
|
|
2552
|
+
// finished run carries would answer the question too late.
|
|
2553
|
+
onTokens: (tokens) =>
|
|
2554
|
+
store.updateRun(runId, { outputTokens: tokens.output, reasoningTokens: tokens.reasoning }),
|
|
2424
2555
|
onKilled: () => {
|
|
2425
2556
|
turnLimit?.close();
|
|
2426
2557
|
turnLimit = undefined;
|
|
@@ -2876,6 +3007,114 @@ export async function handleIssue(
|
|
|
2876
3007
|
* recorded it — merged after all, settled, re-claimed — fails the claim and is
|
|
2877
3008
|
* settled `skipped` rather than woken on stale identity.
|
|
2878
3009
|
*/
|
|
3010
|
+
/**
|
|
3011
|
+
* Record what this host has installed, once per dispatch pass (#919).
|
|
3012
|
+
*
|
|
3013
|
+
* Here rather than at render time, and this is the whole design of the slice:
|
|
3014
|
+
* the read costs three subprocesses, and calling it from the tick took the
|
|
3015
|
+
* tick's own suite from 8.4s to 83.4s while spawning three children every
|
|
3016
|
+
* fifteen minutes to answer a question that changes only when someone installs
|
|
3017
|
+
* something. The dispatch pass is already async and already spawns `gh`, so one
|
|
3018
|
+
* read per pass is free by comparison, and every cheap surface then reads a row.
|
|
3019
|
+
*
|
|
3020
|
+
* Advisory throughout: a probe that throws (no `herdr` on PATH, a `$PATH`
|
|
3021
|
+
* without `omp`) leaves the previous observation in place and logs. A stale
|
|
3022
|
+
* observation is still the truth about the last time anyone could look, and
|
|
3023
|
+
* losing a dispatch pass over a version string would be absurd.
|
|
3024
|
+
*/
|
|
3025
|
+
/**
|
|
3026
|
+
* Make the workspace agree with the live run set, once per pass (#841).
|
|
3027
|
+
*
|
|
3028
|
+
* Runs on every dispatch pass rather than only at startup, because the thing it
|
|
3029
|
+
* repairs — a Herdr restart — is not a conductor event and announces itself
|
|
3030
|
+
* nowhere. It is idempotent by construction: a reconciled fleet reports `intact`
|
|
3031
|
+
* for every live worker and finds no stale panes, so repeated passes converge
|
|
3032
|
+
* instead of accumulating panes or churning them.
|
|
3033
|
+
*
|
|
3034
|
+
* Nothing here can stop a worker. The only mutations are creating a pane, and
|
|
3035
|
+
* releasing one whose run is not live; the authoritative child is never
|
|
3036
|
+
* signalled, and no pane is ever closed.
|
|
3037
|
+
*/
|
|
3038
|
+
function reconcilePanes(d: Deps, project: string, log: (message: string) => void): void {
|
|
3039
|
+
const live = d.store.liveRuns(project);
|
|
3040
|
+
const result = reconcileWorkerPanes(
|
|
3041
|
+
live.map((run) => ({
|
|
3042
|
+
runId: run.id,
|
|
3043
|
+
issue: run.issue,
|
|
3044
|
+
attempt: run.attempt,
|
|
3045
|
+
project,
|
|
3046
|
+
...(run.workerPid === undefined ? {} : { pid: run.workerPid }),
|
|
3047
|
+
...(run.paneId === undefined ? {} : { paneId: run.paneId }),
|
|
3048
|
+
...(run.paneLabel === undefined ? {} : { paneLabel: run.paneLabel }),
|
|
3049
|
+
...(run.sessionFile === undefined ? {} : { sessionFile: run.sessionFile }),
|
|
3050
|
+
})),
|
|
3051
|
+
);
|
|
3052
|
+
if (!result.ok) {
|
|
3053
|
+
// An unreadable workspace is not evidence that anything is stale, so nothing
|
|
3054
|
+
// is released and no run is relabelled — but a fleet that cannot see its own
|
|
3055
|
+
// panes says so, on every pass, rather than going quiet.
|
|
3056
|
+
log(`worker panes not reconciled: ${result.reason}`);
|
|
3057
|
+
return;
|
|
3058
|
+
}
|
|
3059
|
+
for (const outcome of result.outcomes) {
|
|
3060
|
+
switch (outcome.kind) {
|
|
3061
|
+
case "intact":
|
|
3062
|
+
break;
|
|
3063
|
+
case "reassociated":
|
|
3064
|
+
d.store.updateRun(outcome.runId, {
|
|
3065
|
+
paneId: outcome.paneId,
|
|
3066
|
+
paneLabel: outcome.label,
|
|
3067
|
+
paneUnavailable: null,
|
|
3068
|
+
});
|
|
3069
|
+
log(`#${runIssue(live, outcome.runId)} herdr pane re-associated as ${outcome.paneId} after a restart`);
|
|
3070
|
+
break;
|
|
3071
|
+
case "untracked":
|
|
3072
|
+
d.store.updateRun(outcome.runId, { paneId: null, paneLabel: null, paneUnavailable: outcome.reason });
|
|
3073
|
+
log(`#${runIssue(live, outcome.runId)} has no herdr pane: ${outcome.reason}`);
|
|
3074
|
+
break;
|
|
3075
|
+
case "stale-released":
|
|
3076
|
+
log(`released stale herdr pane ${outcome.paneId} (run ${outcome.runId || "unidentified"} is not live)`);
|
|
3077
|
+
break;
|
|
3078
|
+
case "stale-release-failed":
|
|
3079
|
+
log(`stale herdr pane ${outcome.paneId} could not be released: ${outcome.reason}`);
|
|
3080
|
+
break;
|
|
3081
|
+
}
|
|
3082
|
+
}
|
|
3083
|
+
}
|
|
3084
|
+
|
|
3085
|
+
/** The issue a run id belongs to, for a log line a human reads. */
|
|
3086
|
+
function runIssue(live: readonly RunRecord[], runId: string): string {
|
|
3087
|
+
return String(live.find((run) => run.id === runId)?.issue ?? "?");
|
|
3088
|
+
}
|
|
3089
|
+
|
|
3090
|
+
async function recordInstallSurfaces(d: Deps): Promise<void> {
|
|
3091
|
+
if (d.probeInstallSurfaces === undefined) return;
|
|
3092
|
+
try {
|
|
3093
|
+
const surfaces = await d.probeInstallSurfaces();
|
|
3094
|
+
// The telegram peer's versions ride along on the same pass (#961). Read
|
|
3095
|
+
// here rather than in `status` because `status` runs every tick and this
|
|
3096
|
+
// touches the npm registry: one periodic read, rendered from the store as
|
|
3097
|
+
// often as anyone looks. A read that throws leaves the fields absent, which
|
|
3098
|
+
// renders as unverified — never as agreement.
|
|
3099
|
+
const telegram =
|
|
3100
|
+
d.probeTelegramFreshness === undefined ? undefined : await d.probeTelegramFreshness();
|
|
3101
|
+
const tgInstalled = telegram?.surfaces.installed;
|
|
3102
|
+
const tgDaemon = telegram?.surfaces.daemon;
|
|
3103
|
+
const tgPublished = telegram?.surfaces.published;
|
|
3104
|
+
d.store.recordInstallSurfaces({
|
|
3105
|
+
at: Date.now(),
|
|
3106
|
+
cliVersion: surfaces.cliVersion,
|
|
3107
|
+
...(surfaces.ompVersion === undefined ? {} : { ompVersion: surfaces.ompVersion }),
|
|
3108
|
+
...(surfaces.herdrSource === undefined ? {} : { herdrSource: surfaces.herdrSource }),
|
|
3109
|
+
...(tgInstalled?.kind === "version" ? { telegramInstalled: tgInstalled.version } : {}),
|
|
3110
|
+
...(tgDaemon?.kind === "version" ? { telegramDaemon: tgDaemon.version } : {}),
|
|
3111
|
+
...(tgPublished?.kind === "version" ? { telegramPublished: tgPublished.version } : {}),
|
|
3112
|
+
});
|
|
3113
|
+
} catch (err) {
|
|
3114
|
+
log(`install-surface read skipped: ${errText(err)}`);
|
|
3115
|
+
}
|
|
3116
|
+
}
|
|
3117
|
+
|
|
2879
3118
|
export async function dispatchReviewRevisions(d: Deps, pool?: WorkerPool): Promise<void> {
|
|
2880
3119
|
const pending = d.store.pendingReviewRevisions(d.project.name);
|
|
2881
3120
|
if (pending.length === 0) return;
|
|
@@ -2993,6 +3232,350 @@ export async function dispatchReviewRevisions(d: Deps, pool?: WorkerPool): Promi
|
|
|
2993
3232
|
await Promise.allSettled(launches);
|
|
2994
3233
|
}
|
|
2995
3234
|
|
|
3235
|
+
/**
|
|
3236
|
+
* Launch the review-ceiling adjudications this project owes (#932).
|
|
3237
|
+
*
|
|
3238
|
+
* Deliberately its own pass, next to {@link dispatchReviewRevisions} but never
|
|
3239
|
+
* inside it: a revision resumes an implementation worker to change code, an
|
|
3240
|
+
* adjudication opens a fresh read-only session to decide about code. Folding
|
|
3241
|
+
* them together is how "escalate the automation" quietly becomes "resume the
|
|
3242
|
+
* same worker with a different label", which is #873's named silent fake.
|
|
3243
|
+
*
|
|
3244
|
+
* Only `pending` rows are launched. A `running` row belongs to a live
|
|
3245
|
+
* adjudicator — or to one a restart killed, which the startup sweep settles —
|
|
3246
|
+
* so a restart re-reads it and never launches a second adjudicator for one head.
|
|
3247
|
+
*/
|
|
3248
|
+
/**
|
|
3249
|
+
* The turn ceiling for one adjudication (#932).
|
|
3250
|
+
*
|
|
3251
|
+
* Small on purpose, and not the worker cap: a verdict is a read and an answer,
|
|
3252
|
+
* not an implementation. A session still going after this is not deciding — and
|
|
3253
|
+
* the ceiling exists so a stuck adjudicator costs a bounded amount rather than a
|
|
3254
|
+
* worker's whole budget, on work that has already spent its review rounds.
|
|
3255
|
+
*/
|
|
3256
|
+
export const ADJUDICATION_MAX_TURNS = 40;
|
|
3257
|
+
|
|
3258
|
+
export async function dispatchReviewAdjudications(d: Deps, pool?: WorkerPool): Promise<void> {
|
|
3259
|
+
const open = d.store.openReviewAdjudications(d.project.name).filter((a) => a.state === "pending");
|
|
3260
|
+
if (open.length === 0) return;
|
|
3261
|
+
const launches: Promise<void>[] = [];
|
|
3262
|
+
for (const adjudication of open) {
|
|
3263
|
+
if (d.drain?.draining === true) {
|
|
3264
|
+
log(`adjudications held: the daemon is draining (${open.length - launches.length} not launched)`);
|
|
3265
|
+
break;
|
|
3266
|
+
}
|
|
3267
|
+
// The head is re-read before anything launches. A verdict describes the diff
|
|
3268
|
+
// it read, so adjudicating a head the PR has since moved past would produce
|
|
3269
|
+
// a decision about code nobody is merging — settled `stale-head` with both
|
|
3270
|
+
// heads named, never launched.
|
|
3271
|
+
let live: string | undefined;
|
|
3272
|
+
try {
|
|
3273
|
+
live = await d.tracker.prHead(adjudication.prUrl);
|
|
3274
|
+
} catch (err) {
|
|
3275
|
+
log(`#${adjudication.issue} adjudication held: ${adjudication.prUrl} head unreadable (${errText(err)})`);
|
|
3276
|
+
continue;
|
|
3277
|
+
}
|
|
3278
|
+
if (live === undefined) {
|
|
3279
|
+
log(`#${adjudication.issue} adjudication held: ${adjudication.prUrl} head could not be read — retrying next tick`);
|
|
3280
|
+
continue;
|
|
3281
|
+
}
|
|
3282
|
+
if (live.toLowerCase() !== adjudication.headSha.toLowerCase()) {
|
|
3283
|
+
d.store.settleReviewAdjudication(
|
|
3284
|
+
adjudication.id,
|
|
3285
|
+
"stale-head",
|
|
3286
|
+
`recorded ${adjudication.headSha}, live ${live} — the pull request moved before this adjudication launched`,
|
|
3287
|
+
Date.now(),
|
|
3288
|
+
);
|
|
3289
|
+
log(`#${adjudication.issue} adjudication stale-head: recorded ${adjudication.headSha}, live ${live}`);
|
|
3290
|
+
continue;
|
|
3291
|
+
}
|
|
3292
|
+
// The claim IS the single-flight guard: the transition is the row's own
|
|
3293
|
+
// WHERE clause, so two dispatch passes racing one adjudication cannot both
|
|
3294
|
+
// launch — the loser changes nothing and skips.
|
|
3295
|
+
const review = resolveReview(d.project);
|
|
3296
|
+
const provenance = { role: adjudication.role, model: adjudication.role, resolvedAt: Date.now() };
|
|
3297
|
+
if (!d.store.markReviewAdjudicationRunning(adjudication.id, provenance, Date.now())) {
|
|
3298
|
+
log(`#${adjudication.issue} adjudication skipped: another pass already claimed it`);
|
|
3299
|
+
continue;
|
|
3300
|
+
}
|
|
3301
|
+
log(`#${adjudication.issue} adjudication → launching role ${adjudication.role} at ${adjudication.headSha}`);
|
|
3302
|
+
launches.push(handleReviewAdjudication(d, adjudication, review.maxRounds));
|
|
3303
|
+
}
|
|
3304
|
+
if (pool !== undefined) {
|
|
3305
|
+
for (const launch of launches) pool.launch(launch);
|
|
3306
|
+
return;
|
|
3307
|
+
}
|
|
3308
|
+
await Promise.allSettled(launches);
|
|
3309
|
+
}
|
|
3310
|
+
|
|
3311
|
+
/**
|
|
3312
|
+
* Assemble one adjudication's evidence and run it to a durable verdict (#932).
|
|
3313
|
+
*
|
|
3314
|
+
* Every fact comes from a durable row or a live tracker read — never from the
|
|
3315
|
+
* worker's own report, which is the claim under adjudication. The issue body is
|
|
3316
|
+
* carried verbatim because the acceptance criteria ARE the standard being
|
|
3317
|
+
* judged, and every prior round travels WITH its settled outcome: a finding
|
|
3318
|
+
* without its outcome reads as an outstanding complaint even when the worker
|
|
3319
|
+
* fixed it, which is how an adjudicator rejects work that was already corrected.
|
|
3320
|
+
*/
|
|
3321
|
+
export async function handleReviewAdjudication(
|
|
3322
|
+
d: Deps,
|
|
3323
|
+
adjudication: ReviewAdjudicationRecord,
|
|
3324
|
+
maxRounds: number,
|
|
3325
|
+
): Promise<void> {
|
|
3326
|
+
const issue = adjudication.issue;
|
|
3327
|
+
const [title, body, diff, verification] = await Promise.all([
|
|
3328
|
+
d.tracker.getIssue(issue).then(
|
|
3329
|
+
(row) => row?.title,
|
|
3330
|
+
() => undefined,
|
|
3331
|
+
),
|
|
3332
|
+
d.tracker.issueBody(issue).then(
|
|
3333
|
+
(text) => text,
|
|
3334
|
+
() => undefined,
|
|
3335
|
+
),
|
|
3336
|
+
d.tracker.prDiff(adjudication.prUrl).then(
|
|
3337
|
+
(parsed) => parsed,
|
|
3338
|
+
() => undefined,
|
|
3339
|
+
),
|
|
3340
|
+
d.tracker.verifyPr(adjudication.prUrl, adjudication.headSha).then(
|
|
3341
|
+
(result) => result,
|
|
3342
|
+
() => undefined,
|
|
3343
|
+
),
|
|
3344
|
+
]);
|
|
3345
|
+
// Prior rounds, oldest first, keyed by PR so a continuation that inherited the
|
|
3346
|
+
// PR cannot hide its predecessor's findings. The ceiling call's own findings
|
|
3347
|
+
// are appended as the last entry — they are the reason this adjudication
|
|
3348
|
+
// exists, and the durable row is where they were kept precisely so this brief
|
|
3349
|
+
// could carry them.
|
|
3350
|
+
const rounds: AdjudicationRound[] = d.store
|
|
3351
|
+
.reviewRevisionsForPr(d.project.name, adjudication.prUrl)
|
|
3352
|
+
.map((revision) => ({
|
|
3353
|
+
round: revision.round,
|
|
3354
|
+
findings: revision.findings,
|
|
3355
|
+
...(revision.outcome === undefined ? {} : { outcome: revision.outcome }),
|
|
3356
|
+
}));
|
|
3357
|
+
if (adjudication.findings !== undefined) {
|
|
3358
|
+
rounds.push({
|
|
3359
|
+
round: rounds.length + 1,
|
|
3360
|
+
findings: adjudication.findings,
|
|
3361
|
+
outcome: "escalated to this adjudication — no worker round was available",
|
|
3362
|
+
});
|
|
3363
|
+
}
|
|
3364
|
+
const brief = renderAdjudicationBrief({
|
|
3365
|
+
issue,
|
|
3366
|
+
...(title === undefined ? {} : { issueTitle: title }),
|
|
3367
|
+
issueBody: body ?? "(the issue body could not be read — judge the diff against the findings below)",
|
|
3368
|
+
prUrl: adjudication.prUrl,
|
|
3369
|
+
headSha: adjudication.headSha,
|
|
3370
|
+
checks:
|
|
3371
|
+
verification === undefined
|
|
3372
|
+
? "(the check state could not be read)"
|
|
3373
|
+
: `${verification.status} — ${verification.reason}`,
|
|
3374
|
+
diff:
|
|
3375
|
+
diff === undefined
|
|
3376
|
+
? "(the diff could not be read — say so in your reasons rather than guessing)"
|
|
3377
|
+
: renderDiffForAdjudication(diff),
|
|
3378
|
+
rounds,
|
|
3379
|
+
maxRounds,
|
|
3380
|
+
});
|
|
3381
|
+
|
|
3382
|
+
const sessionDir = join(stateDir(), "sessions", `adjudication-${adjudication.id}`);
|
|
3383
|
+
mkdirSync(sessionDir, { recursive: true });
|
|
3384
|
+
const ompSettingsFile = materializeOmpSettings(d.project, sessionDir);
|
|
3385
|
+
const result = await (d.runAdjudicatorImpl ?? runAdjudicator)({
|
|
3386
|
+
brief,
|
|
3387
|
+
// No worktree: an adjudicator has no branch, and everything it needs is in
|
|
3388
|
+
// the brief. The session directory doubles as its confined cwd.
|
|
3389
|
+
cwd: sessionDir,
|
|
3390
|
+
sessionDir,
|
|
3391
|
+
model: adjudication.role,
|
|
3392
|
+
...(ompSettingsFile === undefined ? {} : { ompSettingsFile }),
|
|
3393
|
+
socketPath: join(sessionDir, "ipc.sock"),
|
|
3394
|
+
maxTurns: ADJUDICATION_MAX_TURNS,
|
|
3395
|
+
});
|
|
3396
|
+
|
|
3397
|
+
// What actually ran it, named in the evidence when it differs from the role
|
|
3398
|
+
// that was asked for: the provenance recorded at claim time is conductor's own
|
|
3399
|
+
// resolution (the role it passed to OMP), and the model that wrote the
|
|
3400
|
+
// messages is a later observation only this result carries.
|
|
3401
|
+
const ran =
|
|
3402
|
+
result.model === undefined
|
|
3403
|
+
? ""
|
|
3404
|
+
: `adjudicated by ${result.model}${result.provider === undefined ? "" : ` (${result.provider})`}: `;
|
|
3405
|
+
if (result.verdict === undefined) {
|
|
3406
|
+
// A session that never took a turn did not run: the role could not be
|
|
3407
|
+
// resolved or the harness could not start, and the remedy is configuration
|
|
3408
|
+
// rather than a retry. One that ran and produced nothing usable is a
|
|
3409
|
+
// failure. Neither is ever a clear.
|
|
3410
|
+
const state = result.turns === 0 ? "unavailable-model" : "failed";
|
|
3411
|
+
d.store.settleReviewAdjudication(
|
|
3412
|
+
adjudication.id,
|
|
3413
|
+
state,
|
|
3414
|
+
`${ran}${result.report === "" ? "no verdict and no output" : result.report}`,
|
|
3415
|
+
Date.now(),
|
|
3416
|
+
);
|
|
3417
|
+
log(`#${issue} adjudication ${state} at ${adjudication.headSha}`);
|
|
3418
|
+
return;
|
|
3419
|
+
}
|
|
3420
|
+
// A verdict that names a different head judged something else. Fail closed:
|
|
3421
|
+
// the one-shot is spent on the head it was opened for, and nothing merges on
|
|
3422
|
+
// the strength of a decision about another diff.
|
|
3423
|
+
if (
|
|
3424
|
+
result.verdict.headSha !== undefined &&
|
|
3425
|
+
result.verdict.headSha.toLowerCase() !== adjudication.headSha.toLowerCase()
|
|
3426
|
+
) {
|
|
3427
|
+
d.store.settleReviewAdjudication(
|
|
3428
|
+
adjudication.id,
|
|
3429
|
+
"stale-head",
|
|
3430
|
+
`${ran}the verdict names head ${result.verdict.headSha}, not the adjudicated ${adjudication.headSha}: ${result.verdict.reasons}`,
|
|
3431
|
+
Date.now(),
|
|
3432
|
+
);
|
|
3433
|
+
log(`#${issue} adjudication stale-head: verdict named ${result.verdict.headSha}`);
|
|
3434
|
+
return;
|
|
3435
|
+
}
|
|
3436
|
+
d.store.settleReviewAdjudication(
|
|
3437
|
+
adjudication.id,
|
|
3438
|
+
result.verdict.verdict,
|
|
3439
|
+
`${ran}${result.verdict.reasons}`,
|
|
3440
|
+
Date.now(),
|
|
3441
|
+
);
|
|
3442
|
+
log(`#${issue} adjudication ${result.verdict.verdict} at ${adjudication.headSha}`);
|
|
3443
|
+
}
|
|
3444
|
+
|
|
3445
|
+
/**
|
|
3446
|
+
* Apply the terminal dispositions an adjudication's verdict implies (#876).
|
|
3447
|
+
*
|
|
3448
|
+
* The verdict is a decision; this is what is DONE about it, and the two are
|
|
3449
|
+
* separate rows for exactly that reason. Recording `cleared` while leaving the
|
|
3450
|
+
* merge gate refusing, or `rejected` while leaving the PR open to block the
|
|
3451
|
+
* issue forever, is the issue's named silent fake — so what this pass changes is
|
|
3452
|
+
* tracker and gate state, and the disposition text is only its receipt.
|
|
3453
|
+
*
|
|
3454
|
+
* Idempotent by that receipt: a row whose `disposition` is already recorded is
|
|
3455
|
+
* skipped, so a restart, a retried tick or a partial failure cannot close one PR
|
|
3456
|
+
* twice. Nothing here is destructive before its explanation is durable.
|
|
3457
|
+
*
|
|
3458
|
+
* Neither branch opens a review round, launches another adjudicator, or asks an
|
|
3459
|
+
* operator for anything. Neither touches an attempt or continuation budget
|
|
3460
|
+
* either: a rejected PR closes, and the ORDINARY settle sweep charges that
|
|
3461
|
+
* closure exactly as it charges any PR closed without merging — which is what
|
|
3462
|
+
* keeps this composable with #815's separate corrective-worker path instead of
|
|
3463
|
+
* resetting its counters.
|
|
3464
|
+
*/
|
|
3465
|
+
export async function applyAdjudicationDispositions(d: Deps): Promise<void> {
|
|
3466
|
+
const { project, store } = d;
|
|
3467
|
+
// Terminal rows only, and only those nothing has disposed of yet. Read from
|
|
3468
|
+
// the PRs still in flight: a settled adjudication whose PR is long gone needs
|
|
3469
|
+
// no disposition, and scanning history would re-litigate closed work.
|
|
3470
|
+
const pending = store
|
|
3471
|
+
.activeRuns(project.name)
|
|
3472
|
+
.flatMap((run) => (run.prUrl === undefined ? [] : store.reviewAdjudicationsForPr(project.name, run.prUrl)))
|
|
3473
|
+
.filter((row) => row.settledAt !== undefined && row.disposition === undefined);
|
|
3474
|
+
for (const adjudication of pending) {
|
|
3475
|
+
if (adjudication.state === "cleared") {
|
|
3476
|
+
// The clear re-enters the ORDINARY merge path, and it does so through the
|
|
3477
|
+
// mechanism that path already reads: the #888 gate refuses a head carrying
|
|
3478
|
+
// unresolved review evidence, and #913's clearance is the recorded
|
|
3479
|
+
// disposition that settles it. Reusing it means no second gate, no
|
|
3480
|
+
// adjudication-shaped exception inside `conductor_pr_merge`, and one
|
|
3481
|
+
// audit trail for "this head was cleared, by whom, and why".
|
|
3482
|
+
//
|
|
3483
|
+
// The head is re-read first: a clearance is head-scoped and clears
|
|
3484
|
+
// backwards only, so recording one against a head the PR has moved past
|
|
3485
|
+
// would bless a diff nobody adjudicated.
|
|
3486
|
+
let live: string | undefined;
|
|
3487
|
+
try {
|
|
3488
|
+
live = await d.tracker.prHead(adjudication.prUrl);
|
|
3489
|
+
} catch (err) {
|
|
3490
|
+
log(`#${adjudication.issue} adjudication clearance held: head unreadable (${errText(err)})`);
|
|
3491
|
+
continue;
|
|
3492
|
+
}
|
|
3493
|
+
if (live === undefined) {
|
|
3494
|
+
log(`#${adjudication.issue} adjudication clearance held: head could not be read — retrying next tick`);
|
|
3495
|
+
continue;
|
|
3496
|
+
}
|
|
3497
|
+
if (live.toLowerCase() !== adjudication.headSha.toLowerCase()) {
|
|
3498
|
+
// Visible and non-destructive: the clearance is not recorded, the row
|
|
3499
|
+
// says why, and the PR keeps whatever gate state it has.
|
|
3500
|
+
store.recordReviewAdjudicationDisposition(
|
|
3501
|
+
adjudication.id,
|
|
3502
|
+
`no clearance recorded: the head moved from ${adjudication.headSha} to ${live} after the verdict`,
|
|
3503
|
+
);
|
|
3504
|
+
log(`#${adjudication.issue} adjudication clearance skipped: head moved to ${live}`);
|
|
3505
|
+
continue;
|
|
3506
|
+
}
|
|
3507
|
+
store.recordReviewClearance({
|
|
3508
|
+
project: project.name,
|
|
3509
|
+
prUrl: adjudication.prUrl,
|
|
3510
|
+
headSha: adjudication.headSha,
|
|
3511
|
+
by: `adjudication:${adjudication.role}`,
|
|
3512
|
+
reason: `cleared by the review-ceiling adjudication: ${adjudication.evidence ?? "no evidence recorded"}`,
|
|
3513
|
+
});
|
|
3514
|
+
store.recordReviewAdjudicationDisposition(
|
|
3515
|
+
adjudication.id,
|
|
3516
|
+
`recorded a review clearance at ${adjudication.headSha}; the PR is on the ordinary merge path`,
|
|
3517
|
+
);
|
|
3518
|
+
log(`#${adjudication.issue} adjudication cleared → merge path open at ${adjudication.headSha}`);
|
|
3519
|
+
continue;
|
|
3520
|
+
}
|
|
3521
|
+
// Every non-clear terminal state disposes the same way, and that is
|
|
3522
|
+
// deliberate: `rejected`, `failed`, `unavailable-model` and `stale-head` all
|
|
3523
|
+
// mean "this head is not merging and no further round is coming", and the
|
|
3524
|
+
// blocking artefact is identical in each case — an open PR that occupies the
|
|
3525
|
+
// issue. Only the recorded reason differs.
|
|
3526
|
+
const findings = store
|
|
3527
|
+
.reviewRevisionsForPr(project.name, adjudication.prUrl)
|
|
3528
|
+
.map((revision) => `### Review round ${revision.round} (${revision.outcome ?? "never settled"})
|
|
3529
|
+
|
|
3530
|
+
${revision.findings}`);
|
|
3531
|
+
const comment = [
|
|
3532
|
+
`## Adjudication: ${adjudication.state}`,
|
|
3533
|
+
"",
|
|
3534
|
+
`The review-round ceiling was reached on ${adjudication.headSha}, and the \`${adjudication.role}\` adjudicator ` +
|
|
3535
|
+
`did not clear it. Closing this pull request so the issue is not blocked by work that cannot merge — ` +
|
|
3536
|
+
"nothing here asks anyone to repair it by hand.",
|
|
3537
|
+
"",
|
|
3538
|
+
"### Adjudicator's reasons",
|
|
3539
|
+
"",
|
|
3540
|
+
adjudication.evidence ?? "(none recorded)",
|
|
3541
|
+
...(findings.length === 0 ? [] : ["", "## Preserved review findings", "", ...findings]),
|
|
3542
|
+
"",
|
|
3543
|
+
"The branch itself is untouched and remains in the repository.",
|
|
3544
|
+
].join("\n");
|
|
3545
|
+
const closed = await d.verbActions?.closePr(adjudication.prUrl, comment);
|
|
3546
|
+
if (closed === undefined) {
|
|
3547
|
+
log(`#${adjudication.issue} adjudication disposition held: no action surface is wired`);
|
|
3548
|
+
continue;
|
|
3549
|
+
}
|
|
3550
|
+
if (!closed.ok) {
|
|
3551
|
+
// Retryable, and no receipt written: the next tick tries again rather than
|
|
3552
|
+
// leaving a rejected PR open with a row that claims it was handled.
|
|
3553
|
+
log(`#${adjudication.issue} adjudication disposition held: could not close ${adjudication.prUrl} — ${closed.stderr}`);
|
|
3554
|
+
continue;
|
|
3555
|
+
}
|
|
3556
|
+
store.recordReviewAdjudicationDisposition(
|
|
3557
|
+
adjudication.id,
|
|
3558
|
+
`closed ${adjudication.prUrl} with the adjudicator's reasons and every preserved finding; ` +
|
|
3559
|
+
"the settle sweep releases the issue",
|
|
3560
|
+
);
|
|
3561
|
+
log(`#${adjudication.issue} adjudication ${adjudication.state} → closed ${adjudication.prUrl}`);
|
|
3562
|
+
}
|
|
3563
|
+
}
|
|
3564
|
+
|
|
3565
|
+
/** The diff as one text, from the structured read the settlement audit uses.
|
|
3566
|
+
* `renderAdjudicationBrief` bounds the total; this only flattens it, and says
|
|
3567
|
+
* when the adapter itself had to cut the read short — an adjudicator must know
|
|
3568
|
+
* its silence covers only what it saw. */
|
|
3569
|
+
function renderDiffForAdjudication(diff: PrDiff): string {
|
|
3570
|
+
const files = diff.files.map((file) =>
|
|
3571
|
+
[
|
|
3572
|
+
`--- ${file.status} ${file.previousPath === undefined ? "" : `${file.previousPath} → `}${file.path}`,
|
|
3573
|
+
file.hunks ?? "(no textual hunks: binary, mode-only or pure rename)",
|
|
3574
|
+
].join("\n"),
|
|
3575
|
+
);
|
|
3576
|
+
return [...(diff.truncated ? ["[the tracker truncated this diff read]"] : []), ...files].join("\n\n");
|
|
3577
|
+
}
|
|
3578
|
+
|
|
2996
3579
|
/**
|
|
2997
3580
|
* Resume one review-revision worker: the run row is already claimed
|
|
2998
3581
|
* (`pushed-green` → `running` by the dispatch pass), so this resumes the SAME
|
|
@@ -3113,20 +3696,38 @@ export async function handleReviewRevision(d: Deps, revision: ReviewRevisionReco
|
|
|
3113
3696
|
// stopped anything, the PR is still green and still open, and only the
|
|
3114
3697
|
// wake failed; `settleStopBeforeSession` immediately above covers the
|
|
3115
3698
|
// case where an operator genuinely did. So mirror that branch: restore
|
|
3116
|
-
// `pushed-green` naming the shutdown,
|
|
3117
|
-
// the
|
|
3118
|
-
// — the ordinary settle sweep releases it in the same breath it
|
|
3699
|
+
// `pushed-green` naming the shutdown, and leave the in-progress label
|
|
3700
|
+
// alone — the ordinary settle sweep releases it in the same breath it
|
|
3119
3701
|
// terminalises the row when the PR resolves. Stopping the row here would
|
|
3120
3702
|
// strand it: `settlePushedGreen` sweeps only pushed-* rows and
|
|
3121
3703
|
// classification excludes `stopped`, so the issue would carry neither
|
|
3122
3704
|
// label and nothing would ever revisit the still-green PR.
|
|
3705
|
+
//
|
|
3706
|
+
// The round itself is RE-QUEUED rather than settled `skipped` (#903). A
|
|
3707
|
+
// daemon shutdown is the plainest infrastructure kill there is — no
|
|
3708
|
+
// session started, so no round was worked — and a settled row consumed
|
|
3709
|
+
// one of the PR's rounds through `latestReviewRound`, which is how an
|
|
3710
|
+
// upgrade or a restart used to cost a review round it never spent. The
|
|
3711
|
+
// retry is counted and bounded exactly like any other infra kill, so a
|
|
3712
|
+
// host that shuts down mid-wake forever still escalates.
|
|
3123
3713
|
store.updateRun(runId, {
|
|
3124
3714
|
state: "pushed-green",
|
|
3125
3715
|
endedAt: Date.now(),
|
|
3126
3716
|
lastError: "daemon shutdown began after the review revision claim; the round was not launched",
|
|
3127
3717
|
});
|
|
3128
|
-
store.
|
|
3129
|
-
|
|
3718
|
+
const retry = store.retryReviewRevisionAfterInfra(revision.id, REVIEW_ROUND_INFRA_MAX_RETRIES);
|
|
3719
|
+
if (retry.kind === "exhausted") {
|
|
3720
|
+
store.settleReviewRevision(revision.id, "skipped", Date.now());
|
|
3721
|
+
log(
|
|
3722
|
+
`#${issue} review round ${revision.round} not launched: daemon shutdown began after the claim, and ` +
|
|
3723
|
+
`its infra retries are exhausted (${retry.retries}/${REVIEW_ROUND_INFRA_MAX_RETRIES}) — settled skipped`,
|
|
3724
|
+
);
|
|
3725
|
+
return true;
|
|
3726
|
+
}
|
|
3727
|
+
log(
|
|
3728
|
+
`#${issue} review round ${revision.round} not launched: daemon shutdown began after the claim — ` +
|
|
3729
|
+
`re-queued, retry ${retry.retries}/${REVIEW_ROUND_INFRA_MAX_RETRIES}`,
|
|
3730
|
+
);
|
|
3130
3731
|
return true;
|
|
3131
3732
|
};
|
|
3132
3733
|
|
|
@@ -3286,18 +3887,26 @@ export async function handleReviewRevision(d: Deps, revision: ReviewRevisionReco
|
|
|
3286
3887
|
// totals the run already recorded.
|
|
3287
3888
|
const baseTurns = run.turns;
|
|
3288
3889
|
const baseSpend = run.spendUsd;
|
|
3890
|
+
const baseOutputTokens = run.outputTokens ?? 0;
|
|
3891
|
+
const baseReasoningTokens = run.reasoningTokens ?? 0;
|
|
3289
3892
|
|
|
3290
3893
|
log(`#${issue} review round ${revision.round} → resuming ${branch} (${priorSessionFile})`);
|
|
3291
3894
|
|
|
3292
3895
|
|
|
3293
3896
|
let result: WorkerResult;
|
|
3294
3897
|
try {
|
|
3898
|
+
const runAllowanceUsd = runSpendAllowanceUsd(caps);
|
|
3295
3899
|
result = await runWorker({
|
|
3296
3900
|
brief: renderReviewRevisionPrompt(revision.findings, revision.round),
|
|
3297
3901
|
cwd: worktreePath,
|
|
3298
3902
|
caps,
|
|
3299
3903
|
...(repoSlug === undefined ? {} : { repoSlug }),
|
|
3300
3904
|
maxTurns: () => turnLimit?.maxTurns() ?? run.maxTurns,
|
|
3905
|
+
// A review round meters its own session from zero and gets the same
|
|
3906
|
+
// allowance as any run (#851) — the round is real work with a real
|
|
3907
|
+
// cost, and an unbounded round would reopen the hole for any PR that
|
|
3908
|
+
// was ever returned for revision.
|
|
3909
|
+
...(runAllowanceUsd === null ? {} : { maxSpendUsd: runAllowanceUsd }),
|
|
3301
3910
|
onPauseControl: (control) => {
|
|
3302
3911
|
workerSessionInstalled = true;
|
|
3303
3912
|
workerControl?.install(control);
|
|
@@ -3321,6 +3930,14 @@ export async function handleReviewRevision(d: Deps, revision: ReviewRevisionReco
|
|
|
3321
3930
|
onReleaseBlocked: workerReleaseBlockRecorder(project.name, issue, runId),
|
|
3322
3931
|
onTurn: (n) => store.updateRun(runId, { turns: baseTurns + n }),
|
|
3323
3932
|
onSpend: (usd) => store.updateRun(runId, { spendUsd: baseSpend + usd }),
|
|
3933
|
+
// A revision resumes the SAME run, so its tokens add to the attempt's
|
|
3934
|
+
// totals exactly as its turns and spend do (#518) — resetting them would
|
|
3935
|
+
// make a long deliberation look like a fresh one.
|
|
3936
|
+
onTokens: (tokens) =>
|
|
3937
|
+
store.updateRun(runId, {
|
|
3938
|
+
outputTokens: baseOutputTokens + tokens.output,
|
|
3939
|
+
reasoningTokens: baseReasoningTokens + tokens.reasoning,
|
|
3940
|
+
}),
|
|
3324
3941
|
onKilled: () => {
|
|
3325
3942
|
turnLimit?.close();
|
|
3326
3943
|
turnLimit = undefined;
|
|
@@ -3419,6 +4036,63 @@ export async function handleReviewRevision(d: Deps, revision: ReviewRevisionReco
|
|
|
3419
4036
|
report: finalReport,
|
|
3420
4037
|
...settlement?.patch,
|
|
3421
4038
|
};
|
|
4039
|
+
// #903: an infrastructure kill spends no review round.
|
|
4040
|
+
//
|
|
4041
|
+
// A dispatch that dies before the resumed session takes a turn — a host
|
|
4042
|
+
// permission fault, a spawn failure, a harness that could not start — did
|
|
4043
|
+
// none of the work a round exists to do, yet the pre-#903 path settled the
|
|
4044
|
+
// row `failed` with no retry. During the 2026-08-21/22 outage that turned
|
|
4045
|
+
// one broken mount into a review-ceiling deadlock: three such deaths
|
|
4046
|
+
// consumed the PR's three rounds, `conductor_pr_review` then refused any
|
|
4047
|
+
// further round, and the findings could never be addressed by a worker at
|
|
4048
|
+
// all. Nothing in the loop could recover it.
|
|
4049
|
+
//
|
|
4050
|
+
// So the round is returned to the pending set instead: the same row, the
|
|
4051
|
+
// same round number, the same findings, and the run restored to the exact
|
|
4052
|
+
// `pushed-green` state the verb recorded — which is what the restart
|
|
4053
|
+
// reconcile already does for a round a dying daemon interrupted. The next
|
|
4054
|
+
// dispatch pass resumes the same session. Bounded by
|
|
4055
|
+
// REVIEW_ROUND_INFRA_MAX_RETRIES and counted durably on the row, so a
|
|
4056
|
+
// permanently broken host escalates through the ordinary failed path
|
|
4057
|
+
// instead of retrying every tick forever.
|
|
4058
|
+
const infraKill =
|
|
4059
|
+
state === "failed" || state === "killed"
|
|
4060
|
+
? reviewRoundNeverWorked({
|
|
4061
|
+
turns: result.turns,
|
|
4062
|
+
...(result.prUrl === undefined ? {} : { prUrl: result.prUrl }),
|
|
4063
|
+
...(result.headSha === undefined ? {} : { headSha: result.headSha }),
|
|
4064
|
+
...(settlement?.patch?.salvageSha === undefined
|
|
4065
|
+
? {}
|
|
4066
|
+
: { salvageSha: settlement.patch.salvageSha }),
|
|
4067
|
+
})
|
|
4068
|
+
: undefined;
|
|
4069
|
+
if (infraKill !== undefined) {
|
|
4070
|
+
const retry = store.retryReviewRevisionAfterInfra(revision.id, REVIEW_ROUND_INFRA_MAX_RETRIES);
|
|
4071
|
+
if (retry.kind === "requeued") {
|
|
4072
|
+
const detail =
|
|
4073
|
+
`review round ${revision.round} was killed before its session took a turn ` +
|
|
4074
|
+
`(${infraKill}); the round is re-queued, retry ${retry.retries}/${REVIEW_ROUND_INFRA_MAX_RETRIES}`;
|
|
4075
|
+
store.updateRun(runId, {
|
|
4076
|
+
...terminalPatch,
|
|
4077
|
+
state: "pushed-green",
|
|
4078
|
+
// The transcript the round must still resume is the one the run
|
|
4079
|
+
// recorded, not whatever this dead dispatch opened: a session that
|
|
4080
|
+
// never took a turn has no lineage of its own to inherit.
|
|
4081
|
+
sessionFile: priorSessionFile,
|
|
4082
|
+
report: [`review round ${revision.round}: dispatch killed before turn 1`, "", detail].join("\n"),
|
|
4083
|
+
lastError: detail,
|
|
4084
|
+
});
|
|
4085
|
+
// No label swap and no escalation: nothing failed that a human can
|
|
4086
|
+
// act on yet, the issue keeps its in-progress label, and the next
|
|
4087
|
+
// dispatch pass picks the round up on its own.
|
|
4088
|
+
log(`#${issue} ${detail}`);
|
|
4089
|
+
return;
|
|
4090
|
+
}
|
|
4091
|
+
log(
|
|
4092
|
+
`#${issue} review round ${revision.round} exhausted its infra retries ` +
|
|
4093
|
+
`(${retry.retries}/${REVIEW_ROUND_INFRA_MAX_RETRIES}) — settling it the ordinary way`,
|
|
4094
|
+
);
|
|
4095
|
+
}
|
|
3422
4096
|
if (state === "stopped") {
|
|
3423
4097
|
recordOperatorStop(store, {
|
|
3424
4098
|
project: project.name,
|
|
@@ -3530,6 +4204,35 @@ export async function handleReviewRevision(d: Deps, revision: ReviewRevisionReco
|
|
|
3530
4204
|
publish,
|
|
3531
4205
|
tree: "keep",
|
|
3532
4206
|
});
|
|
4207
|
+
// A crash BEFORE the worker session was installed spent no round (#903):
|
|
4208
|
+
// nothing resumed, nothing was reviewed, and the round is re-queued for
|
|
4209
|
+
// the next pass instead of consuming one of the PR's three. Once the
|
|
4210
|
+
// session is installed the round belongs to the worker, and a crash after
|
|
4211
|
+
// that settles the ordinary way below — including the exhausted case, so
|
|
4212
|
+
// a dispatch that keeps throwing still reaches a human.
|
|
4213
|
+
if (!workerSessionInstalled) {
|
|
4214
|
+
const retry = store.retryReviewRevisionAfterInfra(revision.id, REVIEW_ROUND_INFRA_MAX_RETRIES);
|
|
4215
|
+
if (retry.kind === "requeued") {
|
|
4216
|
+
store.updateRun(runId, {
|
|
4217
|
+
state: "pushed-green",
|
|
4218
|
+
endedAt: Date.now(),
|
|
4219
|
+
sessionFile: priorSessionFile,
|
|
4220
|
+
lastError:
|
|
4221
|
+
`review round ${revision.round} could not be dispatched (${detail}); the round is re-queued, ` +
|
|
4222
|
+
`retry ${retry.retries}/${REVIEW_ROUND_INFRA_MAX_RETRIES}`,
|
|
4223
|
+
...settlement?.patch,
|
|
4224
|
+
});
|
|
4225
|
+
log(
|
|
4226
|
+
`#${issue} review round ${revision.round} re-queued after a pre-session dispatch error — ` +
|
|
4227
|
+
`retry ${retry.retries}/${REVIEW_ROUND_INFRA_MAX_RETRIES}`,
|
|
4228
|
+
);
|
|
4229
|
+
return;
|
|
4230
|
+
}
|
|
4231
|
+
log(
|
|
4232
|
+
`#${issue} review round ${revision.round} exhausted its infra retries ` +
|
|
4233
|
+
`(${retry.retries}/${REVIEW_ROUND_INFRA_MAX_RETRIES}) — failing the round`,
|
|
4234
|
+
);
|
|
4235
|
+
}
|
|
3533
4236
|
store.updateRun(runId, {
|
|
3534
4237
|
state: "failed",
|
|
3535
4238
|
endedAt: Date.now(),
|
|
@@ -4279,16 +4982,26 @@ export interface WorkerPool {
|
|
|
4279
4982
|
drain(): Promise<void>;
|
|
4280
4983
|
}
|
|
4281
4984
|
|
|
4282
|
-
/**
|
|
4283
|
-
|
|
4985
|
+
/**
|
|
4986
|
+
* Keeps background workers alive without making the five-minute tick await them.
|
|
4987
|
+
*
|
|
4988
|
+
* `onSettled` fires as each worker leaves the pool, whichever way it ended
|
|
4989
|
+
* (#878): that is the instant a slot frees, and without it queued work waited
|
|
4990
|
+
* for the next scheduled pass — up to five minutes of idle capacity that reads,
|
|
4991
|
+
* from outside, exactly like a stalled queue. It only *prompts* a pass; every
|
|
4992
|
+
* hold, drain, lane, budget and routing gate is re-evaluated by that pass as
|
|
4993
|
+
* usual.
|
|
4994
|
+
*/
|
|
4995
|
+
export function createWorkerPool(onSettled?: () => void): WorkerPool {
|
|
4284
4996
|
const active = new Set<Promise<void>>();
|
|
4285
4997
|
return {
|
|
4286
4998
|
launch(work) {
|
|
4287
4999
|
active.add(work);
|
|
4288
|
-
void
|
|
4289
|
-
|
|
4290
|
-
()
|
|
4291
|
-
|
|
5000
|
+
const settled = (): void => {
|
|
5001
|
+
active.delete(work);
|
|
5002
|
+
onSettled?.();
|
|
5003
|
+
};
|
|
5004
|
+
void work.then(settled, settled);
|
|
4292
5005
|
},
|
|
4293
5006
|
activeCount: () => active.size,
|
|
4294
5007
|
async drain() {
|
|
@@ -4609,6 +5322,14 @@ export async function tick(d: Deps, workers?: WorkerPool): Promise<void> {
|
|
|
4609
5322
|
} catch (err) {
|
|
4610
5323
|
log(`label reconcile failed: ${errText(err)}`);
|
|
4611
5324
|
}
|
|
5325
|
+
try {
|
|
5326
|
+
// Same phase, same reason as the label reconcile above: a durable row the
|
|
5327
|
+
// tracker has moved past (#964). A separate call because that one iterates
|
|
5328
|
+
// labelled issues, and a promotable issue carries no label.
|
|
5329
|
+
await reconcileGroomingClosures(d);
|
|
5330
|
+
} catch (err) {
|
|
5331
|
+
log(`grooming closure reconcile failed: ${errText(err)}`);
|
|
5332
|
+
}
|
|
4612
5333
|
|
|
4613
5334
|
// Drain the label projection outbox (#201). The maintenance phases above may
|
|
4614
5335
|
// have enqueued ops (settlement releases, recovery requeues, reconciles);
|
|
@@ -4802,6 +5523,28 @@ export async function tick(d: Deps, workers?: WorkerPool): Promise<void> {
|
|
|
4802
5523
|
log(`review revision dispatch failed: ${errText(err)}`);
|
|
4803
5524
|
}
|
|
4804
5525
|
|
|
5526
|
+
// Adjudications are dispatch too (#932), and their own pass: an adjudication
|
|
5527
|
+
// occupies no worker slot and no issue — it opens a fresh read-only session
|
|
5528
|
+
// about a PR whose run has already settled — so it is neither bounded by the
|
|
5529
|
+
// worker cap nor allowed to consume a review round. Placed after revisions so
|
|
5530
|
+
// a PR with a live revision is never adjudicated in the same tick it is being
|
|
5531
|
+
// corrected in; the head re-read inside the pass is the durable guard.
|
|
5532
|
+
try {
|
|
5533
|
+
await dispatchReviewAdjudications(d, workers);
|
|
5534
|
+
} catch (err) {
|
|
5535
|
+
log(`adjudication dispatch failed: ${errText(err)}`);
|
|
5536
|
+
}
|
|
5537
|
+
|
|
5538
|
+
// And what the verdicts imply (#876). Its own pass, after dispatch, because a
|
|
5539
|
+
// disposition acts on a SETTLED adjudication: running it before dispatch would
|
|
5540
|
+
// simply be a tick late, and running it inside dispatch would tie a tracker
|
|
5541
|
+
// mutation to the launch that produced the verdict.
|
|
5542
|
+
try {
|
|
5543
|
+
await applyAdjudicationDispositions(d);
|
|
5544
|
+
} catch (err) {
|
|
5545
|
+
log(`adjudication dispositions failed: ${errText(err)}`);
|
|
5546
|
+
}
|
|
5547
|
+
|
|
4805
5548
|
// route() filters the queue through isEligible() itself, so anything already
|
|
4806
5549
|
// carrying a state label is gone before it gets here.
|
|
4807
5550
|
const ready = await d.tracker.listReady();
|
|
@@ -4820,7 +5563,65 @@ export async function tick(d: Deps, workers?: WorkerPool): Promise<void> {
|
|
|
4820
5563
|
const pending = store.pendingLabelOpsFor(project.name, issue.number);
|
|
4821
5564
|
return pending.length === 0 ? issue : { ...issue, labels: effectiveLabels(issue.labels, pending) };
|
|
4822
5565
|
});
|
|
4823
|
-
|
|
5566
|
+
// Post-unblock stale-list revalidation (#891). `listReady` is a
|
|
5567
|
+
// label-FILTERED search, and GitHub's search index is eventually consistent:
|
|
5568
|
+
// measured 2026-08-22T09:18Z, an `unblock` cleared `agent:failed` and
|
|
5569
|
+
// restored the queue label, `gh issue view` returned the clean label set
|
|
5570
|
+
// immediately, and the very next pass still saw the stale labels, held #889
|
|
5571
|
+
// as `stale-lifecycle` and left the fleet at 0/5 with four spare slots. The
|
|
5572
|
+
// next scheduled pass usually recovers — this is throughput, not correctness
|
|
5573
|
+
// — but an operator-driven unblock that cannot refill the fleet has not done
|
|
5574
|
+
// what it says.
|
|
5575
|
+
//
|
|
5576
|
+
// So a lifecycle label read from the LIST is a suspicion, not a verdict: the
|
|
5577
|
+
// exact per-issue read decides. Bounded to candidates that would otherwise be
|
|
5578
|
+
// dropped for a lifecycle label — never the whole queue — and further to those
|
|
5579
|
+
// whose newest run is not active (a genuinely in-flight issue needs no
|
|
5580
|
+
// revalidation; its own row is the authority) and that are not operator-parked
|
|
5581
|
+
// (a park is a decision, not a stale label).
|
|
5582
|
+
//
|
|
5583
|
+
// An unreadable exact read keeps the stale labels, so the issue stays out of
|
|
5584
|
+
// this pass, and is held as `issue-state-lookup-error` rather than
|
|
5585
|
+
// `stale-lifecycle`: "the tracker could not say" is not evidence of a
|
|
5586
|
+
// residual label, and it must not summon Duty 1 to reconcile a label nobody
|
|
5587
|
+
// has read.
|
|
5588
|
+
const stateLabelSet = new Set(Object.values(project.stateLabels));
|
|
5589
|
+
const suspect = effective.filter(
|
|
5590
|
+
(issue) =>
|
|
5591
|
+
!isEligible(issue, project) &&
|
|
5592
|
+
issue.labels.some((l) => stateLabelSet.has(l)) &&
|
|
5593
|
+
!issue.labels.includes(project.stateLabels.backlog) &&
|
|
5594
|
+
!ACTIVE_STATES.includes(store.latestRun(project.name, issue.number)?.state ?? "merged"),
|
|
5595
|
+
);
|
|
5596
|
+
const revalidated = new Map<number, readonly string[]>();
|
|
5597
|
+
const unreadable = new Set<number>();
|
|
5598
|
+
for (const issue of suspect) {
|
|
5599
|
+
let snapshot: IssueSnapshot | undefined;
|
|
5600
|
+
try {
|
|
5601
|
+
snapshot = await d.tracker.issueSnapshot(issue.number);
|
|
5602
|
+
} catch {
|
|
5603
|
+
snapshot = undefined;
|
|
5604
|
+
}
|
|
5605
|
+
if (snapshot === undefined) {
|
|
5606
|
+
unreadable.add(issue.number);
|
|
5607
|
+
log(`#${issue.number} lifecycle labels could not be revalidated — holding this pass`);
|
|
5608
|
+
continue;
|
|
5609
|
+
}
|
|
5610
|
+
// A closed issue is nobody's candidate; leave the list labels alone and let
|
|
5611
|
+
// the ordinary gates speak.
|
|
5612
|
+
if (snapshot.state === "closed") continue;
|
|
5613
|
+
if (snapshot.labels.join("\u0000") === issue.labels.join("\u0000")) continue;
|
|
5614
|
+
revalidated.set(issue.number, snapshot.labels);
|
|
5615
|
+
log(`#${issue.number} lifecycle labels revalidated: the queue list was stale`);
|
|
5616
|
+
}
|
|
5617
|
+
const verified =
|
|
5618
|
+
revalidated.size === 0
|
|
5619
|
+
? effective
|
|
5620
|
+
: effective.map((issue) => {
|
|
5621
|
+
const live = revalidated.get(issue.number);
|
|
5622
|
+
return live === undefined ? issue : { ...issue, labels: [...live] };
|
|
5623
|
+
});
|
|
5624
|
+
const { routed, unroutable } = route(verified, project);
|
|
4824
5625
|
// route() drops a candidate for two reasons, only one of which is a claim
|
|
4825
5626
|
// question. A lifecycle state label (agent:in-progress/blocked/failed)
|
|
4826
5627
|
// marks a run-owned issue: it is genuinely in flight while its newest run
|
|
@@ -4832,9 +5633,10 @@ export async function tick(d: Deps, workers?: WorkerPool): Promise<void> {
|
|
|
4832
5633
|
// close), which belongs in none of the three populations. So `claimed` is
|
|
4833
5634
|
// defined from actual ownership over genuinely lifecycle-labelled candidates
|
|
4834
5635
|
// rather than as the residual of routing (#228, #611).
|
|
4835
|
-
|
|
4836
|
-
|
|
4837
|
-
|
|
5636
|
+
// Every population below reads the VERIFIED list, so a candidate the exact
|
|
5637
|
+
// read cleared is not simultaneously admitted and counted as held (#891).
|
|
5638
|
+
const dropped = verified.filter(
|
|
5639
|
+
(issue) => !isEligible(issue, project) && issue.labels.some((l) => stateLabelSet.has(l)),
|
|
4838
5640
|
);
|
|
4839
5641
|
// The operator's park label is not a stale lifecycle label: the queue query
|
|
4840
5642
|
// still returns a parked-and-queued issue, route() drops it as ineligible,
|
|
@@ -4843,7 +5645,7 @@ export async function tick(d: Deps, workers?: WorkerPool): Promise<void> {
|
|
|
4843
5645
|
// to reconcile a deliberate decision. Parked is its own population, derived
|
|
4844
5646
|
// from the same isEligible read the gate uses, so the number status renders
|
|
4845
5647
|
// cannot disagree with what admission would hold (#507).
|
|
4846
|
-
const parkedCandidates =
|
|
5648
|
+
const parkedCandidates = verified.filter(
|
|
4847
5649
|
(issue) => !isEligible(issue, project) && issue.labels.includes(project.stateLabels.backlog),
|
|
4848
5650
|
);
|
|
4849
5651
|
const parkedNumbers = new Set(parkedCandidates.map((issue) => issue.number));
|
|
@@ -4858,6 +5660,14 @@ export async function tick(d: Deps, workers?: WorkerPool): Promise<void> {
|
|
|
4858
5660
|
// A park never kills a live run; a parked issue with no run is inventory
|
|
4859
5661
|
// the operator is deliberately holding, not reconciliation work.
|
|
4860
5662
|
parked += 1;
|
|
5663
|
+
} else if (unreadable.has(issue.number)) {
|
|
5664
|
+
// Fail closed, and say which read failed: a tracker that could not answer
|
|
5665
|
+
// is not evidence of a residual label (#891).
|
|
5666
|
+
lifecycleHolds.push({
|
|
5667
|
+
issue: issue.number,
|
|
5668
|
+
reason: "issue-state-lookup-error",
|
|
5669
|
+
detail: "lifecycle labels could not be revalidated against the exact issue read",
|
|
5670
|
+
});
|
|
4861
5671
|
} else {
|
|
4862
5672
|
lifecycleHolds.push({ issue: issue.number, reason: "stale-lifecycle" });
|
|
4863
5673
|
}
|
|
@@ -4959,6 +5769,8 @@ export async function tick(d: Deps, workers?: WorkerPool): Promise<void> {
|
|
|
4959
5769
|
return;
|
|
4960
5770
|
}
|
|
4961
5771
|
recordDispatch(pass.admitted.length, [...routingHolds, ...pass.holds]);
|
|
5772
|
+
await recordInstallSurfaces(d);
|
|
5773
|
+
reconcilePanes(d, project.name, log);
|
|
4962
5774
|
|
|
4963
5775
|
if (pass.admitted.length === 0) return;
|
|
4964
5776
|
|
|
@@ -5452,8 +6264,34 @@ export interface StatusSnapshot {
|
|
|
5452
6264
|
* materialises it, so this is never absent on a real daemon.
|
|
5453
6265
|
*/
|
|
5454
6266
|
review: ReviewPolicy;
|
|
5455
|
-
/** Occupied issues: live workers plus green PRs awaiting a human merge.
|
|
6267
|
+
/** Occupied issues: live workers plus green PRs awaiting a human merge. Both
|
|
6268
|
+
* lifecycle kinds, told apart by {@link StatusSnapshot.leasedRunIds}. */
|
|
5456
6269
|
activeRuns: RunRecord[];
|
|
6270
|
+
/**
|
|
6271
|
+
* The subset of {@link StatusSnapshot.activeRuns} holding an **active
|
|
6272
|
+
* mutation lease** (#898): a live worker, or a dispatched unsettled review
|
|
6273
|
+
* revision. Everything else in `activeRuns` is a worker-free preserved
|
|
6274
|
+
* artifact — durable work awaiting review, merge or recovery, with nothing
|
|
6275
|
+
* writing to it.
|
|
6276
|
+
*
|
|
6277
|
+
* Carried as ids rather than a second run list so the two renderings cannot
|
|
6278
|
+
* disagree about the rows themselves, and derived from `Store.leasedRuns` —
|
|
6279
|
+
* the same query admission's file-lane gate and `conductor_pr_recover` read,
|
|
6280
|
+
* so what status calls a lease is exactly what those two enforce (#899,
|
|
6281
|
+
* #925). A plain array so the dashboard's JSON round-trip preserves it.
|
|
6282
|
+
*/
|
|
6283
|
+
leasedRunIds?: readonly string[];
|
|
6284
|
+
/**
|
|
6285
|
+
* The review-ceiling adjudications an operator needs to see (#874): every
|
|
6286
|
+
* non-terminal one, plus the terminal one for each active run's PR head — so a
|
|
6287
|
+
* cleared or rejected verdict stays visible for exactly as long as the PR it
|
|
6288
|
+
* decided is still in flight, and disappears with it rather than accumulating.
|
|
6289
|
+
*
|
|
6290
|
+
* Keyed by nothing: the records carry their own `prUrl` and `headSha`, which is
|
|
6291
|
+
* how the renderer matches them to a run without inventing a second identity
|
|
6292
|
+
* for a PR.
|
|
6293
|
+
*/
|
|
6294
|
+
reviewAdjudications?: readonly ReviewAdjudicationRecord[];
|
|
5457
6295
|
/**
|
|
5458
6296
|
* runId → live review-revision round, for runs whose revision worker is
|
|
5459
6297
|
* currently dispatched (#692). Derived from the durable `review_revisions`
|
|
@@ -5462,7 +6300,22 @@ export interface StatusSnapshot {
|
|
|
5462
6300
|
* (not a Map) so the dashboard's JSON round-trip of the snapshot preserves
|
|
5463
6301
|
* it byte for byte.
|
|
5464
6302
|
*/
|
|
5465
|
-
|
|
6303
|
+
/**
|
|
6304
|
+
* The live review round per run: its number AND the instant it was dispatched
|
|
6305
|
+
* (#802). The instant is what makes the line honest — a resumed session's
|
|
6306
|
+
* `turns` and `startedAt` are cumulative over the whole attempt, so without a
|
|
6307
|
+
* phase boundary a fresh revision reads as though it had been running for
|
|
6308
|
+
* hours.
|
|
6309
|
+
*/
|
|
6310
|
+
reviewRounds?: Readonly<Record<string, { round: number; dispatchedAt: number }>>;
|
|
6311
|
+
/**
|
|
6312
|
+
* runId → the durable review evidence blocking a merge of the run's
|
|
6313
|
+
* pushed-green PR at its exact recorded head (#888). Derived from the same
|
|
6314
|
+
* `review_revisions` rows `conductor_pr_merge` refuses on, so status can
|
|
6315
|
+
* never present a green PR as merge-ready while the privileged verb will
|
|
6316
|
+
* refuse it. An absent entry means nothing blocks that run's head.
|
|
6317
|
+
*/
|
|
6318
|
+
mergeBlockers?: Readonly<Record<string, ReviewHeadBlocker>>;
|
|
5466
6319
|
/** Newest attempts holding a preserved WIP tip, or a tree that is still the
|
|
5467
6320
|
* only copy of work the daemon could not save. */
|
|
5468
6321
|
salvagedRuns: RunRecord[];
|
|
@@ -5496,6 +6349,20 @@ export interface StatusSnapshot {
|
|
|
5496
6349
|
liveWorkers: number;
|
|
5497
6350
|
runsToday: number;
|
|
5498
6351
|
spendTodayUsd: number;
|
|
6352
|
+
/** The install identities the last dispatch pass recorded (#919). Absent
|
|
6353
|
+
* means no pass has looked yet, which the renderer says out loud rather
|
|
6354
|
+
* than presenting as agreement. */
|
|
6355
|
+
installSurfaces?: InstallSurfaceObservation;
|
|
6356
|
+
/** What live runs have reserved out of today's budget but not yet spent
|
|
6357
|
+
* (#851). Optional so a caller that built a snapshot before reservations
|
|
6358
|
+
* existed renders no figure rather than a fabricated zero. */
|
|
6359
|
+
reservedSpendUsd?: number;
|
|
6360
|
+
/**
|
|
6361
|
+
* Whether the figure above is built on runs that actually reported cost
|
|
6362
|
+
* (#970). Absent when the caller did not judge it; the renderer then says
|
|
6363
|
+
* nothing rather than implying the telemetry is sound.
|
|
6364
|
+
*/
|
|
6365
|
+
spendTelemetry?: SpendTelemetryVerdict;
|
|
5499
6366
|
dispatch?: DispatchSummary;
|
|
5500
6367
|
/**
|
|
5501
6368
|
* Latest plan-allowance verdict, when the caller read one. Optional because
|
|
@@ -5570,12 +6437,50 @@ export function statusSnapshotFromStore(
|
|
|
5570
6437
|
// (#776 review #2).
|
|
5571
6438
|
const live = store.liveRuns(p.name);
|
|
5572
6439
|
const active = store.activeRuns(p.name);
|
|
6440
|
+
// Which of those active rows something is actually writing through (#898).
|
|
6441
|
+
// The same query admission's file-lane gate and `conductor_pr_recover` read,
|
|
6442
|
+
// so status cannot call a row a lease that those two treat as released, or
|
|
6443
|
+
// vice versa (#899, #925).
|
|
6444
|
+
const leasedRunIds = store.leasedRuns(p.name).map((r) => r.id);
|
|
6445
|
+
// Adjudications worth rendering (#874): the open ones always, plus a terminal
|
|
6446
|
+
// verdict for a head an active run is still sitting on. Bounded by the active
|
|
6447
|
+
// set rather than by a history window, so a fleet with a thousand settled
|
|
6448
|
+
// adjudications renders the handful that still describe live work.
|
|
6449
|
+
const openAdjudications = store.openReviewAdjudications(p.name);
|
|
6450
|
+
const adjudicationIds = new Set(openAdjudications.map((a) => a.id));
|
|
6451
|
+
const reviewAdjudications = [...openAdjudications];
|
|
6452
|
+
for (const r of active) {
|
|
6453
|
+
if (r.prUrl === undefined || r.headSha === undefined) continue;
|
|
6454
|
+
const decided = store.reviewAdjudicationForHead(p.name, r.prUrl, r.headSha);
|
|
6455
|
+
if (decided !== undefined && !adjudicationIds.has(decided.id)) {
|
|
6456
|
+
adjudicationIds.add(decided.id);
|
|
6457
|
+
reviewAdjudications.push(decided);
|
|
6458
|
+
}
|
|
6459
|
+
}
|
|
5573
6460
|
// The live review-revision rounds, read from the same durable rows the
|
|
5574
6461
|
// restart recovery uses: a run whose revision is dispatched is read as
|
|
5575
6462
|
// `review-revision N` while its worker is live (#692).
|
|
5576
|
-
const reviewRounds: Record<string, number> = {};
|
|
6463
|
+
const reviewRounds: Record<string, { round: number; dispatchedAt: number }> = {};
|
|
5577
6464
|
for (const revision of store.unsettledReviewRevisions(p.name)) {
|
|
5578
|
-
if (revision.dispatchedAt !== undefined)
|
|
6465
|
+
if (revision.dispatchedAt !== undefined) {
|
|
6466
|
+
reviewRounds[revision.runId] = { round: revision.round, dispatchedAt: revision.dispatchedAt };
|
|
6467
|
+
}
|
|
6468
|
+
}
|
|
6469
|
+
// The exact-head merge blockers (#888), read from the same durable rows the
|
|
6470
|
+
// merge verb consults: a pushed-green run whose PR stands at a head carrying
|
|
6471
|
+
// unresolved review evidence — a round queued there, crashed mid-review
|
|
6472
|
+
// there, or settled `failed` there.
|
|
6473
|
+
const mergeBlockers: Record<string, ReviewHeadBlocker> = {};
|
|
6474
|
+
for (const r of active) {
|
|
6475
|
+
if (r.prUrl === undefined || r.headSha === undefined) continue;
|
|
6476
|
+
const blocker = store.mergeBlockingReviews(p.name, r.prUrl, r.headSha)[0];
|
|
6477
|
+
if (blocker !== undefined) {
|
|
6478
|
+
mergeBlockers[r.id] = {
|
|
6479
|
+
round: blocker.round,
|
|
6480
|
+
state:
|
|
6481
|
+
blocker.settledAt !== undefined ? "failed" : blocker.dispatchedAt !== undefined ? "crashed" : "pending",
|
|
6482
|
+
};
|
|
6483
|
+
}
|
|
5579
6484
|
}
|
|
5580
6485
|
return {
|
|
5581
6486
|
project: p.name,
|
|
@@ -5600,6 +6505,8 @@ export function statusSnapshotFromStore(
|
|
|
5600
6505
|
releaseGrants: resolveReleaseGrants(p),
|
|
5601
6506
|
review: resolveReview(p),
|
|
5602
6507
|
activeRuns: active,
|
|
6508
|
+
leasedRunIds,
|
|
6509
|
+
reviewAdjudications,
|
|
5603
6510
|
reviewRounds,
|
|
5604
6511
|
salvagedRuns: store.salvagedRuns(p.name),
|
|
5605
6512
|
quarantinedRuns: store.quarantinedRuns(p.name),
|
|
@@ -5610,6 +6517,19 @@ export function statusSnapshotFromStore(
|
|
|
5610
6517
|
liveWorkers: live.length,
|
|
5611
6518
|
runsToday: store.runsStartedSince(p.name, since),
|
|
5612
6519
|
spendTodayUsd: store.spendSince(p.name, since),
|
|
6520
|
+
reservedSpendUsd: store.reservedSpendUsd(p.name),
|
|
6521
|
+
// Judged from the store on the same read as the figure above (#970), so the
|
|
6522
|
+
// row that qualifies the spend number cannot disagree with it, and nothing
|
|
6523
|
+
// is probed at render time.
|
|
6524
|
+
spendTelemetry: judgeSpendTelemetry(
|
|
6525
|
+
store.recentSpendSamples?.(p.name, SPEND_SAMPLE_ROWS) ?? [],
|
|
6526
|
+
SPEND_SAMPLE_RUNS,
|
|
6527
|
+
),
|
|
6528
|
+
// Read, never probed: the recorded row is the whole point (#919).
|
|
6529
|
+
...(() => {
|
|
6530
|
+
const observed = store.installSurfaces();
|
|
6531
|
+
return observed === undefined ? {} : { installSurfaces: observed };
|
|
6532
|
+
})(),
|
|
5613
6533
|
...(dispatch === undefined ? {} : { dispatch }),
|
|
5614
6534
|
...(planUsage === undefined ? {} : { planUsage }),
|
|
5615
6535
|
// Written by the tracker's hooks rather than polled, so the renderer does
|
|
@@ -5621,6 +6541,7 @@ export function statusSnapshotFromStore(
|
|
|
5621
6541
|
: { labelOps: { pending: labelOpsPending, oldestAgeMs: now - oldestLabelOpAt } }),
|
|
5622
6542
|
baseHealth: store.baseHealth(p.name),
|
|
5623
6543
|
freezes: store.freezes(p.name),
|
|
6544
|
+
mergeBlockers,
|
|
5624
6545
|
...(orchestratorDown === undefined ? {} : { orchestratorDown }),
|
|
5625
6546
|
};
|
|
5626
6547
|
}
|
|
@@ -5674,8 +6595,15 @@ export function formatDispatchSummary(summary?: DispatchSummary): string {
|
|
|
5674
6595
|
lines.push(` ${hold.reason} ${hold.count}${sample}`);
|
|
5675
6596
|
// A `file-lane` hold groups several issues, each blocked by a different
|
|
5676
6597
|
// file and holder; the grouped line says how many, this says which.
|
|
5677
|
-
|
|
5678
|
-
|
|
6598
|
+
//
|
|
6599
|
+
// Deduplicated for display only, never in the record: a fleet-wide hold
|
|
6600
|
+
// (`credential-class`, #852) gives every held candidate the *same*
|
|
6601
|
+
// sentence, and printing one reason five times reads as five problems.
|
|
6602
|
+
// The persisted details stay index-aligned with `issues` — a reader that
|
|
6603
|
+
// needs "which issue got which detail" still has it.
|
|
6604
|
+
const details = [...new Set(hold.details ?? [])];
|
|
6605
|
+
if (details.length > 0) {
|
|
6606
|
+
lines.push(` ${details.join(" | ")}`);
|
|
5679
6607
|
}
|
|
5680
6608
|
}
|
|
5681
6609
|
}
|
|
@@ -5741,71 +6669,6 @@ export function formatFreezes(freezes: readonly BaseFreeze[]): string[] {
|
|
|
5741
6669
|
}
|
|
5742
6670
|
|
|
5743
6671
|
|
|
5744
|
-
export function formatStatus(s: StatusSnapshot): string {
|
|
5745
|
-
const lines = [
|
|
5746
|
-
`project ${s.project}${s.paused ? " (PAUSED)" : ""}`,
|
|
5747
|
-
`config ${s.configPath}`,
|
|
5748
|
-
`state ${s.stateDir}`,
|
|
5749
|
-
"",
|
|
5750
|
-
...(s.orchestratorDown === undefined ? [] : formatOrchestratorDown(s.orchestratorDown)),
|
|
5751
|
-
"caps",
|
|
5752
|
-
` workers ${s.liveWorkers} / ${s.caps.maxConcurrentWorkers}`,
|
|
5753
|
-
` issues today ${s.runsToday}`,
|
|
5754
|
-
s.caps.dailySpendUsd === null
|
|
5755
|
-
? ` spend today $${s.spendTodayUsd.toFixed(2)} (no daily cap)`
|
|
5756
|
-
: ` spend today $${s.spendTodayUsd.toFixed(2)} / $${s.caps.dailySpendUsd.toFixed(2)}`,
|
|
5757
|
-
// Its own row, never folded into the spend row: they are two independent
|
|
5758
|
-
// controls, and an operator has to be able to see which one stopped the
|
|
5759
|
-
// fleet (#110).
|
|
5760
|
-
` plan usage ${planUsageLine(s.planUsage)}`,
|
|
5761
|
-
` new worker turns ${s.caps.workerMaxTurns}`,
|
|
5762
|
-
...s.turnOverrides.map(
|
|
5763
|
-
({ issue, maxTurns }) => ` turn override #${issue} → ${maxTurns} (next attempt)`,
|
|
5764
|
-
),
|
|
5765
|
-
` worker wall clock ${Math.round(s.caps.workerWallClockMs / 60_000)}m`,
|
|
5766
|
-
` failed attempts ${s.caps.maxAttemptsPerIssue}`,
|
|
5767
|
-
` continuations ${s.caps.maxContinuationsPerIssue}`,
|
|
5768
|
-
"",
|
|
5769
|
-
...formatReleaseGrants(s.releaseGrants),
|
|
5770
|
-
"",
|
|
5771
|
-
formatDispatchSummary(s.dispatch),
|
|
5772
|
-
"",
|
|
5773
|
-
];
|
|
5774
|
-
if (s.activeRuns.length === 0) {
|
|
5775
|
-
lines.push("active runs (none)");
|
|
5776
|
-
} else {
|
|
5777
|
-
lines.push("active runs");
|
|
5778
|
-
for (const r of s.activeRuns) {
|
|
5779
|
-
lines.push(
|
|
5780
|
-
` #${r.issue} ${r.repo} ${r.state} attempt ${r.attempt} ` +
|
|
5781
|
-
`${r.turns}/${r.maxTurns} turns $${r.spendUsd.toFixed(2)} ${r.branch}` +
|
|
5782
|
-
(r.prUrl ? ` ${r.prUrl}` : ""),
|
|
5783
|
-
);
|
|
5784
|
-
// Its escalation was deduplicated the moment it was delivered, so this
|
|
5785
|
-
// line is the only place a flagged run stays visible while its PR waits
|
|
5786
|
-
// for a merge — which is precisely the window the flag is about (#128).
|
|
5787
|
-
const flagged = settlementFlagSummary(r.settlementFlags);
|
|
5788
|
-
if (flagged !== undefined) lines.push(` ${flagged}`);
|
|
5789
|
-
}
|
|
5790
|
-
}
|
|
5791
|
-
lines.push(...formatBaseHealth(s.baseHealth));
|
|
5792
|
-
lines.push(...formatFreezes(s.freezes));
|
|
5793
|
-
lines.push(...formatSalvagedRuns(s.salvagedRuns));
|
|
5794
|
-
lines.push(...formatQuarantinedRuns(s.quarantinedRuns));
|
|
5795
|
-
lines.push(...formatOpenReports(s.openReports));
|
|
5796
|
-
lines.push(...formatVerbLedger(s.verbLedger));
|
|
5797
|
-
// Deploy hint: a restart while workers are live orphans them (salvage runs
|
|
5798
|
-
// first — #35). Prefer pause + drain to zero live workers when you can wait.
|
|
5799
|
-
if (s.liveWorkers > 0) {
|
|
5800
|
-
lines.push(
|
|
5801
|
-
"",
|
|
5802
|
-
`deploy ${s.liveWorkers} live worker(s) — restart salvages dirty trees then orphans the rows; ` +
|
|
5803
|
-
`pause and wait for workers 0/${s.caps.maxConcurrentWorkers} when you can drain instead`,
|
|
5804
|
-
);
|
|
5805
|
-
}
|
|
5806
|
-
return lines.join("\n");
|
|
5807
|
-
}
|
|
5808
|
-
|
|
5809
6672
|
export interface QueuePreview {
|
|
5810
6673
|
project: string;
|
|
5811
6674
|
configPath: string;
|
|
@@ -6194,7 +7057,15 @@ export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
|
|
|
6194
7057
|
store.updateRun(runId, { maxTurns });
|
|
6195
7058
|
});
|
|
6196
7059
|
const workerControls = createWorkerControlRegistry();
|
|
6197
|
-
|
|
7060
|
+
// Created before the pool so a worker's completion can poke the loop it
|
|
7061
|
+
// shares with the interval sleep (#878).
|
|
7062
|
+
const pace = createDispatchPace();
|
|
7063
|
+
const workers = createWorkerPool(() => {
|
|
7064
|
+
// Sticky and coalescing by construction: several workers settling together
|
|
7065
|
+
// produce one pass, and a pass already running absorbs the request rather
|
|
7066
|
+
// than overlapping. No second owner is created and no gate is skipped.
|
|
7067
|
+
pace.requestWake();
|
|
7068
|
+
});
|
|
6198
7069
|
const alive = livingDaemon();
|
|
6199
7070
|
const runtimes: ProjectRuntime[] = [];
|
|
6200
7071
|
|
|
@@ -6241,6 +7112,21 @@ export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
|
|
|
6241
7112
|
: pushRunBranch(project, { repo, runRepoPath: run.worktree, branch });
|
|
6242
7113
|
};
|
|
6243
7114
|
for (const run of await reconcileOrphanedRuns(store, project.name, orphanPublisher)) {
|
|
7115
|
+
// The workspace outlives the process, so a restart inherits panes whose
|
|
7116
|
+
// workers are gone (#842). A session-host child dies with the daemon that
|
|
7117
|
+
// owned its socket, so an orphaned row's pane is authoritatively dead
|
|
7118
|
+
// whatever its recorded pid says — pids are reused, and matching one
|
|
7119
|
+
// would be the mechanism that makes a stranger's process read as a live
|
|
7120
|
+
// worker. Release (never close) so lifecycle authority goes back to
|
|
7121
|
+
// Herdr and the pane stops claiming a worker conductor no longer runs.
|
|
7122
|
+
const orphanPane = releaseOrphanedWorkerPane(run);
|
|
7123
|
+
if (orphanPane.kind === "released") {
|
|
7124
|
+
projectLog(
|
|
7125
|
+
`#${run.issue} herdr pane ${orphanPane.paneId} released: its worker died with the previous daemon`,
|
|
7126
|
+
);
|
|
7127
|
+
} else if (orphanPane.kind === "failed") {
|
|
7128
|
+
projectLog(`#${run.issue} herdr pane ${orphanPane.paneId} release failed: ${orphanPane.reason}`);
|
|
7129
|
+
}
|
|
6244
7130
|
projectLog(
|
|
6245
7131
|
`#${run.issue} orphaned by a previous daemon (attempt ${run.attempt}, was ${run.state}, ` +
|
|
6246
7132
|
`worktree ${run.worktree}) — slot freed; the ${project.stateLabels.inProgress} label stays ` +
|
|
@@ -6365,7 +7251,27 @@ export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
|
|
|
6365
7251
|
cleanup: { next: 0 },
|
|
6366
7252
|
probeCriticalBase: (repo, markers, branch) =>
|
|
6367
7253
|
probeCriticalBase(project, repo, branch, markers),
|
|
7254
|
+
// The same seam `doctor` and `upgrade` read (#904/#919), never a second
|
|
7255
|
+
// implementation. `readHerdr` stays on: a host that runs herdr is the
|
|
7256
|
+
// case where a stale recovery pin matters, and a host without it answers
|
|
7257
|
+
// "absent", which no cheap surface treats as a fault.
|
|
7258
|
+
probeInstallSurfaces: () => inspectSurfaces({ run: runCommand, log, env: process.env }),
|
|
7259
|
+
probeTelegramFreshness: () =>
|
|
7260
|
+
// The module's own reader, so the daemon, `doctor` and `status` cannot
|
|
7261
|
+
// come to disagree about what is installed (#961).
|
|
7262
|
+
checkTelegramFreshness({
|
|
7263
|
+
run: async (cmd, args) => {
|
|
7264
|
+
const r = await runCommand(cmd, args);
|
|
7265
|
+
return { code: r.code, stdout: r.stdout };
|
|
7266
|
+
},
|
|
7267
|
+
}),
|
|
6368
7268
|
probeWorktreeLane: (input) => probeRunLane(input),
|
|
7269
|
+
// The credential-class fence's reader (#852): the runnable probe module,
|
|
7270
|
+
// spawned against the live credential store. Wired once here so the
|
|
7271
|
+
// admission gate and the launch fence ask the same question through the
|
|
7272
|
+
// same transport — two readers would eventually disagree, and the one that
|
|
7273
|
+
// disagreed by passing is the expensive one.
|
|
7274
|
+
probeCredentialClass: (provider) => probeCredentialClass(provider),
|
|
6369
7275
|
// A cross-repo Depends-on prerequisite reads through the same GitHub
|
|
6370
7276
|
// credential/accounting seams as the project tracker — a fresh tracker
|
|
6371
7277
|
// scoped to the referenced repo, reusing the daemon's gh hooks so API
|
|
@@ -6407,6 +7313,31 @@ export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
|
|
|
6407
7313
|
projectLog(`review revision restart recovery failed: ${errText(err)}`);
|
|
6408
7314
|
}
|
|
6409
7315
|
}
|
|
7316
|
+
// Adjudication restart recovery (#932). An adjudication is `running` only
|
|
7317
|
+
// while a session it launched is alive, and that session died with the
|
|
7318
|
+
// previous process — nothing resumes it, because an adjudicator has no
|
|
7319
|
+
// branch, no worktree and no transcript worth continuing: its whole output
|
|
7320
|
+
// is a verdict it never produced.
|
|
7321
|
+
//
|
|
7322
|
+
// So it settles `failed`, which is honest and terminal, rather than being
|
|
7323
|
+
// left `running` forever (a row nothing would ever touch again) or silently
|
|
7324
|
+
// re-queued (a second launch for one head, which the one-shot exists to
|
|
7325
|
+
// prevent). The one shot for that head is spent, and #876's disposition path
|
|
7326
|
+
// reads `failed` exactly as it reads any other non-clear verdict.
|
|
7327
|
+
if (alive === undefined || alive.pid === process.pid) {
|
|
7328
|
+
for (const adjudication of store.openReviewAdjudications(project.name)) {
|
|
7329
|
+
if (adjudication.state !== "running") continue;
|
|
7330
|
+
store.settleReviewAdjudication(
|
|
7331
|
+
adjudication.id,
|
|
7332
|
+
"failed",
|
|
7333
|
+
"the daemon restarted while this adjudication was running; no verdict was produced",
|
|
7334
|
+
Date.now(),
|
|
7335
|
+
);
|
|
7336
|
+
projectLog(
|
|
7337
|
+
`#${adjudication.issue} adjudication failed across restart — its session died with the previous daemon`,
|
|
7338
|
+
);
|
|
7339
|
+
}
|
|
7340
|
+
}
|
|
6410
7341
|
// Startup reconciliation: close an incident carried over from a previous
|
|
6411
7342
|
// process when the orchestrator is up (one recovery notice), or open one
|
|
6412
7343
|
// when it failed to start (one down page). A daemon restarted while still
|
|
@@ -6504,7 +7435,6 @@ export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
|
|
|
6504
7435
|
// The wake surface (#380): `resume` POSTs /wake, `stop` interrupts the same
|
|
6505
7436
|
// sleep. A wake request is sticky and coalescing — several resumes in quick
|
|
6506
7437
|
// succession produce one prompt pass, never an overlapping one.
|
|
6507
|
-
const pace = createDispatchPace();
|
|
6508
7438
|
const stop = (): void => {
|
|
6509
7439
|
if (stopping) return;
|
|
6510
7440
|
stopping = true;
|