omp-conductor 0.18.1 → 0.19.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +106 -41
- package/REFERENCE.md +866 -31
- package/agents/to-spec.md +6 -2
- package/package.json +1 -1
- package/schema/config.schema.json +32 -1
- package/src/admission.ts +212 -26
- package/src/arm-challenge.ts +250 -57
- package/src/ask.ts +288 -1
- package/src/briefs/orchestrator.md +27 -13
- package/src/briefs/to-spec.md +6 -2
- package/src/cli.ts +127 -2
- package/src/command-help.ts +9 -1
- package/src/command-manifest.ts +52 -8
- package/src/commands/arm.ts +6 -2
- package/src/commands/context.ts +2 -0
- package/src/commands/intake.ts +4 -19
- package/src/commands/message.ts +26 -2
- package/src/commands/reconcile-units.ts +104 -0
- package/src/commands/release-composition.ts +232 -0
- package/src/commands/resume.ts +2 -27
- package/src/commands/setup.ts +101 -16
- package/src/commands/stats.ts +11 -30
- package/src/commands/tail.ts +31 -1
- package/src/commands/upgrade.ts +20 -3
- package/src/commands/verb.ts +2 -1
- package/src/commands/watch.ts +4 -17
- package/src/config-schema.ts +38 -6
- package/src/config.ts +103 -8
- package/src/credential-class.ts +366 -0
- package/src/daemon.ts +1368 -529
- package/src/dashboard/app.js +504 -2
- package/src/dashboard/controls.ts +336 -0
- package/src/dashboard/index.html +30 -0
- package/src/dashboard/server.ts +271 -30
- package/src/dashboard/style.css +116 -0
- package/src/dashboard/transcript.ts +173 -0
- package/src/decisions.ts +19 -11
- package/src/doctor.ts +431 -148
- package/src/escalate.ts +22 -11
- package/src/failure-class.ts +59 -0
- package/src/fleet.ts +587 -230
- package/src/host.ts +6 -455
- package/src/omp-settings.ts +19 -0
- package/src/omp.ts +40 -56
- package/src/orchestrator-tick.ts +564 -121
- package/src/pause.ts +233 -0
- package/src/session-host.ts +6 -41
- package/src/settlement.ts +159 -2
- package/src/setup-answers.ts +97 -0
- package/src/setup-host.ts +343 -1160
- package/src/setup-install.ts +204 -27
- package/src/setup-wizard.ts +252 -51
- package/src/setup.ts +87 -4
- package/src/spend-telemetry.ts +117 -0
- package/src/stats.ts +35 -0
- package/src/status-render.ts +485 -19
- package/src/store.ts +1229 -55
- package/src/telegram-freshness.ts +269 -0
- package/src/to-spec.ts +50 -2
- package/src/types.ts +759 -10
- package/src/unblock.ts +22 -0
- package/src/unit-reconcile.ts +303 -0
- package/src/upgrade-verify.ts +8 -1
- package/src/upgrade.ts +299 -12
- package/src/verbs/actions.ts +124 -10
- package/src/verbs/protocol.ts +70 -2
- package/src/verbs/server.ts +485 -11
- package/src/wake.ts +48 -0
- package/src/worker.ts +401 -14
package/src/daemon.ts
CHANGED
|
@@ -36,20 +36,38 @@ import {
|
|
|
36
36
|
import { createEscalator, escalationIssueRef } from "./escalate.ts";
|
|
37
37
|
import { pendingCodeGraph, probeCodeGraph, type CodeGraphHealth } from "./graph-health.ts";
|
|
38
38
|
import { graphHint } from "./graph.ts";
|
|
39
|
-
import {
|
|
40
|
-
hostConstraintsNotice,
|
|
41
|
-
resolveWorkerIdentity,
|
|
42
|
-
WORKER_ACCOUNT,
|
|
43
|
-
type WorkerIdentity,
|
|
44
|
-
type WorkerIdentityResolution,
|
|
45
|
-
} from "./host.ts";
|
|
39
|
+
import { hostConstraintsNotice } from "./host.ts";
|
|
46
40
|
import { acquireOnceLease, healthCheck, livingDaemon, probeUnit, SYSTEMD_UNIT } from "./lifecycle.ts";
|
|
47
41
|
import { requestImmediateTick, resolveTickConfigCwd, STALL_MARKER_FILE } from "./orchestrator-tick.ts";
|
|
48
|
-
import { runDoctor } from "./doctor.ts";
|
|
42
|
+
import { SPEND_SAMPLE_ROWS, SPEND_SAMPLE_RUNS, runDoctor } from "./doctor.ts";
|
|
43
|
+
import { judgeSpendTelemetry, type SpendTelemetryVerdict } from "./spend-telemetry.ts";
|
|
44
|
+
import {
|
|
45
|
+
clearPauseIfUnchanged,
|
|
46
|
+
isPaused,
|
|
47
|
+
pausedAt,
|
|
48
|
+
pausedPath,
|
|
49
|
+
pauseInstance,
|
|
50
|
+
pauseInstanceAt,
|
|
51
|
+
pauseProvenance,
|
|
52
|
+
pauseSourceToken,
|
|
53
|
+
setPaused,
|
|
54
|
+
} from "./pause.ts";
|
|
49
55
|
import { appendJournal, launchTransientUnit, readUpgradeJournal } from "./upgrade-journal.ts";
|
|
50
56
|
import { runCommand, verifyPendingUpgrade, type UpgradeVerifyDeps } from "./upgrade-verify.ts";
|
|
57
|
+
import { inspectSurfaces, type InstalledSurfaces } from "./upgrade.ts";
|
|
58
|
+
import { checkTelegramFreshness, type TelegramFreshness } from "./telegram-freshness.ts";
|
|
51
59
|
import { processStartTimeMs } from "./upgrade-verify.ts";
|
|
52
|
-
import {
|
|
60
|
+
import {
|
|
61
|
+
fleetLayers,
|
|
62
|
+
herdrPaneOmpStarts,
|
|
63
|
+
openWorkerPane,
|
|
64
|
+
reconcileWorkerPanes,
|
|
65
|
+
releaseOrphanedWorkerPane,
|
|
66
|
+
releaseWorkerPane,
|
|
67
|
+
reportWorkerPaneState,
|
|
68
|
+
resolveHerdrSession,
|
|
69
|
+
type WorkerPaneOutcome,
|
|
70
|
+
} from "./fleet.ts";
|
|
53
71
|
import { startOrchestrator } from "./orchestrator.ts";
|
|
54
72
|
import type { OrchestratorHandle } from "./orchestrator.ts";
|
|
55
73
|
import {
|
|
@@ -77,11 +95,18 @@ import {
|
|
|
77
95
|
effectiveModel,
|
|
78
96
|
hasContinuationBudget,
|
|
79
97
|
hasFailedAttemptBudget,
|
|
98
|
+
runSpendAllowanceUsd,
|
|
99
|
+
startOfToday,
|
|
80
100
|
laneEcho,
|
|
81
101
|
} from "./admission.ts";
|
|
82
102
|
import type { Admission, AdmissionHold } from "./admission.ts";
|
|
83
103
|
import type { EffectiveModel, FileLane } from "./types.ts";
|
|
84
104
|
import { log, errText, safeEscalate } from "./log.ts";
|
|
105
|
+
import {
|
|
106
|
+
oauthFenceVerdict,
|
|
107
|
+
probeCredentialClass,
|
|
108
|
+
type CredentialClassProbeResult,
|
|
109
|
+
} from "./credential-class.ts";
|
|
85
110
|
import {
|
|
86
111
|
adoptSalvagedPrs,
|
|
87
112
|
classifyAndRecover,
|
|
@@ -91,6 +116,7 @@ import {
|
|
|
91
116
|
reactToProviderCredit,
|
|
92
117
|
readSessionError,
|
|
93
118
|
reconcileOrphanedRuns,
|
|
119
|
+
reconcileGroomingClosures,
|
|
94
120
|
reconcileStaleLabels,
|
|
95
121
|
recordOperatorStop,
|
|
96
122
|
settlePushedGreen,
|
|
@@ -99,7 +125,13 @@ import {
|
|
|
99
125
|
} from "./settlement.ts";
|
|
100
126
|
import { materializeOmpSettings } from "./omp-settings.ts";
|
|
101
127
|
import { evaluateDecisionConditions, probeNpmVersion, probeRateLimitReset } from "./decisions.ts";
|
|
102
|
-
import {
|
|
128
|
+
import {
|
|
129
|
+
infraLogSignature,
|
|
130
|
+
infraSignatureVersion,
|
|
131
|
+
providerCreditRefusal,
|
|
132
|
+
providerTransientFault,
|
|
133
|
+
reviewRoundNeverWorked,
|
|
134
|
+
} from "./failure-class.ts";
|
|
103
135
|
import {
|
|
104
136
|
DEFAULT_MODEL_FALLBACK_THRESHOLD,
|
|
105
137
|
fallbackClause,
|
|
@@ -118,10 +150,11 @@ import {
|
|
|
118
150
|
utcDay,
|
|
119
151
|
} from "./store.ts";
|
|
120
152
|
import { GhPrMissingError, GraphqlBreaker, makeTracker, type RateLimitStatus } from "./tracker/github.ts";
|
|
121
|
-
import { DEFAULT_REPORT_POLICY, RELEASE_SHAPES } from "./types.ts";
|
|
153
|
+
import { DEFAULT_REPORT_POLICY, RELEASE_SHAPES, REVIEW_ROUND_INFRA_MAX_RETRIES } from "./types.ts";
|
|
122
154
|
import type {
|
|
123
155
|
BaseFreeze,
|
|
124
156
|
BaseHealth,
|
|
157
|
+
InstallSurfaceObservation,
|
|
125
158
|
AdmissionHoldReason,
|
|
126
159
|
Caps,
|
|
127
160
|
ConductorConfig,
|
|
@@ -131,6 +164,8 @@ import type {
|
|
|
131
164
|
InterruptCategory,
|
|
132
165
|
IssueComment,
|
|
133
166
|
IssueSnapshot,
|
|
167
|
+
PrDiff,
|
|
168
|
+
ReviewAdjudicationRecord,
|
|
134
169
|
MergedPrInfo,
|
|
135
170
|
OpenCloser,
|
|
136
171
|
ReleaseShape,
|
|
@@ -147,6 +182,7 @@ import type {
|
|
|
147
182
|
FailureClass,
|
|
148
183
|
HostConstraints,
|
|
149
184
|
RecoveryAction,
|
|
185
|
+
ReviewHeadBlocker,
|
|
150
186
|
ReviewPolicy,
|
|
151
187
|
ReviewRevisionOutcome,
|
|
152
188
|
ReviewRevisionRecord,
|
|
@@ -160,14 +196,19 @@ import type {
|
|
|
160
196
|
WorkflowRun,
|
|
161
197
|
} from "./types.ts";
|
|
162
198
|
import {
|
|
199
|
+
type AdjudicationRound,
|
|
200
|
+
type AdjudicatorOpts,
|
|
201
|
+
type AdjudicationResult,
|
|
163
202
|
type KilledBy,
|
|
164
203
|
type WorkerPauseControl,
|
|
165
204
|
type WorkerPausePhase,
|
|
166
205
|
type WorkerResult,
|
|
167
206
|
type RunWorkerDeps,
|
|
168
207
|
ORPHAN_RESUME_PROMPT,
|
|
208
|
+
renderAdjudicationBrief,
|
|
169
209
|
renderBrief,
|
|
170
210
|
renderReviewRevisionPrompt,
|
|
211
|
+
runAdjudicator,
|
|
171
212
|
runWorker,
|
|
172
213
|
} from "./worker.ts";
|
|
173
214
|
import {
|
|
@@ -286,33 +327,6 @@ interface Deps {
|
|
|
286
327
|
workerControls: WorkerControlRegistry;
|
|
287
328
|
/** Session seam for lifecycle integration tests; production uses the real harness. */
|
|
288
329
|
workerDeps?: RunWorkerDeps;
|
|
289
|
-
/**
|
|
290
|
-
* Resolves the dedicated worker identity (#798) — the account worker sessions
|
|
291
|
-
* run under, with its uid/gid/home, the transition launcher and a live harness
|
|
292
|
-
* binding (#828).
|
|
293
|
-
*
|
|
294
|
-
* Called **at every worker launch**, never cached: every input is host state
|
|
295
|
-
* that can change under a running daemon. A reboot can start this service
|
|
296
|
-
* before systemd has mounted the harness binding, and an operator can install
|
|
297
|
-
* the account or re-run `setup host` at any time — a verdict taken once at
|
|
298
|
-
* startup would hold the whole fleet closed until somebody thought to restart
|
|
299
|
-
* the daemon, which is the outage #828 exists to end rather than relocate.
|
|
300
|
-
* Resolution is a handful of `stat` calls and one `/etc/passwd` read; a
|
|
301
|
-
* dispatch does far more than that before it reaches this gate.
|
|
302
|
-
*
|
|
303
|
-
* Every worker dispatch refuses to launch on an unresolved identity (fail
|
|
304
|
-
* closed — an unbound worker is indistinguishable from an operator shell),
|
|
305
|
-
* and this is how the dispatcher secures the run's sockets and grants the
|
|
306
|
-
* run's paths to the account. Absent entirely, dispatch fails closed naming
|
|
307
|
-
* the missing account.
|
|
308
|
-
*/
|
|
309
|
-
workerIdentity?: () => WorkerIdentityResolution;
|
|
310
|
-
/**
|
|
311
|
-
* Re-owns a run's working paths (worktree + session dir) under the worker
|
|
312
|
-
* identity before the session launches. Wired by `runDaemon` to the
|
|
313
|
-
* recursive chown; a test fixture leaves it unset so no test chowns.
|
|
314
|
-
*/
|
|
315
|
-
grantWorkerPaths?: (identity: WorkerIdentity, worktreePath: string, sessionDir: string) => void;
|
|
316
330
|
integrity: IntegrityGate;
|
|
317
331
|
stall: StallGate;
|
|
318
332
|
/**
|
|
@@ -365,6 +379,17 @@ interface Deps {
|
|
|
365
379
|
* probe the way `criticalBase` does.
|
|
366
380
|
*/
|
|
367
381
|
probeWorktreeLane?: RunLaneProbe;
|
|
382
|
+
/**
|
|
383
|
+
* Reads one provider's current credential class from the harness, out of
|
|
384
|
+
* process (#852). Wired by `runDaemon` to {@link probeCredentialClass}; a test
|
|
385
|
+
* injects a fake.
|
|
386
|
+
*
|
|
387
|
+
* Absent, both fences refuse a project that declares `requireOauthProviders` —
|
|
388
|
+
* this one fails CLOSED, unlike `probeWorktreeLane`, because an unverified
|
|
389
|
+
* credential class costs exactly what a wrong one costs. A project declaring
|
|
390
|
+
* nothing never calls it, so an unwired test dispatches as it always has.
|
|
391
|
+
*/
|
|
392
|
+
probeCredentialClass?: (provider: string) => Promise<CredentialClassProbeResult>;
|
|
368
393
|
/**
|
|
369
394
|
* Reads one issue's tracker state in a repository the admission tracker is
|
|
370
395
|
* not bound to — the cross-repo Depends-on interlock (#420). Wired by
|
|
@@ -379,6 +404,24 @@ interface Deps {
|
|
|
379
404
|
* body fails that branch closed rather than synthesising a cycle.
|
|
380
405
|
*/
|
|
381
406
|
probeBodyIn?: (repo: string, issue: number) => Promise<string | undefined>;
|
|
407
|
+
/**
|
|
408
|
+
* Reads the three install identities this host carries (#919). Wired by
|
|
409
|
+
* `runDaemon` to `inspectSurfaces`, the same seam `doctor` and `upgrade`
|
|
410
|
+
* read, so nothing re-implements the probe. A test injects its own; absent,
|
|
411
|
+
* the pass records nothing and every cheap surface honestly says "not
|
|
412
|
+
* observed yet" rather than claiming agreement.
|
|
413
|
+
*/
|
|
414
|
+
probeInstallSurfaces?: () => Promise<InstalledSurfaces>;
|
|
415
|
+
/** The `omp-telegram` install/daemon/published triple, read on the same
|
|
416
|
+
* periodic pass as the surfaces above (#961). */
|
|
417
|
+
probeTelegramFreshness?: () => Promise<TelegramFreshness>;
|
|
418
|
+
/**
|
|
419
|
+
* Runs one review-ceiling adjudication (#932). Production is
|
|
420
|
+
* {@link runAdjudicator}; a test injects a fake so the assertion can be the
|
|
421
|
+
* launch arguments and the assembled brief — which is what this pass actually
|
|
422
|
+
* produces — rather than a stored flag.
|
|
423
|
+
*/
|
|
424
|
+
runAdjudicatorImpl?: (opts: AdjudicatorOpts) => Promise<AdjudicationResult>;
|
|
382
425
|
}
|
|
383
426
|
|
|
384
427
|
/**
|
|
@@ -408,6 +451,7 @@ export function verbDeps(d: Pick<Deps, "project" | "store" | "tracker" | "verbAc
|
|
|
408
451
|
log,
|
|
409
452
|
now: () => Date.now(),
|
|
410
453
|
chain: { readBaseChain },
|
|
454
|
+
lane: { probeRunLane },
|
|
411
455
|
};
|
|
412
456
|
}
|
|
413
457
|
|
|
@@ -500,121 +544,6 @@ export function checkStall(gate: StallGate, marker: string, now = Date.now()): S
|
|
|
500
544
|
* could destroy work an operator would rather read first — the same refusal to
|
|
501
545
|
* guess that the recovery plugin is built on.
|
|
502
546
|
*/
|
|
503
|
-
// --------------------------------------------------------- worker identity (#798) --
|
|
504
|
-
|
|
505
|
-
/**
|
|
506
|
-
* Re-own a run's working paths (worktree, session dir) under the worker
|
|
507
|
-
* identity before the session launches. The daemon runs this as root; the
|
|
508
|
-
* worker identity is granted its own run paths by ownership, never by
|
|
509
|
-
* loosened modes on the daemon's.
|
|
510
|
-
*
|
|
511
|
-
* The walk never re-owns through a path a worker-uid process could
|
|
512
|
-
* re-resolve: every ownership change is `fchownSync` on a descriptor opened
|
|
513
|
-
* with `O_NOFOLLOW`, and every descriptor is verified — via `/proc/self/fd`'s
|
|
514
|
-
* kernel-resolved path — to still sit under the directory the walk opened.
|
|
515
|
-
* A directory entry raced into a symlink therefore either fails `O_NOFOLLOW`
|
|
516
|
-
* at the final component or resolves outside the verified parent and is
|
|
517
|
-
* skipped; it can never carry the chown to an external target. Symlinks
|
|
518
|
-
* within the tree are left untouched (the worker manages entries through its
|
|
519
|
-
* parent directories, and git recreates links on checkout), as are anything
|
|
520
|
-
* unopenable — fifos, sockets, devices — which git never creates. A missing
|
|
521
|
-
* or racing entry is not this dispatch's problem — the next dispatch re-owns
|
|
522
|
-
* whatever survives; a tree already owned by the worker identity — the
|
|
523
|
-
* resume and review-revision re-ownership of a tree a prior dispatch granted
|
|
524
|
-
* — is the only tree a worker-uid process could have modified, and is
|
|
525
|
-
* skipped outright rather than walked.
|
|
526
|
-
*/
|
|
527
|
-
export function chownRecursive(root: string, uid: number, gid: number): void {
|
|
528
|
-
try {
|
|
529
|
-
const current = lstatSync(root);
|
|
530
|
-
if (current.uid === uid && current.gid === gid) return;
|
|
531
|
-
} catch {
|
|
532
|
-
// A missing or racing root is the caller's own existence check; the next
|
|
533
|
-
// dispatch re-owns whatever survives.
|
|
534
|
-
return;
|
|
535
|
-
}
|
|
536
|
-
const rootFd = openNoFollowDir(root);
|
|
537
|
-
try {
|
|
538
|
-
walkDir(rootFd, root, uid, gid);
|
|
539
|
-
} finally {
|
|
540
|
-
try {
|
|
541
|
-
closeSync(rootFd);
|
|
542
|
-
} catch {
|
|
543
|
-
// Already closed by a raced-away walk; nothing to do.
|
|
544
|
-
}
|
|
545
|
-
}
|
|
546
|
-
}
|
|
547
|
-
|
|
548
|
-
/** Open a directory without following a final-component symlink. */
|
|
549
|
-
function openNoFollowDir(path: string): number {
|
|
550
|
-
return openSync(path, constants.O_RDONLY | constants.O_DIRECTORY | constants.O_NOFOLLOW);
|
|
551
|
-
}
|
|
552
|
-
|
|
553
|
-
/** The kernel-resolved path of an open descriptor: what the object actually is. */
|
|
554
|
-
function fdRealPath(fd: number): string | undefined {
|
|
555
|
-
try {
|
|
556
|
-
return readlinkSync(`/proc/self/fd/${fd}`);
|
|
557
|
-
} catch {
|
|
558
|
-
return undefined;
|
|
559
|
-
}
|
|
560
|
-
}
|
|
561
|
-
|
|
562
|
-
function walkDir(dirFd: number, dirPath: string, uid: number, gid: number): void {
|
|
563
|
-
try {
|
|
564
|
-
fchownSync(dirFd, uid, gid);
|
|
565
|
-
} catch {
|
|
566
|
-
// Raced away; the next dispatch re-owns what survives.
|
|
567
|
-
}
|
|
568
|
-
const dirReal = fdRealPath(dirFd);
|
|
569
|
-
if (dirReal === undefined) return;
|
|
570
|
-
let entries: Dirent[];
|
|
571
|
-
try {
|
|
572
|
-
entries = readdirSync(dirPath, { withFileTypes: true });
|
|
573
|
-
} catch {
|
|
574
|
-
return;
|
|
575
|
-
}
|
|
576
|
-
for (const entry of entries) {
|
|
577
|
-
const childPath = join(dirPath, entry.name);
|
|
578
|
-
try {
|
|
579
|
-
if (entry.isDirectory()) {
|
|
580
|
-
const childFd = openNoFollowDir(childPath);
|
|
581
|
-
try {
|
|
582
|
-
// The entry is genuinely beneath the directory this fd owns only
|
|
583
|
-
// when the kernel resolves the opened object to a path under it.
|
|
584
|
-
// Anything else — a name swapped to a symlink, a foreign listing
|
|
585
|
-
// read through a replaced parent — is refused, never re-resolved.
|
|
586
|
-
const childReal = fdRealPath(childFd);
|
|
587
|
-
if (childReal === undefined || !childReal.startsWith(`${dirReal}/`)) continue;
|
|
588
|
-
walkDir(childFd, childPath, uid, gid);
|
|
589
|
-
} finally {
|
|
590
|
-
try {
|
|
591
|
-
closeSync(childFd);
|
|
592
|
-
} catch {
|
|
593
|
-
// Raced away; nothing to close.
|
|
594
|
-
}
|
|
595
|
-
}
|
|
596
|
-
continue;
|
|
597
|
-
}
|
|
598
|
-
// Symlinks stay untouched (never followed, never chowned), and so do
|
|
599
|
-
// entries a read-only open cannot name safely (fifos, sockets, devices).
|
|
600
|
-
if (entry.isSymbolicLink() || !entry.isFile()) continue;
|
|
601
|
-
const fd = openSync(childPath, constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK);
|
|
602
|
-
try {
|
|
603
|
-
const real = fdRealPath(fd);
|
|
604
|
-
if (real !== undefined && real.startsWith(`${dirReal}/`)) fchownSync(fd, uid, gid);
|
|
605
|
-
} finally {
|
|
606
|
-
try {
|
|
607
|
-
closeSync(fd);
|
|
608
|
-
} catch {
|
|
609
|
-
// Raced away; nothing to close.
|
|
610
|
-
}
|
|
611
|
-
}
|
|
612
|
-
} catch {
|
|
613
|
-
// O_NOFOLLOW refusal (a symlink swapped onto the name), or gone: nothing to chown.
|
|
614
|
-
}
|
|
615
|
-
}
|
|
616
|
-
}
|
|
617
|
-
|
|
618
547
|
export async function watchOrchestrator(d: Deps, now = Date.now()): Promise<void> {
|
|
619
548
|
const marker = join(stateDir(), STALL_MARKER_FILE);
|
|
620
549
|
const repeat = d.stall.paged;
|
|
@@ -664,180 +593,21 @@ export async function watchOrchestrator(d: Deps, now = Date.now()): Promise<void
|
|
|
664
593
|
markPaged(d.stall, delivered, now);
|
|
665
594
|
}
|
|
666
595
|
|
|
667
|
-
|
|
668
|
-
|
|
669
|
-
|
|
670
|
-
|
|
671
|
-
|
|
672
|
-
|
|
673
|
-
|
|
674
|
-
|
|
675
|
-
|
|
676
|
-
|
|
677
|
-
|
|
678
|
-
|
|
679
|
-
|
|
680
|
-
|
|
681
|
-
|
|
682
|
-
export function isPaused(project?: string): boolean {
|
|
683
|
-
return activePausePaths(project).length !== 0;
|
|
684
|
-
}
|
|
685
|
-
|
|
686
|
-
/**
|
|
687
|
-
* The epoch-ms timestamp at which the current pause began. A project pause also
|
|
688
|
-
* observes the legacy bare sentinel, which pauses every project. If any active
|
|
689
|
-
* sentinel is unreadable or unparseable, the timestamp is unknown so callers
|
|
690
|
-
* continue to fail closed.
|
|
691
|
-
*/
|
|
692
|
-
export function pausedAt(project?: string): number | undefined {
|
|
693
|
-
const paths = activePausePaths(project);
|
|
694
|
-
if (paths.length === 0) return undefined;
|
|
695
|
-
const times: number[] = [];
|
|
696
|
-
for (const path of paths) {
|
|
697
|
-
try {
|
|
698
|
-
const first = readFileSync(path, "utf8").split("\n")[0]?.trim();
|
|
699
|
-
if (first === undefined || first === "") return undefined;
|
|
700
|
-
const time = Date.parse(first);
|
|
701
|
-
if (Number.isNaN(time)) return undefined;
|
|
702
|
-
times.push(time);
|
|
703
|
-
} catch {
|
|
704
|
-
return undefined;
|
|
705
|
-
}
|
|
706
|
-
}
|
|
707
|
-
return Math.min(...times);
|
|
708
|
-
}
|
|
709
|
-
|
|
710
|
-
/**
|
|
711
|
-
* Who paused the project and why. Per-project provenance wins when both its
|
|
712
|
-
* sentinel and the legacy all-project sentinel are active.
|
|
713
|
-
*/
|
|
714
|
-
export function pauseProvenance(
|
|
715
|
-
project?: string,
|
|
716
|
-
): { source: string; reason?: string } | undefined {
|
|
717
|
-
const paths =
|
|
718
|
-
project === undefined
|
|
719
|
-
? [pausedPath()]
|
|
720
|
-
: [pausedPath(project), pausedPath()];
|
|
721
|
-
const path = paths.find((candidate) => existsSync(candidate));
|
|
722
|
-
if (path === undefined) return undefined;
|
|
723
|
-
try {
|
|
724
|
-
const second = readFileSync(path, "utf8").split("\n")[1];
|
|
725
|
-
if (second === undefined) return undefined;
|
|
726
|
-
const match = /^source=(\S+)(?: reason="(.*)")?$/.exec(second.trim());
|
|
727
|
-
if (match === null) return undefined;
|
|
728
|
-
const source = match[1]!;
|
|
729
|
-
const reason = match[2];
|
|
730
|
-
return { source, ...(reason === undefined ? {} : { reason }) };
|
|
731
|
-
} catch {
|
|
732
|
-
return undefined;
|
|
733
|
-
}
|
|
734
|
-
}
|
|
735
|
-
|
|
736
|
-
/**
|
|
737
|
-
* One pause sentinel FILE read as an instance identity: who set it, why, and
|
|
738
|
-
* the creation instant, all from that exact file. An unreadable or malformed
|
|
739
|
-
* file is undefined — the caller may treat it as absence.
|
|
740
|
-
*/
|
|
741
|
-
function pauseInstanceAt(
|
|
742
|
-
path: string,
|
|
743
|
-
): { source: string; reason?: string; since: number } | undefined {
|
|
744
|
-
try {
|
|
745
|
-
const [line1, line2] = readFileSync(path, "utf8").split("\n");
|
|
746
|
-
const since = Date.parse(line1?.trim() ?? "");
|
|
747
|
-
if (!Number.isFinite(since)) return undefined;
|
|
748
|
-
if (line2 === undefined) return undefined;
|
|
749
|
-
const match = /^source=(\S+)(?: reason="(.*)")?$/.exec(line2.trim());
|
|
750
|
-
if (match === null) return undefined;
|
|
751
|
-
const source = match[1]!;
|
|
752
|
-
const reason = match[2];
|
|
753
|
-
return { source, since, ...(reason === undefined ? {} : { reason }) };
|
|
754
|
-
} catch {
|
|
755
|
-
return undefined;
|
|
756
|
-
}
|
|
757
|
-
}
|
|
758
|
-
|
|
759
|
-
/**
|
|
760
|
-
* One pause sentinel read as a single identity: who set it, why, and the
|
|
761
|
-
* creation instant, all from the SAME file that was selected. Unlike pairing
|
|
762
|
-
* {@link pauseProvenance} with {@link pausedAt} — which can describe
|
|
763
|
-
* different files when a project pause coexists with the legacy global
|
|
764
|
-
* sentinel, letting a stale global timestamp mask a recreated project pause —
|
|
765
|
-
* this reads provenance and timestamp from one sentinel, so a caller can prove
|
|
766
|
-
* "the pause I set still exists" instead of "some pause with the same labels
|
|
767
|
-
* still exists" (#377). Per-project sentinel wins, like {@link pauseProvenance}.
|
|
768
|
-
*/
|
|
769
|
-
export function pauseInstance(
|
|
770
|
-
project?: string,
|
|
771
|
-
): { source: string; reason?: string; since: number } | undefined {
|
|
772
|
-
const paths =
|
|
773
|
-
project === undefined
|
|
774
|
-
? [pausedPath()]
|
|
775
|
-
: [pausedPath(project), pausedPath()];
|
|
776
|
-
const path = paths.find((candidate) => existsSync(candidate));
|
|
777
|
-
if (path === undefined) return undefined;
|
|
778
|
-
return pauseInstanceAt(path);
|
|
779
|
-
}
|
|
780
|
-
|
|
781
|
-
/**
|
|
782
|
-
* Compare-and-clear one pause sentinel (#780 review): remove `path` only while
|
|
783
|
-
* it still holds exactly the `expected` instance — same source, same reason,
|
|
784
|
-
* same creation instant — as read by {@link pauseInstance}. A hold or pause
|
|
785
|
-
* that replaced or recreated the sentinel between the read and the clear is a
|
|
786
|
-
* newer instance (writes always re-stamp `since`), so it is never destroyed:
|
|
787
|
-
* returning false keeps the newer fence in force. Scoped strictly to one
|
|
788
|
-
* caller-provided path, so auto-expiry can clear the per-project spend-cap
|
|
789
|
-
* sentinel without ever touching the legacy global sentinel.
|
|
790
|
-
*/
|
|
791
|
-
export function clearPauseIfUnchanged(
|
|
792
|
-
path: string,
|
|
793
|
-
expected: { source: string; reason?: string; since: number },
|
|
794
|
-
): boolean {
|
|
795
|
-
const current = pauseInstanceAt(path);
|
|
796
|
-
if (current === undefined) return false;
|
|
797
|
-
if (
|
|
798
|
-
current.source !== expected.source ||
|
|
799
|
-
current.since !== expected.since ||
|
|
800
|
-
current.reason !== expected.reason
|
|
801
|
-
) {
|
|
802
|
-
return false;
|
|
803
|
-
}
|
|
804
|
-
rmSync(path, { force: true });
|
|
805
|
-
return true;
|
|
806
|
-
}
|
|
807
|
-
|
|
808
|
-
/**
|
|
809
|
-
* The pause sentinel's `source=` token, proven from a verb. The sentinel's
|
|
810
|
-
* source line is read back as a single `\S+` token (see {@link pauseInstance}
|
|
811
|
-
* and {@link pauseProvenance}), so a verb that contains a space (`setup host`)
|
|
812
|
-
* is unrepresentable verbatim and must be encoded before it reaches disk —
|
|
813
|
-
* otherwise the fence cannot prove its own pause and refuses forever (#552).
|
|
814
|
-
* Spaces become `-`; the human-readable verb is preserved in the sentinel's
|
|
815
|
-
* `reason=` instead.
|
|
816
|
-
*/
|
|
817
|
-
export function pauseSourceToken(verb: string): string {
|
|
818
|
-
return verb.trim().replace(/\s+/g, "-");
|
|
819
|
-
}
|
|
820
|
-
|
|
821
|
-
export function setPaused(
|
|
822
|
-
v: boolean,
|
|
823
|
-
why?: { source: string; reason?: string },
|
|
824
|
-
project?: string,
|
|
825
|
-
): void {
|
|
826
|
-
const path = pausedPath(project);
|
|
827
|
-
if (v) {
|
|
828
|
-
mkdirSync(dirname(path), { recursive: true });
|
|
829
|
-
const line1 = `${new Date().toISOString()}\n`;
|
|
830
|
-
if (why === undefined) {
|
|
831
|
-
writeFileSync(path, line1);
|
|
832
|
-
} else {
|
|
833
|
-
const reason = why.reason === undefined ? "" : ` reason="${why.reason.replaceAll('"', "")}"`;
|
|
834
|
-
writeFileSync(path, `${line1}source=${why.source}${reason}\n`);
|
|
835
|
-
}
|
|
836
|
-
} else {
|
|
837
|
-
rmSync(path, { force: true });
|
|
838
|
-
if (project !== undefined) rmSync(pausedPath(), { force: true });
|
|
839
|
-
}
|
|
840
|
-
}
|
|
596
|
+
// ------------------------------------------------------------------- pause
|
|
597
|
+
// The sentinel itself lives in `pause.ts` (#938): this module imports
|
|
598
|
+
// `doctor.ts`, so a health check that needs to read a fence cannot import back
|
|
599
|
+
// here. Re-exported so every existing `from "./daemon.ts"` import still
|
|
600
|
+
// resolves — the move is structural, not a change of surface.
|
|
601
|
+
export {
|
|
602
|
+
clearPauseIfUnchanged,
|
|
603
|
+
isPaused,
|
|
604
|
+
pausedAt,
|
|
605
|
+
pausedPath,
|
|
606
|
+
pauseInstance,
|
|
607
|
+
pauseProvenance,
|
|
608
|
+
pauseSourceToken,
|
|
609
|
+
setPaused,
|
|
610
|
+
};
|
|
841
611
|
|
|
842
612
|
// ------------------------------------------------------------------- drain
|
|
843
613
|
// (#484 slice 1) A project drain is a durable, self-expiring admission fence:
|
|
@@ -1238,19 +1008,6 @@ export function markPaged(
|
|
|
1238
1008
|
|
|
1239
1009
|
// ---------------------------------------------------------------------- helpers
|
|
1240
1010
|
|
|
1241
|
-
/**
|
|
1242
|
-
* Local midnight, matching how a human reads "today".
|
|
1243
|
-
*
|
|
1244
|
-
* ponytail: a rolling 24h window would be fairer to a run that started at
|
|
1245
|
-
* 23:50, but midnight is what someone checking a morning spend report expects.
|
|
1246
|
-
* Upgrade path is a `capWindow: "day" | "rolling24h"` config key.
|
|
1247
|
-
*/
|
|
1248
|
-
function startOfToday(): number {
|
|
1249
|
-
const d = new Date();
|
|
1250
|
-
d.setHours(0, 0, 0, 0);
|
|
1251
|
-
return d.getTime();
|
|
1252
|
-
}
|
|
1253
|
-
|
|
1254
1011
|
/**
|
|
1255
1012
|
* `owner/repo` for `gh`, derived from the clone URL.
|
|
1256
1013
|
*
|
|
@@ -1448,6 +1205,7 @@ function swapLabel(store: Store, projectName: string, issue: number, from: strin
|
|
|
1448
1205
|
function endedBy(killedBy: KilledBy | undefined): string {
|
|
1449
1206
|
if (killedBy === "turns") return "killed by the turns cap";
|
|
1450
1207
|
if (killedBy === "wallclock") return "killed by the wall-clock cap";
|
|
1208
|
+
if (killedBy === "spend") return "killed by the per-run spend cap";
|
|
1451
1209
|
return "killed by a failed run";
|
|
1452
1210
|
}
|
|
1453
1211
|
|
|
@@ -1653,7 +1411,19 @@ export interface WorkerControlSlot {
|
|
|
1653
1411
|
}
|
|
1654
1412
|
|
|
1655
1413
|
export interface WorkerControlRegistry {
|
|
1656
|
-
|
|
1414
|
+
/**
|
|
1415
|
+
* `onPhase` is called after a pause/resume/stop that actually changed this
|
|
1416
|
+
* worker's phase (#842) — the authoritative transition, from the same control
|
|
1417
|
+
* that performed it. It exists so a surface outside this registry (the run's
|
|
1418
|
+
* Herdr representation) can follow the phase without polling and without
|
|
1419
|
+
* inventing a second notion of "paused".
|
|
1420
|
+
*/
|
|
1421
|
+
open(
|
|
1422
|
+
project: string,
|
|
1423
|
+
issue: number,
|
|
1424
|
+
runId: string,
|
|
1425
|
+
onPhase?: (phase: WorkerPausePhase) => void,
|
|
1426
|
+
): WorkerControlSlot;
|
|
1657
1427
|
pause(project: string, issue: number): Promise<WorkerControlResult>;
|
|
1658
1428
|
resume(project: string, issue: number): WorkerControlResult;
|
|
1659
1429
|
stop(project: string, issue: number, reason: string): Promise<WorkerControlResult>;
|
|
@@ -1671,12 +1441,13 @@ export function createWorkerControlRegistry(): WorkerControlRegistry {
|
|
|
1671
1441
|
stopReason?: string;
|
|
1672
1442
|
stopError?: string;
|
|
1673
1443
|
finished: PromiseWithResolvers<void>;
|
|
1444
|
+
onPhase?: (phase: WorkerPausePhase) => void;
|
|
1674
1445
|
}
|
|
1675
1446
|
|
|
1676
1447
|
const active = new Map<string, Entry>();
|
|
1677
1448
|
const key = (project: string, issue: number): string => `${project}\0${issue}`;
|
|
1678
1449
|
return {
|
|
1679
|
-
open(project, issue, runId) {
|
|
1450
|
+
open(project, issue, runId, onPhase) {
|
|
1680
1451
|
const k = key(project, issue);
|
|
1681
1452
|
if (active.has(k)) throw new Error(`#${issue} already has a live worker controller`);
|
|
1682
1453
|
const entry: Entry = {
|
|
@@ -1684,6 +1455,7 @@ export function createWorkerControlRegistry(): WorkerControlRegistry {
|
|
|
1684
1455
|
issue,
|
|
1685
1456
|
runId,
|
|
1686
1457
|
finished: Promise.withResolvers<void>(),
|
|
1458
|
+
...(onPhase === undefined ? {} : { onPhase }),
|
|
1687
1459
|
};
|
|
1688
1460
|
active.set(k, entry);
|
|
1689
1461
|
return {
|
|
@@ -1709,7 +1481,9 @@ export function createWorkerControlRegistry(): WorkerControlRegistry {
|
|
|
1709
1481
|
if (entry?.control === undefined) return { kind: "not-active" };
|
|
1710
1482
|
try {
|
|
1711
1483
|
await entry.control.pause();
|
|
1712
|
-
|
|
1484
|
+
const phase = entry.control.phase();
|
|
1485
|
+
entry.onPhase?.(phase);
|
|
1486
|
+
return { kind: "ok", runId: entry.runId, phase };
|
|
1713
1487
|
} catch (err) {
|
|
1714
1488
|
return {
|
|
1715
1489
|
kind: "refused",
|
|
@@ -1723,7 +1497,9 @@ export function createWorkerControlRegistry(): WorkerControlRegistry {
|
|
|
1723
1497
|
if (entry?.control === undefined) return { kind: "not-active" };
|
|
1724
1498
|
try {
|
|
1725
1499
|
entry.control.resume();
|
|
1726
|
-
|
|
1500
|
+
const phase = entry.control.phase();
|
|
1501
|
+
entry.onPhase?.(phase);
|
|
1502
|
+
return { kind: "ok", runId: entry.runId, phase };
|
|
1727
1503
|
} catch (err) {
|
|
1728
1504
|
return {
|
|
1729
1505
|
kind: "refused",
|
|
@@ -1768,17 +1544,147 @@ export function createWorkerControlRegistry(): WorkerControlRegistry {
|
|
|
1768
1544
|
};
|
|
1769
1545
|
}
|
|
1770
1546
|
|
|
1547
|
+
/**
|
|
1548
|
+
* The daemon half of a worker's `pushed-green` claim (#85's contact with
|
|
1549
|
+
* reality; #782's publication attribution).
|
|
1550
|
+
*
|
|
1551
|
+
* A yield is a transport, not proof of GitHub side effects: `claim` is
|
|
1552
|
+
* caller-supplied text and the worker adapter turns it into a green result
|
|
1553
|
+
* without the daemon ever having seen a mediated publication verb for it — the
|
|
1554
|
+
* shape that let the #777 incident guess a non-existent PR. The tracker read
|
|
1555
|
+
* at the end proves the PR is open and green at the exact head; the evidence
|
|
1556
|
+
* gate here proves that PR is *this run's own mediated work* first.
|
|
1557
|
+
*
|
|
1558
|
+
* Two things are asked, in cost order:
|
|
1559
|
+
*
|
|
1560
|
+
* - the claimed URL must equal the one this run's row records, which is never
|
|
1561
|
+
* a worker-supplied string at the moment a live run is verified —
|
|
1562
|
+
* conductor_pr_create (create or adoption) writes it on the run, the daemon
|
|
1563
|
+
* seeds it at claim from a terminal predecessor on the same branch (#434),
|
|
1564
|
+
* and orchestrator-only recovery writes it for settled runs;
|
|
1565
|
+
* - and the claimed head must have a mediated publisher: an allowed
|
|
1566
|
+
* `conductor_push` on this exact run that published it on this run's branch
|
|
1567
|
+
* (the verb only ever pushes `refs/heads/<branch>`), an allowed
|
|
1568
|
+
* `conductor_pr_create` on this run for this exact PR, or — for a run that
|
|
1569
|
+
* published nothing new — the live tip of this run's own branch.
|
|
1570
|
+
*
|
|
1571
|
+
* That last path is the one the row cannot supply. It used to be served by
|
|
1572
|
+
* comparing the claim against `run.prUrl`/`run.headSha`, and both are
|
|
1573
|
+
* worker-tainted upstream: terminal settlement writes a worker's reported pair
|
|
1574
|
+
* onto the row *before* any verification, and continuation inheritance
|
|
1575
|
+
* validates only the predecessor's metadata and open state — so a turn-capped
|
|
1576
|
+
* attempt that reported someone else's real green PR had that pair inherited
|
|
1577
|
+
* and re-presented as its own evidence. Asking the tracker for the branch tip
|
|
1578
|
+
* removes the worker from the loop entirely: `refs/heads/<branch>` is a ref
|
|
1579
|
+
* only a mediated `conductor_push` or a mediated `conductor_pr_update_branch`
|
|
1580
|
+
* can move, so its live commit is daemon provenance no reported string can
|
|
1581
|
+
* forge. It is also why a legitimate mediated base-branch update now passes:
|
|
1582
|
+
* that server-side merge creates a head no `conductor_push` ever published and
|
|
1583
|
+
* the row still carries the older one, which the recorded-pair test rejected.
|
|
1584
|
+
*
|
|
1585
|
+
* Binding to the *exact current run* is what makes an old attempt, a different
|
|
1586
|
+
* branch, or #806's orchestrator-only settled-run recovery invisible here: the
|
|
1587
|
+
* ledger query is run-scoped by `runId`, and recoveries store no runId at all.
|
|
1588
|
+
* That query asks for the run's complete history rather than the ledger's
|
|
1589
|
+
* newest rows — publication evidence sits at the *start* of a run, and a
|
|
1590
|
+
* review-revision round or a burst of refused mutations pushed it past the
|
|
1591
|
+
* default page, turning a verified push into a definitive false failure.
|
|
1592
|
+
*
|
|
1593
|
+
* A claim that fails this gate is a definitive failure — never a retryable
|
|
1594
|
+
* `pushed-pending` — because a guessed URL is not something a later tick is
|
|
1595
|
+
* waiting on. A claim the gate could not *read* is the opposite: an unreadable
|
|
1596
|
+
* branch tip is #781's transient outage, so it stays retryable rather than
|
|
1597
|
+
* burning an attempt on a flaky read.
|
|
1598
|
+
*/
|
|
1771
1599
|
export async function verifyPushedGreenClaim(
|
|
1772
|
-
tracker: Pick<Tracker, "verifyPr">,
|
|
1600
|
+
tracker: Pick<Tracker, "verifyPr" | "branchHead">,
|
|
1773
1601
|
claim: Pick<WorkerResult, "prUrl" | "headSha">,
|
|
1602
|
+
publication: {
|
|
1603
|
+
project: string;
|
|
1604
|
+
issue: number;
|
|
1605
|
+
runId: string;
|
|
1606
|
+
/** The branch conductor routed this run onto. `conductor_push` publishes
|
|
1607
|
+
* exactly `refs/heads/<branch>` and refuses any other ref. */
|
|
1608
|
+
branch: string;
|
|
1609
|
+
/** The identity `tracker.branchHead` reads the live tip with: the
|
|
1610
|
+
* canonical `owner/repo` when the routed clone URL carries one, else the
|
|
1611
|
+
* routed repository name. A tracker that cannot resolve it answers
|
|
1612
|
+
* undefined, which stays retryable rather than definitive. */
|
|
1613
|
+
repo: string;
|
|
1614
|
+
store: Pick<Store, "verbLedger" | "getRun">;
|
|
1615
|
+
},
|
|
1774
1616
|
): Promise<{
|
|
1775
1617
|
state: "pushed-green" | "pushed-pending" | "failed";
|
|
1776
1618
|
reason?: string;
|
|
1777
1619
|
}> {
|
|
1778
|
-
|
|
1620
|
+
const { prUrl, headSha } = claim;
|
|
1621
|
+
if (prUrl === undefined || headSha === undefined) {
|
|
1779
1622
|
return { state: "failed", reason: "Worker did not report a PR URL and observed head SHA" };
|
|
1780
1623
|
}
|
|
1781
|
-
|
|
1624
|
+
// Exact current run, never "some publication for the issue": the query is
|
|
1625
|
+
// scoped to this runId plus project/issue, so a previous attempt's verbs and
|
|
1626
|
+
// the orchestrator's recovery verbs are invisible here. Unbounded on
|
|
1627
|
+
// purpose — see the note above about evidence ageing off the newest page.
|
|
1628
|
+
const ledger = publication.store.verbLedger(publication.project, {
|
|
1629
|
+
runId: publication.runId,
|
|
1630
|
+
issue: publication.issue,
|
|
1631
|
+
limit: Number.MAX_SAFE_INTEGER,
|
|
1632
|
+
});
|
|
1633
|
+
const run = publication.store.getRun(publication.runId);
|
|
1634
|
+
const branchRef = `refs/heads/${publication.branch}`;
|
|
1635
|
+
if (run?.prUrl !== prUrl) {
|
|
1636
|
+
return {
|
|
1637
|
+
state: "failed",
|
|
1638
|
+
reason:
|
|
1639
|
+
"Pushed-green claim has no mediated publication evidence: the claimed PR is not this run's " +
|
|
1640
|
+
"recorded PR",
|
|
1641
|
+
};
|
|
1642
|
+
}
|
|
1643
|
+
const pushedThisHead = ledger.some(
|
|
1644
|
+
(entry) =>
|
|
1645
|
+
entry.decision === "allowed" &&
|
|
1646
|
+
entry.verb === "conductor_push" &&
|
|
1647
|
+
entry.sha === headSha &&
|
|
1648
|
+
entry.detail.includes(branchRef),
|
|
1649
|
+
);
|
|
1650
|
+
// Bound to the claimed PR, not merely to "a create happened": both allowed
|
|
1651
|
+
// details name the URL they produced (`opened <url> …`, `adopted <url> …`),
|
|
1652
|
+
// so an unrelated create on this run cannot vouch for another PR.
|
|
1653
|
+
const createdHere = ledger.some(
|
|
1654
|
+
(entry) =>
|
|
1655
|
+
entry.decision === "allowed" &&
|
|
1656
|
+
entry.verb === "conductor_pr_create" &&
|
|
1657
|
+
entry.detail.includes(prUrl),
|
|
1658
|
+
);
|
|
1659
|
+
|
|
1660
|
+
if (!pushedThisHead && !createdHere) {
|
|
1661
|
+
// Nothing this run published carries the claimed head, so the only
|
|
1662
|
+
// remaining evidence is the branch itself: a continuation or review round
|
|
1663
|
+
// that pushed nothing, or a head a mediated base-branch update produced.
|
|
1664
|
+
let tip: string | undefined;
|
|
1665
|
+
try {
|
|
1666
|
+
tip = await tracker.branchHead(publication.repo, publication.branch);
|
|
1667
|
+
} catch {
|
|
1668
|
+
tip = undefined;
|
|
1669
|
+
}
|
|
1670
|
+
if (tip === undefined) {
|
|
1671
|
+
return {
|
|
1672
|
+
state: "pushed-pending",
|
|
1673
|
+
reason: `Live head of ${branchRef} unavailable; retrying`,
|
|
1674
|
+
};
|
|
1675
|
+
}
|
|
1676
|
+
if (tip !== headSha) {
|
|
1677
|
+
return {
|
|
1678
|
+
state: "failed",
|
|
1679
|
+
reason:
|
|
1680
|
+
"Pushed-green claim has no mediated publication evidence: this run's conductor_push / " +
|
|
1681
|
+
`conductor_pr_create ledger does not cover the claimed head, and ${branchRef} is at ${tip}, ` +
|
|
1682
|
+
"not the claimed head",
|
|
1683
|
+
};
|
|
1684
|
+
}
|
|
1685
|
+
}
|
|
1686
|
+
|
|
1687
|
+
const verification = await tracker.verifyPr(prUrl, headSha);
|
|
1782
1688
|
if (verification === undefined) {
|
|
1783
1689
|
return { state: "pushed-pending", reason: "GitHub PR verification unavailable; retrying" };
|
|
1784
1690
|
}
|
|
@@ -1932,33 +1838,6 @@ function orphanResumeVerdict(
|
|
|
1932
1838
|
return { kind: "resume", prior };
|
|
1933
1839
|
}
|
|
1934
1840
|
|
|
1935
|
-
/**
|
|
1936
|
-
* The worker identity for one launch, resolved now — or a throw naming the host
|
|
1937
|
-
* change that is missing (#798/#828).
|
|
1938
|
-
*
|
|
1939
|
-
* Every input is host state a running daemon does not control: the account can
|
|
1940
|
-
* be created after startup, and systemd can bring this service up before it has
|
|
1941
|
-
* mounted the harness binding a worker resolves through. So the verdict is taken
|
|
1942
|
-
* per launch. A daemon that cached one at startup would hold the whole fleet
|
|
1943
|
-
* closed on a boot race until somebody restarted it by hand — the outage #828
|
|
1944
|
-
* exists to end, not to relocate.
|
|
1945
|
-
*
|
|
1946
|
-
* The throw lands in the caller's dispatch catch, which settles the run failed
|
|
1947
|
-
* with this reason and escalates. It classifies as a start failure, so a host
|
|
1948
|
-
* fault charges the issue no implementation attempt.
|
|
1949
|
-
*/
|
|
1950
|
-
function launchIdentity(d: Pick<Deps, "workerIdentity">, launching: string): WorkerIdentity {
|
|
1951
|
-
const resolution: WorkerIdentityResolution = d.workerIdentity?.() ?? {
|
|
1952
|
-
ok: false,
|
|
1953
|
-
reason: `the ${WORKER_ACCOUNT} account is not installed on this host`,
|
|
1954
|
-
};
|
|
1955
|
-
if (resolution.ok) return resolution.identity;
|
|
1956
|
-
throw new Error(
|
|
1957
|
-
`worker identity unavailable: ${resolution.reason} — refusing to launch an unbound ${launching}; ` +
|
|
1958
|
-
"run `omp-conductor setup host` to install the dedicated worker identity",
|
|
1959
|
-
);
|
|
1960
|
-
}
|
|
1961
|
-
|
|
1962
1841
|
export async function handleIssue(
|
|
1963
1842
|
d: Deps,
|
|
1964
1843
|
r: Routed,
|
|
@@ -1981,6 +1860,32 @@ export async function handleIssue(
|
|
|
1981
1860
|
let turnLimit: TurnLimitController | undefined;
|
|
1982
1861
|
let workerControl: WorkerControlSlot | undefined;
|
|
1983
1862
|
let workerSessionInstalled = false;
|
|
1863
|
+
// The run's Herdr representation (#840), hoisted for the same reason the verb
|
|
1864
|
+
// listener is: whoever tears the run down has to hand the pane's lifecycle
|
|
1865
|
+
// authority back, and a representation that outlives its child is a worker the
|
|
1866
|
+
// workspace still shows as live.
|
|
1867
|
+
let workerPane: Extract<WorkerPaneOutcome, { kind: "tracked" }> | undefined;
|
|
1868
|
+
// Herdr orders lifecycle reports by `seq`, so this run owns one counter and
|
|
1869
|
+
// every report takes the next value (#842). A shared or restarted counter would
|
|
1870
|
+
// let a late report overwrite a newer state — the pane would then show `working`
|
|
1871
|
+
// for a worker that has already blocked.
|
|
1872
|
+
let paneSeq = 0;
|
|
1873
|
+
const nextPaneSeq = (): number => (paneSeq += 1);
|
|
1874
|
+
/**
|
|
1875
|
+
* Project one authoritative transition onto the pane (#842).
|
|
1876
|
+
*
|
|
1877
|
+
* The input is always a durable run transition or a typed session event —
|
|
1878
|
+
* never the pane's own output, which is display and nothing more. A failure is
|
|
1879
|
+
* logged, not raised: the workspace lagging is not worth failing a run over.
|
|
1880
|
+
*/
|
|
1881
|
+
const projectPaneState = (state: "working" | "idle" | "blocked" | "unknown", message?: string): void => {
|
|
1882
|
+
if (workerPane === undefined) return;
|
|
1883
|
+
const reported = reportWorkerPaneState(workerPane.paneId, workerPane.label, state, {
|
|
1884
|
+
seq: nextPaneSeq(),
|
|
1885
|
+
...(message === undefined ? {} : { message }),
|
|
1886
|
+
});
|
|
1887
|
+
if (!reported.ok) log(`#${issue} herdr pane state ${state} not reported: ${reported.reason}`);
|
|
1888
|
+
};
|
|
1984
1889
|
// The run's own repository. Hoisted for the same reason `worktreePath` is —
|
|
1985
1890
|
// the catch and finally paths have to publish the branch.
|
|
1986
1891
|
let runRepo: RunRepoRef | undefined;
|
|
@@ -2097,6 +2002,73 @@ export async function handleIssue(
|
|
|
2097
2002
|
return true;
|
|
2098
2003
|
};
|
|
2099
2004
|
|
|
2005
|
+
/**
|
|
2006
|
+
* The launch half of the credential-class fence (#852).
|
|
2007
|
+
*
|
|
2008
|
+
* Admission already held every candidate while a required provider billed to
|
|
2009
|
+
* the wrong credential, but a grant can be disabled in the window *between*
|
|
2010
|
+
* that check and this spawn — a refresh failing, an operator revoking, a token
|
|
2011
|
+
* expiring — and that window is exactly what the incident's acceptance
|
|
2012
|
+
* criterion names. So the same question is asked again here, as the last thing
|
|
2013
|
+
* before the session exists, and the run is closed rather than launched.
|
|
2014
|
+
*
|
|
2015
|
+
* Settled the same way the stop and drain fences settle: the claim closes, the
|
|
2016
|
+
* tree is salvaged, nothing is launched. It is deliberately NOT a failed
|
|
2017
|
+
* attempt against the issue's budget — the issue is not what is wrong, the
|
|
2018
|
+
* host's credentials are, and charging an attempt for it would exhaust an
|
|
2019
|
+
* issue while the operator re-authenticates.
|
|
2020
|
+
*/
|
|
2021
|
+
const settleCredentialClassBeforeSession = async (): Promise<boolean> => {
|
|
2022
|
+
const required = project.requireOauthProviders ?? [];
|
|
2023
|
+
if (required.length === 0 || run === undefined || workerSessionInstalled) return false;
|
|
2024
|
+
const fence = await oauthFenceVerdict(
|
|
2025
|
+
required,
|
|
2026
|
+
d.probeCredentialClass ??
|
|
2027
|
+
(async (provider) => ({
|
|
2028
|
+
ok: false,
|
|
2029
|
+
reason: `no credential probe is wired for ${provider}`,
|
|
2030
|
+
})),
|
|
2031
|
+
);
|
|
2032
|
+
if (fence.ok) return false;
|
|
2033
|
+
turnLimit?.close();
|
|
2034
|
+
turnLimit = undefined;
|
|
2035
|
+
const settlement =
|
|
2036
|
+
worktreePath === undefined
|
|
2037
|
+
? undefined
|
|
2038
|
+
: await settleWorktree({
|
|
2039
|
+
issue,
|
|
2040
|
+
attempt,
|
|
2041
|
+
ending: "not launched: a required subscription credential is unavailable",
|
|
2042
|
+
worktree: worktreePath,
|
|
2043
|
+
branch,
|
|
2044
|
+
publish,
|
|
2045
|
+
tree: "remove",
|
|
2046
|
+
mirrorPath,
|
|
2047
|
+
});
|
|
2048
|
+
recordOperatorStop(store, {
|
|
2049
|
+
project: project.name,
|
|
2050
|
+
issue,
|
|
2051
|
+
runId: run.id,
|
|
2052
|
+
inProgress,
|
|
2053
|
+
reason: `required subscription credential unavailable: ${fence.reason}`,
|
|
2054
|
+
patch: {
|
|
2055
|
+
endedAt: Date.now(),
|
|
2056
|
+
turns: run.turns,
|
|
2057
|
+
spendUsd: run.spendUsd,
|
|
2058
|
+
worktree: worktreePath ?? run.worktree,
|
|
2059
|
+
report: [
|
|
2060
|
+
"This run was closed instead of launched: a provider this project requires to bill",
|
|
2061
|
+
"to its subscription would have billed to something else.",
|
|
2062
|
+
fence.reason,
|
|
2063
|
+
...(settlement?.lines ?? []),
|
|
2064
|
+
].join("\n"),
|
|
2065
|
+
...settlement?.patch,
|
|
2066
|
+
},
|
|
2067
|
+
});
|
|
2068
|
+
log(`#${issue} not launched: ${fence.reason}`);
|
|
2069
|
+
return true;
|
|
2070
|
+
};
|
|
2071
|
+
|
|
2100
2072
|
try {
|
|
2101
2073
|
// The claim-side of the stop fence (#374): the run row is the boundary the
|
|
2102
2074
|
// shutdown drain waits on, so the claim itself refuses once the daemon is
|
|
@@ -2179,13 +2151,27 @@ export async function handleIssue(
|
|
|
2179
2151
|
// (#286) keeps its semantics in both cases — it is the same resolution,
|
|
2180
2152
|
// one different primary.
|
|
2181
2153
|
const declaredModel = admittedModel?.model;
|
|
2154
|
+
// #807: a chain that already bought its one model escalation dispatches on
|
|
2155
|
+
// that stronger selector, outranking both the declaration and the project
|
|
2156
|
+
// default — the settlement that wrote the marker did so precisely because
|
|
2157
|
+
// the previous tier spun to a cap with nothing to show. Read from the
|
|
2158
|
+
// store, so it is sticky to this issue's chain and a fresh issue (no
|
|
2159
|
+
// marker) resolves exactly as it always has. The provider chain still
|
|
2160
|
+
// layers on top: an escalated run whose provider then aborts fails over
|
|
2161
|
+
// normally, because escalation only ever changes the primary.
|
|
2162
|
+
const escalation = store.modelEscalation(project.name, issue);
|
|
2163
|
+
const primaryModel = escalation?.model ?? declaredModel ?? project.workerModel;
|
|
2182
2164
|
const choice = resolveDispatchModel({
|
|
2183
|
-
workerModel:
|
|
2165
|
+
workerModel: primaryModel,
|
|
2184
2166
|
modelFallbacks: project.modelFallbacks,
|
|
2185
2167
|
threshold: project.modelFallbackThreshold ?? DEFAULT_MODEL_FALLBACK_THRESHOLD,
|
|
2186
2168
|
streak: chainFacts.streak,
|
|
2187
2169
|
});
|
|
2188
|
-
const clause =
|
|
2170
|
+
const clause =
|
|
2171
|
+
fallbackClause(choice, chainFacts, primaryModel) ??
|
|
2172
|
+
(escalation === undefined
|
|
2173
|
+
? undefined
|
|
2174
|
+
: `on ${escalation.model} — this chain's one model escalation, after ${escalation.failureClass}`);
|
|
2189
2175
|
|
|
2190
2176
|
// #536: an orphan-clean requeue resumes the interrupted session instead of
|
|
2191
2177
|
// re-reading the repo from zero. Decided here, before this attempt's row
|
|
@@ -2238,6 +2224,12 @@ export async function handleIssue(
|
|
|
2238
2224
|
// enforced and the brief rendered, never a re-parse. Absent for a run
|
|
2239
2225
|
// with no declaration (fail open), exactly as it was admitted.
|
|
2240
2226
|
lane: admittedLane,
|
|
2227
|
+
// #851: the allowance admission reserved for this run, recorded on the
|
|
2228
|
+
// row before anything launches. Derived from the same caps the gate
|
|
2229
|
+
// read, so the reservation the pass enforced and the one the row holds
|
|
2230
|
+
// are one value; `undefined` (NULL) when the fleet has no spend cap at
|
|
2231
|
+
// all, which is the truth rather than "reserved nothing".
|
|
2232
|
+
spendReservedUsd: runSpendAllowanceUsd(caps) ?? undefined,
|
|
2241
2233
|
});
|
|
2242
2234
|
// #434: carry a terminal predecessor's open PR onto the continuation row.
|
|
2243
2235
|
// The claim itself stays synchronous — `handleIssue` claims at the top of
|
|
@@ -2272,17 +2264,24 @@ export async function handleIssue(
|
|
|
2272
2264
|
store.enqueueLabelOps(project.name, [{ issue, op: "add", label: inProgress }]);
|
|
2273
2265
|
claimed = true;
|
|
2274
2266
|
turnLimit = d.turnLimits.open(project.name, issue, runId, maxTurns);
|
|
2275
|
-
|
|
2267
|
+
// The pane follows the phase the registry actually reached (#842) — never a
|
|
2268
|
+
// guess made at the call site, and never the pane's own output read back.
|
|
2269
|
+
workerControl = d.workerControls.open(project.name, issue, runId, (phase) => {
|
|
2270
|
+
projectPaneState(
|
|
2271
|
+
phase === "running" ? "working" : "idle",
|
|
2272
|
+
phase === "running" ? "resumed" : `${phase} by the operator`,
|
|
2273
|
+
);
|
|
2274
|
+
});
|
|
2276
2275
|
if (await settleStopBeforeSession()) return;
|
|
2277
2276
|
if (await settleDrainBeforeSession()) return;
|
|
2277
|
+
// The credential fence, twice, for the same reason the stop fence is
|
|
2278
|
+
// consulted more than once (#852). Here it is the cheap one: a run whose
|
|
2279
|
+
// required subscription credential is already unavailable is closed before
|
|
2280
|
+
// anything clones a mirror or cuts a worktree, because provisioning for a
|
|
2281
|
+
// launch that cannot happen is pure waste. The second call, immediately
|
|
2282
|
+
// before the spawn, is the one that closes the admission-to-session window.
|
|
2283
|
+
if (await settleCredentialClassBeforeSession()) return;
|
|
2278
2284
|
|
|
2279
|
-
// The worker identity is this run's launch gate (#798): a worker session
|
|
2280
|
-
// that cannot be launched under the dedicated unprivileged account is
|
|
2281
|
-
// indistinguishable from an operator shell, so dispatch refuses before any
|
|
2282
|
-
// tree or session is created. Resolved here rather than read off a startup
|
|
2283
|
-
// verdict, so a host that gained its account — or its harness binding
|
|
2284
|
-
// (#828) — after the daemon came up dispatches on the next tick.
|
|
2285
|
-
const identity = launchIdentity(d, "worker session");
|
|
2286
2285
|
|
|
2287
2286
|
// A run's tree is <workspaceRoot>/<issue> and addRunRepo refuses to reuse
|
|
2288
2287
|
// an existing path, so a retry — or a tree kept from a failed attempt — has
|
|
@@ -2366,10 +2365,6 @@ export async function handleIssue(
|
|
|
2366
2365
|
},
|
|
2367
2366
|
{
|
|
2368
2367
|
...(d.verbPeerReader === undefined ? {} : { peerReader: d.verbPeerReader }),
|
|
2369
|
-
// A worker-run channel is secured to the worker identity (#798): the
|
|
2370
|
-
// child that must connect to it runs as that uid, and the peer verdict
|
|
2371
|
-
// expects that uid on the wire rather than the daemon's.
|
|
2372
|
-
channelOwner: { uid: identity.uid, gid: identity.gid },
|
|
2373
2368
|
},
|
|
2374
2369
|
);
|
|
2375
2370
|
if (await settleStopBeforeSession()) return;
|
|
@@ -2384,7 +2379,10 @@ export async function handleIssue(
|
|
|
2384
2379
|
// Recorded before the launch, so even a run killed mid-flight leaves the
|
|
2385
2380
|
// model it chose on its row. Only a chain-configured project writes the
|
|
2386
2381
|
// column: absent `modelFallbacks` must preserve today's rows byte for byte.
|
|
2387
|
-
|
|
2382
|
+
// An escalated chain (#807) writes it too — the selector it dispatched on
|
|
2383
|
+
// is the whole provenance of that recovery, and its harness-resolved model
|
|
2384
|
+
// lands beside it in `resolvedModel` when the run settles.
|
|
2385
|
+
if ((chainConfigured || escalation !== undefined) && choice.model !== undefined) {
|
|
2388
2386
|
store.updateRun(runId, { model: choice.model });
|
|
2389
2387
|
}
|
|
2390
2388
|
|
|
@@ -2433,24 +2431,27 @@ export async function handleIssue(
|
|
|
2433
2431
|
}
|
|
2434
2432
|
if (await settleStopBeforeSession()) return;
|
|
2435
2433
|
if (await settleDrainBeforeSession()) return;
|
|
2434
|
+
// Last, so it is the newest fact anything has about the credentials — a
|
|
2435
|
+
// grant disabled since admission is caught here rather than paid for.
|
|
2436
|
+
if (await settleCredentialClassBeforeSession()) return;
|
|
2436
2437
|
|
|
2437
2438
|
const repoSlug = githubRepo(r.repo.cloneUrl);
|
|
2438
2439
|
|
|
2439
|
-
// The run's working paths are granted to the worker identity by ownership
|
|
2440
|
-
// (#798) — the worktree the session edits and the session directory it
|
|
2441
|
-
// writes its transcript and settings into. The chown lands here, after
|
|
2442
|
-
// provisioning and the verb socket, so the session starts on a tree it
|
|
2443
|
-
// owns; the worker identity never inherits anything from root's trees.
|
|
2444
|
-
d.grantWorkerPaths?.(identity, worktreePath, sessionDir);
|
|
2445
2440
|
|
|
2446
2441
|
let result: WorkerResult;
|
|
2447
2442
|
try {
|
|
2443
|
+
const runAllowanceUsd = runSpendAllowanceUsd(caps);
|
|
2448
2444
|
result = await runWorker({
|
|
2449
2445
|
brief,
|
|
2450
2446
|
cwd: worktreePath,
|
|
2451
2447
|
caps,
|
|
2452
2448
|
...(repoSlug === undefined ? {} : { repoSlug }),
|
|
2453
2449
|
maxTurns: () => turnLimit?.maxTurns() ?? maxTurns,
|
|
2450
|
+
// #851: the allowance this run reserved at admission is also its live
|
|
2451
|
+
// ceiling. Without it the reservation would only bound how many runs
|
|
2452
|
+
// start, not what one of them spends, and the day's total would be
|
|
2453
|
+
// unbounded again.
|
|
2454
|
+
...(runAllowanceUsd === null ? {} : { maxSpendUsd: runAllowanceUsd }),
|
|
2454
2455
|
onPauseControl: (control) => {
|
|
2455
2456
|
workerSessionInstalled = true;
|
|
2456
2457
|
workerControl?.install(control);
|
|
@@ -2469,10 +2470,72 @@ export async function handleIssue(
|
|
|
2469
2470
|
onSpawn: (pid) => {
|
|
2470
2471
|
verbListener?.bindPid(pid);
|
|
2471
2472
|
},
|
|
2473
|
+
// The workspace representation of this exact child (#840). Best effort by
|
|
2474
|
+
// construction: a worker that cannot be *shown* is still a worker, so a
|
|
2475
|
+
// failure is logged with its reason and the run proceeds. What a failure
|
|
2476
|
+
// should mean for the launch — fail closed, or a named degraded state —
|
|
2477
|
+
// is #841's policy and is deliberately not decided here. Nothing is
|
|
2478
|
+
// claimed silently: either the pane id is logged, or the reason is.
|
|
2479
|
+
pane: {
|
|
2480
|
+
open: (pid) => {
|
|
2481
|
+
// Adopt before creating (#842): a run that already carries a pane
|
|
2482
|
+
// identity gets that pane reported against, never a second one. The
|
|
2483
|
+
// durable row is what makes this survive the process that opened it,
|
|
2484
|
+
// and duplicate-prevention is exactly what it buys — a re-entered
|
|
2485
|
+
// launch for one run must not leave two panes claiming it.
|
|
2486
|
+
const recorded = store.getRun(runId);
|
|
2487
|
+
if (recorded?.paneId !== undefined && recorded.paneLabel !== undefined) {
|
|
2488
|
+
workerPane = { kind: "tracked", paneId: recorded.paneId, label: recorded.paneLabel, pid };
|
|
2489
|
+
store.updateRun(runId, { workerPid: pid });
|
|
2490
|
+
const state = reportWorkerPaneState(recorded.paneId, recorded.paneLabel, "working", {
|
|
2491
|
+
seq: nextPaneSeq(),
|
|
2492
|
+
});
|
|
2493
|
+
log(
|
|
2494
|
+
state.ok
|
|
2495
|
+
? `#${issue} herdr pane ${recorded.paneId} re-reported for pid ${pid}`
|
|
2496
|
+
: `#${issue} herdr pane ${recorded.paneId} could not be re-reported: ${state.reason}`,
|
|
2497
|
+
);
|
|
2498
|
+
return;
|
|
2499
|
+
}
|
|
2500
|
+
const outcome = openWorkerPane({
|
|
2501
|
+
project: project.name,
|
|
2502
|
+
issue,
|
|
2503
|
+
attempt,
|
|
2504
|
+
runId,
|
|
2505
|
+
pid,
|
|
2506
|
+
...(recorded?.sessionFile === undefined ? {} : { sessionFile: recorded.sessionFile }),
|
|
2507
|
+
});
|
|
2508
|
+
if (outcome.kind === "tracked") {
|
|
2509
|
+
workerPane = outcome;
|
|
2510
|
+
// Durable before it is announced: a pane the store does not know
|
|
2511
|
+
// about is a pane a restart cannot reconcile.
|
|
2512
|
+
store.updateRun(runId, {
|
|
2513
|
+
workerPid: pid,
|
|
2514
|
+
paneId: outcome.paneId,
|
|
2515
|
+
paneLabel: outcome.label,
|
|
2516
|
+
// Cleared, because there is now a pane: a leftover reason beside a
|
|
2517
|
+
// tracked run would keep `status` saying degraded forever (#841).
|
|
2518
|
+
paneUnavailable: null,
|
|
2519
|
+
});
|
|
2520
|
+
log(`#${issue} herdr pane ${outcome.paneId} (${outcome.label}) tracks pid ${pid}`);
|
|
2521
|
+
} else {
|
|
2522
|
+
// Named, never silent (#841): the run proceeds untracked, and the
|
|
2523
|
+
// reason is durable so `status` can say the fleet is running blind
|
|
2524
|
+
// rather than looking identical to a fleet with no workers.
|
|
2525
|
+
store.updateRun(runId, { workerPid: pid, paneUnavailable: outcome.reason });
|
|
2526
|
+
log(`#${issue} no herdr pane: ${outcome.reason}`);
|
|
2527
|
+
}
|
|
2528
|
+
},
|
|
2529
|
+
release: () => {
|
|
2530
|
+
if (workerPane === undefined) return;
|
|
2531
|
+
const released = releaseWorkerPane(workerPane.paneId, workerPane.label, { seq: nextPaneSeq() });
|
|
2532
|
+
if (!released.ok) log(`#${issue} herdr pane release failed: ${released.reason}`);
|
|
2533
|
+
workerPane = undefined;
|
|
2534
|
+
},
|
|
2535
|
+
},
|
|
2472
2536
|
onChildLog: (line) => {
|
|
2473
2537
|
log(`#${issue} ${line}`);
|
|
2474
2538
|
},
|
|
2475
|
-
workerIdentity: identity,
|
|
2476
2539
|
...(choice.model === undefined ? {} : { model: choice.model }),
|
|
2477
2540
|
// The fleet-owned omp settings overlay (#537): the staged YAML the
|
|
2478
2541
|
// session loads through `Settings.init({ configFiles: [<path>] })` —
|
|
@@ -2484,6 +2547,11 @@ export async function handleIssue(
|
|
|
2484
2547
|
onReleaseBlocked: workerReleaseBlockRecorder(project.name, issue, runId),
|
|
2485
2548
|
onTurn: (n) => store.updateRun(runId, { turns: n }),
|
|
2486
2549
|
onSpend: (usd) => store.updateRun(runId, { spendUsd: usd }),
|
|
2550
|
+
// #518: recorded live, not at settlement — the moment this explains is
|
|
2551
|
+
// forty minutes before the wall-clock cap fires, so a number only a
|
|
2552
|
+
// finished run carries would answer the question too late.
|
|
2553
|
+
onTokens: (tokens) =>
|
|
2554
|
+
store.updateRun(runId, { outputTokens: tokens.output, reasoningTokens: tokens.reasoning }),
|
|
2487
2555
|
onKilled: () => {
|
|
2488
2556
|
turnLimit?.close();
|
|
2489
2557
|
turnLimit = undefined;
|
|
@@ -2530,7 +2598,14 @@ export async function handleIssue(
|
|
|
2530
2598
|
|
|
2531
2599
|
const verified: { state: RunState; reason?: string } =
|
|
2532
2600
|
result.state === "pushed-green"
|
|
2533
|
-
? await verifyPushedGreenClaim(tracker, result
|
|
2601
|
+
? await verifyPushedGreenClaim(tracker, result, {
|
|
2602
|
+
project: project.name,
|
|
2603
|
+
issue,
|
|
2604
|
+
runId,
|
|
2605
|
+
branch,
|
|
2606
|
+
repo: repoSlug ?? r.repo.name,
|
|
2607
|
+
store,
|
|
2608
|
+
})
|
|
2534
2609
|
: { state: result.state };
|
|
2535
2610
|
const state = verified.state;
|
|
2536
2611
|
|
|
@@ -2932,6 +3007,114 @@ export async function handleIssue(
|
|
|
2932
3007
|
* recorded it — merged after all, settled, re-claimed — fails the claim and is
|
|
2933
3008
|
* settled `skipped` rather than woken on stale identity.
|
|
2934
3009
|
*/
|
|
3010
|
+
/**
|
|
3011
|
+
* Record what this host has installed, once per dispatch pass (#919).
|
|
3012
|
+
*
|
|
3013
|
+
* Here rather than at render time, and this is the whole design of the slice:
|
|
3014
|
+
* the read costs three subprocesses, and calling it from the tick took the
|
|
3015
|
+
* tick's own suite from 8.4s to 83.4s while spawning three children every
|
|
3016
|
+
* fifteen minutes to answer a question that changes only when someone installs
|
|
3017
|
+
* something. The dispatch pass is already async and already spawns `gh`, so one
|
|
3018
|
+
* read per pass is free by comparison, and every cheap surface then reads a row.
|
|
3019
|
+
*
|
|
3020
|
+
* Advisory throughout: a probe that throws (no `herdr` on PATH, a `$PATH`
|
|
3021
|
+
* without `omp`) leaves the previous observation in place and logs. A stale
|
|
3022
|
+
* observation is still the truth about the last time anyone could look, and
|
|
3023
|
+
* losing a dispatch pass over a version string would be absurd.
|
|
3024
|
+
*/
|
|
3025
|
+
/**
|
|
3026
|
+
* Make the workspace agree with the live run set, once per pass (#841).
|
|
3027
|
+
*
|
|
3028
|
+
* Runs on every dispatch pass rather than only at startup, because the thing it
|
|
3029
|
+
* repairs — a Herdr restart — is not a conductor event and announces itself
|
|
3030
|
+
* nowhere. It is idempotent by construction: a reconciled fleet reports `intact`
|
|
3031
|
+
* for every live worker and finds no stale panes, so repeated passes converge
|
|
3032
|
+
* instead of accumulating panes or churning them.
|
|
3033
|
+
*
|
|
3034
|
+
* Nothing here can stop a worker. The only mutations are creating a pane, and
|
|
3035
|
+
* releasing one whose run is not live; the authoritative child is never
|
|
3036
|
+
* signalled, and no pane is ever closed.
|
|
3037
|
+
*/
|
|
3038
|
+
function reconcilePanes(d: Deps, project: string, log: (message: string) => void): void {
|
|
3039
|
+
const live = d.store.liveRuns(project);
|
|
3040
|
+
const result = reconcileWorkerPanes(
|
|
3041
|
+
live.map((run) => ({
|
|
3042
|
+
runId: run.id,
|
|
3043
|
+
issue: run.issue,
|
|
3044
|
+
attempt: run.attempt,
|
|
3045
|
+
project,
|
|
3046
|
+
...(run.workerPid === undefined ? {} : { pid: run.workerPid }),
|
|
3047
|
+
...(run.paneId === undefined ? {} : { paneId: run.paneId }),
|
|
3048
|
+
...(run.paneLabel === undefined ? {} : { paneLabel: run.paneLabel }),
|
|
3049
|
+
...(run.sessionFile === undefined ? {} : { sessionFile: run.sessionFile }),
|
|
3050
|
+
})),
|
|
3051
|
+
);
|
|
3052
|
+
if (!result.ok) {
|
|
3053
|
+
// An unreadable workspace is not evidence that anything is stale, so nothing
|
|
3054
|
+
// is released and no run is relabelled — but a fleet that cannot see its own
|
|
3055
|
+
// panes says so, on every pass, rather than going quiet.
|
|
3056
|
+
log(`worker panes not reconciled: ${result.reason}`);
|
|
3057
|
+
return;
|
|
3058
|
+
}
|
|
3059
|
+
for (const outcome of result.outcomes) {
|
|
3060
|
+
switch (outcome.kind) {
|
|
3061
|
+
case "intact":
|
|
3062
|
+
break;
|
|
3063
|
+
case "reassociated":
|
|
3064
|
+
d.store.updateRun(outcome.runId, {
|
|
3065
|
+
paneId: outcome.paneId,
|
|
3066
|
+
paneLabel: outcome.label,
|
|
3067
|
+
paneUnavailable: null,
|
|
3068
|
+
});
|
|
3069
|
+
log(`#${runIssue(live, outcome.runId)} herdr pane re-associated as ${outcome.paneId} after a restart`);
|
|
3070
|
+
break;
|
|
3071
|
+
case "untracked":
|
|
3072
|
+
d.store.updateRun(outcome.runId, { paneId: null, paneLabel: null, paneUnavailable: outcome.reason });
|
|
3073
|
+
log(`#${runIssue(live, outcome.runId)} has no herdr pane: ${outcome.reason}`);
|
|
3074
|
+
break;
|
|
3075
|
+
case "stale-released":
|
|
3076
|
+
log(`released stale herdr pane ${outcome.paneId} (run ${outcome.runId || "unidentified"} is not live)`);
|
|
3077
|
+
break;
|
|
3078
|
+
case "stale-release-failed":
|
|
3079
|
+
log(`stale herdr pane ${outcome.paneId} could not be released: ${outcome.reason}`);
|
|
3080
|
+
break;
|
|
3081
|
+
}
|
|
3082
|
+
}
|
|
3083
|
+
}
|
|
3084
|
+
|
|
3085
|
+
/** The issue a run id belongs to, for a log line a human reads. */
|
|
3086
|
+
function runIssue(live: readonly RunRecord[], runId: string): string {
|
|
3087
|
+
return String(live.find((run) => run.id === runId)?.issue ?? "?");
|
|
3088
|
+
}
|
|
3089
|
+
|
|
3090
|
+
async function recordInstallSurfaces(d: Deps): Promise<void> {
|
|
3091
|
+
if (d.probeInstallSurfaces === undefined) return;
|
|
3092
|
+
try {
|
|
3093
|
+
const surfaces = await d.probeInstallSurfaces();
|
|
3094
|
+
// The telegram peer's versions ride along on the same pass (#961). Read
|
|
3095
|
+
// here rather than in `status` because `status` runs every tick and this
|
|
3096
|
+
// touches the npm registry: one periodic read, rendered from the store as
|
|
3097
|
+
// often as anyone looks. A read that throws leaves the fields absent, which
|
|
3098
|
+
// renders as unverified — never as agreement.
|
|
3099
|
+
const telegram =
|
|
3100
|
+
d.probeTelegramFreshness === undefined ? undefined : await d.probeTelegramFreshness();
|
|
3101
|
+
const tgInstalled = telegram?.surfaces.installed;
|
|
3102
|
+
const tgDaemon = telegram?.surfaces.daemon;
|
|
3103
|
+
const tgPublished = telegram?.surfaces.published;
|
|
3104
|
+
d.store.recordInstallSurfaces({
|
|
3105
|
+
at: Date.now(),
|
|
3106
|
+
cliVersion: surfaces.cliVersion,
|
|
3107
|
+
...(surfaces.ompVersion === undefined ? {} : { ompVersion: surfaces.ompVersion }),
|
|
3108
|
+
...(surfaces.herdrSource === undefined ? {} : { herdrSource: surfaces.herdrSource }),
|
|
3109
|
+
...(tgInstalled?.kind === "version" ? { telegramInstalled: tgInstalled.version } : {}),
|
|
3110
|
+
...(tgDaemon?.kind === "version" ? { telegramDaemon: tgDaemon.version } : {}),
|
|
3111
|
+
...(tgPublished?.kind === "version" ? { telegramPublished: tgPublished.version } : {}),
|
|
3112
|
+
});
|
|
3113
|
+
} catch (err) {
|
|
3114
|
+
log(`install-surface read skipped: ${errText(err)}`);
|
|
3115
|
+
}
|
|
3116
|
+
}
|
|
3117
|
+
|
|
2935
3118
|
export async function dispatchReviewRevisions(d: Deps, pool?: WorkerPool): Promise<void> {
|
|
2936
3119
|
const pending = d.store.pendingReviewRevisions(d.project.name);
|
|
2937
3120
|
if (pending.length === 0) return;
|
|
@@ -3049,6 +3232,350 @@ export async function dispatchReviewRevisions(d: Deps, pool?: WorkerPool): Promi
|
|
|
3049
3232
|
await Promise.allSettled(launches);
|
|
3050
3233
|
}
|
|
3051
3234
|
|
|
3235
|
+
/**
|
|
3236
|
+
* Launch the review-ceiling adjudications this project owes (#932).
|
|
3237
|
+
*
|
|
3238
|
+
* Deliberately its own pass, next to {@link dispatchReviewRevisions} but never
|
|
3239
|
+
* inside it: a revision resumes an implementation worker to change code, an
|
|
3240
|
+
* adjudication opens a fresh read-only session to decide about code. Folding
|
|
3241
|
+
* them together is how "escalate the automation" quietly becomes "resume the
|
|
3242
|
+
* same worker with a different label", which is #873's named silent fake.
|
|
3243
|
+
*
|
|
3244
|
+
* Only `pending` rows are launched. A `running` row belongs to a live
|
|
3245
|
+
* adjudicator — or to one a restart killed, which the startup sweep settles —
|
|
3246
|
+
* so a restart re-reads it and never launches a second adjudicator for one head.
|
|
3247
|
+
*/
|
|
3248
|
+
/**
|
|
3249
|
+
* The turn ceiling for one adjudication (#932).
|
|
3250
|
+
*
|
|
3251
|
+
* Small on purpose, and not the worker cap: a verdict is a read and an answer,
|
|
3252
|
+
* not an implementation. A session still going after this is not deciding — and
|
|
3253
|
+
* the ceiling exists so a stuck adjudicator costs a bounded amount rather than a
|
|
3254
|
+
* worker's whole budget, on work that has already spent its review rounds.
|
|
3255
|
+
*/
|
|
3256
|
+
export const ADJUDICATION_MAX_TURNS = 40;
|
|
3257
|
+
|
|
3258
|
+
export async function dispatchReviewAdjudications(d: Deps, pool?: WorkerPool): Promise<void> {
|
|
3259
|
+
const open = d.store.openReviewAdjudications(d.project.name).filter((a) => a.state === "pending");
|
|
3260
|
+
if (open.length === 0) return;
|
|
3261
|
+
const launches: Promise<void>[] = [];
|
|
3262
|
+
for (const adjudication of open) {
|
|
3263
|
+
if (d.drain?.draining === true) {
|
|
3264
|
+
log(`adjudications held: the daemon is draining (${open.length - launches.length} not launched)`);
|
|
3265
|
+
break;
|
|
3266
|
+
}
|
|
3267
|
+
// The head is re-read before anything launches. A verdict describes the diff
|
|
3268
|
+
// it read, so adjudicating a head the PR has since moved past would produce
|
|
3269
|
+
// a decision about code nobody is merging — settled `stale-head` with both
|
|
3270
|
+
// heads named, never launched.
|
|
3271
|
+
let live: string | undefined;
|
|
3272
|
+
try {
|
|
3273
|
+
live = await d.tracker.prHead(adjudication.prUrl);
|
|
3274
|
+
} catch (err) {
|
|
3275
|
+
log(`#${adjudication.issue} adjudication held: ${adjudication.prUrl} head unreadable (${errText(err)})`);
|
|
3276
|
+
continue;
|
|
3277
|
+
}
|
|
3278
|
+
if (live === undefined) {
|
|
3279
|
+
log(`#${adjudication.issue} adjudication held: ${adjudication.prUrl} head could not be read — retrying next tick`);
|
|
3280
|
+
continue;
|
|
3281
|
+
}
|
|
3282
|
+
if (live.toLowerCase() !== adjudication.headSha.toLowerCase()) {
|
|
3283
|
+
d.store.settleReviewAdjudication(
|
|
3284
|
+
adjudication.id,
|
|
3285
|
+
"stale-head",
|
|
3286
|
+
`recorded ${adjudication.headSha}, live ${live} — the pull request moved before this adjudication launched`,
|
|
3287
|
+
Date.now(),
|
|
3288
|
+
);
|
|
3289
|
+
log(`#${adjudication.issue} adjudication stale-head: recorded ${adjudication.headSha}, live ${live}`);
|
|
3290
|
+
continue;
|
|
3291
|
+
}
|
|
3292
|
+
// The claim IS the single-flight guard: the transition is the row's own
|
|
3293
|
+
// WHERE clause, so two dispatch passes racing one adjudication cannot both
|
|
3294
|
+
// launch — the loser changes nothing and skips.
|
|
3295
|
+
const review = resolveReview(d.project);
|
|
3296
|
+
const provenance = { role: adjudication.role, model: adjudication.role, resolvedAt: Date.now() };
|
|
3297
|
+
if (!d.store.markReviewAdjudicationRunning(adjudication.id, provenance, Date.now())) {
|
|
3298
|
+
log(`#${adjudication.issue} adjudication skipped: another pass already claimed it`);
|
|
3299
|
+
continue;
|
|
3300
|
+
}
|
|
3301
|
+
log(`#${adjudication.issue} adjudication → launching role ${adjudication.role} at ${adjudication.headSha}`);
|
|
3302
|
+
launches.push(handleReviewAdjudication(d, adjudication, review.maxRounds));
|
|
3303
|
+
}
|
|
3304
|
+
if (pool !== undefined) {
|
|
3305
|
+
for (const launch of launches) pool.launch(launch);
|
|
3306
|
+
return;
|
|
3307
|
+
}
|
|
3308
|
+
await Promise.allSettled(launches);
|
|
3309
|
+
}
|
|
3310
|
+
|
|
3311
|
+
/**
|
|
3312
|
+
* Assemble one adjudication's evidence and run it to a durable verdict (#932).
|
|
3313
|
+
*
|
|
3314
|
+
* Every fact comes from a durable row or a live tracker read — never from the
|
|
3315
|
+
* worker's own report, which is the claim under adjudication. The issue body is
|
|
3316
|
+
* carried verbatim because the acceptance criteria ARE the standard being
|
|
3317
|
+
* judged, and every prior round travels WITH its settled outcome: a finding
|
|
3318
|
+
* without its outcome reads as an outstanding complaint even when the worker
|
|
3319
|
+
* fixed it, which is how an adjudicator rejects work that was already corrected.
|
|
3320
|
+
*/
|
|
3321
|
+
export async function handleReviewAdjudication(
|
|
3322
|
+
d: Deps,
|
|
3323
|
+
adjudication: ReviewAdjudicationRecord,
|
|
3324
|
+
maxRounds: number,
|
|
3325
|
+
): Promise<void> {
|
|
3326
|
+
const issue = adjudication.issue;
|
|
3327
|
+
const [title, body, diff, verification] = await Promise.all([
|
|
3328
|
+
d.tracker.getIssue(issue).then(
|
|
3329
|
+
(row) => row?.title,
|
|
3330
|
+
() => undefined,
|
|
3331
|
+
),
|
|
3332
|
+
d.tracker.issueBody(issue).then(
|
|
3333
|
+
(text) => text,
|
|
3334
|
+
() => undefined,
|
|
3335
|
+
),
|
|
3336
|
+
d.tracker.prDiff(adjudication.prUrl).then(
|
|
3337
|
+
(parsed) => parsed,
|
|
3338
|
+
() => undefined,
|
|
3339
|
+
),
|
|
3340
|
+
d.tracker.verifyPr(adjudication.prUrl, adjudication.headSha).then(
|
|
3341
|
+
(result) => result,
|
|
3342
|
+
() => undefined,
|
|
3343
|
+
),
|
|
3344
|
+
]);
|
|
3345
|
+
// Prior rounds, oldest first, keyed by PR so a continuation that inherited the
|
|
3346
|
+
// PR cannot hide its predecessor's findings. The ceiling call's own findings
|
|
3347
|
+
// are appended as the last entry — they are the reason this adjudication
|
|
3348
|
+
// exists, and the durable row is where they were kept precisely so this brief
|
|
3349
|
+
// could carry them.
|
|
3350
|
+
const rounds: AdjudicationRound[] = d.store
|
|
3351
|
+
.reviewRevisionsForPr(d.project.name, adjudication.prUrl)
|
|
3352
|
+
.map((revision) => ({
|
|
3353
|
+
round: revision.round,
|
|
3354
|
+
findings: revision.findings,
|
|
3355
|
+
...(revision.outcome === undefined ? {} : { outcome: revision.outcome }),
|
|
3356
|
+
}));
|
|
3357
|
+
if (adjudication.findings !== undefined) {
|
|
3358
|
+
rounds.push({
|
|
3359
|
+
round: rounds.length + 1,
|
|
3360
|
+
findings: adjudication.findings,
|
|
3361
|
+
outcome: "escalated to this adjudication — no worker round was available",
|
|
3362
|
+
});
|
|
3363
|
+
}
|
|
3364
|
+
const brief = renderAdjudicationBrief({
|
|
3365
|
+
issue,
|
|
3366
|
+
...(title === undefined ? {} : { issueTitle: title }),
|
|
3367
|
+
issueBody: body ?? "(the issue body could not be read — judge the diff against the findings below)",
|
|
3368
|
+
prUrl: adjudication.prUrl,
|
|
3369
|
+
headSha: adjudication.headSha,
|
|
3370
|
+
checks:
|
|
3371
|
+
verification === undefined
|
|
3372
|
+
? "(the check state could not be read)"
|
|
3373
|
+
: `${verification.status} — ${verification.reason}`,
|
|
3374
|
+
diff:
|
|
3375
|
+
diff === undefined
|
|
3376
|
+
? "(the diff could not be read — say so in your reasons rather than guessing)"
|
|
3377
|
+
: renderDiffForAdjudication(diff),
|
|
3378
|
+
rounds,
|
|
3379
|
+
maxRounds,
|
|
3380
|
+
});
|
|
3381
|
+
|
|
3382
|
+
const sessionDir = join(stateDir(), "sessions", `adjudication-${adjudication.id}`);
|
|
3383
|
+
mkdirSync(sessionDir, { recursive: true });
|
|
3384
|
+
const ompSettingsFile = materializeOmpSettings(d.project, sessionDir);
|
|
3385
|
+
const result = await (d.runAdjudicatorImpl ?? runAdjudicator)({
|
|
3386
|
+
brief,
|
|
3387
|
+
// No worktree: an adjudicator has no branch, and everything it needs is in
|
|
3388
|
+
// the brief. The session directory doubles as its confined cwd.
|
|
3389
|
+
cwd: sessionDir,
|
|
3390
|
+
sessionDir,
|
|
3391
|
+
model: adjudication.role,
|
|
3392
|
+
...(ompSettingsFile === undefined ? {} : { ompSettingsFile }),
|
|
3393
|
+
socketPath: join(sessionDir, "ipc.sock"),
|
|
3394
|
+
maxTurns: ADJUDICATION_MAX_TURNS,
|
|
3395
|
+
});
|
|
3396
|
+
|
|
3397
|
+
// What actually ran it, named in the evidence when it differs from the role
|
|
3398
|
+
// that was asked for: the provenance recorded at claim time is conductor's own
|
|
3399
|
+
// resolution (the role it passed to OMP), and the model that wrote the
|
|
3400
|
+
// messages is a later observation only this result carries.
|
|
3401
|
+
const ran =
|
|
3402
|
+
result.model === undefined
|
|
3403
|
+
? ""
|
|
3404
|
+
: `adjudicated by ${result.model}${result.provider === undefined ? "" : ` (${result.provider})`}: `;
|
|
3405
|
+
if (result.verdict === undefined) {
|
|
3406
|
+
// A session that never took a turn did not run: the role could not be
|
|
3407
|
+
// resolved or the harness could not start, and the remedy is configuration
|
|
3408
|
+
// rather than a retry. One that ran and produced nothing usable is a
|
|
3409
|
+
// failure. Neither is ever a clear.
|
|
3410
|
+
const state = result.turns === 0 ? "unavailable-model" : "failed";
|
|
3411
|
+
d.store.settleReviewAdjudication(
|
|
3412
|
+
adjudication.id,
|
|
3413
|
+
state,
|
|
3414
|
+
`${ran}${result.report === "" ? "no verdict and no output" : result.report}`,
|
|
3415
|
+
Date.now(),
|
|
3416
|
+
);
|
|
3417
|
+
log(`#${issue} adjudication ${state} at ${adjudication.headSha}`);
|
|
3418
|
+
return;
|
|
3419
|
+
}
|
|
3420
|
+
// A verdict that names a different head judged something else. Fail closed:
|
|
3421
|
+
// the one-shot is spent on the head it was opened for, and nothing merges on
|
|
3422
|
+
// the strength of a decision about another diff.
|
|
3423
|
+
if (
|
|
3424
|
+
result.verdict.headSha !== undefined &&
|
|
3425
|
+
result.verdict.headSha.toLowerCase() !== adjudication.headSha.toLowerCase()
|
|
3426
|
+
) {
|
|
3427
|
+
d.store.settleReviewAdjudication(
|
|
3428
|
+
adjudication.id,
|
|
3429
|
+
"stale-head",
|
|
3430
|
+
`${ran}the verdict names head ${result.verdict.headSha}, not the adjudicated ${adjudication.headSha}: ${result.verdict.reasons}`,
|
|
3431
|
+
Date.now(),
|
|
3432
|
+
);
|
|
3433
|
+
log(`#${issue} adjudication stale-head: verdict named ${result.verdict.headSha}`);
|
|
3434
|
+
return;
|
|
3435
|
+
}
|
|
3436
|
+
d.store.settleReviewAdjudication(
|
|
3437
|
+
adjudication.id,
|
|
3438
|
+
result.verdict.verdict,
|
|
3439
|
+
`${ran}${result.verdict.reasons}`,
|
|
3440
|
+
Date.now(),
|
|
3441
|
+
);
|
|
3442
|
+
log(`#${issue} adjudication ${result.verdict.verdict} at ${adjudication.headSha}`);
|
|
3443
|
+
}
|
|
3444
|
+
|
|
3445
|
+
/**
|
|
3446
|
+
* Apply the terminal dispositions an adjudication's verdict implies (#876).
|
|
3447
|
+
*
|
|
3448
|
+
* The verdict is a decision; this is what is DONE about it, and the two are
|
|
3449
|
+
* separate rows for exactly that reason. Recording `cleared` while leaving the
|
|
3450
|
+
* merge gate refusing, or `rejected` while leaving the PR open to block the
|
|
3451
|
+
* issue forever, is the issue's named silent fake — so what this pass changes is
|
|
3452
|
+
* tracker and gate state, and the disposition text is only its receipt.
|
|
3453
|
+
*
|
|
3454
|
+
* Idempotent by that receipt: a row whose `disposition` is already recorded is
|
|
3455
|
+
* skipped, so a restart, a retried tick or a partial failure cannot close one PR
|
|
3456
|
+
* twice. Nothing here is destructive before its explanation is durable.
|
|
3457
|
+
*
|
|
3458
|
+
* Neither branch opens a review round, launches another adjudicator, or asks an
|
|
3459
|
+
* operator for anything. Neither touches an attempt or continuation budget
|
|
3460
|
+
* either: a rejected PR closes, and the ORDINARY settle sweep charges that
|
|
3461
|
+
* closure exactly as it charges any PR closed without merging — which is what
|
|
3462
|
+
* keeps this composable with #815's separate corrective-worker path instead of
|
|
3463
|
+
* resetting its counters.
|
|
3464
|
+
*/
|
|
3465
|
+
export async function applyAdjudicationDispositions(d: Deps): Promise<void> {
|
|
3466
|
+
const { project, store } = d;
|
|
3467
|
+
// Terminal rows only, and only those nothing has disposed of yet. Read from
|
|
3468
|
+
// the PRs still in flight: a settled adjudication whose PR is long gone needs
|
|
3469
|
+
// no disposition, and scanning history would re-litigate closed work.
|
|
3470
|
+
const pending = store
|
|
3471
|
+
.activeRuns(project.name)
|
|
3472
|
+
.flatMap((run) => (run.prUrl === undefined ? [] : store.reviewAdjudicationsForPr(project.name, run.prUrl)))
|
|
3473
|
+
.filter((row) => row.settledAt !== undefined && row.disposition === undefined);
|
|
3474
|
+
for (const adjudication of pending) {
|
|
3475
|
+
if (adjudication.state === "cleared") {
|
|
3476
|
+
// The clear re-enters the ORDINARY merge path, and it does so through the
|
|
3477
|
+
// mechanism that path already reads: the #888 gate refuses a head carrying
|
|
3478
|
+
// unresolved review evidence, and #913's clearance is the recorded
|
|
3479
|
+
// disposition that settles it. Reusing it means no second gate, no
|
|
3480
|
+
// adjudication-shaped exception inside `conductor_pr_merge`, and one
|
|
3481
|
+
// audit trail for "this head was cleared, by whom, and why".
|
|
3482
|
+
//
|
|
3483
|
+
// The head is re-read first: a clearance is head-scoped and clears
|
|
3484
|
+
// backwards only, so recording one against a head the PR has moved past
|
|
3485
|
+
// would bless a diff nobody adjudicated.
|
|
3486
|
+
let live: string | undefined;
|
|
3487
|
+
try {
|
|
3488
|
+
live = await d.tracker.prHead(adjudication.prUrl);
|
|
3489
|
+
} catch (err) {
|
|
3490
|
+
log(`#${adjudication.issue} adjudication clearance held: head unreadable (${errText(err)})`);
|
|
3491
|
+
continue;
|
|
3492
|
+
}
|
|
3493
|
+
if (live === undefined) {
|
|
3494
|
+
log(`#${adjudication.issue} adjudication clearance held: head could not be read — retrying next tick`);
|
|
3495
|
+
continue;
|
|
3496
|
+
}
|
|
3497
|
+
if (live.toLowerCase() !== adjudication.headSha.toLowerCase()) {
|
|
3498
|
+
// Visible and non-destructive: the clearance is not recorded, the row
|
|
3499
|
+
// says why, and the PR keeps whatever gate state it has.
|
|
3500
|
+
store.recordReviewAdjudicationDisposition(
|
|
3501
|
+
adjudication.id,
|
|
3502
|
+
`no clearance recorded: the head moved from ${adjudication.headSha} to ${live} after the verdict`,
|
|
3503
|
+
);
|
|
3504
|
+
log(`#${adjudication.issue} adjudication clearance skipped: head moved to ${live}`);
|
|
3505
|
+
continue;
|
|
3506
|
+
}
|
|
3507
|
+
store.recordReviewClearance({
|
|
3508
|
+
project: project.name,
|
|
3509
|
+
prUrl: adjudication.prUrl,
|
|
3510
|
+
headSha: adjudication.headSha,
|
|
3511
|
+
by: `adjudication:${adjudication.role}`,
|
|
3512
|
+
reason: `cleared by the review-ceiling adjudication: ${adjudication.evidence ?? "no evidence recorded"}`,
|
|
3513
|
+
});
|
|
3514
|
+
store.recordReviewAdjudicationDisposition(
|
|
3515
|
+
adjudication.id,
|
|
3516
|
+
`recorded a review clearance at ${adjudication.headSha}; the PR is on the ordinary merge path`,
|
|
3517
|
+
);
|
|
3518
|
+
log(`#${adjudication.issue} adjudication cleared → merge path open at ${adjudication.headSha}`);
|
|
3519
|
+
continue;
|
|
3520
|
+
}
|
|
3521
|
+
// Every non-clear terminal state disposes the same way, and that is
|
|
3522
|
+
// deliberate: `rejected`, `failed`, `unavailable-model` and `stale-head` all
|
|
3523
|
+
// mean "this head is not merging and no further round is coming", and the
|
|
3524
|
+
// blocking artefact is identical in each case — an open PR that occupies the
|
|
3525
|
+
// issue. Only the recorded reason differs.
|
|
3526
|
+
const findings = store
|
|
3527
|
+
.reviewRevisionsForPr(project.name, adjudication.prUrl)
|
|
3528
|
+
.map((revision) => `### Review round ${revision.round} (${revision.outcome ?? "never settled"})
|
|
3529
|
+
|
|
3530
|
+
${revision.findings}`);
|
|
3531
|
+
const comment = [
|
|
3532
|
+
`## Adjudication: ${adjudication.state}`,
|
|
3533
|
+
"",
|
|
3534
|
+
`The review-round ceiling was reached on ${adjudication.headSha}, and the \`${adjudication.role}\` adjudicator ` +
|
|
3535
|
+
`did not clear it. Closing this pull request so the issue is not blocked by work that cannot merge — ` +
|
|
3536
|
+
"nothing here asks anyone to repair it by hand.",
|
|
3537
|
+
"",
|
|
3538
|
+
"### Adjudicator's reasons",
|
|
3539
|
+
"",
|
|
3540
|
+
adjudication.evidence ?? "(none recorded)",
|
|
3541
|
+
...(findings.length === 0 ? [] : ["", "## Preserved review findings", "", ...findings]),
|
|
3542
|
+
"",
|
|
3543
|
+
"The branch itself is untouched and remains in the repository.",
|
|
3544
|
+
].join("\n");
|
|
3545
|
+
const closed = await d.verbActions?.closePr(adjudication.prUrl, comment);
|
|
3546
|
+
if (closed === undefined) {
|
|
3547
|
+
log(`#${adjudication.issue} adjudication disposition held: no action surface is wired`);
|
|
3548
|
+
continue;
|
|
3549
|
+
}
|
|
3550
|
+
if (!closed.ok) {
|
|
3551
|
+
// Retryable, and no receipt written: the next tick tries again rather than
|
|
3552
|
+
// leaving a rejected PR open with a row that claims it was handled.
|
|
3553
|
+
log(`#${adjudication.issue} adjudication disposition held: could not close ${adjudication.prUrl} — ${closed.stderr}`);
|
|
3554
|
+
continue;
|
|
3555
|
+
}
|
|
3556
|
+
store.recordReviewAdjudicationDisposition(
|
|
3557
|
+
adjudication.id,
|
|
3558
|
+
`closed ${adjudication.prUrl} with the adjudicator's reasons and every preserved finding; ` +
|
|
3559
|
+
"the settle sweep releases the issue",
|
|
3560
|
+
);
|
|
3561
|
+
log(`#${adjudication.issue} adjudication ${adjudication.state} → closed ${adjudication.prUrl}`);
|
|
3562
|
+
}
|
|
3563
|
+
}
|
|
3564
|
+
|
|
3565
|
+
/** The diff as one text, from the structured read the settlement audit uses.
|
|
3566
|
+
* `renderAdjudicationBrief` bounds the total; this only flattens it, and says
|
|
3567
|
+
* when the adapter itself had to cut the read short — an adjudicator must know
|
|
3568
|
+
* its silence covers only what it saw. */
|
|
3569
|
+
function renderDiffForAdjudication(diff: PrDiff): string {
|
|
3570
|
+
const files = diff.files.map((file) =>
|
|
3571
|
+
[
|
|
3572
|
+
`--- ${file.status} ${file.previousPath === undefined ? "" : `${file.previousPath} → `}${file.path}`,
|
|
3573
|
+
file.hunks ?? "(no textual hunks: binary, mode-only or pure rename)",
|
|
3574
|
+
].join("\n"),
|
|
3575
|
+
);
|
|
3576
|
+
return [...(diff.truncated ? ["[the tracker truncated this diff read]"] : []), ...files].join("\n\n");
|
|
3577
|
+
}
|
|
3578
|
+
|
|
3052
3579
|
/**
|
|
3053
3580
|
* Resume one review-revision worker: the run row is already claimed
|
|
3054
3581
|
* (`pushed-green` → `running` by the dispatch pass), so this resumes the SAME
|
|
@@ -3169,20 +3696,38 @@ export async function handleReviewRevision(d: Deps, revision: ReviewRevisionReco
|
|
|
3169
3696
|
// stopped anything, the PR is still green and still open, and only the
|
|
3170
3697
|
// wake failed; `settleStopBeforeSession` immediately above covers the
|
|
3171
3698
|
// case where an operator genuinely did. So mirror that branch: restore
|
|
3172
|
-
// `pushed-green` naming the shutdown,
|
|
3173
|
-
// the
|
|
3174
|
-
// — the ordinary settle sweep releases it in the same breath it
|
|
3699
|
+
// `pushed-green` naming the shutdown, and leave the in-progress label
|
|
3700
|
+
// alone — the ordinary settle sweep releases it in the same breath it
|
|
3175
3701
|
// terminalises the row when the PR resolves. Stopping the row here would
|
|
3176
3702
|
// strand it: `settlePushedGreen` sweeps only pushed-* rows and
|
|
3177
3703
|
// classification excludes `stopped`, so the issue would carry neither
|
|
3178
3704
|
// label and nothing would ever revisit the still-green PR.
|
|
3705
|
+
//
|
|
3706
|
+
// The round itself is RE-QUEUED rather than settled `skipped` (#903). A
|
|
3707
|
+
// daemon shutdown is the plainest infrastructure kill there is — no
|
|
3708
|
+
// session started, so no round was worked — and a settled row consumed
|
|
3709
|
+
// one of the PR's rounds through `latestReviewRound`, which is how an
|
|
3710
|
+
// upgrade or a restart used to cost a review round it never spent. The
|
|
3711
|
+
// retry is counted and bounded exactly like any other infra kill, so a
|
|
3712
|
+
// host that shuts down mid-wake forever still escalates.
|
|
3179
3713
|
store.updateRun(runId, {
|
|
3180
3714
|
state: "pushed-green",
|
|
3181
3715
|
endedAt: Date.now(),
|
|
3182
3716
|
lastError: "daemon shutdown began after the review revision claim; the round was not launched",
|
|
3183
3717
|
});
|
|
3184
|
-
store.
|
|
3185
|
-
|
|
3718
|
+
const retry = store.retryReviewRevisionAfterInfra(revision.id, REVIEW_ROUND_INFRA_MAX_RETRIES);
|
|
3719
|
+
if (retry.kind === "exhausted") {
|
|
3720
|
+
store.settleReviewRevision(revision.id, "skipped", Date.now());
|
|
3721
|
+
log(
|
|
3722
|
+
`#${issue} review round ${revision.round} not launched: daemon shutdown began after the claim, and ` +
|
|
3723
|
+
`its infra retries are exhausted (${retry.retries}/${REVIEW_ROUND_INFRA_MAX_RETRIES}) — settled skipped`,
|
|
3724
|
+
);
|
|
3725
|
+
return true;
|
|
3726
|
+
}
|
|
3727
|
+
log(
|
|
3728
|
+
`#${issue} review round ${revision.round} not launched: daemon shutdown began after the claim — ` +
|
|
3729
|
+
`re-queued, retry ${retry.retries}/${REVIEW_ROUND_INFRA_MAX_RETRIES}`,
|
|
3730
|
+
);
|
|
3186
3731
|
return true;
|
|
3187
3732
|
};
|
|
3188
3733
|
|
|
@@ -3215,14 +3760,7 @@ export async function handleReviewRevision(d: Deps, revision: ReviewRevisionReco
|
|
|
3215
3760
|
if (await settleStopBeforeSession()) return;
|
|
3216
3761
|
if (await settleDrainBeforeSession()) return;
|
|
3217
3762
|
|
|
3218
|
-
// The same launch gate as a fresh claim (#798): a revision worker is a
|
|
3219
|
-
// worker, and an unbound one is an operator shell — refuse before any tree
|
|
3220
|
-
// or session is created. Thrown inside the try so the ordinary dispatch
|
|
3221
|
-
// catch settles it: the run row is returned to terminal `failed` with the
|
|
3222
|
-
// reason, the revision row is settled with it, and the still-green PR stays
|
|
3223
|
-
// open for a healthy retry.
|
|
3224
3763
|
try {
|
|
3225
|
-
const identity = launchIdentity(d, "review-revision worker");
|
|
3226
3764
|
// Reattach the run's own branch at the same per-issue path the run used:
|
|
3227
3765
|
// a `pushed-green` settle removed the worktree, so provisioning is the
|
|
3228
3766
|
// same continuation reattach as a normal re-claim. A capped/failed run's
|
|
@@ -3337,7 +3875,6 @@ export async function handleReviewRevision(d: Deps, revision: ReviewRevisionReco
|
|
|
3337
3875
|
},
|
|
3338
3876
|
{
|
|
3339
3877
|
...(d.verbPeerReader === undefined ? {} : { peerReader: d.verbPeerReader }),
|
|
3340
|
-
channelOwner: { uid: identity.uid, gid: identity.gid },
|
|
3341
3878
|
},
|
|
3342
3879
|
);
|
|
3343
3880
|
if (await settleStopBeforeSession()) return;
|
|
@@ -3350,22 +3887,26 @@ export async function handleReviewRevision(d: Deps, revision: ReviewRevisionReco
|
|
|
3350
3887
|
// totals the run already recorded.
|
|
3351
3888
|
const baseTurns = run.turns;
|
|
3352
3889
|
const baseSpend = run.spendUsd;
|
|
3890
|
+
const baseOutputTokens = run.outputTokens ?? 0;
|
|
3891
|
+
const baseReasoningTokens = run.reasoningTokens ?? 0;
|
|
3353
3892
|
|
|
3354
3893
|
log(`#${issue} review round ${revision.round} → resuming ${branch} (${priorSessionFile})`);
|
|
3355
3894
|
|
|
3356
|
-
// The revision worker gets its own tree ownership, exactly as a fresh
|
|
3357
|
-
// claim does (#798): the resumed worktree and session directory belong to
|
|
3358
|
-
// the worker identity before its session starts.
|
|
3359
|
-
d.grantWorkerPaths?.(identity, worktreePath, sessionDir);
|
|
3360
3895
|
|
|
3361
3896
|
let result: WorkerResult;
|
|
3362
3897
|
try {
|
|
3898
|
+
const runAllowanceUsd = runSpendAllowanceUsd(caps);
|
|
3363
3899
|
result = await runWorker({
|
|
3364
3900
|
brief: renderReviewRevisionPrompt(revision.findings, revision.round),
|
|
3365
3901
|
cwd: worktreePath,
|
|
3366
3902
|
caps,
|
|
3367
3903
|
...(repoSlug === undefined ? {} : { repoSlug }),
|
|
3368
3904
|
maxTurns: () => turnLimit?.maxTurns() ?? run.maxTurns,
|
|
3905
|
+
// A review round meters its own session from zero and gets the same
|
|
3906
|
+
// allowance as any run (#851) — the round is real work with a real
|
|
3907
|
+
// cost, and an unbounded round would reopen the hole for any PR that
|
|
3908
|
+
// was ever returned for revision.
|
|
3909
|
+
...(runAllowanceUsd === null ? {} : { maxSpendUsd: runAllowanceUsd }),
|
|
3369
3910
|
onPauseControl: (control) => {
|
|
3370
3911
|
workerSessionInstalled = true;
|
|
3371
3912
|
workerControl?.install(control);
|
|
@@ -3382,7 +3923,6 @@ export async function handleReviewRevision(d: Deps, revision: ReviewRevisionReco
|
|
|
3382
3923
|
onChildLog: (line) => {
|
|
3383
3924
|
log(`#${issue} ${line}`);
|
|
3384
3925
|
},
|
|
3385
|
-
workerIdentity: identity,
|
|
3386
3926
|
// The continuation stays on the model the green run used (#286).
|
|
3387
3927
|
...(run.model === undefined ? {} : { model: run.model }),
|
|
3388
3928
|
...(ompSettingsFile === undefined ? {} : { ompSettingsFile }),
|
|
@@ -3390,6 +3930,14 @@ export async function handleReviewRevision(d: Deps, revision: ReviewRevisionReco
|
|
|
3390
3930
|
onReleaseBlocked: workerReleaseBlockRecorder(project.name, issue, runId),
|
|
3391
3931
|
onTurn: (n) => store.updateRun(runId, { turns: baseTurns + n }),
|
|
3392
3932
|
onSpend: (usd) => store.updateRun(runId, { spendUsd: baseSpend + usd }),
|
|
3933
|
+
// A revision resumes the SAME run, so its tokens add to the attempt's
|
|
3934
|
+
// totals exactly as its turns and spend do (#518) — resetting them would
|
|
3935
|
+
// make a long deliberation look like a fresh one.
|
|
3936
|
+
onTokens: (tokens) =>
|
|
3937
|
+
store.updateRun(runId, {
|
|
3938
|
+
outputTokens: baseOutputTokens + tokens.output,
|
|
3939
|
+
reasoningTokens: baseReasoningTokens + tokens.reasoning,
|
|
3940
|
+
}),
|
|
3393
3941
|
onKilled: () => {
|
|
3394
3942
|
turnLimit?.close();
|
|
3395
3943
|
turnLimit = undefined;
|
|
@@ -3418,7 +3966,14 @@ export async function handleReviewRevision(d: Deps, revision: ReviewRevisionReco
|
|
|
3418
3966
|
|
|
3419
3967
|
const verified: { state: RunState; reason?: string } =
|
|
3420
3968
|
result.state === "pushed-green"
|
|
3421
|
-
? await verifyPushedGreenClaim(tracker, result
|
|
3969
|
+
? await verifyPushedGreenClaim(tracker, result, {
|
|
3970
|
+
project: project.name,
|
|
3971
|
+
issue,
|
|
3972
|
+
runId,
|
|
3973
|
+
branch,
|
|
3974
|
+
repo: repoSlug ?? repo.name,
|
|
3975
|
+
store,
|
|
3976
|
+
})
|
|
3422
3977
|
: { state: result.state };
|
|
3423
3978
|
const state = verified.state;
|
|
3424
3979
|
|
|
@@ -3481,6 +4036,63 @@ export async function handleReviewRevision(d: Deps, revision: ReviewRevisionReco
|
|
|
3481
4036
|
report: finalReport,
|
|
3482
4037
|
...settlement?.patch,
|
|
3483
4038
|
};
|
|
4039
|
+
// #903: an infrastructure kill spends no review round.
|
|
4040
|
+
//
|
|
4041
|
+
// A dispatch that dies before the resumed session takes a turn — a host
|
|
4042
|
+
// permission fault, a spawn failure, a harness that could not start — did
|
|
4043
|
+
// none of the work a round exists to do, yet the pre-#903 path settled the
|
|
4044
|
+
// row `failed` with no retry. During the 2026-08-21/22 outage that turned
|
|
4045
|
+
// one broken mount into a review-ceiling deadlock: three such deaths
|
|
4046
|
+
// consumed the PR's three rounds, `conductor_pr_review` then refused any
|
|
4047
|
+
// further round, and the findings could never be addressed by a worker at
|
|
4048
|
+
// all. Nothing in the loop could recover it.
|
|
4049
|
+
//
|
|
4050
|
+
// So the round is returned to the pending set instead: the same row, the
|
|
4051
|
+
// same round number, the same findings, and the run restored to the exact
|
|
4052
|
+
// `pushed-green` state the verb recorded — which is what the restart
|
|
4053
|
+
// reconcile already does for a round a dying daemon interrupted. The next
|
|
4054
|
+
// dispatch pass resumes the same session. Bounded by
|
|
4055
|
+
// REVIEW_ROUND_INFRA_MAX_RETRIES and counted durably on the row, so a
|
|
4056
|
+
// permanently broken host escalates through the ordinary failed path
|
|
4057
|
+
// instead of retrying every tick forever.
|
|
4058
|
+
const infraKill =
|
|
4059
|
+
state === "failed" || state === "killed"
|
|
4060
|
+
? reviewRoundNeverWorked({
|
|
4061
|
+
turns: result.turns,
|
|
4062
|
+
...(result.prUrl === undefined ? {} : { prUrl: result.prUrl }),
|
|
4063
|
+
...(result.headSha === undefined ? {} : { headSha: result.headSha }),
|
|
4064
|
+
...(settlement?.patch?.salvageSha === undefined
|
|
4065
|
+
? {}
|
|
4066
|
+
: { salvageSha: settlement.patch.salvageSha }),
|
|
4067
|
+
})
|
|
4068
|
+
: undefined;
|
|
4069
|
+
if (infraKill !== undefined) {
|
|
4070
|
+
const retry = store.retryReviewRevisionAfterInfra(revision.id, REVIEW_ROUND_INFRA_MAX_RETRIES);
|
|
4071
|
+
if (retry.kind === "requeued") {
|
|
4072
|
+
const detail =
|
|
4073
|
+
`review round ${revision.round} was killed before its session took a turn ` +
|
|
4074
|
+
`(${infraKill}); the round is re-queued, retry ${retry.retries}/${REVIEW_ROUND_INFRA_MAX_RETRIES}`;
|
|
4075
|
+
store.updateRun(runId, {
|
|
4076
|
+
...terminalPatch,
|
|
4077
|
+
state: "pushed-green",
|
|
4078
|
+
// The transcript the round must still resume is the one the run
|
|
4079
|
+
// recorded, not whatever this dead dispatch opened: a session that
|
|
4080
|
+
// never took a turn has no lineage of its own to inherit.
|
|
4081
|
+
sessionFile: priorSessionFile,
|
|
4082
|
+
report: [`review round ${revision.round}: dispatch killed before turn 1`, "", detail].join("\n"),
|
|
4083
|
+
lastError: detail,
|
|
4084
|
+
});
|
|
4085
|
+
// No label swap and no escalation: nothing failed that a human can
|
|
4086
|
+
// act on yet, the issue keeps its in-progress label, and the next
|
|
4087
|
+
// dispatch pass picks the round up on its own.
|
|
4088
|
+
log(`#${issue} ${detail}`);
|
|
4089
|
+
return;
|
|
4090
|
+
}
|
|
4091
|
+
log(
|
|
4092
|
+
`#${issue} review round ${revision.round} exhausted its infra retries ` +
|
|
4093
|
+
`(${retry.retries}/${REVIEW_ROUND_INFRA_MAX_RETRIES}) — settling it the ordinary way`,
|
|
4094
|
+
);
|
|
4095
|
+
}
|
|
3484
4096
|
if (state === "stopped") {
|
|
3485
4097
|
recordOperatorStop(store, {
|
|
3486
4098
|
project: project.name,
|
|
@@ -3592,6 +4204,35 @@ export async function handleReviewRevision(d: Deps, revision: ReviewRevisionReco
|
|
|
3592
4204
|
publish,
|
|
3593
4205
|
tree: "keep",
|
|
3594
4206
|
});
|
|
4207
|
+
// A crash BEFORE the worker session was installed spent no round (#903):
|
|
4208
|
+
// nothing resumed, nothing was reviewed, and the round is re-queued for
|
|
4209
|
+
// the next pass instead of consuming one of the PR's three. Once the
|
|
4210
|
+
// session is installed the round belongs to the worker, and a crash after
|
|
4211
|
+
// that settles the ordinary way below — including the exhausted case, so
|
|
4212
|
+
// a dispatch that keeps throwing still reaches a human.
|
|
4213
|
+
if (!workerSessionInstalled) {
|
|
4214
|
+
const retry = store.retryReviewRevisionAfterInfra(revision.id, REVIEW_ROUND_INFRA_MAX_RETRIES);
|
|
4215
|
+
if (retry.kind === "requeued") {
|
|
4216
|
+
store.updateRun(runId, {
|
|
4217
|
+
state: "pushed-green",
|
|
4218
|
+
endedAt: Date.now(),
|
|
4219
|
+
sessionFile: priorSessionFile,
|
|
4220
|
+
lastError:
|
|
4221
|
+
`review round ${revision.round} could not be dispatched (${detail}); the round is re-queued, ` +
|
|
4222
|
+
`retry ${retry.retries}/${REVIEW_ROUND_INFRA_MAX_RETRIES}`,
|
|
4223
|
+
...settlement?.patch,
|
|
4224
|
+
});
|
|
4225
|
+
log(
|
|
4226
|
+
`#${issue} review round ${revision.round} re-queued after a pre-session dispatch error — ` +
|
|
4227
|
+
`retry ${retry.retries}/${REVIEW_ROUND_INFRA_MAX_RETRIES}`,
|
|
4228
|
+
);
|
|
4229
|
+
return;
|
|
4230
|
+
}
|
|
4231
|
+
log(
|
|
4232
|
+
`#${issue} review round ${revision.round} exhausted its infra retries ` +
|
|
4233
|
+
`(${retry.retries}/${REVIEW_ROUND_INFRA_MAX_RETRIES}) — failing the round`,
|
|
4234
|
+
);
|
|
4235
|
+
}
|
|
3595
4236
|
store.updateRun(runId, {
|
|
3596
4237
|
state: "failed",
|
|
3597
4238
|
endedAt: Date.now(),
|
|
@@ -4341,16 +4982,26 @@ export interface WorkerPool {
|
|
|
4341
4982
|
drain(): Promise<void>;
|
|
4342
4983
|
}
|
|
4343
4984
|
|
|
4344
|
-
/**
|
|
4345
|
-
|
|
4985
|
+
/**
|
|
4986
|
+
* Keeps background workers alive without making the five-minute tick await them.
|
|
4987
|
+
*
|
|
4988
|
+
* `onSettled` fires as each worker leaves the pool, whichever way it ended
|
|
4989
|
+
* (#878): that is the instant a slot frees, and without it queued work waited
|
|
4990
|
+
* for the next scheduled pass — up to five minutes of idle capacity that reads,
|
|
4991
|
+
* from outside, exactly like a stalled queue. It only *prompts* a pass; every
|
|
4992
|
+
* hold, drain, lane, budget and routing gate is re-evaluated by that pass as
|
|
4993
|
+
* usual.
|
|
4994
|
+
*/
|
|
4995
|
+
export function createWorkerPool(onSettled?: () => void): WorkerPool {
|
|
4346
4996
|
const active = new Set<Promise<void>>();
|
|
4347
4997
|
return {
|
|
4348
4998
|
launch(work) {
|
|
4349
4999
|
active.add(work);
|
|
4350
|
-
void
|
|
4351
|
-
|
|
4352
|
-
()
|
|
4353
|
-
|
|
5000
|
+
const settled = (): void => {
|
|
5001
|
+
active.delete(work);
|
|
5002
|
+
onSettled?.();
|
|
5003
|
+
};
|
|
5004
|
+
void work.then(settled, settled);
|
|
4354
5005
|
},
|
|
4355
5006
|
activeCount: () => active.size,
|
|
4356
5007
|
async drain() {
|
|
@@ -4671,6 +5322,14 @@ export async function tick(d: Deps, workers?: WorkerPool): Promise<void> {
|
|
|
4671
5322
|
} catch (err) {
|
|
4672
5323
|
log(`label reconcile failed: ${errText(err)}`);
|
|
4673
5324
|
}
|
|
5325
|
+
try {
|
|
5326
|
+
// Same phase, same reason as the label reconcile above: a durable row the
|
|
5327
|
+
// tracker has moved past (#964). A separate call because that one iterates
|
|
5328
|
+
// labelled issues, and a promotable issue carries no label.
|
|
5329
|
+
await reconcileGroomingClosures(d);
|
|
5330
|
+
} catch (err) {
|
|
5331
|
+
log(`grooming closure reconcile failed: ${errText(err)}`);
|
|
5332
|
+
}
|
|
4674
5333
|
|
|
4675
5334
|
// Drain the label projection outbox (#201). The maintenance phases above may
|
|
4676
5335
|
// have enqueued ops (settlement releases, recovery requeues, reconciles);
|
|
@@ -4864,6 +5523,28 @@ export async function tick(d: Deps, workers?: WorkerPool): Promise<void> {
|
|
|
4864
5523
|
log(`review revision dispatch failed: ${errText(err)}`);
|
|
4865
5524
|
}
|
|
4866
5525
|
|
|
5526
|
+
// Adjudications are dispatch too (#932), and their own pass: an adjudication
|
|
5527
|
+
// occupies no worker slot and no issue — it opens a fresh read-only session
|
|
5528
|
+
// about a PR whose run has already settled — so it is neither bounded by the
|
|
5529
|
+
// worker cap nor allowed to consume a review round. Placed after revisions so
|
|
5530
|
+
// a PR with a live revision is never adjudicated in the same tick it is being
|
|
5531
|
+
// corrected in; the head re-read inside the pass is the durable guard.
|
|
5532
|
+
try {
|
|
5533
|
+
await dispatchReviewAdjudications(d, workers);
|
|
5534
|
+
} catch (err) {
|
|
5535
|
+
log(`adjudication dispatch failed: ${errText(err)}`);
|
|
5536
|
+
}
|
|
5537
|
+
|
|
5538
|
+
// And what the verdicts imply (#876). Its own pass, after dispatch, because a
|
|
5539
|
+
// disposition acts on a SETTLED adjudication: running it before dispatch would
|
|
5540
|
+
// simply be a tick late, and running it inside dispatch would tie a tracker
|
|
5541
|
+
// mutation to the launch that produced the verdict.
|
|
5542
|
+
try {
|
|
5543
|
+
await applyAdjudicationDispositions(d);
|
|
5544
|
+
} catch (err) {
|
|
5545
|
+
log(`adjudication dispositions failed: ${errText(err)}`);
|
|
5546
|
+
}
|
|
5547
|
+
|
|
4867
5548
|
// route() filters the queue through isEligible() itself, so anything already
|
|
4868
5549
|
// carrying a state label is gone before it gets here.
|
|
4869
5550
|
const ready = await d.tracker.listReady();
|
|
@@ -4882,7 +5563,65 @@ export async function tick(d: Deps, workers?: WorkerPool): Promise<void> {
|
|
|
4882
5563
|
const pending = store.pendingLabelOpsFor(project.name, issue.number);
|
|
4883
5564
|
return pending.length === 0 ? issue : { ...issue, labels: effectiveLabels(issue.labels, pending) };
|
|
4884
5565
|
});
|
|
4885
|
-
|
|
5566
|
+
// Post-unblock stale-list revalidation (#891). `listReady` is a
|
|
5567
|
+
// label-FILTERED search, and GitHub's search index is eventually consistent:
|
|
5568
|
+
// measured 2026-08-22T09:18Z, an `unblock` cleared `agent:failed` and
|
|
5569
|
+
// restored the queue label, `gh issue view` returned the clean label set
|
|
5570
|
+
// immediately, and the very next pass still saw the stale labels, held #889
|
|
5571
|
+
// as `stale-lifecycle` and left the fleet at 0/5 with four spare slots. The
|
|
5572
|
+
// next scheduled pass usually recovers — this is throughput, not correctness
|
|
5573
|
+
// — but an operator-driven unblock that cannot refill the fleet has not done
|
|
5574
|
+
// what it says.
|
|
5575
|
+
//
|
|
5576
|
+
// So a lifecycle label read from the LIST is a suspicion, not a verdict: the
|
|
5577
|
+
// exact per-issue read decides. Bounded to candidates that would otherwise be
|
|
5578
|
+
// dropped for a lifecycle label — never the whole queue — and further to those
|
|
5579
|
+
// whose newest run is not active (a genuinely in-flight issue needs no
|
|
5580
|
+
// revalidation; its own row is the authority) and that are not operator-parked
|
|
5581
|
+
// (a park is a decision, not a stale label).
|
|
5582
|
+
//
|
|
5583
|
+
// An unreadable exact read keeps the stale labels, so the issue stays out of
|
|
5584
|
+
// this pass, and is held as `issue-state-lookup-error` rather than
|
|
5585
|
+
// `stale-lifecycle`: "the tracker could not say" is not evidence of a
|
|
5586
|
+
// residual label, and it must not summon Duty 1 to reconcile a label nobody
|
|
5587
|
+
// has read.
|
|
5588
|
+
const stateLabelSet = new Set(Object.values(project.stateLabels));
|
|
5589
|
+
const suspect = effective.filter(
|
|
5590
|
+
(issue) =>
|
|
5591
|
+
!isEligible(issue, project) &&
|
|
5592
|
+
issue.labels.some((l) => stateLabelSet.has(l)) &&
|
|
5593
|
+
!issue.labels.includes(project.stateLabels.backlog) &&
|
|
5594
|
+
!ACTIVE_STATES.includes(store.latestRun(project.name, issue.number)?.state ?? "merged"),
|
|
5595
|
+
);
|
|
5596
|
+
const revalidated = new Map<number, readonly string[]>();
|
|
5597
|
+
const unreadable = new Set<number>();
|
|
5598
|
+
for (const issue of suspect) {
|
|
5599
|
+
let snapshot: IssueSnapshot | undefined;
|
|
5600
|
+
try {
|
|
5601
|
+
snapshot = await d.tracker.issueSnapshot(issue.number);
|
|
5602
|
+
} catch {
|
|
5603
|
+
snapshot = undefined;
|
|
5604
|
+
}
|
|
5605
|
+
if (snapshot === undefined) {
|
|
5606
|
+
unreadable.add(issue.number);
|
|
5607
|
+
log(`#${issue.number} lifecycle labels could not be revalidated — holding this pass`);
|
|
5608
|
+
continue;
|
|
5609
|
+
}
|
|
5610
|
+
// A closed issue is nobody's candidate; leave the list labels alone and let
|
|
5611
|
+
// the ordinary gates speak.
|
|
5612
|
+
if (snapshot.state === "closed") continue;
|
|
5613
|
+
if (snapshot.labels.join("\u0000") === issue.labels.join("\u0000")) continue;
|
|
5614
|
+
revalidated.set(issue.number, snapshot.labels);
|
|
5615
|
+
log(`#${issue.number} lifecycle labels revalidated: the queue list was stale`);
|
|
5616
|
+
}
|
|
5617
|
+
const verified =
|
|
5618
|
+
revalidated.size === 0
|
|
5619
|
+
? effective
|
|
5620
|
+
: effective.map((issue) => {
|
|
5621
|
+
const live = revalidated.get(issue.number);
|
|
5622
|
+
return live === undefined ? issue : { ...issue, labels: [...live] };
|
|
5623
|
+
});
|
|
5624
|
+
const { routed, unroutable } = route(verified, project);
|
|
4886
5625
|
// route() drops a candidate for two reasons, only one of which is a claim
|
|
4887
5626
|
// question. A lifecycle state label (agent:in-progress/blocked/failed)
|
|
4888
5627
|
// marks a run-owned issue: it is genuinely in flight while its newest run
|
|
@@ -4894,9 +5633,10 @@ export async function tick(d: Deps, workers?: WorkerPool): Promise<void> {
|
|
|
4894
5633
|
// close), which belongs in none of the three populations. So `claimed` is
|
|
4895
5634
|
// defined from actual ownership over genuinely lifecycle-labelled candidates
|
|
4896
5635
|
// rather than as the residual of routing (#228, #611).
|
|
4897
|
-
|
|
4898
|
-
|
|
4899
|
-
|
|
5636
|
+
// Every population below reads the VERIFIED list, so a candidate the exact
|
|
5637
|
+
// read cleared is not simultaneously admitted and counted as held (#891).
|
|
5638
|
+
const dropped = verified.filter(
|
|
5639
|
+
(issue) => !isEligible(issue, project) && issue.labels.some((l) => stateLabelSet.has(l)),
|
|
4900
5640
|
);
|
|
4901
5641
|
// The operator's park label is not a stale lifecycle label: the queue query
|
|
4902
5642
|
// still returns a parked-and-queued issue, route() drops it as ineligible,
|
|
@@ -4905,7 +5645,7 @@ export async function tick(d: Deps, workers?: WorkerPool): Promise<void> {
|
|
|
4905
5645
|
// to reconcile a deliberate decision. Parked is its own population, derived
|
|
4906
5646
|
// from the same isEligible read the gate uses, so the number status renders
|
|
4907
5647
|
// cannot disagree with what admission would hold (#507).
|
|
4908
|
-
const parkedCandidates =
|
|
5648
|
+
const parkedCandidates = verified.filter(
|
|
4909
5649
|
(issue) => !isEligible(issue, project) && issue.labels.includes(project.stateLabels.backlog),
|
|
4910
5650
|
);
|
|
4911
5651
|
const parkedNumbers = new Set(parkedCandidates.map((issue) => issue.number));
|
|
@@ -4920,6 +5660,14 @@ export async function tick(d: Deps, workers?: WorkerPool): Promise<void> {
|
|
|
4920
5660
|
// A park never kills a live run; a parked issue with no run is inventory
|
|
4921
5661
|
// the operator is deliberately holding, not reconciliation work.
|
|
4922
5662
|
parked += 1;
|
|
5663
|
+
} else if (unreadable.has(issue.number)) {
|
|
5664
|
+
// Fail closed, and say which read failed: a tracker that could not answer
|
|
5665
|
+
// is not evidence of a residual label (#891).
|
|
5666
|
+
lifecycleHolds.push({
|
|
5667
|
+
issue: issue.number,
|
|
5668
|
+
reason: "issue-state-lookup-error",
|
|
5669
|
+
detail: "lifecycle labels could not be revalidated against the exact issue read",
|
|
5670
|
+
});
|
|
4923
5671
|
} else {
|
|
4924
5672
|
lifecycleHolds.push({ issue: issue.number, reason: "stale-lifecycle" });
|
|
4925
5673
|
}
|
|
@@ -5021,6 +5769,8 @@ export async function tick(d: Deps, workers?: WorkerPool): Promise<void> {
|
|
|
5021
5769
|
return;
|
|
5022
5770
|
}
|
|
5023
5771
|
recordDispatch(pass.admitted.length, [...routingHolds, ...pass.holds]);
|
|
5772
|
+
await recordInstallSurfaces(d);
|
|
5773
|
+
reconcilePanes(d, project.name, log);
|
|
5024
5774
|
|
|
5025
5775
|
if (pass.admitted.length === 0) return;
|
|
5026
5776
|
|
|
@@ -5514,8 +6264,34 @@ export interface StatusSnapshot {
|
|
|
5514
6264
|
* materialises it, so this is never absent on a real daemon.
|
|
5515
6265
|
*/
|
|
5516
6266
|
review: ReviewPolicy;
|
|
5517
|
-
/** Occupied issues: live workers plus green PRs awaiting a human merge.
|
|
6267
|
+
/** Occupied issues: live workers plus green PRs awaiting a human merge. Both
|
|
6268
|
+
* lifecycle kinds, told apart by {@link StatusSnapshot.leasedRunIds}. */
|
|
5518
6269
|
activeRuns: RunRecord[];
|
|
6270
|
+
/**
|
|
6271
|
+
* The subset of {@link StatusSnapshot.activeRuns} holding an **active
|
|
6272
|
+
* mutation lease** (#898): a live worker, or a dispatched unsettled review
|
|
6273
|
+
* revision. Everything else in `activeRuns` is a worker-free preserved
|
|
6274
|
+
* artifact — durable work awaiting review, merge or recovery, with nothing
|
|
6275
|
+
* writing to it.
|
|
6276
|
+
*
|
|
6277
|
+
* Carried as ids rather than a second run list so the two renderings cannot
|
|
6278
|
+
* disagree about the rows themselves, and derived from `Store.leasedRuns` —
|
|
6279
|
+
* the same query admission's file-lane gate and `conductor_pr_recover` read,
|
|
6280
|
+
* so what status calls a lease is exactly what those two enforce (#899,
|
|
6281
|
+
* #925). A plain array so the dashboard's JSON round-trip preserves it.
|
|
6282
|
+
*/
|
|
6283
|
+
leasedRunIds?: readonly string[];
|
|
6284
|
+
/**
|
|
6285
|
+
* The review-ceiling adjudications an operator needs to see (#874): every
|
|
6286
|
+
* non-terminal one, plus the terminal one for each active run's PR head — so a
|
|
6287
|
+
* cleared or rejected verdict stays visible for exactly as long as the PR it
|
|
6288
|
+
* decided is still in flight, and disappears with it rather than accumulating.
|
|
6289
|
+
*
|
|
6290
|
+
* Keyed by nothing: the records carry their own `prUrl` and `headSha`, which is
|
|
6291
|
+
* how the renderer matches them to a run without inventing a second identity
|
|
6292
|
+
* for a PR.
|
|
6293
|
+
*/
|
|
6294
|
+
reviewAdjudications?: readonly ReviewAdjudicationRecord[];
|
|
5519
6295
|
/**
|
|
5520
6296
|
* runId → live review-revision round, for runs whose revision worker is
|
|
5521
6297
|
* currently dispatched (#692). Derived from the durable `review_revisions`
|
|
@@ -5524,7 +6300,22 @@ export interface StatusSnapshot {
|
|
|
5524
6300
|
* (not a Map) so the dashboard's JSON round-trip of the snapshot preserves
|
|
5525
6301
|
* it byte for byte.
|
|
5526
6302
|
*/
|
|
5527
|
-
|
|
6303
|
+
/**
|
|
6304
|
+
* The live review round per run: its number AND the instant it was dispatched
|
|
6305
|
+
* (#802). The instant is what makes the line honest — a resumed session's
|
|
6306
|
+
* `turns` and `startedAt` are cumulative over the whole attempt, so without a
|
|
6307
|
+
* phase boundary a fresh revision reads as though it had been running for
|
|
6308
|
+
* hours.
|
|
6309
|
+
*/
|
|
6310
|
+
reviewRounds?: Readonly<Record<string, { round: number; dispatchedAt: number }>>;
|
|
6311
|
+
/**
|
|
6312
|
+
* runId → the durable review evidence blocking a merge of the run's
|
|
6313
|
+
* pushed-green PR at its exact recorded head (#888). Derived from the same
|
|
6314
|
+
* `review_revisions` rows `conductor_pr_merge` refuses on, so status can
|
|
6315
|
+
* never present a green PR as merge-ready while the privileged verb will
|
|
6316
|
+
* refuse it. An absent entry means nothing blocks that run's head.
|
|
6317
|
+
*/
|
|
6318
|
+
mergeBlockers?: Readonly<Record<string, ReviewHeadBlocker>>;
|
|
5528
6319
|
/** Newest attempts holding a preserved WIP tip, or a tree that is still the
|
|
5529
6320
|
* only copy of work the daemon could not save. */
|
|
5530
6321
|
salvagedRuns: RunRecord[];
|
|
@@ -5558,6 +6349,20 @@ export interface StatusSnapshot {
|
|
|
5558
6349
|
liveWorkers: number;
|
|
5559
6350
|
runsToday: number;
|
|
5560
6351
|
spendTodayUsd: number;
|
|
6352
|
+
/** The install identities the last dispatch pass recorded (#919). Absent
|
|
6353
|
+
* means no pass has looked yet, which the renderer says out loud rather
|
|
6354
|
+
* than presenting as agreement. */
|
|
6355
|
+
installSurfaces?: InstallSurfaceObservation;
|
|
6356
|
+
/** What live runs have reserved out of today's budget but not yet spent
|
|
6357
|
+
* (#851). Optional so a caller that built a snapshot before reservations
|
|
6358
|
+
* existed renders no figure rather than a fabricated zero. */
|
|
6359
|
+
reservedSpendUsd?: number;
|
|
6360
|
+
/**
|
|
6361
|
+
* Whether the figure above is built on runs that actually reported cost
|
|
6362
|
+
* (#970). Absent when the caller did not judge it; the renderer then says
|
|
6363
|
+
* nothing rather than implying the telemetry is sound.
|
|
6364
|
+
*/
|
|
6365
|
+
spendTelemetry?: SpendTelemetryVerdict;
|
|
5561
6366
|
dispatch?: DispatchSummary;
|
|
5562
6367
|
/**
|
|
5563
6368
|
* Latest plan-allowance verdict, when the caller read one. Optional because
|
|
@@ -5632,12 +6437,50 @@ export function statusSnapshotFromStore(
|
|
|
5632
6437
|
// (#776 review #2).
|
|
5633
6438
|
const live = store.liveRuns(p.name);
|
|
5634
6439
|
const active = store.activeRuns(p.name);
|
|
6440
|
+
// Which of those active rows something is actually writing through (#898).
|
|
6441
|
+
// The same query admission's file-lane gate and `conductor_pr_recover` read,
|
|
6442
|
+
// so status cannot call a row a lease that those two treat as released, or
|
|
6443
|
+
// vice versa (#899, #925).
|
|
6444
|
+
const leasedRunIds = store.leasedRuns(p.name).map((r) => r.id);
|
|
6445
|
+
// Adjudications worth rendering (#874): the open ones always, plus a terminal
|
|
6446
|
+
// verdict for a head an active run is still sitting on. Bounded by the active
|
|
6447
|
+
// set rather than by a history window, so a fleet with a thousand settled
|
|
6448
|
+
// adjudications renders the handful that still describe live work.
|
|
6449
|
+
const openAdjudications = store.openReviewAdjudications(p.name);
|
|
6450
|
+
const adjudicationIds = new Set(openAdjudications.map((a) => a.id));
|
|
6451
|
+
const reviewAdjudications = [...openAdjudications];
|
|
6452
|
+
for (const r of active) {
|
|
6453
|
+
if (r.prUrl === undefined || r.headSha === undefined) continue;
|
|
6454
|
+
const decided = store.reviewAdjudicationForHead(p.name, r.prUrl, r.headSha);
|
|
6455
|
+
if (decided !== undefined && !adjudicationIds.has(decided.id)) {
|
|
6456
|
+
adjudicationIds.add(decided.id);
|
|
6457
|
+
reviewAdjudications.push(decided);
|
|
6458
|
+
}
|
|
6459
|
+
}
|
|
5635
6460
|
// The live review-revision rounds, read from the same durable rows the
|
|
5636
6461
|
// restart recovery uses: a run whose revision is dispatched is read as
|
|
5637
6462
|
// `review-revision N` while its worker is live (#692).
|
|
5638
|
-
const reviewRounds: Record<string, number> = {};
|
|
6463
|
+
const reviewRounds: Record<string, { round: number; dispatchedAt: number }> = {};
|
|
5639
6464
|
for (const revision of store.unsettledReviewRevisions(p.name)) {
|
|
5640
|
-
if (revision.dispatchedAt !== undefined)
|
|
6465
|
+
if (revision.dispatchedAt !== undefined) {
|
|
6466
|
+
reviewRounds[revision.runId] = { round: revision.round, dispatchedAt: revision.dispatchedAt };
|
|
6467
|
+
}
|
|
6468
|
+
}
|
|
6469
|
+
// The exact-head merge blockers (#888), read from the same durable rows the
|
|
6470
|
+
// merge verb consults: a pushed-green run whose PR stands at a head carrying
|
|
6471
|
+
// unresolved review evidence — a round queued there, crashed mid-review
|
|
6472
|
+
// there, or settled `failed` there.
|
|
6473
|
+
const mergeBlockers: Record<string, ReviewHeadBlocker> = {};
|
|
6474
|
+
for (const r of active) {
|
|
6475
|
+
if (r.prUrl === undefined || r.headSha === undefined) continue;
|
|
6476
|
+
const blocker = store.mergeBlockingReviews(p.name, r.prUrl, r.headSha)[0];
|
|
6477
|
+
if (blocker !== undefined) {
|
|
6478
|
+
mergeBlockers[r.id] = {
|
|
6479
|
+
round: blocker.round,
|
|
6480
|
+
state:
|
|
6481
|
+
blocker.settledAt !== undefined ? "failed" : blocker.dispatchedAt !== undefined ? "crashed" : "pending",
|
|
6482
|
+
};
|
|
6483
|
+
}
|
|
5641
6484
|
}
|
|
5642
6485
|
return {
|
|
5643
6486
|
project: p.name,
|
|
@@ -5662,6 +6505,8 @@ export function statusSnapshotFromStore(
|
|
|
5662
6505
|
releaseGrants: resolveReleaseGrants(p),
|
|
5663
6506
|
review: resolveReview(p),
|
|
5664
6507
|
activeRuns: active,
|
|
6508
|
+
leasedRunIds,
|
|
6509
|
+
reviewAdjudications,
|
|
5665
6510
|
reviewRounds,
|
|
5666
6511
|
salvagedRuns: store.salvagedRuns(p.name),
|
|
5667
6512
|
quarantinedRuns: store.quarantinedRuns(p.name),
|
|
@@ -5672,6 +6517,19 @@ export function statusSnapshotFromStore(
|
|
|
5672
6517
|
liveWorkers: live.length,
|
|
5673
6518
|
runsToday: store.runsStartedSince(p.name, since),
|
|
5674
6519
|
spendTodayUsd: store.spendSince(p.name, since),
|
|
6520
|
+
reservedSpendUsd: store.reservedSpendUsd(p.name),
|
|
6521
|
+
// Judged from the store on the same read as the figure above (#970), so the
|
|
6522
|
+
// row that qualifies the spend number cannot disagree with it, and nothing
|
|
6523
|
+
// is probed at render time.
|
|
6524
|
+
spendTelemetry: judgeSpendTelemetry(
|
|
6525
|
+
store.recentSpendSamples?.(p.name, SPEND_SAMPLE_ROWS) ?? [],
|
|
6526
|
+
SPEND_SAMPLE_RUNS,
|
|
6527
|
+
),
|
|
6528
|
+
// Read, never probed: the recorded row is the whole point (#919).
|
|
6529
|
+
...(() => {
|
|
6530
|
+
const observed = store.installSurfaces();
|
|
6531
|
+
return observed === undefined ? {} : { installSurfaces: observed };
|
|
6532
|
+
})(),
|
|
5675
6533
|
...(dispatch === undefined ? {} : { dispatch }),
|
|
5676
6534
|
...(planUsage === undefined ? {} : { planUsage }),
|
|
5677
6535
|
// Written by the tracker's hooks rather than polled, so the renderer does
|
|
@@ -5683,6 +6541,7 @@ export function statusSnapshotFromStore(
|
|
|
5683
6541
|
: { labelOps: { pending: labelOpsPending, oldestAgeMs: now - oldestLabelOpAt } }),
|
|
5684
6542
|
baseHealth: store.baseHealth(p.name),
|
|
5685
6543
|
freezes: store.freezes(p.name),
|
|
6544
|
+
mergeBlockers,
|
|
5686
6545
|
...(orchestratorDown === undefined ? {} : { orchestratorDown }),
|
|
5687
6546
|
};
|
|
5688
6547
|
}
|
|
@@ -5736,8 +6595,15 @@ export function formatDispatchSummary(summary?: DispatchSummary): string {
|
|
|
5736
6595
|
lines.push(` ${hold.reason} ${hold.count}${sample}`);
|
|
5737
6596
|
// A `file-lane` hold groups several issues, each blocked by a different
|
|
5738
6597
|
// file and holder; the grouped line says how many, this says which.
|
|
5739
|
-
|
|
5740
|
-
|
|
6598
|
+
//
|
|
6599
|
+
// Deduplicated for display only, never in the record: a fleet-wide hold
|
|
6600
|
+
// (`credential-class`, #852) gives every held candidate the *same*
|
|
6601
|
+
// sentence, and printing one reason five times reads as five problems.
|
|
6602
|
+
// The persisted details stay index-aligned with `issues` — a reader that
|
|
6603
|
+
// needs "which issue got which detail" still has it.
|
|
6604
|
+
const details = [...new Set(hold.details ?? [])];
|
|
6605
|
+
if (details.length > 0) {
|
|
6606
|
+
lines.push(` ${details.join(" | ")}`);
|
|
5741
6607
|
}
|
|
5742
6608
|
}
|
|
5743
6609
|
}
|
|
@@ -5803,71 +6669,6 @@ export function formatFreezes(freezes: readonly BaseFreeze[]): string[] {
|
|
|
5803
6669
|
}
|
|
5804
6670
|
|
|
5805
6671
|
|
|
5806
|
-
export function formatStatus(s: StatusSnapshot): string {
|
|
5807
|
-
const lines = [
|
|
5808
|
-
`project ${s.project}${s.paused ? " (PAUSED)" : ""}`,
|
|
5809
|
-
`config ${s.configPath}`,
|
|
5810
|
-
`state ${s.stateDir}`,
|
|
5811
|
-
"",
|
|
5812
|
-
...(s.orchestratorDown === undefined ? [] : formatOrchestratorDown(s.orchestratorDown)),
|
|
5813
|
-
"caps",
|
|
5814
|
-
` workers ${s.liveWorkers} / ${s.caps.maxConcurrentWorkers}`,
|
|
5815
|
-
` issues today ${s.runsToday}`,
|
|
5816
|
-
s.caps.dailySpendUsd === null
|
|
5817
|
-
? ` spend today $${s.spendTodayUsd.toFixed(2)} (no daily cap)`
|
|
5818
|
-
: ` spend today $${s.spendTodayUsd.toFixed(2)} / $${s.caps.dailySpendUsd.toFixed(2)}`,
|
|
5819
|
-
// Its own row, never folded into the spend row: they are two independent
|
|
5820
|
-
// controls, and an operator has to be able to see which one stopped the
|
|
5821
|
-
// fleet (#110).
|
|
5822
|
-
` plan usage ${planUsageLine(s.planUsage)}`,
|
|
5823
|
-
` new worker turns ${s.caps.workerMaxTurns}`,
|
|
5824
|
-
...s.turnOverrides.map(
|
|
5825
|
-
({ issue, maxTurns }) => ` turn override #${issue} → ${maxTurns} (next attempt)`,
|
|
5826
|
-
),
|
|
5827
|
-
` worker wall clock ${Math.round(s.caps.workerWallClockMs / 60_000)}m`,
|
|
5828
|
-
` failed attempts ${s.caps.maxAttemptsPerIssue}`,
|
|
5829
|
-
` continuations ${s.caps.maxContinuationsPerIssue}`,
|
|
5830
|
-
"",
|
|
5831
|
-
...formatReleaseGrants(s.releaseGrants),
|
|
5832
|
-
"",
|
|
5833
|
-
formatDispatchSummary(s.dispatch),
|
|
5834
|
-
"",
|
|
5835
|
-
];
|
|
5836
|
-
if (s.activeRuns.length === 0) {
|
|
5837
|
-
lines.push("active runs (none)");
|
|
5838
|
-
} else {
|
|
5839
|
-
lines.push("active runs");
|
|
5840
|
-
for (const r of s.activeRuns) {
|
|
5841
|
-
lines.push(
|
|
5842
|
-
` #${r.issue} ${r.repo} ${r.state} attempt ${r.attempt} ` +
|
|
5843
|
-
`${r.turns}/${r.maxTurns} turns $${r.spendUsd.toFixed(2)} ${r.branch}` +
|
|
5844
|
-
(r.prUrl ? ` ${r.prUrl}` : ""),
|
|
5845
|
-
);
|
|
5846
|
-
// Its escalation was deduplicated the moment it was delivered, so this
|
|
5847
|
-
// line is the only place a flagged run stays visible while its PR waits
|
|
5848
|
-
// for a merge — which is precisely the window the flag is about (#128).
|
|
5849
|
-
const flagged = settlementFlagSummary(r.settlementFlags);
|
|
5850
|
-
if (flagged !== undefined) lines.push(` ${flagged}`);
|
|
5851
|
-
}
|
|
5852
|
-
}
|
|
5853
|
-
lines.push(...formatBaseHealth(s.baseHealth));
|
|
5854
|
-
lines.push(...formatFreezes(s.freezes));
|
|
5855
|
-
lines.push(...formatSalvagedRuns(s.salvagedRuns));
|
|
5856
|
-
lines.push(...formatQuarantinedRuns(s.quarantinedRuns));
|
|
5857
|
-
lines.push(...formatOpenReports(s.openReports));
|
|
5858
|
-
lines.push(...formatVerbLedger(s.verbLedger));
|
|
5859
|
-
// Deploy hint: a restart while workers are live orphans them (salvage runs
|
|
5860
|
-
// first — #35). Prefer pause + drain to zero live workers when you can wait.
|
|
5861
|
-
if (s.liveWorkers > 0) {
|
|
5862
|
-
lines.push(
|
|
5863
|
-
"",
|
|
5864
|
-
`deploy ${s.liveWorkers} live worker(s) — restart salvages dirty trees then orphans the rows; ` +
|
|
5865
|
-
`pause and wait for workers 0/${s.caps.maxConcurrentWorkers} when you can drain instead`,
|
|
5866
|
-
);
|
|
5867
|
-
}
|
|
5868
|
-
return lines.join("\n");
|
|
5869
|
-
}
|
|
5870
|
-
|
|
5871
6672
|
export interface QueuePreview {
|
|
5872
6673
|
project: string;
|
|
5873
6674
|
configPath: string;
|
|
@@ -6250,32 +7051,21 @@ export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
|
|
|
6250
7051
|
const store = openStore(dbPath());
|
|
6251
7052
|
const verbPeerReader = peerCredentialReader();
|
|
6252
7053
|
const verbDir = ensureVerbSocketDir(stateDir());
|
|
6253
|
-
// The worker identity (#798). Probed once here for the startup banner only —
|
|
6254
|
-
// every dispatch re-resolves it through {@link launchIdentity}, because the
|
|
6255
|
-
// account, setpriv and the harness binding (#828) are all host state that can
|
|
6256
|
-
// arrive after this process did. A host that cannot establish it keeps its
|
|
6257
|
-
// control plane running and fails each worker launch closed with the reason,
|
|
6258
|
-
// so the operator hears a concrete host change instead of a fleet that
|
|
6259
|
-
// silently ran workers as root.
|
|
6260
|
-
const identityAtStartup = resolveWorkerIdentity();
|
|
6261
|
-
if (identityAtStartup.ok) {
|
|
6262
|
-
log(
|
|
6263
|
-
`worker identity: ${identityAtStartup.identity.account} uid=${identityAtStartup.identity.uid} ` +
|
|
6264
|
-
`gid=${identityAtStartup.identity.gid} home=${identityAtStartup.identity.home}`,
|
|
6265
|
-
);
|
|
6266
|
-
} else {
|
|
6267
|
-
log(
|
|
6268
|
-
"worker identity unavailable — worker dispatch fails closed until the host provides it " +
|
|
6269
|
-
`(re-checked at every launch, so no restart is needed once it does): ${identityAtStartup.reason}`,
|
|
6270
|
-
);
|
|
6271
|
-
}
|
|
6272
7054
|
const integrity: IntegrityGate = { baseline: packageManifest(), paged: false };
|
|
6273
7055
|
const usage = sharedUsageSource();
|
|
6274
7056
|
const turnLimits = createTurnLimitRegistry((runId, maxTurns) => {
|
|
6275
7057
|
store.updateRun(runId, { maxTurns });
|
|
6276
7058
|
});
|
|
6277
7059
|
const workerControls = createWorkerControlRegistry();
|
|
6278
|
-
|
|
7060
|
+
// Created before the pool so a worker's completion can poke the loop it
|
|
7061
|
+
// shares with the interval sleep (#878).
|
|
7062
|
+
const pace = createDispatchPace();
|
|
7063
|
+
const workers = createWorkerPool(() => {
|
|
7064
|
+
// Sticky and coalescing by construction: several workers settling together
|
|
7065
|
+
// produce one pass, and a pass already running absorbs the request rather
|
|
7066
|
+
// than overlapping. No second owner is created and no gate is skipped.
|
|
7067
|
+
pace.requestWake();
|
|
7068
|
+
});
|
|
6279
7069
|
const alive = livingDaemon();
|
|
6280
7070
|
const runtimes: ProjectRuntime[] = [];
|
|
6281
7071
|
|
|
@@ -6322,6 +7112,21 @@ export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
|
|
|
6322
7112
|
: pushRunBranch(project, { repo, runRepoPath: run.worktree, branch });
|
|
6323
7113
|
};
|
|
6324
7114
|
for (const run of await reconcileOrphanedRuns(store, project.name, orphanPublisher)) {
|
|
7115
|
+
// The workspace outlives the process, so a restart inherits panes whose
|
|
7116
|
+
// workers are gone (#842). A session-host child dies with the daemon that
|
|
7117
|
+
// owned its socket, so an orphaned row's pane is authoritatively dead
|
|
7118
|
+
// whatever its recorded pid says — pids are reused, and matching one
|
|
7119
|
+
// would be the mechanism that makes a stranger's process read as a live
|
|
7120
|
+
// worker. Release (never close) so lifecycle authority goes back to
|
|
7121
|
+
// Herdr and the pane stops claiming a worker conductor no longer runs.
|
|
7122
|
+
const orphanPane = releaseOrphanedWorkerPane(run);
|
|
7123
|
+
if (orphanPane.kind === "released") {
|
|
7124
|
+
projectLog(
|
|
7125
|
+
`#${run.issue} herdr pane ${orphanPane.paneId} released: its worker died with the previous daemon`,
|
|
7126
|
+
);
|
|
7127
|
+
} else if (orphanPane.kind === "failed") {
|
|
7128
|
+
projectLog(`#${run.issue} herdr pane ${orphanPane.paneId} release failed: ${orphanPane.reason}`);
|
|
7129
|
+
}
|
|
6325
7130
|
projectLog(
|
|
6326
7131
|
`#${run.issue} orphaned by a previous daemon (attempt ${run.attempt}, was ${run.state}, ` +
|
|
6327
7132
|
`worktree ${run.worktree}) — slot freed; the ${project.stateLabels.inProgress} label stays ` +
|
|
@@ -6446,17 +7251,27 @@ export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
|
|
|
6446
7251
|
cleanup: { next: 0 },
|
|
6447
7252
|
probeCriticalBase: (repo, markers, branch) =>
|
|
6448
7253
|
probeCriticalBase(project, repo, branch, markers),
|
|
7254
|
+
// The same seam `doctor` and `upgrade` read (#904/#919), never a second
|
|
7255
|
+
// implementation. `readHerdr` stays on: a host that runs herdr is the
|
|
7256
|
+
// case where a stale recovery pin matters, and a host without it answers
|
|
7257
|
+
// "absent", which no cheap surface treats as a fault.
|
|
7258
|
+
probeInstallSurfaces: () => inspectSurfaces({ run: runCommand, log, env: process.env }),
|
|
7259
|
+
probeTelegramFreshness: () =>
|
|
7260
|
+
// The module's own reader, so the daemon, `doctor` and `status` cannot
|
|
7261
|
+
// come to disagree about what is installed (#961).
|
|
7262
|
+
checkTelegramFreshness({
|
|
7263
|
+
run: async (cmd, args) => {
|
|
7264
|
+
const r = await runCommand(cmd, args);
|
|
7265
|
+
return { code: r.code, stdout: r.stdout };
|
|
7266
|
+
},
|
|
7267
|
+
}),
|
|
6449
7268
|
probeWorktreeLane: (input) => probeRunLane(input),
|
|
6450
|
-
// The
|
|
6451
|
-
|
|
6452
|
-
//
|
|
6453
|
-
//
|
|
6454
|
-
//
|
|
6455
|
-
|
|
6456
|
-
grantWorkerPaths: (identity, worktreePath, sessionDir) => {
|
|
6457
|
-
chownRecursive(worktreePath, identity.uid, identity.gid);
|
|
6458
|
-
chownRecursive(sessionDir, identity.uid, identity.gid);
|
|
6459
|
-
},
|
|
7269
|
+
// The credential-class fence's reader (#852): the runnable probe module,
|
|
7270
|
+
// spawned against the live credential store. Wired once here so the
|
|
7271
|
+
// admission gate and the launch fence ask the same question through the
|
|
7272
|
+
// same transport — two readers would eventually disagree, and the one that
|
|
7273
|
+
// disagreed by passing is the expensive one.
|
|
7274
|
+
probeCredentialClass: (provider) => probeCredentialClass(provider),
|
|
6460
7275
|
// A cross-repo Depends-on prerequisite reads through the same GitHub
|
|
6461
7276
|
// credential/accounting seams as the project tracker — a fresh tracker
|
|
6462
7277
|
// scoped to the referenced repo, reusing the daemon's gh hooks so API
|
|
@@ -6498,6 +7313,31 @@ export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
|
|
|
6498
7313
|
projectLog(`review revision restart recovery failed: ${errText(err)}`);
|
|
6499
7314
|
}
|
|
6500
7315
|
}
|
|
7316
|
+
// Adjudication restart recovery (#932). An adjudication is `running` only
|
|
7317
|
+
// while a session it launched is alive, and that session died with the
|
|
7318
|
+
// previous process — nothing resumes it, because an adjudicator has no
|
|
7319
|
+
// branch, no worktree and no transcript worth continuing: its whole output
|
|
7320
|
+
// is a verdict it never produced.
|
|
7321
|
+
//
|
|
7322
|
+
// So it settles `failed`, which is honest and terminal, rather than being
|
|
7323
|
+
// left `running` forever (a row nothing would ever touch again) or silently
|
|
7324
|
+
// re-queued (a second launch for one head, which the one-shot exists to
|
|
7325
|
+
// prevent). The one shot for that head is spent, and #876's disposition path
|
|
7326
|
+
// reads `failed` exactly as it reads any other non-clear verdict.
|
|
7327
|
+
if (alive === undefined || alive.pid === process.pid) {
|
|
7328
|
+
for (const adjudication of store.openReviewAdjudications(project.name)) {
|
|
7329
|
+
if (adjudication.state !== "running") continue;
|
|
7330
|
+
store.settleReviewAdjudication(
|
|
7331
|
+
adjudication.id,
|
|
7332
|
+
"failed",
|
|
7333
|
+
"the daemon restarted while this adjudication was running; no verdict was produced",
|
|
7334
|
+
Date.now(),
|
|
7335
|
+
);
|
|
7336
|
+
projectLog(
|
|
7337
|
+
`#${adjudication.issue} adjudication failed across restart — its session died with the previous daemon`,
|
|
7338
|
+
);
|
|
7339
|
+
}
|
|
7340
|
+
}
|
|
6501
7341
|
// Startup reconciliation: close an incident carried over from a previous
|
|
6502
7342
|
// process when the orchestrator is up (one recovery notice), or open one
|
|
6503
7343
|
// when it failed to start (one down page). A daemon restarted while still
|
|
@@ -6595,7 +7435,6 @@ export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
|
|
|
6595
7435
|
// The wake surface (#380): `resume` POSTs /wake, `stop` interrupts the same
|
|
6596
7436
|
// sleep. A wake request is sticky and coalescing — several resumes in quick
|
|
6597
7437
|
// succession produce one prompt pass, never an overlapping one.
|
|
6598
|
-
const pace = createDispatchPace();
|
|
6599
7438
|
const stop = (): void => {
|
|
6600
7439
|
if (stopping) return;
|
|
6601
7440
|
stopping = true;
|