omp-conductor 0.17.1 → 0.18.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +34 -0
- package/REFERENCE.md +71 -17
- package/agents/to-spec.md +90 -0
- package/package.json +2 -1
- package/schema/config.schema.json +53 -1
- package/src/admission.ts +308 -76
- package/src/ask.ts +307 -10
- package/src/backups.ts +2 -2
- package/src/board.ts +17 -3
- package/src/briefs/orchestrator.md +43 -14
- package/src/briefs/to-spec.md +84 -0
- package/src/briefs/worker.md +37 -19
- package/src/cli.ts +2 -0
- package/src/command-help.ts +19 -1
- package/src/command-manifest.ts +27 -2
- package/src/commands/context.ts +1 -0
- package/src/commands/drain.ts +176 -0
- package/src/commands/extend.ts +6 -10
- package/src/commands/status.ts +5 -1
- package/src/commands/watch.ts +110 -3
- package/src/commands/worker.ts +9 -10
- package/src/config-schema.ts +57 -0
- package/src/config.ts +102 -2
- package/src/daemon.ts +1220 -1517
- package/src/dashboard/app.js +4 -1
- package/src/dashboard/server.ts +5 -2
- package/src/decisions.ts +279 -16
- package/src/depends-on.ts +261 -1
- package/src/diff-flags.ts +425 -1
- package/src/digest-schedule.ts +37 -0
- package/src/doctor.ts +52 -0
- package/src/escalate.ts +9 -3
- package/src/failure-class.ts +43 -4
- package/src/fleet.ts +166 -24
- package/src/gitops.ts +188 -81
- package/src/graph-health.ts +55 -8
- package/src/graph.ts +379 -69
- package/src/harness-loader.ts +59 -0
- package/src/host.ts +567 -2
- package/src/lifecycle.ts +158 -6
- package/src/omp.ts +269 -20
- package/src/orchestrator-tick.ts +1489 -26
- package/src/orchestrator.ts +12 -0
- package/src/privileged.ts +1 -4
- package/src/release-policy.ts +503 -9
- package/src/routing.ts +11 -3
- package/src/session-host.ts +115 -5
- package/src/settlement.ts +1780 -0
- package/src/setup-host.ts +1205 -6
- package/src/setup-install.ts +119 -30
- package/src/setup-wizard.ts +88 -2
- package/src/setup.ts +119 -13
- package/src/shell.ts +15 -0
- package/src/status-render.ts +100 -11
- package/src/store.ts +519 -45
- package/src/to-spec.ts +387 -0
- package/src/tracker/github.ts +150 -14
- package/src/types.ts +470 -16
- package/src/upgrade-verify.ts +209 -2
- package/src/upgrade.ts +175 -1
- package/src/verbs/protocol.ts +39 -0
- package/src/verbs/server.ts +770 -40
- package/src/verbs/socket.ts +24 -5
- package/src/worker.ts +239 -9
- package/src/worktree.ts +142 -18
package/src/daemon.ts
CHANGED
|
@@ -8,35 +8,48 @@
|
|
|
8
8
|
* here and enforced before anything is claimed.
|
|
9
9
|
*/
|
|
10
10
|
import { createHash } from "node:crypto";
|
|
11
|
-
import { chmodSync, existsSync, mkdirSync, readdirSync, readFileSync, renameSync, rmSync, statSync, writeFileSync } from "node:fs";
|
|
11
|
+
import { chmodSync, closeSync, constants, existsSync, fchownSync, lstatSync, mkdirSync, openSync, readdirSync, readFileSync, readlinkSync, renameSync, rmSync, statSync, writeFileSync, type Dirent } from "node:fs";
|
|
12
12
|
import { dirname, join, relative } from "node:path";
|
|
13
13
|
import {
|
|
14
14
|
configPath,
|
|
15
|
+
dbBackupDirFor,
|
|
15
16
|
findProject,
|
|
16
17
|
loadConfig,
|
|
17
18
|
resolveCaps,
|
|
18
19
|
resolveReleaseGrants,
|
|
20
|
+
resolveReview,
|
|
19
21
|
stateDir,
|
|
20
22
|
} from "./config.ts";
|
|
21
23
|
import { availabilityState, type AvailabilityState } from "./availability.ts";
|
|
22
24
|
import {
|
|
23
|
-
UNREADABLE_TREE_FLAG,
|
|
24
|
-
analyseSettlement,
|
|
25
|
-
deriveChangedLine,
|
|
26
25
|
formatSettlementFlags,
|
|
27
26
|
settlementFlagSummary,
|
|
28
27
|
withDerivedChangedLine,
|
|
29
28
|
} from "./diff-flags.ts";
|
|
30
|
-
import {
|
|
29
|
+
import {
|
|
30
|
+
dbSnapshotDue,
|
|
31
|
+
dbSnapshotMarkerKey,
|
|
32
|
+
digestScheduleState,
|
|
33
|
+
localDayKey,
|
|
34
|
+
type DigestScheduleState,
|
|
35
|
+
} from "./digest-schedule.ts";
|
|
31
36
|
import { createEscalator, escalationIssueRef } from "./escalate.ts";
|
|
32
37
|
import { pendingCodeGraph, probeCodeGraph, type CodeGraphHealth } from "./graph-health.ts";
|
|
33
38
|
import { graphHint } from "./graph.ts";
|
|
39
|
+
import {
|
|
40
|
+
hostConstraintsNotice,
|
|
41
|
+
resolveWorkerIdentity,
|
|
42
|
+
WORKER_ACCOUNT,
|
|
43
|
+
type WorkerIdentity,
|
|
44
|
+
type WorkerIdentityResolution,
|
|
45
|
+
} from "./host.ts";
|
|
34
46
|
import { acquireOnceLease, healthCheck, livingDaemon, probeUnit, SYSTEMD_UNIT } from "./lifecycle.ts";
|
|
35
47
|
import { requestImmediateTick, resolveTickConfigCwd, STALL_MARKER_FILE } from "./orchestrator-tick.ts";
|
|
36
48
|
import { runDoctor } from "./doctor.ts";
|
|
37
49
|
import { appendJournal, launchTransientUnit, readUpgradeJournal } from "./upgrade-journal.ts";
|
|
38
50
|
import { runCommand, verifyPendingUpgrade, type UpgradeVerifyDeps } from "./upgrade-verify.ts";
|
|
39
|
-
import {
|
|
51
|
+
import { processStartTimeMs } from "./upgrade-verify.ts";
|
|
52
|
+
import { fleetLayers, herdrPaneOmpStarts, resolveHerdrSession } from "./fleet.ts";
|
|
40
53
|
import { startOrchestrator } from "./orchestrator.ts";
|
|
41
54
|
import type { OrchestratorHandle } from "./orchestrator.ts";
|
|
42
55
|
import {
|
|
@@ -56,39 +69,66 @@ import {
|
|
|
56
69
|
type GateShape,
|
|
57
70
|
type ReleaseBlockContext,
|
|
58
71
|
} from "./release-policy.ts";
|
|
59
|
-
import { branchName, effectiveLabels, route } from "./routing.ts";
|
|
72
|
+
import { branchName, effectiveLabels, isEligible, route } from "./routing.ts";
|
|
60
73
|
import type { Routed, UnroutableReason } from "./routing.ts";
|
|
61
74
|
import {
|
|
62
75
|
admitCandidates,
|
|
63
76
|
effectiveLane,
|
|
77
|
+
effectiveModel,
|
|
64
78
|
hasContinuationBudget,
|
|
65
79
|
hasFailedAttemptBudget,
|
|
80
|
+
laneEcho,
|
|
66
81
|
} from "./admission.ts";
|
|
67
|
-
import type { Admission, AdmissionHold
|
|
82
|
+
import type { Admission, AdmissionHold } from "./admission.ts";
|
|
83
|
+
import type { EffectiveModel, FileLane } from "./types.ts";
|
|
68
84
|
import { log, errText, safeEscalate } from "./log.ts";
|
|
85
|
+
import {
|
|
86
|
+
adoptSalvagedPrs,
|
|
87
|
+
classifyAndRecover,
|
|
88
|
+
collectSettlementFlags,
|
|
89
|
+
formatQuarantinedRuns,
|
|
90
|
+
formatSalvagedRuns,
|
|
91
|
+
reactToProviderCredit,
|
|
92
|
+
readSessionError,
|
|
93
|
+
reconcileOrphanedRuns,
|
|
94
|
+
reconcileStaleLabels,
|
|
95
|
+
recordOperatorStop,
|
|
96
|
+
settlePushedGreen,
|
|
97
|
+
settleWorktree,
|
|
98
|
+
swapToQueue,
|
|
99
|
+
} from "./settlement.ts";
|
|
69
100
|
import { materializeOmpSettings } from "./omp-settings.ts";
|
|
70
101
|
import { evaluateDecisionConditions, probeNpmVersion, probeRateLimitReset } from "./decisions.ts";
|
|
71
|
-
import {
|
|
102
|
+
import { infraLogSignature, infraSignatureVersion, providerCreditRefusal, providerTransientFault } from "./failure-class.ts";
|
|
72
103
|
import {
|
|
73
104
|
DEFAULT_MODEL_FALLBACK_THRESHOLD,
|
|
74
105
|
fallbackClause,
|
|
75
|
-
formatModelsTried,
|
|
76
|
-
modelsTried,
|
|
77
106
|
providerFailureFacts,
|
|
78
107
|
resolveDispatchModel,
|
|
79
108
|
} from "./model-fallback.ts";
|
|
80
109
|
import { projectLabels } from "./label-projection.ts";
|
|
81
|
-
import {
|
|
82
|
-
|
|
110
|
+
import {
|
|
111
|
+
DB_SNAPSHOT_RETENTION,
|
|
112
|
+
ACTIVE_STATES,
|
|
113
|
+
dbPath,
|
|
114
|
+
LIVE_STATES,
|
|
115
|
+
openStore,
|
|
116
|
+
pruneDbSnapshots,
|
|
117
|
+
snapshotDb,
|
|
118
|
+
utcDay,
|
|
119
|
+
} from "./store.ts";
|
|
120
|
+
import { GhPrMissingError, GraphqlBreaker, makeTracker, type RateLimitStatus } from "./tracker/github.ts";
|
|
83
121
|
import { DEFAULT_REPORT_POLICY, RELEASE_SHAPES } from "./types.ts";
|
|
84
122
|
import type {
|
|
85
123
|
BaseFreeze,
|
|
86
124
|
BaseHealth,
|
|
87
125
|
AdmissionHoldReason,
|
|
88
126
|
Caps,
|
|
127
|
+
ConductorConfig,
|
|
89
128
|
DispatchSummary,
|
|
90
129
|
DigestBacklog,
|
|
91
130
|
Escalation,
|
|
131
|
+
InterruptCategory,
|
|
92
132
|
IssueComment,
|
|
93
133
|
IssueSnapshot,
|
|
94
134
|
MergedPrInfo,
|
|
@@ -96,13 +136,18 @@ import type {
|
|
|
96
136
|
ReleaseShape,
|
|
97
137
|
OrchestratorIncident,
|
|
98
138
|
PrState,
|
|
139
|
+
PrVerification,
|
|
99
140
|
ProjectConfig,
|
|
100
141
|
ReadyIssue,
|
|
142
|
+
ReportScope,
|
|
143
|
+
ReportingPolicy,
|
|
101
144
|
RepoTarget,
|
|
102
145
|
ReportRecord,
|
|
103
146
|
ResolvedGrants,
|
|
104
147
|
FailureClass,
|
|
148
|
+
HostConstraints,
|
|
105
149
|
RecoveryAction,
|
|
150
|
+
ReviewPolicy,
|
|
106
151
|
ReviewRevisionOutcome,
|
|
107
152
|
ReviewRevisionRecord,
|
|
108
153
|
RunRecord,
|
|
@@ -139,8 +184,8 @@ import {
|
|
|
139
184
|
import { githubVerbActions } from "./verbs/actions.ts";
|
|
140
185
|
import { formatVerbLedger, STATUS_LEDGER_SCAN } from "./verbs/ledger.ts";
|
|
141
186
|
import {
|
|
187
|
+
isHeadMismatch,
|
|
142
188
|
listenVerbChannel,
|
|
143
|
-
PR_LOOKUP_WINDOW_MS,
|
|
144
189
|
type VerbActions,
|
|
145
190
|
type VerbDeps,
|
|
146
191
|
type VerbListener,
|
|
@@ -180,26 +225,7 @@ const GRAPH_HEALTH_INTERVAL_MS = 60_000;
|
|
|
180
225
|
* report an operator is waiting on must not sit in the outbox for the length of
|
|
181
226
|
* a poll interval, and delivery is owed even while claiming is paused (#123). */
|
|
182
227
|
const REPORT_DELIVERY_INTERVAL_MS = 30_000;
|
|
183
|
-
|
|
184
|
-
* retries — but only a bounded number of times. Three strikes for one issue
|
|
185
|
-
* means the mirror itself is broken, not unlucky, and the sweep escalates
|
|
186
|
-
* instead of burning a turn-0 run per tick forever (#168, #177). */
|
|
187
|
-
const DISPATCH_INFRA_MAX_STRIKES = 3;
|
|
188
|
-
/** A provider-transient requeue (stream stalled mid-run) is retried, but only a
|
|
189
|
-
* bounded number of times: three aborted streams for one issue means the
|
|
190
|
-
* provider itself is degraded, not unlucky, and the sweep escalates to a
|
|
191
|
-
* human instead of requeueing into a down provider forever (#220). */
|
|
192
|
-
const PROVIDER_TRANSIENT_MAX_STRIKES = 3;
|
|
193
|
-
/** A provider-capacity requeue (sustained in-session rate limiting) is retried,
|
|
194
|
-
* but only a bounded number of times: three throttled runs for one issue mean
|
|
195
|
-
* the provider is at capacity, not unlucky, and the sweep escalates to a human
|
|
196
|
-
* instead of requeueing into a throttled provider forever (#573). The issue's
|
|
197
|
-
* own chain moves onto its next model per strike (via {@link FAILOVER_CLASSES}),
|
|
198
|
-
* so a bounded chain is exhaustible; this caps the unbounded no-chain case. */
|
|
199
|
-
const PROVIDER_CAPACITY_MAX_STRIKES = 3;
|
|
200
|
-
/** Terminal rows inspected for a salvaged PR per tick. GitHub provenance reads
|
|
201
|
-
* are maintenance, but a backlog must not turn one tick into an API burst. */
|
|
202
|
-
const SALVAGED_PR_ADOPTION_BATCH = 10;
|
|
228
|
+
|
|
203
229
|
const DEFAULT_PORT = 8787;
|
|
204
230
|
const BRIEF_TEMPLATE_PATH = join(import.meta.dir, "briefs", "worker.md");
|
|
205
231
|
|
|
@@ -233,6 +259,10 @@ export interface DrainSignal {
|
|
|
233
259
|
interface Deps {
|
|
234
260
|
project: ProjectConfig;
|
|
235
261
|
caps: Caps;
|
|
262
|
+
/** The typed host-constraints block (#721), re-resolved at the tick
|
|
263
|
+
* boundary like `project`/`caps`, and rendered into every brief the tick
|
|
264
|
+
* dispatches. Optional so tests that never exercise it can omit it. */
|
|
265
|
+
host?: HostConstraints;
|
|
236
266
|
tracker: Tracker;
|
|
237
267
|
store: Store;
|
|
238
268
|
/**
|
|
@@ -256,6 +286,33 @@ interface Deps {
|
|
|
256
286
|
workerControls: WorkerControlRegistry;
|
|
257
287
|
/** Session seam for lifecycle integration tests; production uses the real harness. */
|
|
258
288
|
workerDeps?: RunWorkerDeps;
|
|
289
|
+
/**
|
|
290
|
+
* Resolves the dedicated worker identity (#798) — the account worker sessions
|
|
291
|
+
* run under, with its uid/gid/home, the transition launcher and a live harness
|
|
292
|
+
* binding (#828).
|
|
293
|
+
*
|
|
294
|
+
* Called **at every worker launch**, never cached: every input is host state
|
|
295
|
+
* that can change under a running daemon. A reboot can start this service
|
|
296
|
+
* before systemd has mounted the harness binding, and an operator can install
|
|
297
|
+
* the account or re-run `setup host` at any time — a verdict taken once at
|
|
298
|
+
* startup would hold the whole fleet closed until somebody thought to restart
|
|
299
|
+
* the daemon, which is the outage #828 exists to end rather than relocate.
|
|
300
|
+
* Resolution is a handful of `stat` calls and one `/etc/passwd` read; a
|
|
301
|
+
* dispatch does far more than that before it reaches this gate.
|
|
302
|
+
*
|
|
303
|
+
* Every worker dispatch refuses to launch on an unresolved identity (fail
|
|
304
|
+
* closed — an unbound worker is indistinguishable from an operator shell),
|
|
305
|
+
* and this is how the dispatcher secures the run's sockets and grants the
|
|
306
|
+
* run's paths to the account. Absent entirely, dispatch fails closed naming
|
|
307
|
+
* the missing account.
|
|
308
|
+
*/
|
|
309
|
+
workerIdentity?: () => WorkerIdentityResolution;
|
|
310
|
+
/**
|
|
311
|
+
* Re-owns a run's working paths (worktree + session dir) under the worker
|
|
312
|
+
* identity before the session launches. Wired by `runDaemon` to the
|
|
313
|
+
* recursive chown; a test fixture leaves it unset so no test chowns.
|
|
314
|
+
*/
|
|
315
|
+
grantWorkerPaths?: (identity: WorkerIdentity, worktreePath: string, sessionDir: string) => void;
|
|
259
316
|
integrity: IntegrityGate;
|
|
260
317
|
stall: StallGate;
|
|
261
318
|
/**
|
|
@@ -315,6 +372,13 @@ interface Deps {
|
|
|
315
372
|
* admission fails a routed cross-repo prerequisite closed.
|
|
316
373
|
*/
|
|
317
374
|
probeIssueIn?: (repo: string, issue: number) => Promise<IssueSnapshot | undefined>;
|
|
375
|
+
/**
|
|
376
|
+
* Reads one issue's BODY in a repository the admission tracker is not bound
|
|
377
|
+
* to — the dependency-graph cycle pass (#421). Wired by `runDaemon` to a
|
|
378
|
+
* repo-scoped tracker; a test injects a fake. Absent, a routed reachable
|
|
379
|
+
* body fails that branch closed rather than synthesising a cycle.
|
|
380
|
+
*/
|
|
381
|
+
probeBodyIn?: (repo: string, issue: number) => Promise<string | undefined>;
|
|
318
382
|
}
|
|
319
383
|
|
|
320
384
|
/**
|
|
@@ -436,6 +500,121 @@ export function checkStall(gate: StallGate, marker: string, now = Date.now()): S
|
|
|
436
500
|
* could destroy work an operator would rather read first — the same refusal to
|
|
437
501
|
* guess that the recovery plugin is built on.
|
|
438
502
|
*/
|
|
503
|
+
// --------------------------------------------------------- worker identity (#798) --
|
|
504
|
+
|
|
505
|
+
/**
|
|
506
|
+
* Re-own a run's working paths (worktree, session dir) under the worker
|
|
507
|
+
* identity before the session launches. The daemon runs this as root; the
|
|
508
|
+
* worker identity is granted its own run paths by ownership, never by
|
|
509
|
+
* loosened modes on the daemon's.
|
|
510
|
+
*
|
|
511
|
+
* The walk never re-owns through a path a worker-uid process could
|
|
512
|
+
* re-resolve: every ownership change is `fchownSync` on a descriptor opened
|
|
513
|
+
* with `O_NOFOLLOW`, and every descriptor is verified — via `/proc/self/fd`'s
|
|
514
|
+
* kernel-resolved path — to still sit under the directory the walk opened.
|
|
515
|
+
* A directory entry raced into a symlink therefore either fails `O_NOFOLLOW`
|
|
516
|
+
* at the final component or resolves outside the verified parent and is
|
|
517
|
+
* skipped; it can never carry the chown to an external target. Symlinks
|
|
518
|
+
* within the tree are left untouched (the worker manages entries through its
|
|
519
|
+
* parent directories, and git recreates links on checkout), as are anything
|
|
520
|
+
* unopenable — fifos, sockets, devices — which git never creates. A missing
|
|
521
|
+
* or racing entry is not this dispatch's problem — the next dispatch re-owns
|
|
522
|
+
* whatever survives; a tree already owned by the worker identity — the
|
|
523
|
+
* resume and review-revision re-ownership of a tree a prior dispatch granted
|
|
524
|
+
* — is the only tree a worker-uid process could have modified, and is
|
|
525
|
+
* skipped outright rather than walked.
|
|
526
|
+
*/
|
|
527
|
+
export function chownRecursive(root: string, uid: number, gid: number): void {
|
|
528
|
+
try {
|
|
529
|
+
const current = lstatSync(root);
|
|
530
|
+
if (current.uid === uid && current.gid === gid) return;
|
|
531
|
+
} catch {
|
|
532
|
+
// A missing or racing root is the caller's own existence check; the next
|
|
533
|
+
// dispatch re-owns whatever survives.
|
|
534
|
+
return;
|
|
535
|
+
}
|
|
536
|
+
const rootFd = openNoFollowDir(root);
|
|
537
|
+
try {
|
|
538
|
+
walkDir(rootFd, root, uid, gid);
|
|
539
|
+
} finally {
|
|
540
|
+
try {
|
|
541
|
+
closeSync(rootFd);
|
|
542
|
+
} catch {
|
|
543
|
+
// Already closed by a raced-away walk; nothing to do.
|
|
544
|
+
}
|
|
545
|
+
}
|
|
546
|
+
}
|
|
547
|
+
|
|
548
|
+
/** Open a directory without following a final-component symlink. */
|
|
549
|
+
function openNoFollowDir(path: string): number {
|
|
550
|
+
return openSync(path, constants.O_RDONLY | constants.O_DIRECTORY | constants.O_NOFOLLOW);
|
|
551
|
+
}
|
|
552
|
+
|
|
553
|
+
/** The kernel-resolved path of an open descriptor: what the object actually is. */
|
|
554
|
+
function fdRealPath(fd: number): string | undefined {
|
|
555
|
+
try {
|
|
556
|
+
return readlinkSync(`/proc/self/fd/${fd}`);
|
|
557
|
+
} catch {
|
|
558
|
+
return undefined;
|
|
559
|
+
}
|
|
560
|
+
}
|
|
561
|
+
|
|
562
|
+
function walkDir(dirFd: number, dirPath: string, uid: number, gid: number): void {
|
|
563
|
+
try {
|
|
564
|
+
fchownSync(dirFd, uid, gid);
|
|
565
|
+
} catch {
|
|
566
|
+
// Raced away; the next dispatch re-owns what survives.
|
|
567
|
+
}
|
|
568
|
+
const dirReal = fdRealPath(dirFd);
|
|
569
|
+
if (dirReal === undefined) return;
|
|
570
|
+
let entries: Dirent[];
|
|
571
|
+
try {
|
|
572
|
+
entries = readdirSync(dirPath, { withFileTypes: true });
|
|
573
|
+
} catch {
|
|
574
|
+
return;
|
|
575
|
+
}
|
|
576
|
+
for (const entry of entries) {
|
|
577
|
+
const childPath = join(dirPath, entry.name);
|
|
578
|
+
try {
|
|
579
|
+
if (entry.isDirectory()) {
|
|
580
|
+
const childFd = openNoFollowDir(childPath);
|
|
581
|
+
try {
|
|
582
|
+
// The entry is genuinely beneath the directory this fd owns only
|
|
583
|
+
// when the kernel resolves the opened object to a path under it.
|
|
584
|
+
// Anything else — a name swapped to a symlink, a foreign listing
|
|
585
|
+
// read through a replaced parent — is refused, never re-resolved.
|
|
586
|
+
const childReal = fdRealPath(childFd);
|
|
587
|
+
if (childReal === undefined || !childReal.startsWith(`${dirReal}/`)) continue;
|
|
588
|
+
walkDir(childFd, childPath, uid, gid);
|
|
589
|
+
} finally {
|
|
590
|
+
try {
|
|
591
|
+
closeSync(childFd);
|
|
592
|
+
} catch {
|
|
593
|
+
// Raced away; nothing to close.
|
|
594
|
+
}
|
|
595
|
+
}
|
|
596
|
+
continue;
|
|
597
|
+
}
|
|
598
|
+
// Symlinks stay untouched (never followed, never chowned), and so do
|
|
599
|
+
// entries a read-only open cannot name safely (fifos, sockets, devices).
|
|
600
|
+
if (entry.isSymbolicLink() || !entry.isFile()) continue;
|
|
601
|
+
const fd = openSync(childPath, constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK);
|
|
602
|
+
try {
|
|
603
|
+
const real = fdRealPath(fd);
|
|
604
|
+
if (real !== undefined && real.startsWith(`${dirReal}/`)) fchownSync(fd, uid, gid);
|
|
605
|
+
} finally {
|
|
606
|
+
try {
|
|
607
|
+
closeSync(fd);
|
|
608
|
+
} catch {
|
|
609
|
+
// Raced away; nothing to close.
|
|
610
|
+
}
|
|
611
|
+
}
|
|
612
|
+
} catch {
|
|
613
|
+
// O_NOFOLLOW refusal (a symlink swapped onto the name), or gone: nothing to chown.
|
|
614
|
+
}
|
|
615
|
+
}
|
|
616
|
+
}
|
|
617
|
+
|
|
439
618
|
export async function watchOrchestrator(d: Deps, now = Date.now()): Promise<void> {
|
|
440
619
|
const marker = join(stateDir(), STALL_MARKER_FILE);
|
|
441
620
|
const repeat = d.stall.paged;
|
|
@@ -554,6 +733,29 @@ export function pauseProvenance(
|
|
|
554
733
|
}
|
|
555
734
|
}
|
|
556
735
|
|
|
736
|
+
/**
|
|
737
|
+
* One pause sentinel FILE read as an instance identity: who set it, why, and
|
|
738
|
+
* the creation instant, all from that exact file. An unreadable or malformed
|
|
739
|
+
* file is undefined — the caller may treat it as absence.
|
|
740
|
+
*/
|
|
741
|
+
function pauseInstanceAt(
|
|
742
|
+
path: string,
|
|
743
|
+
): { source: string; reason?: string; since: number } | undefined {
|
|
744
|
+
try {
|
|
745
|
+
const [line1, line2] = readFileSync(path, "utf8").split("\n");
|
|
746
|
+
const since = Date.parse(line1?.trim() ?? "");
|
|
747
|
+
if (!Number.isFinite(since)) return undefined;
|
|
748
|
+
if (line2 === undefined) return undefined;
|
|
749
|
+
const match = /^source=(\S+)(?: reason="(.*)")?$/.exec(line2.trim());
|
|
750
|
+
if (match === null) return undefined;
|
|
751
|
+
const source = match[1]!;
|
|
752
|
+
const reason = match[2];
|
|
753
|
+
return { source, since, ...(reason === undefined ? {} : { reason }) };
|
|
754
|
+
} catch {
|
|
755
|
+
return undefined;
|
|
756
|
+
}
|
|
757
|
+
}
|
|
758
|
+
|
|
557
759
|
/**
|
|
558
760
|
* One pause sentinel read as a single identity: who set it, why, and the
|
|
559
761
|
* creation instant, all from the SAME file that was selected. Unlike pairing
|
|
@@ -573,19 +775,34 @@ export function pauseInstance(
|
|
|
573
775
|
: [pausedPath(project), pausedPath()];
|
|
574
776
|
const path = paths.find((candidate) => existsSync(candidate));
|
|
575
777
|
if (path === undefined) return undefined;
|
|
576
|
-
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
|
|
587
|
-
|
|
778
|
+
return pauseInstanceAt(path);
|
|
779
|
+
}
|
|
780
|
+
|
|
781
|
+
/**
|
|
782
|
+
* Compare-and-clear one pause sentinel (#780 review): remove `path` only while
|
|
783
|
+
* it still holds exactly the `expected` instance — same source, same reason,
|
|
784
|
+
* same creation instant — as read by {@link pauseInstance}. A hold or pause
|
|
785
|
+
* that replaced or recreated the sentinel between the read and the clear is a
|
|
786
|
+
* newer instance (writes always re-stamp `since`), so it is never destroyed:
|
|
787
|
+
* returning false keeps the newer fence in force. Scoped strictly to one
|
|
788
|
+
* caller-provided path, so auto-expiry can clear the per-project spend-cap
|
|
789
|
+
* sentinel without ever touching the legacy global sentinel.
|
|
790
|
+
*/
|
|
791
|
+
export function clearPauseIfUnchanged(
|
|
792
|
+
path: string,
|
|
793
|
+
expected: { source: string; reason?: string; since: number },
|
|
794
|
+
): boolean {
|
|
795
|
+
const current = pauseInstanceAt(path);
|
|
796
|
+
if (current === undefined) return false;
|
|
797
|
+
if (
|
|
798
|
+
current.source !== expected.source ||
|
|
799
|
+
current.since !== expected.since ||
|
|
800
|
+
current.reason !== expected.reason
|
|
801
|
+
) {
|
|
802
|
+
return false;
|
|
588
803
|
}
|
|
804
|
+
rmSync(path, { force: true });
|
|
805
|
+
return true;
|
|
589
806
|
}
|
|
590
807
|
|
|
591
808
|
/**
|
|
@@ -622,6 +839,173 @@ export function setPaused(
|
|
|
622
839
|
}
|
|
623
840
|
}
|
|
624
841
|
|
|
842
|
+
// ------------------------------------------------------------------- drain
|
|
843
|
+
// (#484 slice 1) A project drain is a durable, self-expiring admission fence:
|
|
844
|
+
// the same boundary as a pause — settlement above it, nothing claimed below —
|
|
845
|
+
// but recorded with an absolute deadline, so an orchestrator crash can never
|
|
846
|
+
// strand admission. The record is a JSON file under the state directory (like
|
|
847
|
+
// the pause sentinel), scoped to exactly one configured project, and replaced
|
|
848
|
+
// atomically on create. Human CLI wording and release-verb coupling are later
|
|
849
|
+
// #484 children.
|
|
850
|
+
|
|
851
|
+
/** The persisted shape of one project drain. Every field is validated when a
|
|
852
|
+
* record is read — a record that cannot be trusted is never a fence. */
|
|
853
|
+
export interface DrainRecord {
|
|
854
|
+
/** The configured project this drain fences; must match the reader. */
|
|
855
|
+
project: string;
|
|
856
|
+
/** ISO instant the drain intent was recorded. */
|
|
857
|
+
createdAt: string;
|
|
858
|
+
/** Absolute ISO deadline: admission resumes automatically at or after it. */
|
|
859
|
+
expiresAt: string;
|
|
860
|
+
/** Purpose recorded at creation (release window, maintenance…). */
|
|
861
|
+
reason?: string;
|
|
862
|
+
}
|
|
863
|
+
|
|
864
|
+
/** Create-time options for {@link createDrain}. */
|
|
865
|
+
export interface CreateDrainOptions {
|
|
866
|
+
/** Absolute epoch-ms deadline — a drain must always expire on its own. */
|
|
867
|
+
expiresAt: number;
|
|
868
|
+
/** Purpose, persisted on the record and shown in structured status. */
|
|
869
|
+
reason?: string;
|
|
870
|
+
}
|
|
871
|
+
|
|
872
|
+
/** Why a drain record could not be trusted, named deterministically. */
|
|
873
|
+
export type DrainProblem =
|
|
874
|
+
| "unparseable-json"
|
|
875
|
+
| "invalid-record"
|
|
876
|
+
| "invalid-project"
|
|
877
|
+
| "invalid-created-at"
|
|
878
|
+
| "invalid-expires-at"
|
|
879
|
+
| "expiry-not-future"
|
|
880
|
+
| "invalid-reason";
|
|
881
|
+
|
|
882
|
+
/** The verdict of one drain read. `error` is the malformed-record case: the
|
|
883
|
+
* caller may fail its pass closed, and the dispatch-side
|
|
884
|
+
* {@link consumeDrain} removes the record in the same transition, so it can
|
|
885
|
+
* never become a permanent drain. */
|
|
886
|
+
export type DrainVerdict =
|
|
887
|
+
| { kind: "active"; drain: DrainRecord }
|
|
888
|
+
| { kind: "inactive" }
|
|
889
|
+
| { kind: "error"; problem: DrainProblem };
|
|
890
|
+
|
|
891
|
+
/** Where this project's drain record lives. Project-scoped by construction:
|
|
892
|
+
* a record present at one project's path never fences another project. */
|
|
893
|
+
export function drainPath(project: string): string {
|
|
894
|
+
return join(stateDir(), `drain-${project}.json`);
|
|
895
|
+
}
|
|
896
|
+
|
|
897
|
+
/**
|
|
898
|
+
* Records a bounded drain intent for `project`, replacing any prior drain of
|
|
899
|
+
* the same project atomically (tmp + rename, exactly like the admission ack).
|
|
900
|
+
* The record is a file, so it survives orchestrator and daemon loss; the
|
|
901
|
+
* absolute `expiresAt` is what stops it from ever stranding admission.
|
|
902
|
+
*/
|
|
903
|
+
export function createDrain(
|
|
904
|
+
project: string,
|
|
905
|
+
opts: CreateDrainOptions,
|
|
906
|
+
now = Date.now(),
|
|
907
|
+
): void {
|
|
908
|
+
if (project === "") {
|
|
909
|
+
throw new Error("drain project must not be empty");
|
|
910
|
+
}
|
|
911
|
+
if (!Number.isFinite(opts.expiresAt) || opts.expiresAt <= now) {
|
|
912
|
+
throw new Error("drain expiresAt must be a finite epoch-ms timestamp in the future");
|
|
913
|
+
}
|
|
914
|
+
if (opts.reason !== undefined && typeof opts.reason !== "string") {
|
|
915
|
+
throw new Error("drain reason must be a string");
|
|
916
|
+
}
|
|
917
|
+
const record: DrainRecord = {
|
|
918
|
+
project,
|
|
919
|
+
createdAt: new Date(now).toISOString(),
|
|
920
|
+
expiresAt: new Date(opts.expiresAt).toISOString(),
|
|
921
|
+
...(opts.reason === undefined ? {} : { reason: opts.reason }),
|
|
922
|
+
};
|
|
923
|
+
const path = drainPath(project);
|
|
924
|
+
mkdirSync(dirname(path), { recursive: true });
|
|
925
|
+
const tmp = `${path}.${process.pid.toString(36)}.${Date.now().toString(36)}.tmp`;
|
|
926
|
+
writeFileSync(tmp, `${JSON.stringify(record, null, 2)}\n`);
|
|
927
|
+
renameSync(tmp, path);
|
|
928
|
+
}
|
|
929
|
+
|
|
930
|
+
/**
|
|
931
|
+
* One drained state read with no side effects — this never touches the record
|
|
932
|
+
* on disk. That purity is what lets observational surfaces live off it: a
|
|
933
|
+
* status/dashboard/health read must not consume a malformed record before a
|
|
934
|
+
* dispatch pass fails closed on it, or the pass would read absent and admit.
|
|
935
|
+
* The dispatch side performs the actual cleanup transition through
|
|
936
|
+
* {@link consumeDrain}. Callers that only need to *see* the state (including
|
|
937
|
+
* the claim path, which refuses but must not unbind its own pass) use this.
|
|
938
|
+
*/
|
|
939
|
+
export function readDrain(project: string, now = Date.now()): DrainVerdict {
|
|
940
|
+
const path = drainPath(project);
|
|
941
|
+
if (!existsSync(path)) return { kind: "inactive" };
|
|
942
|
+
let raw: unknown;
|
|
943
|
+
try {
|
|
944
|
+
raw = JSON.parse(readFileSync(path, "utf8"));
|
|
945
|
+
} catch {
|
|
946
|
+
return { kind: "error", problem: "unparseable-json" };
|
|
947
|
+
}
|
|
948
|
+
if (raw === null || typeof raw !== "object" || Array.isArray(raw)) {
|
|
949
|
+
return { kind: "error", problem: "invalid-record" };
|
|
950
|
+
}
|
|
951
|
+
const rec = raw as Record<string, unknown>;
|
|
952
|
+
if (typeof rec["project"] !== "string" || rec["project"] === "") {
|
|
953
|
+
return { kind: "error", problem: "invalid-project" };
|
|
954
|
+
}
|
|
955
|
+
if (typeof rec["createdAt"] !== "string" || Number.isNaN(Date.parse(rec["createdAt"]))) {
|
|
956
|
+
return { kind: "error", problem: "invalid-created-at" };
|
|
957
|
+
}
|
|
958
|
+
if (typeof rec["expiresAt"] !== "string" || Number.isNaN(Date.parse(rec["expiresAt"]))) {
|
|
959
|
+
return { kind: "error", problem: "invalid-expires-at" };
|
|
960
|
+
}
|
|
961
|
+
if (Date.parse(rec["expiresAt"] as string) <= Date.parse(rec["createdAt"] as string)) {
|
|
962
|
+
return { kind: "error", problem: "expiry-not-future" };
|
|
963
|
+
}
|
|
964
|
+
const reason = rec["reason"];
|
|
965
|
+
if (reason !== undefined && typeof reason !== "string") {
|
|
966
|
+
return { kind: "error", problem: "invalid-reason" };
|
|
967
|
+
}
|
|
968
|
+
if (rec["project"] !== project) {
|
|
969
|
+
// A record persisted at this project's path but naming another project is
|
|
970
|
+
// either a copy or a rename mishap; it fences nobody (that project's drain
|
|
971
|
+
// lives at its own path).
|
|
972
|
+
return { kind: "inactive" };
|
|
973
|
+
}
|
|
974
|
+
if (Date.parse(rec["expiresAt"] as string) <= now) {
|
|
975
|
+
return { kind: "inactive" };
|
|
976
|
+
}
|
|
977
|
+
const drain: DrainRecord = {
|
|
978
|
+
project: rec["project"],
|
|
979
|
+
createdAt: rec["createdAt"],
|
|
980
|
+
expiresAt: rec["expiresAt"],
|
|
981
|
+
...(reason === undefined ? {} : { reason }),
|
|
982
|
+
};
|
|
983
|
+
return { kind: "active", drain };
|
|
984
|
+
}
|
|
985
|
+
|
|
986
|
+
/**
|
|
987
|
+
* The dispatch-side drain read: the same verdict as {@link readDrain}, and the
|
|
988
|
+
* one place a stale or untrustworthy record is cleared as a side effect —
|
|
989
|
+
* expired and wrong-project records are removed (bounded stale-state cleanup
|
|
990
|
+
* on the next pass), and a malformed record is removed in the very transition
|
|
991
|
+
* that fails the pass closed, so it can never become an unbounded permanent
|
|
992
|
+
* drain. Only dispatch callers use this: an observational read here would let
|
|
993
|
+
* a status/health reader consume the malformed marker before the pass that
|
|
994
|
+
* must fail closed on it ever ran.
|
|
995
|
+
*/
|
|
996
|
+
export function consumeDrain(project: string, now = Date.now()): DrainVerdict {
|
|
997
|
+
const verdict = readDrain(project, now);
|
|
998
|
+
if (verdict.kind !== "active") {
|
|
999
|
+
rmSync(drainPath(project), { force: true });
|
|
1000
|
+
}
|
|
1001
|
+
return verdict;
|
|
1002
|
+
}
|
|
1003
|
+
|
|
1004
|
+
/** Removes this project's drain, idempotently — a second cancel is a no-op. */
|
|
1005
|
+
export function cancelDrain(project: string): void {
|
|
1006
|
+
rmSync(drainPath(project), { force: true });
|
|
1007
|
+
}
|
|
1008
|
+
|
|
625
1009
|
// ----------------------------------------------------------------- admission
|
|
626
1010
|
// acknowledgement (#651, review #3)
|
|
627
1011
|
//
|
|
@@ -991,6 +1375,36 @@ function renderDiscussion(comments: IssueComment[] | "unread", lane?: FileLane):
|
|
|
991
1375
|
return lines.join("\n");
|
|
992
1376
|
}
|
|
993
1377
|
|
|
1378
|
+
/**
|
|
1379
|
+
* The parsed file lane as a brief section (#724): the file list the gate will
|
|
1380
|
+
* enforce, or the explicit fail-open note — the same `laneEcho` the promotion
|
|
1381
|
+
* verb prints, so the author and the worker read one parse. Deliberately only
|
|
1382
|
+
* the parse, never the source line: the prose already renders in the body or
|
|
1383
|
+
* the discussion, and in #720's case the prose is exactly what looked
|
|
1384
|
+
* reasonable to a human while parsed greedily.
|
|
1385
|
+
*/
|
|
1386
|
+
function laneBlock(lane: FileLane | undefined): string {
|
|
1387
|
+
const echo = laneEcho(lane);
|
|
1388
|
+
const body = lane === undefined ? `_${echo}_` : `\`${echo}\``;
|
|
1389
|
+
return ["## File lane (as parsed)", "", body, "", ""].join("\n");
|
|
1390
|
+
}
|
|
1391
|
+
|
|
1392
|
+
/**
|
|
1393
|
+
* The declared model as a brief section (#535): the selector the dispatch
|
|
1394
|
+
* will launch on — the same parse admission carried, never a second read —
|
|
1395
|
+
* or the explicit fail-open note, so the worker reads the run's model on the
|
|
1396
|
+
* brief itself rather than inferring it. Deliberately only the selector,
|
|
1397
|
+
* never the source line: the declaration's prose already renders in the body
|
|
1398
|
+
* or Discussion.
|
|
1399
|
+
*/
|
|
1400
|
+
function modelBlock(model: EffectiveModel | undefined): string {
|
|
1401
|
+
const body =
|
|
1402
|
+
model === undefined
|
|
1403
|
+
? "_no model declared — the project's workerModel (or harness default) is in effect_"
|
|
1404
|
+
: `\`${model.model}\``;
|
|
1405
|
+
return ["## Model (as parsed)", "", body, "", ""].join("\n");
|
|
1406
|
+
}
|
|
1407
|
+
|
|
994
1408
|
/**
|
|
995
1409
|
* What an orphan-resumed worker is told about the file lane on top of the
|
|
996
1410
|
* continuation notice (#608). The original brief already in the transcript
|
|
@@ -1008,6 +1422,8 @@ function resumeLaneBlock(lane: FileLane): string {
|
|
|
1008
1422
|
"",
|
|
1009
1423
|
lane.source,
|
|
1010
1424
|
"",
|
|
1425
|
+
`Parsed files: ${laneEcho(lane)}.`,
|
|
1426
|
+
"",
|
|
1011
1427
|
].join("\n");
|
|
1012
1428
|
}
|
|
1013
1429
|
|
|
@@ -1024,211 +1440,6 @@ function swapLabel(store: Store, projectName: string, issue: number, from: strin
|
|
|
1024
1440
|
{ issue, op: "remove", label: from },
|
|
1025
1441
|
]);
|
|
1026
1442
|
}
|
|
1027
|
-
/**
|
|
1028
|
-
* Persist the operator-stop transition before releasing its live controller.
|
|
1029
|
-
* The row is terminal first, then its in-progress label is removed through the
|
|
1030
|
-
* same durable projection outbox as every other lifecycle transition.
|
|
1031
|
-
*/
|
|
1032
|
-
export function recordOperatorStop(
|
|
1033
|
-
store: Pick<Store, "updateRun" | "enqueueLabelOps">,
|
|
1034
|
-
args: {
|
|
1035
|
-
project: string;
|
|
1036
|
-
issue: number;
|
|
1037
|
-
runId: string;
|
|
1038
|
-
inProgress: string;
|
|
1039
|
-
reason: string;
|
|
1040
|
-
patch: Partial<RunRecord>;
|
|
1041
|
-
},
|
|
1042
|
-
): void {
|
|
1043
|
-
store.updateRun(args.runId, {
|
|
1044
|
-
...args.patch,
|
|
1045
|
-
state: "stopped",
|
|
1046
|
-
lastError: `operator stopped: ${args.reason}`,
|
|
1047
|
-
});
|
|
1048
|
-
store.enqueueLabelOps(args.project, [
|
|
1049
|
-
{ issue: args.issue, op: "remove", label: args.inProgress },
|
|
1050
|
-
]);
|
|
1051
|
-
}
|
|
1052
|
-
|
|
1053
|
-
|
|
1054
|
-
/**
|
|
1055
|
-
* The escalator throws when no transport is configured or Telegram rejects, and
|
|
1056
|
-
* only records the dedup marker on success. A page that cannot be delivered
|
|
1057
|
-
* must not take the tick down with it — log it and let the next tick retry.
|
|
1058
|
-
*
|
|
1059
|
-
* Returns whether it actually went out, because "page once" and "page once
|
|
1060
|
-
* *successfully*" are different promises: a caller that latches a once-only
|
|
1061
|
-
* gate on the attempt turns one failed delivery into permanent silence about a
|
|
1062
|
-
* condition that is still true.
|
|
1063
|
-
*/
|
|
1064
|
-
|
|
1065
|
-
async function reactToProviderCredit(
|
|
1066
|
-
d: Deps,
|
|
1067
|
-
issue: number,
|
|
1068
|
-
message: string,
|
|
1069
|
-
sessionFile: string | undefined,
|
|
1070
|
-
): Promise<void> {
|
|
1071
|
-
const { project } = d;
|
|
1072
|
-
const alreadyPaused = isPaused(project.name);
|
|
1073
|
-
if (!alreadyPaused) setPaused(true, { source: "provider-credit", reason: message }, project.name);
|
|
1074
|
-
log(
|
|
1075
|
-
`#${issue} provider refused for credit — dispatch ${alreadyPaused ? "remains paused" : "paused"}: ${message}`,
|
|
1076
|
-
);
|
|
1077
|
-
// Fleet-scoped and run-independent on purpose. The notification ledger
|
|
1078
|
-
// dedupes on `project:issue:tier:summary`, so `NO_ISSUE` plus a summary
|
|
1079
|
-
// carrying no run or attempt pages once for the fleet, not once per run.
|
|
1080
|
-
await safeEscalate(d, {
|
|
1081
|
-
tier: 2,
|
|
1082
|
-
category: "fleet-stopped",
|
|
1083
|
-
project: project.name,
|
|
1084
|
-
issue: NO_ISSUE,
|
|
1085
|
-
summary: `Model provider refused for credit — ${project.name} is paused`,
|
|
1086
|
-
detail: [
|
|
1087
|
-
message,
|
|
1088
|
-
"",
|
|
1089
|
-
"No implementation attempt was charged: this is a billing state, not a",
|
|
1090
|
-
"failed implementation. Each affected issue keeps its queue label and",
|
|
1091
|
-
"re-dispatches on `omp-conductor resume` once the provider has credit.",
|
|
1092
|
-
`Session: ${sessionFile ?? "(no transcript)"}`,
|
|
1093
|
-
].join("\n"),
|
|
1094
|
-
});
|
|
1095
|
-
}
|
|
1096
|
-
|
|
1097
|
-
/**
|
|
1098
|
-
* What a salvage attempt contributes to the escalation: where the work went, or
|
|
1099
|
-
* that it went nowhere. Split from the effects below for the same reason
|
|
1100
|
-
* `checkIntegrity` is — this wording is the whole thing a human acts on, so it
|
|
1101
|
-
* is worth a test holding it, and the sha in it is the only pointer to work
|
|
1102
|
-
* that no longer has any other copy.
|
|
1103
|
-
*
|
|
1104
|
-
* `retained` is not cosmetic. These lines used to promise a tree "kept for
|
|
1105
|
-
* inspection" unconditionally, which was true only because salvage ran only on
|
|
1106
|
-
* the paths that keep one. A blocked run's tree is removed the moment its work
|
|
1107
|
-
* is safely on the branch, and sending an operator to a path this process just
|
|
1108
|
-
* deleted is the same class of mistake as #118 itself.
|
|
1109
|
-
*/
|
|
1110
|
-
export function salvageLines(
|
|
1111
|
-
outcome: SalvageOutcome,
|
|
1112
|
-
worktree: string,
|
|
1113
|
-
retained: boolean,
|
|
1114
|
-
): string[] {
|
|
1115
|
-
const fate = retained
|
|
1116
|
-
? `Worktree kept for inspection: ${worktree}`
|
|
1117
|
-
: `Worktree removed: ${worktree}`;
|
|
1118
|
-
|
|
1119
|
-
if (outcome.kind === "nothing") return [`${fate} — nothing uncommitted to salvage`];
|
|
1120
|
-
|
|
1121
|
-
if (outcome.kind === "failed") {
|
|
1122
|
-
return [
|
|
1123
|
-
`WIP SALVAGE FAILED: ${outcome.error}`,
|
|
1124
|
-
`Uncommitted work in ${worktree} is the only copy of it, so the tree was kept.`,
|
|
1125
|
-
"This issue is held out of dispatch until the tree is recovered by hand and",
|
|
1126
|
-
"`omp-conductor unblock <n> --force` records that you accepted it.",
|
|
1127
|
-
];
|
|
1128
|
-
}
|
|
1129
|
-
|
|
1130
|
-
const where =
|
|
1131
|
-
`WIP committed to ${outcome.branch} @ ${outcome.sha}` +
|
|
1132
|
-
(outcome.pushed
|
|
1133
|
-
? " and pushed — the work outlives this worktree"
|
|
1134
|
-
: ` but NOT pushed (${outcome.pushError ?? "no reason given"}) — it lives only in this host's mirror`);
|
|
1135
|
-
// Manifest belongs in the escalation too: opening the commit is how the
|
|
1136
|
-
// orchestrator talked itself into scrubbing a worker tree (#38).
|
|
1137
|
-
const n = outcome.files.length;
|
|
1138
|
-
const count = `${n} file${n === 1 ? "" : "s"}`;
|
|
1139
|
-
const manifest =
|
|
1140
|
-
outcome.newPaths.length === 0
|
|
1141
|
-
? `${count} (all modifications to tracked paths)`
|
|
1142
|
-
: `${count}; new: ${outcome.newPaths.slice(0, 12).join(", ")}${
|
|
1143
|
-
outcome.newPaths.length > 12 ? `, … +${outcome.newPaths.length - 12} more` : ""
|
|
1144
|
-
}`;
|
|
1145
|
-
return [where, manifest, fate];
|
|
1146
|
-
}
|
|
1147
|
-
|
|
1148
|
-
/** Everything a settled run has to record and say about its worktree. */
|
|
1149
|
-
export interface WorktreeSettlement {
|
|
1150
|
-
outcome: SalvageOutcome;
|
|
1151
|
-
/** Whether the tree still exists now the run is over. */
|
|
1152
|
-
retained: boolean;
|
|
1153
|
-
/** Escalation lines naming where the work went. */
|
|
1154
|
-
lines: string[];
|
|
1155
|
-
/** Row fields recording the durable ref, or the failure that blocks a re-claim. */
|
|
1156
|
-
patch: Pick<RunRecord, "salvageSha" | "salvageError">;
|
|
1157
|
-
}
|
|
1158
|
-
|
|
1159
|
-
/**
|
|
1160
|
-
* Decides what becomes of a finished run's worktree: save the work, then keep
|
|
1161
|
-
* or remove the tree, then say which.
|
|
1162
|
-
*
|
|
1163
|
-
* One function because the two halves are one decision and splitting them is
|
|
1164
|
-
* how #118 happened — the removal at the end of dispatch had no idea whether
|
|
1165
|
-
* anything had been saved, and the salvage at the top of the failure branch had
|
|
1166
|
-
* no idea the blocked branch fell through to a `--force` removal.
|
|
1167
|
-
*
|
|
1168
|
-
* A salvage that *fails* retains the tree whatever the caller asked for. There
|
|
1169
|
-
* was real work, git refused to commit it, and the tree is now the only copy in
|
|
1170
|
-
* existence: deleting it on schedule would be the data loss this whole path
|
|
1171
|
-
* exists to prevent. The issue is held out of dispatch until an operator says
|
|
1172
|
-
* otherwise, because the next attempt's `worktree remove --force` would finish
|
|
1173
|
-
* the job (see `admitCandidates`).
|
|
1174
|
-
*
|
|
1175
|
-
* Exported so a test can drive the real decision against a real git tree.
|
|
1176
|
-
*/
|
|
1177
|
-
export async function settleWorktree(
|
|
1178
|
-
args: {
|
|
1179
|
-
issue: number;
|
|
1180
|
-
attempt: number;
|
|
1181
|
-
/** Clause for the commit subject: "killed by the turns cap", "blocked …". */
|
|
1182
|
-
ending: string;
|
|
1183
|
-
worktree: string;
|
|
1184
|
-
/** The run's branch, so the pre-removal publish names the right ref. */
|
|
1185
|
-
branch: string;
|
|
1186
|
-
/**
|
|
1187
|
-
* Publishes the run branch on the privileged side. Required rather than
|
|
1188
|
-
* optional: the run's commits live in a repository of its own,
|
|
1189
|
-
* so a removal that did not publish first would delete the only copy —
|
|
1190
|
-
* which is #121's data loss with one extra step. `undefined` is a visible
|
|
1191
|
-
* decision at the call site, never an omission.
|
|
1192
|
-
*/
|
|
1193
|
-
publish: RunPublisher | undefined;
|
|
1194
|
-
} & (
|
|
1195
|
-
| /** Terminal-failure and orphan trees are evidence, and are kept even when clean. */
|
|
1196
|
-
{ tree: "keep" }
|
|
1197
|
-
| { tree: "remove"; mirrorPath: string }
|
|
1198
|
-
),
|
|
1199
|
-
): Promise<WorktreeSettlement> {
|
|
1200
|
-
const { issue, attempt, ending, worktree, branch, publish } = args;
|
|
1201
|
-
const outcome = await salvageWip(worktree, issue, attempt, ending, publish);
|
|
1202
|
-
const retained = args.tree === "keep" || outcome.kind === "failed";
|
|
1203
|
-
if (!retained && args.tree === "remove") {
|
|
1204
|
-
// Before the removal, always — not only when salvage found something. A run
|
|
1205
|
-
// that *committed* and could not publish has its work in its own repository
|
|
1206
|
-
// and nowhere else, and salvage never sees a committed tree because it is
|
|
1207
|
-
// clean. The mirror fetch inside `publish` is what preserves it; the push
|
|
1208
|
-
// to GitHub can fail (no network, protected ref) and the work still lives.
|
|
1209
|
-
const published = await publish?.(branch);
|
|
1210
|
-
if (published !== undefined && !published.ok) {
|
|
1211
|
-
log(`#${issue} publish before removal failed, work is preserved in the mirror: ${published.stderr}`);
|
|
1212
|
-
}
|
|
1213
|
-
await removeWorktree(args.mirrorPath, worktree);
|
|
1214
|
-
}
|
|
1215
|
-
|
|
1216
|
-
const lines = salvageLines(outcome, worktree, retained);
|
|
1217
|
-
log(`#${issue} salvage: ${lines.join(" ")}`);
|
|
1218
|
-
return {
|
|
1219
|
-
outcome,
|
|
1220
|
-
retained,
|
|
1221
|
-
lines,
|
|
1222
|
-
patch:
|
|
1223
|
-
outcome.kind === "salvaged"
|
|
1224
|
-
? { salvageSha: outcome.sha }
|
|
1225
|
-
: outcome.kind === "failed"
|
|
1226
|
-
? { salvageError: outcome.error }
|
|
1227
|
-
: {},
|
|
1228
|
-
};
|
|
1229
|
-
}
|
|
1230
|
-
|
|
1231
|
-
|
|
1232
1443
|
/**
|
|
1233
1444
|
* How a run's end is named — in the salvage commit, and to whoever reads it.
|
|
1234
1445
|
* The whole clause, not a bare reason: a graceful block was not killed by
|
|
@@ -1271,6 +1482,22 @@ export async function buildBrief(
|
|
|
1271
1482
|
* resolved from the rendered thread itself.
|
|
1272
1483
|
*/
|
|
1273
1484
|
lane?: FileLane;
|
|
1485
|
+
/**
|
|
1486
|
+
* The effective `Model:` declaration admission resolved for this
|
|
1487
|
+
* candidate (#535). When carried, the brief echoes exactly this
|
|
1488
|
+
* selector — the same value dispatch launches on — instead of
|
|
1489
|
+
* recomputing from the dispatch-time comment read, so the run's model is
|
|
1490
|
+
* one value on every surface. Absent (unit-level callers), the model is
|
|
1491
|
+
* resolved from the rendered thread itself.
|
|
1492
|
+
*/
|
|
1493
|
+
model?: EffectiveModel;
|
|
1494
|
+
/**
|
|
1495
|
+
* The typed host-constraints block the brief renders (#721), re-read with
|
|
1496
|
+
* the config at the tick boundary so an operator edit applies on the next
|
|
1497
|
+
* dispatch. Absent renders no host-constraints section at all — the
|
|
1498
|
+
* brief is byte-for-byte what it always was.
|
|
1499
|
+
*/
|
|
1500
|
+
host?: HostConstraints;
|
|
1274
1501
|
} = {},
|
|
1275
1502
|
): Promise<string> {
|
|
1276
1503
|
// Read per dispatch rather than caching: editing the brief then takes effect
|
|
@@ -1315,6 +1542,16 @@ export async function buildBrief(
|
|
|
1315
1542
|
// discussion budget, keeping the gate and the worker-visible brief on one
|
|
1316
1543
|
// lane (#608).
|
|
1317
1544
|
const comments = opts.comments ?? [];
|
|
1545
|
+
// The effective lane admission resolved (or resolves) for this candidate:
|
|
1546
|
+
// the carried admission snapshot when dispatch has one, else the thread
|
|
1547
|
+
// itself. Both the discussion renderer and the parsed-lane section draw on
|
|
1548
|
+
// the same value, so the brief shows one lane on every surface (#608, #724).
|
|
1549
|
+
const lane = opts.lane ?? (comments === "unread" ? undefined : effectiveLane(r.issue.body, comments));
|
|
1550
|
+
// The effective model declaration admission resolved (or resolves) for this
|
|
1551
|
+
// candidate: the carried admission snapshot when dispatch has one, else the
|
|
1552
|
+
// thread itself. Dispatch launches on the same value, so the brief shows
|
|
1553
|
+
// one model on every surface (#535).
|
|
1554
|
+
const model = opts.model ?? (comments === "unread" ? undefined : effectiveModel(r.issue.body, comments));
|
|
1318
1555
|
return renderBrief(template, {
|
|
1319
1556
|
ISSUE_NUMBER: String(r.issue.number),
|
|
1320
1557
|
ISSUE_TITLE: r.issue.title,
|
|
@@ -1323,19 +1560,24 @@ export async function buildBrief(
|
|
|
1323
1560
|
BRANCH: branch,
|
|
1324
1561
|
WORKTREE: worktree,
|
|
1325
1562
|
ACCEPTANCE_CRITERIA: acceptanceCriteria(r.issue),
|
|
1326
|
-
ISSUE_COMMENTS: renderDiscussion(
|
|
1327
|
-
|
|
1328
|
-
|
|
1329
|
-
),
|
|
1563
|
+
ISSUE_COMMENTS: renderDiscussion(comments, lane),
|
|
1564
|
+
FILE_LANE: laneBlock(lane),
|
|
1565
|
+
MODEL: modelBlock(model),
|
|
1330
1566
|
GATES: gatesBlock(r.repo),
|
|
1331
1567
|
// The guarded shell suites and their sanctioned parse-only check, derived
|
|
1332
1568
|
// from SHARED_HOST_SCRIPTS so the brief and the refusal share one list
|
|
1333
1569
|
// (#687). Non-empty whenever the guard has paths, so the placeholder line
|
|
1334
1570
|
// always renders to a line.
|
|
1335
1571
|
SHARED_HOST_NOTICE: sharedHostBriefNotice(),
|
|
1336
|
-
//
|
|
1337
|
-
//
|
|
1338
|
-
//
|
|
1572
|
+
// The typed host-constraints paragraph (#721): derived cores/RAM folded
|
|
1573
|
+
// into the operator's description, the non-interactive PATH, and the
|
|
1574
|
+
// routed repo's convention. Empty when the config names none — no
|
|
1575
|
+
// section, no placeholder text.
|
|
1576
|
+
HOST_CONSTRAINTS: hostConstraintsNotice(opts.host, repoSlug(r.repo)),
|
|
1577
|
+
// The brief's code-graph paragraph: the exact `project` key for a
|
|
1578
|
+
// configured repo, or an explicit "no graph" statement for an
|
|
1579
|
+
// unconfigured one — never silence, because a worker that knows there is
|
|
1580
|
+
// no graph stops looking for it.
|
|
1339
1581
|
GRAPH_HINT: graphHint(r.repo),
|
|
1340
1582
|
CONTINUATION: continuation,
|
|
1341
1583
|
});
|
|
@@ -1344,178 +1586,6 @@ export async function buildBrief(
|
|
|
1344
1586
|
// ------------------------------------------------------------------- one issue
|
|
1345
1587
|
|
|
1346
1588
|
|
|
1347
|
-
/** The failure classes `countContinuations` deliberately does not charge — the
|
|
1348
|
-
* inverted copy of its exclusions, kept beside the breakdown that consumes it
|
|
1349
|
-
* so the two can only drift together (#439). `orphan-clean` is absent on
|
|
1350
|
-
* purpose: daemon orphans consume the continuation budget, which is exactly
|
|
1351
|
-
* why the requeue side has to respect the ceiling instead of racing it. */
|
|
1352
|
-
const NON_CONTINUATION_CLASSES: Partial<Record<FailureClass, true>> = {
|
|
1353
|
-
"admin-kill": true,
|
|
1354
|
-
"settlement-stuck": true,
|
|
1355
|
-
"env-start-failure": true,
|
|
1356
|
-
"dispatch-infra": true,
|
|
1357
|
-
"provider-credit": true,
|
|
1358
|
-
"provider-transient": true,
|
|
1359
|
-
"provider-capacity": true,
|
|
1360
|
-
};
|
|
1361
|
-
|
|
1362
|
-
/** How one issue spent its continuation budget, grouped by failure class —
|
|
1363
|
-
* the exact rows `continuationsFor` charges, so an exhaustion escalation
|
|
1364
|
-
* reports the same budget it says is spent. `unclassified` groups rows that
|
|
1365
|
-
* charged before the class was written (a pre-upgrade NULL). */
|
|
1366
|
-
function continuationBreakdown(runs: readonly RunRecord[]): Map<string, number> {
|
|
1367
|
-
const perClass = new Map<string, number>();
|
|
1368
|
-
for (const r of runs) {
|
|
1369
|
-
const chargedAsKilledOrOrphaned =
|
|
1370
|
-
(r.state === "killed" || r.state === "orphaned" || r.state === "blocked") &&
|
|
1371
|
-
(r.failureClass === undefined || NON_CONTINUATION_CLASSES[r.failureClass] === undefined);
|
|
1372
|
-
const chargedAsReturned = r.state === "failed" && r.failureClass === "returned-for-revision";
|
|
1373
|
-
if (!chargedAsKilledOrOrphaned && !chargedAsReturned) continue;
|
|
1374
|
-
const cls = r.failureClass ?? "unclassified";
|
|
1375
|
-
perClass.set(cls, (perClass.get(cls) ?? 0) + 1);
|
|
1376
|
-
}
|
|
1377
|
-
return perClass;
|
|
1378
|
-
}
|
|
1379
|
-
|
|
1380
|
-
/** The newest attempt's preserved work, if any, so an exhaustion escalation can
|
|
1381
|
-
* say whether continuing is worthwhile: `salvageSha`/`headSha`/`prUrl` are the
|
|
1382
|
-
* three artifacts a run can leave, and all null on a branch nothing reached. */
|
|
1383
|
-
function newestContinuableRun(runs: readonly RunRecord[]): RunRecord | undefined {
|
|
1384
|
-
const newestFirst = [...runs].reverse();
|
|
1385
|
-
return newestFirst.find(
|
|
1386
|
-
(r) => r.salvageSha !== undefined || r.headSha !== undefined || r.prUrl !== undefined,
|
|
1387
|
-
);
|
|
1388
|
-
}
|
|
1389
|
-
|
|
1390
|
-
/** The fenced-block info string that marks an exhaustion postmortem comment, so
|
|
1391
|
-
* a grooming scout re-slicing the issue can find and parse the whole block by
|
|
1392
|
-
* grepping for it. */
|
|
1393
|
-
export const POSTMORTEM_MARKER = "conductor-postmortem";
|
|
1394
|
-
|
|
1395
|
-
/** How one issue's attempt chain failed, as "3× ci-deterministic, 1× …" — the
|
|
1396
|
-
* digest shape for naming what the exhaustion was. Groups every row by its
|
|
1397
|
-
* failure class, whether or not it charged the continuation budget, because
|
|
1398
|
-
* the postmortem tells the whole story and not just the budget half (#290). */
|
|
1399
|
-
export function attemptClassBreakdown(runs: readonly RunRecord[]): string {
|
|
1400
|
-
const perClass = new Map<string, number>();
|
|
1401
|
-
for (const r of runs) {
|
|
1402
|
-
const cls = r.failureClass ?? "unclassified";
|
|
1403
|
-
perClass.set(cls, (perClass.get(cls) ?? 0) + 1);
|
|
1404
|
-
}
|
|
1405
|
-
if (perClass.size === 0) return "unclassified";
|
|
1406
|
-
return Array.from(perClass, ([cls, n]) => `${n}× ${cls}`).join(", ");
|
|
1407
|
-
}
|
|
1408
|
-
|
|
1409
|
-
/** Wall-clock duration of one run as a compact human string ("45m", "1h30m"). */
|
|
1410
|
-
export function humanDuration(ms: number): string {
|
|
1411
|
-
const seconds = Math.max(0, Math.round(ms / 1_000));
|
|
1412
|
-
if (seconds < 60) return `${seconds}s`;
|
|
1413
|
-
const minutes = Math.round(seconds / 60);
|
|
1414
|
-
if (minutes < 60) return `${minutes}m`;
|
|
1415
|
-
const hours = Math.floor(minutes / 60);
|
|
1416
|
-
const rest = minutes % 60;
|
|
1417
|
-
return rest === 0 ? `${hours}h` : `${hours}h${rest}m`;
|
|
1418
|
-
}
|
|
1419
|
-
|
|
1420
|
-
/** Flatten and bound a run's last error to one greppable table line. */
|
|
1421
|
-
export function oneLineBrief(text: string | undefined): string | undefined {
|
|
1422
|
-
if (text === undefined || text.trim() === "") return undefined;
|
|
1423
|
-
const flat = text.replace(/\s+/g, " ").trim();
|
|
1424
|
-
return flat.length > 90 ? `${flat.slice(0, 89)}…` : flat;
|
|
1425
|
-
}
|
|
1426
|
-
|
|
1427
|
-
/**
|
|
1428
|
-
* The exhaustion postmortem block: one greppable fenced block covering every
|
|
1429
|
-
* attempt in the chain — continuation rows included — with per-attempt turns,
|
|
1430
|
-
* wall clock, failure class and a one-line last error, the explicit salvage
|
|
1431
|
-
* state, the spend total and the transcript paths for local inspection.
|
|
1432
|
-
*
|
|
1433
|
-
* Pure so the tests hold the shape, not the transport: the writer below owns
|
|
1434
|
-
* the once-only guarantee, this owns what "once" looks like.
|
|
1435
|
-
*/
|
|
1436
|
-
export function formatExhaustionPostmortem(args: {
|
|
1437
|
-
issue: number;
|
|
1438
|
-
runs: readonly RunRecord[];
|
|
1439
|
-
reason: string;
|
|
1440
|
-
}): string {
|
|
1441
|
-
const { issue, runs, reason } = args;
|
|
1442
|
-
const artifact = newestContinuableRun(runs);
|
|
1443
|
-
const totalSpend = runs.reduce((sum, r) => sum + r.spendUsd, 0);
|
|
1444
|
-
const spend = `$${totalSpend.toFixed(2)}`;
|
|
1445
|
-
const attemptLines = runs.map((r) => {
|
|
1446
|
-
const wall = r.endedAt === undefined ? "—" : humanDuration(r.endedAt - r.startedAt);
|
|
1447
|
-
const cls = r.failureClass ?? "unclassified";
|
|
1448
|
-
const error = oneLineBrief(r.lastError) ?? "—";
|
|
1449
|
-
return (
|
|
1450
|
-
` attempt ${r.attempt} ${r.state.padEnd(12)} turns ${r.turns}/${r.maxTurns} ` +
|
|
1451
|
-
`${cls.padEnd(24)} ${wall.padStart(4)} last error: ${error}`
|
|
1452
|
-
);
|
|
1453
|
-
});
|
|
1454
|
-
const salvage =
|
|
1455
|
-
artifact === undefined
|
|
1456
|
-
? "Salvaged WIP: absent — no attempt preserved a branch, head SHA or pull request."
|
|
1457
|
-
: `Salvaged WIP: present — branch ${artifact.branch} at ${artifact.headSha ?? artifact.salvageSha}` +
|
|
1458
|
-
`${artifact.prUrl === undefined ? "" : ` (PR ${artifact.prUrl})`}.`;
|
|
1459
|
-
return [
|
|
1460
|
-
`\`\`\`${POSTMORTEM_MARKER}`,
|
|
1461
|
-
`#${issue} exhausted: ${attemptClassBreakdown(runs)}`,
|
|
1462
|
-
reason,
|
|
1463
|
-
"",
|
|
1464
|
-
`Attempts (${runs.length} total, ${spend} spend):`,
|
|
1465
|
-
...attemptLines,
|
|
1466
|
-
salvage,
|
|
1467
|
-
`Spend total: ${spend} across ${runs.length} attempts.`,
|
|
1468
|
-
"Transcripts:",
|
|
1469
|
-
...runs.map((r) => (r.sessionFile === undefined ? " (none)" : ` ${r.sessionFile}`)),
|
|
1470
|
-
"```",
|
|
1471
|
-
].join("\n");
|
|
1472
|
-
}
|
|
1473
|
-
|
|
1474
|
-
/** Dedupe key prefix for the exhaustion postmortem comment, per issue. */
|
|
1475
|
-
function postmortemDedupeKey(project: string, issue: number): string {
|
|
1476
|
-
return `${project}:postmortem:${issue}`;
|
|
1477
|
-
}
|
|
1478
|
-
|
|
1479
|
-
/**
|
|
1480
|
-
* The exhaustion postmortem: written exactly once per issue, at the point the
|
|
1481
|
-
* continuation budget is spent and the issue is settled toward a human.
|
|
1482
|
-
*
|
|
1483
|
-
* The comment and the material event are both gated by the store's notification
|
|
1484
|
-
* ledger — the same idempotence guard the escalator uses — so re-settling an
|
|
1485
|
-
* already-postmortemed issue posts nothing and records nothing. A body-string
|
|
1486
|
-
* match on the issue would be the wrong guard: an issue re-scoped and re-run
|
|
1487
|
-
* would still carry the old block, and the guarantee asked of this is "decided
|
|
1488
|
-
* once", not "deduped against what is already written".
|
|
1489
|
-
*
|
|
1490
|
-
* The digest must name the exhaustion even if the comment write fails, so the
|
|
1491
|
-
* material event is recorded before the write and unconditionally (the ledger
|
|
1492
|
-
* is append-only, and the gate above already ran once). The comment failure is
|
|
1493
|
-
* logged rather than taking the sweep down with it.
|
|
1494
|
-
*/
|
|
1495
|
-
async function postExhaustionPostmortem(d: Deps, run: RunRecord, reason: string): Promise<void> {
|
|
1496
|
-
const { project, tracker, store } = d;
|
|
1497
|
-
const key = postmortemDedupeKey(project.name, run.issue);
|
|
1498
|
-
if (store.wasNotified(key)) return;
|
|
1499
|
-
const runs = store.runsForIssue(project.name, run.issue);
|
|
1500
|
-
const body = formatExhaustionPostmortem({ issue: run.issue, runs, reason });
|
|
1501
|
-
const occurredAt = Date.now();
|
|
1502
|
-
store.recordMaterialEvent({
|
|
1503
|
-
project: project.name,
|
|
1504
|
-
category: "exhaustion",
|
|
1505
|
-
summary: `#${run.issue} exhausted: ${attemptClassBreakdown(runs)}`,
|
|
1506
|
-
evidence: body,
|
|
1507
|
-
occurredAt,
|
|
1508
|
-
recordedAt: occurredAt,
|
|
1509
|
-
});
|
|
1510
|
-
try {
|
|
1511
|
-
await tracker.comment(run.issue, body);
|
|
1512
|
-
store.markNotified(key);
|
|
1513
|
-
log(`#${run.issue} posted exhaustion postmortem (${runs.length} attempts)`);
|
|
1514
|
-
} catch (err) {
|
|
1515
|
-
log(`#${run.issue} postmortem comment could not be posted (${errText(err)})`);
|
|
1516
|
-
}
|
|
1517
|
-
}
|
|
1518
|
-
|
|
1519
1589
|
export type ExtendTurnLimitResult =
|
|
1520
1590
|
| { kind: "extended"; runId: string; maxTurns: number }
|
|
1521
1591
|
| { kind: "not-increase"; runId: string; maxTurns: number }
|
|
@@ -1719,48 +1789,9 @@ export async function verifyPushedGreenClaim(
|
|
|
1719
1789
|
};
|
|
1720
1790
|
}
|
|
1721
1791
|
|
|
1722
|
-
/** What the settlement audit of one green run produced: the advisory flags
|
|
1723
|
-
* (test weakening only — the file-list disclosure is derived, not flagged),
|
|
1724
|
-
* whether the diff was cut short, and the `changed:` line composed from the
|
|
1725
|
-
* PR's own diff. */
|
|
1726
|
-
export interface SettlementAuditResult {
|
|
1727
|
-
flags: SettlementFlag[];
|
|
1728
|
-
truncated: boolean;
|
|
1729
|
-
/** The `changed:` file list derived from the PR's diff, present whenever the
|
|
1730
|
-
* diff could be read. Absent means the tree could not be read, and `flags`
|
|
1731
|
-
* then carries exactly {@link UNREADABLE_TREE_FLAG}. */
|
|
1732
|
-
changedLine?: string;
|
|
1733
|
-
}
|
|
1734
|
-
|
|
1735
1792
|
/**
|
|
1736
|
-
*
|
|
1737
|
-
*
|
|
1738
|
-
* The thin half of the split #85 established: this fetches, {@link
|
|
1739
|
-
* analyseSettlement} decides. It runs beside {@link verifyPushedGreenClaim} and
|
|
1740
|
-
* shares none of its authority — that function decides a run's state, this one
|
|
1741
|
-
* cannot, by construction. It returns evidence and the caller appends it.
|
|
1742
|
-
*
|
|
1743
|
-
* A diff that cannot be read is a finding (`changed-line-missing`), never an
|
|
1744
|
-
* empty flag list: nothing was derived and nothing was checked, and that must
|
|
1745
|
-
* not read as a clean bill.
|
|
1746
|
-
*/
|
|
1747
|
-
export async function collectSettlementFlags(
|
|
1748
|
-
tracker: Pick<Tracker, "prDiff">,
|
|
1749
|
-
claim: { prUrl?: string; issueText: string },
|
|
1750
|
-
): Promise<SettlementAuditResult> {
|
|
1751
|
-
if (claim.prUrl === undefined) return { flags: [UNREADABLE_TREE_FLAG], truncated: false };
|
|
1752
|
-
const diff = await tracker.prDiff(claim.prUrl);
|
|
1753
|
-
if (diff === undefined) return { flags: [UNREADABLE_TREE_FLAG], truncated: false };
|
|
1754
|
-
return {
|
|
1755
|
-
flags: analyseSettlement({ issueText: claim.issueText, diff }),
|
|
1756
|
-
truncated: diff.truncated,
|
|
1757
|
-
changedLine: deriveChangedLine(diff),
|
|
1758
|
-
};
|
|
1759
|
-
}
|
|
1760
|
-
|
|
1761
|
-
/**
|
|
1762
|
-
* One attempt at one issue, from claim to terminal state. Everything is inside
|
|
1763
|
-
* a single try/catch so that a bad issue costs its own run and nothing else.
|
|
1793
|
+
* One attempt at one issue, from claim to terminal state. Everything is inside
|
|
1794
|
+
* a single try/catch so that a bad issue costs its own run and nothing else.
|
|
1764
1795
|
*/
|
|
1765
1796
|
/**
|
|
1766
1797
|
* Whether a turns-cap kill is handed straight back to the queue.
|
|
@@ -1901,11 +1932,39 @@ function orphanResumeVerdict(
|
|
|
1901
1932
|
return { kind: "resume", prior };
|
|
1902
1933
|
}
|
|
1903
1934
|
|
|
1935
|
+
/**
|
|
1936
|
+
* The worker identity for one launch, resolved now — or a throw naming the host
|
|
1937
|
+
* change that is missing (#798/#828).
|
|
1938
|
+
*
|
|
1939
|
+
* Every input is host state a running daemon does not control: the account can
|
|
1940
|
+
* be created after startup, and systemd can bring this service up before it has
|
|
1941
|
+
* mounted the harness binding a worker resolves through. So the verdict is taken
|
|
1942
|
+
* per launch. A daemon that cached one at startup would hold the whole fleet
|
|
1943
|
+
* closed on a boot race until somebody restarted it by hand — the outage #828
|
|
1944
|
+
* exists to end, not to relocate.
|
|
1945
|
+
*
|
|
1946
|
+
* The throw lands in the caller's dispatch catch, which settles the run failed
|
|
1947
|
+
* with this reason and escalates. It classifies as a start failure, so a host
|
|
1948
|
+
* fault charges the issue no implementation attempt.
|
|
1949
|
+
*/
|
|
1950
|
+
function launchIdentity(d: Pick<Deps, "workerIdentity">, launching: string): WorkerIdentity {
|
|
1951
|
+
const resolution: WorkerIdentityResolution = d.workerIdentity?.() ?? {
|
|
1952
|
+
ok: false,
|
|
1953
|
+
reason: `the ${WORKER_ACCOUNT} account is not installed on this host`,
|
|
1954
|
+
};
|
|
1955
|
+
if (resolution.ok) return resolution.identity;
|
|
1956
|
+
throw new Error(
|
|
1957
|
+
`worker identity unavailable: ${resolution.reason} — refusing to launch an unbound ${launching}; ` +
|
|
1958
|
+
"run `omp-conductor setup host` to install the dedicated worker identity",
|
|
1959
|
+
);
|
|
1960
|
+
}
|
|
1961
|
+
|
|
1904
1962
|
export async function handleIssue(
|
|
1905
1963
|
d: Deps,
|
|
1906
1964
|
r: Routed,
|
|
1907
1965
|
attempt: number,
|
|
1908
1966
|
admittedLane?: FileLane,
|
|
1967
|
+
admittedModel?: EffectiveModel,
|
|
1909
1968
|
): Promise<void> {
|
|
1910
1969
|
const { project, caps, tracker, store } = d;
|
|
1911
1970
|
const issue = r.issue.number;
|
|
@@ -2067,6 +2126,24 @@ export async function handleIssue(
|
|
|
2067
2126
|
return;
|
|
2068
2127
|
}
|
|
2069
2128
|
|
|
2129
|
+
// The claim-side of the project drain fence (#484): the tick's gate sits
|
|
2130
|
+
// above routing, so a drain created after that gate can still land
|
|
2131
|
+
// mid-pass — the claim re-checks and refuses while a fresh drain is in
|
|
2132
|
+
// force. An invalid record fails closed the same way. The read here is
|
|
2133
|
+
// observational on purpose: the claim refuses but must not consume the
|
|
2134
|
+
// marker, or the first refused claim would unbind the rest of its own
|
|
2135
|
+
// pass; the tick's consumeDrain removes it on the next pass, so no claim
|
|
2136
|
+
// path can ever be blocked permanently by it.
|
|
2137
|
+
const claimDrain = readDrain(d.project.name);
|
|
2138
|
+
if (claimDrain.kind === "active" || claimDrain.kind === "error") {
|
|
2139
|
+
log(
|
|
2140
|
+
claimDrain.kind === "error"
|
|
2141
|
+
? `#${issue} not claimed: drain record invalid (${claimDrain.problem})`
|
|
2142
|
+
: `#${issue} not claimed: project drain in effect`,
|
|
2143
|
+
);
|
|
2144
|
+
return;
|
|
2145
|
+
}
|
|
2146
|
+
|
|
2070
2147
|
// Claim on the STORE first, before anything that can fail. The run row —
|
|
2071
2148
|
// not the label — is the crash-safe guard against double dispatch: rows
|
|
2072
2149
|
// are local, written before any network call, and the startup orphan
|
|
@@ -2094,13 +2171,21 @@ export async function handleIssue(
|
|
|
2094
2171
|
// unconfigured project — and today's dispatch is byte for byte what it
|
|
2095
2172
|
// has always been.
|
|
2096
2173
|
const chainConfigured = (project.modelFallbacks?.length ?? 0) > 0;
|
|
2174
|
+
// A `Model:` declaration admission resolved for this candidate (#535) is
|
|
2175
|
+
// this issue's workerModel: the orchestrator names a tier when it
|
|
2176
|
+
// promotes, and dispatch launches on that selector exactly as if the
|
|
2177
|
+
// project's `workerModel` were the declared value. Absent a declaration,
|
|
2178
|
+
// today's `project.workerModel` is unchanged, and the failover chain
|
|
2179
|
+
// (#286) keeps its semantics in both cases — it is the same resolution,
|
|
2180
|
+
// one different primary.
|
|
2181
|
+
const declaredModel = admittedModel?.model;
|
|
2097
2182
|
const choice = resolveDispatchModel({
|
|
2098
|
-
workerModel: project.workerModel,
|
|
2183
|
+
workerModel: declaredModel ?? project.workerModel,
|
|
2099
2184
|
modelFallbacks: project.modelFallbacks,
|
|
2100
2185
|
threshold: project.modelFallbackThreshold ?? DEFAULT_MODEL_FALLBACK_THRESHOLD,
|
|
2101
2186
|
streak: chainFacts.streak,
|
|
2102
2187
|
});
|
|
2103
|
-
const clause = fallbackClause(choice, chainFacts, project.workerModel);
|
|
2188
|
+
const clause = fallbackClause(choice, chainFacts, declaredModel ?? project.workerModel);
|
|
2104
2189
|
|
|
2105
2190
|
// #536: an orphan-clean requeue resumes the interrupted session instead of
|
|
2106
2191
|
// re-reading the repo from zero. Decided here, before this attempt's row
|
|
@@ -2142,6 +2227,17 @@ export async function handleIssue(
|
|
|
2142
2227
|
spendUsd: 0,
|
|
2143
2228
|
maxTurns: caps.workerMaxTurns,
|
|
2144
2229
|
startedAt: Date.now(),
|
|
2230
|
+
// #567: the orphan-clean attempt this claim continues, when the verdict
|
|
2231
|
+
// above fired. `undefined` for a fresh dispatch — the store maps that to
|
|
2232
|
+
// NULL, so a fresh row simply never carries the field.
|
|
2233
|
+
resumedFromRunId: resuming?.id,
|
|
2234
|
+
// #744: the file-lane declaration admission resolved for this candidate
|
|
2235
|
+
// is persisted on the row, so lane occupancy survives across dispatch
|
|
2236
|
+
// passes — a later pass knows what this run *intends* to touch, not only
|
|
2237
|
+
// what it has touched so far. This is the exact `Admission.lane` the gate
|
|
2238
|
+
// enforced and the brief rendered, never a re-parse. Absent for a run
|
|
2239
|
+
// with no declaration (fail open), exactly as it was admitted.
|
|
2240
|
+
lane: admittedLane,
|
|
2145
2241
|
});
|
|
2146
2242
|
// #434: carry a terminal predecessor's open PR onto the continuation row.
|
|
2147
2243
|
// The claim itself stays synchronous — `handleIssue` claims at the top of
|
|
@@ -2180,6 +2276,14 @@ export async function handleIssue(
|
|
|
2180
2276
|
if (await settleStopBeforeSession()) return;
|
|
2181
2277
|
if (await settleDrainBeforeSession()) return;
|
|
2182
2278
|
|
|
2279
|
+
// The worker identity is this run's launch gate (#798): a worker session
|
|
2280
|
+
// that cannot be launched under the dedicated unprivileged account is
|
|
2281
|
+
// indistinguishable from an operator shell, so dispatch refuses before any
|
|
2282
|
+
// tree or session is created. Resolved here rather than read off a startup
|
|
2283
|
+
// verdict, so a host that gained its account — or its harness binding
|
|
2284
|
+
// (#828) — after the daemon came up dispatches on the next tick.
|
|
2285
|
+
const identity = launchIdentity(d, "worker session");
|
|
2286
|
+
|
|
2183
2287
|
// A run's tree is <workspaceRoot>/<issue> and addRunRepo refuses to reuse
|
|
2184
2288
|
// an existing path, so a retry — or a tree kept from a failed attempt — has
|
|
2185
2289
|
// to be cleared first. Both helpers are pure path math, and removeWorktree
|
|
@@ -2260,7 +2364,13 @@ export async function handleIssue(
|
|
|
2260
2364
|
runRepoPath: worktreePath,
|
|
2261
2365
|
branch,
|
|
2262
2366
|
},
|
|
2263
|
-
{
|
|
2367
|
+
{
|
|
2368
|
+
...(d.verbPeerReader === undefined ? {} : { peerReader: d.verbPeerReader }),
|
|
2369
|
+
// A worker-run channel is secured to the worker identity (#798): the
|
|
2370
|
+
// child that must connect to it runs as that uid, and the peer verdict
|
|
2371
|
+
// expects that uid on the wire rather than the daemon's.
|
|
2372
|
+
channelOwner: { uid: identity.uid, gid: identity.gid },
|
|
2373
|
+
},
|
|
2264
2374
|
);
|
|
2265
2375
|
if (await settleStopBeforeSession()) return;
|
|
2266
2376
|
if (await settleDrainBeforeSession()) return;
|
|
@@ -2317,6 +2427,8 @@ export async function handleIssue(
|
|
|
2317
2427
|
: {}),
|
|
2318
2428
|
comments,
|
|
2319
2429
|
lane: admittedLane,
|
|
2430
|
+
model: admittedModel,
|
|
2431
|
+
host: d.host,
|
|
2320
2432
|
});
|
|
2321
2433
|
}
|
|
2322
2434
|
if (await settleStopBeforeSession()) return;
|
|
@@ -2324,6 +2436,13 @@ export async function handleIssue(
|
|
|
2324
2436
|
|
|
2325
2437
|
const repoSlug = githubRepo(r.repo.cloneUrl);
|
|
2326
2438
|
|
|
2439
|
+
// The run's working paths are granted to the worker identity by ownership
|
|
2440
|
+
// (#798) — the worktree the session edits and the session directory it
|
|
2441
|
+
// writes its transcript and settings into. The chown lands here, after
|
|
2442
|
+
// provisioning and the verb socket, so the session starts on a tree it
|
|
2443
|
+
// owns; the worker identity never inherits anything from root's trees.
|
|
2444
|
+
d.grantWorkerPaths?.(identity, worktreePath, sessionDir);
|
|
2445
|
+
|
|
2327
2446
|
let result: WorkerResult;
|
|
2328
2447
|
try {
|
|
2329
2448
|
result = await runWorker({
|
|
@@ -2353,6 +2472,7 @@ export async function handleIssue(
|
|
|
2353
2472
|
onChildLog: (line) => {
|
|
2354
2473
|
log(`#${issue} ${line}`);
|
|
2355
2474
|
},
|
|
2475
|
+
workerIdentity: identity,
|
|
2356
2476
|
...(choice.model === undefined ? {} : { model: choice.model }),
|
|
2357
2477
|
// The fleet-owned omp settings overlay (#537): the staged YAML the
|
|
2358
2478
|
// session loads through `Settings.init({ configFiles: [<path>] })` —
|
|
@@ -2432,6 +2552,15 @@ export async function handleIssue(
|
|
|
2432
2552
|
? await collectSettlementFlags(tracker, {
|
|
2433
2553
|
prUrl: result.prUrl,
|
|
2434
2554
|
issueText: `${r.issue.title}\n${r.issue.body}`,
|
|
2555
|
+
// The claimed-proof check compares the PR's Verified commands
|
|
2556
|
+
// against what this run's session actually recorded.
|
|
2557
|
+
sessionFile: result.sessionFile,
|
|
2558
|
+
// The effective file lane admission resolved for this run at
|
|
2559
|
+
// dispatch — the same value the brief rendered, pre-dispatch
|
|
2560
|
+
// comment declarations included (#608, #744). The audit flags a
|
|
2561
|
+
// diff that escapes it, so a widened lane is named on evidence
|
|
2562
|
+
// rather than found by reading the PR's file list by hand (#739).
|
|
2563
|
+
lane: admittedLane,
|
|
2435
2564
|
})
|
|
2436
2565
|
: undefined;
|
|
2437
2566
|
if (result.state === "pushed-green" && audit?.truncated) {
|
|
@@ -2541,6 +2670,10 @@ export async function handleIssue(
|
|
|
2541
2670
|
...(result.prUrl === undefined ? {} : { prUrl: result.prUrl }),
|
|
2542
2671
|
...(result.headSha === undefined ? {} : { headSha: result.headSha }),
|
|
2543
2672
|
sessionFile: result.sessionFile,
|
|
2673
|
+
// The code-graph session observation (#726): what the run's own session
|
|
2674
|
+
// registry held at start, persisted with the rest of the run's facts.
|
|
2675
|
+
// Absent only when the session surface did not record one.
|
|
2676
|
+
...(result.graphTools === undefined ? {} : { graphTools: result.graphTools }),
|
|
2544
2677
|
// Every terminal state persists the worker's report — with the `changed:`
|
|
2545
2678
|
// file list derived from the PR's diff where one could be read — not
|
|
2546
2679
|
// just a green push: a stopped attempt's partial report is still part of
|
|
@@ -2618,7 +2751,7 @@ export async function handleIssue(
|
|
|
2618
2751
|
});
|
|
2619
2752
|
|
|
2620
2753
|
if (providerCredit !== undefined) {
|
|
2621
|
-
await reactToProviderCredit(d, issue, providerCredit, result.sessionFile);
|
|
2754
|
+
await reactToProviderCredit({ project: d.project, escalate: (e) => d.escalate(e), isPaused, setPaused }, issue, providerCredit, result.sessionFile);
|
|
2622
2755
|
swapToQueue(d, issue, inProgress);
|
|
2623
2756
|
} else if (continueTurns) {
|
|
2624
2757
|
// Requeue as one ordered pair: the in-progress removal before the
|
|
@@ -2812,6 +2945,91 @@ export async function dispatchReviewRevisions(d: Deps, pool?: WorkerPool): Promi
|
|
|
2812
2945
|
);
|
|
2813
2946
|
break;
|
|
2814
2947
|
}
|
|
2948
|
+
// A capped/failed run has no pushed-green settle sweep keeping its row
|
|
2949
|
+
// honest (#795 review round 1): nothing transitions a `failed` / `killed`
|
|
2950
|
+
// row when its PR merges or closes, so a PR that changed after the verb
|
|
2951
|
+
// recorded the round would otherwise be claimed and a worker resumed
|
|
2952
|
+
// against a dead PR. The settled-green origin keeps its own interlock
|
|
2953
|
+
// (the settle sweep flips the row and the claim below refuses it), so
|
|
2954
|
+
// only terminal-origin rounds re-read the reviewed PR fact here: the
|
|
2955
|
+
// round dispatches only while the PR is still open at the exact reviewed
|
|
2956
|
+
// head.
|
|
2957
|
+
//
|
|
2958
|
+
// The skip decision is decisive-fact only (review round 2): a definitively
|
|
2959
|
+
// missing PR (`GhPrMissingError` — a corroborated 404, #779), a PR that
|
|
2960
|
+
// is definitively not open, and a head that has definitively MOVED are
|
|
2961
|
+
// settled skipped, exactly like any other row that moved. Everything
|
|
2962
|
+
// transient or unreadable — a bare 404 that could not be corroborated, an
|
|
2963
|
+
// undefined answer, a pending check run, a same-head red check that a
|
|
2964
|
+
// rerun may clear, a throwing tracker — stays PENDING for the next tick:
|
|
2965
|
+
// settling it would permanently discard the orchestrator's findings and
|
|
2966
|
+
// consume the round.
|
|
2967
|
+
const origin = d.store.getRun(revision.runId)?.state;
|
|
2968
|
+
if (origin === "failed" || origin === "killed") {
|
|
2969
|
+
let prState: PrState | undefined;
|
|
2970
|
+
try {
|
|
2971
|
+
prState = await d.tracker.prState(revision.prUrl);
|
|
2972
|
+
} catch (err) {
|
|
2973
|
+
if (err instanceof GhPrMissingError) {
|
|
2974
|
+
d.store.settleReviewRevision(revision.id, "skipped", Date.now());
|
|
2975
|
+
log(
|
|
2976
|
+
`#${revision.issue} review round ${revision.round} skipped: ${revision.prUrl} does not exist — the reviewed PR is gone`,
|
|
2977
|
+
);
|
|
2978
|
+
continue;
|
|
2979
|
+
}
|
|
2980
|
+
log(
|
|
2981
|
+
`#${revision.issue} review round ${revision.round} held: PR state re-check failed (${errText(err)}) — retrying next tick`,
|
|
2982
|
+
);
|
|
2983
|
+
continue;
|
|
2984
|
+
}
|
|
2985
|
+
if (prState === undefined) {
|
|
2986
|
+
log(
|
|
2987
|
+
`#${revision.issue} review round ${revision.round} held: ${revision.prUrl} state could not be re-read — retrying next tick`,
|
|
2988
|
+
);
|
|
2989
|
+
continue;
|
|
2990
|
+
}
|
|
2991
|
+
if (prState !== "open") {
|
|
2992
|
+
d.store.settleReviewRevision(revision.id, "skipped", Date.now());
|
|
2993
|
+
log(
|
|
2994
|
+
`#${revision.issue} review round ${revision.round} skipped: ${revision.prUrl} is ${prState}, not open — a review round resumes an open PR only`,
|
|
2995
|
+
);
|
|
2996
|
+
continue;
|
|
2997
|
+
}
|
|
2998
|
+
let verification: PrVerification | undefined;
|
|
2999
|
+
try {
|
|
3000
|
+
verification = await d.tracker.verifyPr(revision.prUrl, revision.headSha);
|
|
3001
|
+
} catch (err) {
|
|
3002
|
+
log(
|
|
3003
|
+
`#${revision.issue} review round ${revision.round} held: the reviewed-head check failed (${errText(err)}) — retrying next tick`,
|
|
3004
|
+
);
|
|
3005
|
+
continue;
|
|
3006
|
+
}
|
|
3007
|
+
if (verification === undefined || verification.status === "pending") {
|
|
3008
|
+
log(
|
|
3009
|
+
`#${revision.issue} review round ${revision.round} held: ${revision.prUrl} at ${revision.headSha} is ` +
|
|
3010
|
+
`${verification?.status ?? "unverifiable"} — retrying next tick`,
|
|
3011
|
+
);
|
|
3012
|
+
continue;
|
|
3013
|
+
}
|
|
3014
|
+
if (verification.status === "failed" && isHeadMismatch(verification.reason)) {
|
|
3015
|
+
d.store.settleReviewRevision(revision.id, "skipped", Date.now());
|
|
3016
|
+
log(
|
|
3017
|
+
`#${revision.issue} review round ${revision.round} skipped: ${revision.prUrl} no longer stands at ` +
|
|
3018
|
+
`${revision.headSha} (${verification.reason}) — the reviewed head moved`,
|
|
3019
|
+
);
|
|
3020
|
+
continue;
|
|
3021
|
+
}
|
|
3022
|
+
// Anything else — a same-head red check a rerun may clear, an unknown
|
|
3023
|
+
// verdict — is not definitive: the round stays pending for the next
|
|
3024
|
+
// tick rather than discarding the findings.
|
|
3025
|
+
if (verification.status !== "green") {
|
|
3026
|
+
log(
|
|
3027
|
+
`#${revision.issue} review round ${revision.round} held: ${revision.prUrl} at ${revision.headSha} is ` +
|
|
3028
|
+
`${verification.status} — retrying next tick`,
|
|
3029
|
+
);
|
|
3030
|
+
continue;
|
|
3031
|
+
}
|
|
3032
|
+
}
|
|
2815
3033
|
if (!d.store.claimRunForReview(revision.runId)) {
|
|
2816
3034
|
d.store.settleReviewRevision(revision.id, "skipped", Date.now());
|
|
2817
3035
|
log(
|
|
@@ -2972,18 +3190,84 @@ export async function handleReviewRevision(d: Deps, revision: ReviewRevisionReco
|
|
|
2972
3190
|
// and shutdown fences cover the whole wake window, exactly as they do for a
|
|
2973
3191
|
// fresh claim in `handleIssue` (#374). The original run's entries were
|
|
2974
3192
|
// closed at its settle, so reopening by issue is safe.
|
|
3193
|
+
//
|
|
3194
|
+
// #747: the round gets its own bounded allowance instead of the original
|
|
3195
|
+
// attempt's remainder. The resumed session continues the run's own counter
|
|
3196
|
+
// — the attempt's spent turns travel with the session — so a ceiling of
|
|
3197
|
+
// `run.maxTurns` would leave the round only `run.maxTurns - run.turns`,
|
|
3198
|
+
// and returning a PR from a near-cap run could not finish. Open at the
|
|
3199
|
+
// run's ceiling, then raise it by the round's own allowance — the run's
|
|
3200
|
+
// ceiling again, capped by the project's hard ceiling — through the same
|
|
3201
|
+
// override path `omp-conductor extend` uses, so the raise is persisted on
|
|
3202
|
+
// the row (the run's budget stays explainable in status) while `turns`
|
|
3203
|
+
// keeps accumulating the run's true cost: nothing is reset.
|
|
2975
3204
|
turnLimit = d.turnLimits.open(project.name, issue, runId, run.maxTurns);
|
|
3205
|
+
const revisionAllowance = Math.min(run.maxTurns, caps.workerMaxTurnsCeiling);
|
|
3206
|
+
const revisionCeiling = run.turns + revisionAllowance;
|
|
3207
|
+
const revisionRaise = d.turnLimits.extend(project.name, issue, revisionCeiling);
|
|
3208
|
+
if (revisionRaise.kind === "extended") {
|
|
3209
|
+
log(
|
|
3210
|
+
`#${issue} review round ${revision.round} turn ceiling ${run.maxTurns} → ${revisionCeiling} ` +
|
|
3211
|
+
`(+${revisionAllowance} round allowance on ${run.turns} spent)`,
|
|
3212
|
+
);
|
|
3213
|
+
}
|
|
2976
3214
|
workerControl = d.workerControls.open(project.name, issue, runId);
|
|
2977
3215
|
if (await settleStopBeforeSession()) return;
|
|
2978
3216
|
if (await settleDrainBeforeSession()) return;
|
|
2979
3217
|
|
|
3218
|
+
// The same launch gate as a fresh claim (#798): a revision worker is a
|
|
3219
|
+
// worker, and an unbound one is an operator shell — refuse before any tree
|
|
3220
|
+
// or session is created. Thrown inside the try so the ordinary dispatch
|
|
3221
|
+
// catch settles it: the run row is returned to terminal `failed` with the
|
|
3222
|
+
// reason, the revision row is settled with it, and the still-green PR stays
|
|
3223
|
+
// open for a healthy retry.
|
|
2980
3224
|
try {
|
|
3225
|
+
const identity = launchIdentity(d, "review-revision worker");
|
|
2981
3226
|
// Reattach the run's own branch at the same per-issue path the run used:
|
|
2982
|
-
//
|
|
2983
|
-
//
|
|
2984
|
-
//
|
|
2985
|
-
//
|
|
3227
|
+
// a `pushed-green` settle removed the worktree, so provisioning is the
|
|
3228
|
+
// same continuation reattach as a normal re-claim. A capped/failed run's
|
|
3229
|
+
// tree was RETAINED at its terminal settle (`tree: "keep"`, #795) — the
|
|
3230
|
+
// reviewed PR is green, so the branch already holds the work and the kept
|
|
3231
|
+
// tree holds nothing the branch does not, unless its salvage failed. Clear
|
|
3232
|
+
// it the way restart recovery does (#692): re-attempt the salvage through
|
|
3233
|
+
// `tree: "remove"`, and refuse the removal if that still fails — the tree
|
|
3234
|
+
// is then the only copy of work and is never destroyed. A provisioning
|
|
3235
|
+
// failure restores the row — the PR is still green and still open, and
|
|
3236
|
+
// only the wake failed — and says so.
|
|
2986
3237
|
try {
|
|
3238
|
+
if (run.worktree !== "" && existsSync(run.worktree)) {
|
|
3239
|
+
const keptSettlement = await settleWorktree({
|
|
3240
|
+
issue: run.issue,
|
|
3241
|
+
attempt: run.attempt,
|
|
3242
|
+
ending: `review round ${revision.round} resume`,
|
|
3243
|
+
worktree: run.worktree,
|
|
3244
|
+
branch: run.branch,
|
|
3245
|
+
publish: (branch) => pushRunBranch(project, { repo, runRepoPath: run.worktree, branch }),
|
|
3246
|
+
tree: "remove",
|
|
3247
|
+
mirrorPath,
|
|
3248
|
+
});
|
|
3249
|
+
if (keptSettlement.retained) {
|
|
3250
|
+
log(
|
|
3251
|
+
`#${issue} review round ${revision.round} retained worktree ${run.worktree}: ` +
|
|
3252
|
+
"its salvage failed, so the tree is the only copy of work and will not be removed",
|
|
3253
|
+
);
|
|
3254
|
+
store.updateRun(runId, {
|
|
3255
|
+
state: "pushed-green",
|
|
3256
|
+
endedAt: Date.now(),
|
|
3257
|
+
lastError: `review round ${revision.round} could not clear the run's retained worktree: ${keptSettlement.lines.join(" ")}`,
|
|
3258
|
+
});
|
|
3259
|
+
store.settleReviewRevision(revision.id, "skipped", Date.now());
|
|
3260
|
+
await safeEscalate(d, {
|
|
3261
|
+
tier: 1,
|
|
3262
|
+
project: project.name,
|
|
3263
|
+
issue,
|
|
3264
|
+
runId,
|
|
3265
|
+
summary: `#${issue} review round ${revision.round} could not be dispatched — the run's retained worktree is the only copy of work`,
|
|
3266
|
+
detail: [revision.prUrl, "", ...keptSettlement.lines].join("\n"),
|
|
3267
|
+
});
|
|
3268
|
+
return;
|
|
3269
|
+
}
|
|
3270
|
+
}
|
|
2987
3271
|
const provisioned = await addRunRepo(repo, project.mirrorRoot, project.workspaceRoot, issue, branch);
|
|
2988
3272
|
worktreePath = provisioned.path;
|
|
2989
3273
|
runRepo = { repo, runRepoPath: worktreePath, branch };
|
|
@@ -3051,7 +3335,10 @@ export async function handleReviewRevision(d: Deps, revision: ReviewRevisionReco
|
|
|
3051
3335
|
runRepoPath: worktreePath,
|
|
3052
3336
|
branch,
|
|
3053
3337
|
},
|
|
3054
|
-
{
|
|
3338
|
+
{
|
|
3339
|
+
...(d.verbPeerReader === undefined ? {} : { peerReader: d.verbPeerReader }),
|
|
3340
|
+
channelOwner: { uid: identity.uid, gid: identity.gid },
|
|
3341
|
+
},
|
|
3055
3342
|
);
|
|
3056
3343
|
if (await settleStopBeforeSession()) return;
|
|
3057
3344
|
if (await settleDrainBeforeSession()) return;
|
|
@@ -3066,6 +3353,11 @@ export async function handleReviewRevision(d: Deps, revision: ReviewRevisionReco
|
|
|
3066
3353
|
|
|
3067
3354
|
log(`#${issue} review round ${revision.round} → resuming ${branch} (${priorSessionFile})`);
|
|
3068
3355
|
|
|
3356
|
+
// The revision worker gets its own tree ownership, exactly as a fresh
|
|
3357
|
+
// claim does (#798): the resumed worktree and session directory belong to
|
|
3358
|
+
// the worker identity before its session starts.
|
|
3359
|
+
d.grantWorkerPaths?.(identity, worktreePath, sessionDir);
|
|
3360
|
+
|
|
3069
3361
|
let result: WorkerResult;
|
|
3070
3362
|
try {
|
|
3071
3363
|
result = await runWorker({
|
|
@@ -3090,6 +3382,7 @@ export async function handleReviewRevision(d: Deps, revision: ReviewRevisionReco
|
|
|
3090
3382
|
onChildLog: (line) => {
|
|
3091
3383
|
log(`#${issue} ${line}`);
|
|
3092
3384
|
},
|
|
3385
|
+
workerIdentity: identity,
|
|
3093
3386
|
// The continuation stays on the model the green run used (#286).
|
|
3094
3387
|
...(run.model === undefined ? {} : { model: run.model }),
|
|
3095
3388
|
...(ompSettingsFile === undefined ? {} : { ompSettingsFile }),
|
|
@@ -3184,6 +3477,7 @@ export async function handleReviewRevision(d: Deps, revision: ReviewRevisionReco
|
|
|
3184
3477
|
...(result.prUrl === undefined ? {} : { prUrl: result.prUrl }),
|
|
3185
3478
|
...(result.headSha === undefined ? {} : { headSha: result.headSha }),
|
|
3186
3479
|
sessionFile: result.sessionFile,
|
|
3480
|
+
...(result.graphTools === undefined ? {} : { graphTools: result.graphTools }),
|
|
3187
3481
|
report: finalReport,
|
|
3188
3482
|
...settlement?.patch,
|
|
3189
3483
|
};
|
|
@@ -3392,8 +3686,12 @@ export async function reconcileCrashedReviewRevisions(d: Deps): Promise<ReviewRe
|
|
|
3392
3686
|
}
|
|
3393
3687
|
// A revision the previous daemon never claimed: the run is still a
|
|
3394
3688
|
// settled green row and the round is still pending, so the ordinary
|
|
3395
|
-
// dispatch pass wakes it on the next tick untouched.
|
|
3396
|
-
|
|
3689
|
+
// dispatch pass wakes it on the next tick untouched. Same for a
|
|
3690
|
+
// capped/failed run whose review round is pending (#795): a `failed` /
|
|
3691
|
+
// `killed` row is terminal too — no process died on this round, the
|
|
3692
|
+
// revision was never claimed, and the ordinary dispatch pass claims it
|
|
3693
|
+
// exactly like a pending round on a pushed-green row.
|
|
3694
|
+
if (run.state === "pushed-green" || run.state === "failed" || run.state === "killed") continue;
|
|
3397
3695
|
// The run a previous daemon claimed for this round and died on: the
|
|
3398
3696
|
// orphan sweep just marked it `orphaned` (salvaging the tree to the
|
|
3399
3697
|
// branch), so restore it to the reviewable state the verb recorded.
|
|
@@ -3528,119 +3826,6 @@ export async function reconcileCrashedReviewRevisions(d: Deps): Promise<ReviewRe
|
|
|
3528
3826
|
}
|
|
3529
3827
|
|
|
3530
3828
|
// ------------------------------------------------------------------- settlement
|
|
3531
|
-
|
|
3532
|
-
/** What a resolved PR turns its `pushed-green` row into. */
|
|
3533
|
-
export interface Settlement {
|
|
3534
|
-
state: "merged" | "failed";
|
|
3535
|
-
/** The log line after `#<n> settled: `, and — for a rejection — the row's own
|
|
3536
|
-
* `lastError`, because a `failed` row whose worker succeeded has to say so. */
|
|
3537
|
-
reason: string;
|
|
3538
|
-
}
|
|
3539
|
-
|
|
3540
|
-
/**
|
|
3541
|
-
* What one `pushed-green` row becomes now its PR has an answer, or undefined to
|
|
3542
|
-
* leave the row exactly as it is.
|
|
3543
|
-
*
|
|
3544
|
-
* A `pushed-green` row is the only one nothing ever revisited: the worker is
|
|
3545
|
-
* finished, `reconcileOrphanedRuns` only settles rows that held a process, and
|
|
3546
|
-
* `merged` went unwritten from day one. So they accumulated — three of them on
|
|
3547
|
-
* the reference fleet on 2026-08-07, every PR merged and every issue closed,
|
|
3548
|
-
* with `/healthz` still reporting three active runs and their issues
|
|
3549
|
-
* permanently unclaimable, because the busy set *is* the active set (#18).
|
|
3550
|
-
*
|
|
3551
|
-
* The mapping, and why each answer is the only honest one:
|
|
3552
|
-
*
|
|
3553
|
-
* - `merged` — the work landed. That is what `merged` was reserved for.
|
|
3554
|
-
* - `closed` — a human read the work and said no. Leaving it `pushed-green`
|
|
3555
|
-
* forever is a lie; `failed` records that it did not land and releases the
|
|
3556
|
-
* busy guard, so an issue a human re-queues can be attempted again. A row
|
|
3557
|
-
* that had reached `pushed-green` or `pushed-pending` is classified
|
|
3558
|
-
* `returned-for-revision` at settlement. A review decision asks for another
|
|
3559
|
-
* implementation pass, not a failure, so it consumes the continuation budget
|
|
3560
|
-
* instead of the failed-attempt budget.
|
|
3561
|
-
* - `open`, and undefined — nothing changes. Undefined is "could not tell": a
|
|
3562
|
-
* flaky network, a revoked token, a deleted PR. Settling on it would record a
|
|
3563
|
-
* merge that never happened, and the next tick asks again for free. An
|
|
3564
|
-
* ambiguous answer must never settle a row.
|
|
3565
|
-
*/
|
|
3566
|
-
export function settlementFor(pr: PrState | undefined, prUrl: string): Settlement | undefined {
|
|
3567
|
-
if (pr === "merged") return { state: "merged", reason: `${prUrl} merged` };
|
|
3568
|
-
if (pr === "closed") return { state: "failed", reason: `${prUrl} closed without merging` };
|
|
3569
|
-
return undefined;
|
|
3570
|
-
}
|
|
3571
|
-
|
|
3572
|
-
/**
|
|
3573
|
-
* Records the in-progress label's release as a projection op (#201).
|
|
3574
|
-
*
|
|
3575
|
-
* Settlement used to write only half of what it knew. On 2026-08-09 that cost
|
|
3576
|
-
* the reference fleet two issues in one night: veltro#331 settled to `failed`
|
|
3577
|
-
* at 23:53Z once the orchestrator closed veltro#332 unmerged, and veltro#344
|
|
3578
|
-
* settled to `merged` at 05:54Z once chad#452 squash-merged. Both rows left the
|
|
3579
|
-
* active set correctly; an authoritative `gh issue view` on each afterwards
|
|
3580
|
-
* still showed `agent:in-progress` — permanently unclaimable with no supported
|
|
3581
|
-
* way back (#18).
|
|
3582
|
-
*
|
|
3583
|
-
* The outbox makes the row transition and the label one fact again: the
|
|
3584
|
-
* removal is enqueued in the same breath as the row is terminalised, the
|
|
3585
|
-
* projector applies it with unbounded retry, and while it is pending the
|
|
3586
|
-
* eligibility overlay treats the label as already gone. A tracker that refuses
|
|
3587
|
-
* the write (403, rate limit) can no longer strand the row — that is `#184`
|
|
3588
|
-
* and `#198` closed. No `pushed-*` row is ever written terminal with its
|
|
3589
|
-
* label release owed but unrecorded, because enqueueing is a local store write
|
|
3590
|
-
* that cannot fail on the tracker.
|
|
3591
|
-
*
|
|
3592
|
-
* Synchronous. The op is durable the moment this returns.
|
|
3593
|
-
*/
|
|
3594
|
-
export function releaseInProgress(
|
|
3595
|
-
d: Pick<Deps, "project" | "store">,
|
|
3596
|
-
issue: number,
|
|
3597
|
-
why: string,
|
|
3598
|
-
): void {
|
|
3599
|
-
const label = d.project.stateLabels.inProgress;
|
|
3600
|
-
d.store.enqueueLabelOps(d.project.name, [{ issue, op: "remove", label }]);
|
|
3601
|
-
log(`#${issue} released ${label} (queued): ${why}`);
|
|
3602
|
-
}
|
|
3603
|
-
|
|
3604
|
-
/**
|
|
3605
|
-
* Asks the tracker about every `pushed-green` PR and settles the ones that
|
|
3606
|
-
* resolved.
|
|
3607
|
-
*
|
|
3608
|
-
* Effects at the call site, decision in {@link settlementFor} — the same split
|
|
3609
|
-
* as `checkIntegrity`/`watchOrchestrator`. Exported like `admitCandidates`
|
|
3610
|
-
* rather than kept private, because half of what has to hold is about the sweep
|
|
3611
|
-
* and not the mapping: that a row without a PR costs no API call, and that one
|
|
3612
|
-
* unreachable PR does not stop the others from settling.
|
|
3613
|
-
*
|
|
3614
|
-
* The label is released too, which reverses what this function first promised.
|
|
3615
|
-
* It used to leave tracker labels alone exactly as {@link reconcileOrphanedRuns}
|
|
3616
|
-
* does, reasoning that a merge closes the issue anyway and that deciding what an
|
|
3617
|
-
* issue's labels should say next is the orchestrator's drain duty. There turned
|
|
3618
|
-
* out to be no such path: on 2026-08-09 two settled rows left their issues
|
|
3619
|
-
* carrying `agent:in-progress` forever, with the brief forbidding the
|
|
3620
|
-
* orchestrator from touching it and `unblock` declining to (see
|
|
3621
|
-
* {@link releaseInProgress}). The row transition and the label are one fact, and
|
|
3622
|
-
* writing half of it is the whole of that bug.
|
|
3623
|
-
*
|
|
3624
|
-
* Releasing it is safe here specifically because of what these rows are. A
|
|
3625
|
-
* `pushed-green` or `pushed-pending` row has no process behind it — its worker
|
|
3626
|
-
* exited and its worktree is gone — so a terminal answer about its PR proves no
|
|
3627
|
-
* worker owns the issue, and the duplicate-dispatch interlock the label exists
|
|
3628
|
-
* for is spent. {@link reconcileOrphanedRuns} still leaves labels alone for the
|
|
3629
|
-
* opposite reason: an orphaned `running` row is work nobody has read yet. And
|
|
3630
|
-
* the brief's rule stays absolute, because this is a daemon-owned write through
|
|
3631
|
-
* the same Tracker port the dispatcher claimed the issue with — orphan detection
|
|
3632
|
-
* is only trustworthy while every state label on the tracker came from this
|
|
3633
|
-
* package.
|
|
3634
|
-
*
|
|
3635
|
-
* The two writes are ordered label-then-row, and the order is load-bearing. This
|
|
3636
|
-
* sweep is the only thing that revisits a `pushed-*` row, so the terminal state
|
|
3637
|
-
* is also the row's exit from it: written first, a tracker that then failed on
|
|
3638
|
-
* the label would leave `agent:in-progress` with nothing left to retry it — #18
|
|
3639
|
-
* exactly, in the last window able to reach it. Writing the label first makes
|
|
3640
|
-
* failure cost a repeated `gh` call on the next tick instead, and the row stays
|
|
3641
|
-
* in the busy set throughout, so no second worker can be sent at the issue while
|
|
3642
|
-
* it waits.
|
|
3643
|
-
*/
|
|
3644
3829
|
const BASE_CHECK_BATCH = 20;
|
|
3645
3830
|
const BASE_CHECK_WINDOW_MS = 24 * 60 * 60 * 1_000;
|
|
3646
3831
|
const BASE_STATUS_WINDOW_MS = 7 * 24 * 60 * 60 * 1_000;
|
|
@@ -3948,220 +4133,6 @@ export async function watchBaseHealth(
|
|
|
3948
4133
|
}
|
|
3949
4134
|
}
|
|
3950
4135
|
|
|
3951
|
-
export async function settlePushedGreen(
|
|
3952
|
-
d: Pick<Deps, "project" | "tracker" | "store">,
|
|
3953
|
-
): Promise<number> {
|
|
3954
|
-
const { project, tracker, store } = d;
|
|
3955
|
-
// Runs the sweep resolved by terminalising the row. Every terminal write
|
|
3956
|
-
// below increments it; the tick attributes it as `settled` on the pass's
|
|
3957
|
-
// dispatch record, so a held pass is visible as work done, not just as a
|
|
3958
|
-
// clock that moved (#497).
|
|
3959
|
-
let settled = 0;
|
|
3960
|
-
// Filtered from the active set rather than asked for with a new query: active
|
|
3961
|
-
// is live workers plus these, so the list is bounded by the worker cap plus
|
|
3962
|
-
// the number of PRs awaiting a merge — a handful, by construction. A fleet
|
|
3963
|
-
// where that is not a handful has a merge problem, not a dispatch one.
|
|
3964
|
-
const pending = store
|
|
3965
|
-
.activeRuns(project.name)
|
|
3966
|
-
.filter((r) => r.state === "pushed-green" || r.state === "pushed-pending");
|
|
3967
|
-
|
|
3968
|
-
for (const run of pending) {
|
|
3969
|
-
// Nothing to ask about. A pushed result requires a PR, so a malformed row
|
|
3970
|
-
// must not buy a `gh` call every five minutes forever.
|
|
3971
|
-
if (run.prUrl === undefined) continue;
|
|
3972
|
-
|
|
3973
|
-
let pr: PrState | undefined;
|
|
3974
|
-
try {
|
|
3975
|
-
pr = await tracker.prState(run.prUrl);
|
|
3976
|
-
} catch (err) {
|
|
3977
|
-
// Per row, like admission's held candidate. The GitHub adapter already
|
|
3978
|
-
// answers undefined instead of throwing, so this catch is the port's
|
|
3979
|
-
// contract rather than that adapter's behaviour — and a tracker that does
|
|
3980
|
-
// throw must cost its own row, not the whole sweep.
|
|
3981
|
-
log(`#${run.issue} not settled: PR state lookup failed (${errText(err)}) — retrying next tick`);
|
|
3982
|
-
continue;
|
|
3983
|
-
}
|
|
3984
|
-
|
|
3985
|
-
const settlement = settlementFor(pr, run.prUrl);
|
|
3986
|
-
if (settlement !== undefined) {
|
|
3987
|
-
// A mediated merge enters a second, bounded observation phase. Record the
|
|
3988
|
-
// exact merge commit before the row leaves the active set; if GitHub cannot
|
|
3989
|
-
// supply it yet, retry this settlement next tick rather than create a
|
|
3990
|
-
// merged row whose base result can never be attributed.
|
|
3991
|
-
let merged: MergedPrInfo | undefined;
|
|
3992
|
-
if (settlement.state === "merged") {
|
|
3993
|
-
try {
|
|
3994
|
-
merged = await tracker.mergedPrInfo(run.prUrl);
|
|
3995
|
-
} catch (err) {
|
|
3996
|
-
log(`#${run.issue} not settled: merge identity lookup failed (${errText(err)}) — retrying next tick`);
|
|
3997
|
-
continue;
|
|
3998
|
-
}
|
|
3999
|
-
if (merged === undefined) {
|
|
4000
|
-
log(`#${run.issue} not settled: merge identity unavailable — retrying next tick`);
|
|
4001
|
-
continue;
|
|
4002
|
-
}
|
|
4003
|
-
}
|
|
4004
|
-
|
|
4005
|
-
// The label removal and the terminal row are one fact again (#201): the
|
|
4006
|
-
// release is enqueued — a durable local write that cannot fail on the
|
|
4007
|
-
// tracker — in the same breath as the row is terminalised.
|
|
4008
|
-
releaseInProgress(d, run.issue, settlement.reason);
|
|
4009
|
-
const patch: Partial<RunRecord> = {
|
|
4010
|
-
state: settlement.state,
|
|
4011
|
-
endedAt: Date.now(),
|
|
4012
|
-
...(merged === undefined
|
|
4013
|
-
? {}
|
|
4014
|
-
: {
|
|
4015
|
-
mergeSha: merged.mergeSha,
|
|
4016
|
-
baseRef: merged.baseRef,
|
|
4017
|
-
baseCheck: "pending",
|
|
4018
|
-
}),
|
|
4019
|
-
};
|
|
4020
|
-
if (settlement.state === "failed") {
|
|
4021
|
-
patch.lastError = settlement.reason;
|
|
4022
|
-
patch.failureClass = "returned-for-revision";
|
|
4023
|
-
patch.recoveryAction = "none";
|
|
4024
|
-
}
|
|
4025
|
-
store.updateRun(run.id, patch);
|
|
4026
|
-
settled += 1;
|
|
4027
|
-
log(`#${run.issue} settled: ${settlement.reason}`);
|
|
4028
|
-
continue;
|
|
4029
|
-
}
|
|
4030
|
-
|
|
4031
|
-
if (run.state !== "pushed-pending" || pr !== "open" || run.headSha === undefined) continue;
|
|
4032
|
-
let verification;
|
|
4033
|
-
try {
|
|
4034
|
-
verification = await tracker.verifyPr(run.prUrl, run.headSha);
|
|
4035
|
-
} catch (err) {
|
|
4036
|
-
log(`#${run.issue} checks not settled (${errText(err)}) — retrying next tick`);
|
|
4037
|
-
continue;
|
|
4038
|
-
}
|
|
4039
|
-
if (verification === undefined) continue;
|
|
4040
|
-
if (verification.status === "green") {
|
|
4041
|
-
store.updateRun(run.id, { state: "pushed-green", lastError: null });
|
|
4042
|
-
log(`#${run.issue} checks settled: ${verification.reason}`);
|
|
4043
|
-
} else if (verification.status === "failed") {
|
|
4044
|
-
// Equally terminal, so the release is enqueued before the row writes,
|
|
4045
|
-
// for the same reason as the settlement branch above (see there). The
|
|
4046
|
-
// green branch releases nothing — that row is still awaiting a merge,
|
|
4047
|
-
// and its live PR is exactly the work the label must keep guarding.
|
|
4048
|
-
releaseInProgress(d, run.issue, verification.reason);
|
|
4049
|
-
store.updateRun(run.id, { state: "failed", lastError: verification.reason });
|
|
4050
|
-
settled += 1;
|
|
4051
|
-
log(`#${run.issue} checks failed: ${verification.reason}`);
|
|
4052
|
-
} else {
|
|
4053
|
-
store.updateRun(run.id, { lastError: verification.reason });
|
|
4054
|
-
}
|
|
4055
|
-
}
|
|
4056
|
-
return settled;
|
|
4057
|
-
}
|
|
4058
|
-
|
|
4059
|
-
const ADOPTABLE_PR_STATES: Partial<Record<RunState, true>> = {
|
|
4060
|
-
failed: true,
|
|
4061
|
-
killed: true,
|
|
4062
|
-
orphaned: true,
|
|
4063
|
-
blocked: true,
|
|
4064
|
-
};
|
|
4065
|
-
|
|
4066
|
-
/**
|
|
4067
|
-
* Reattaches a recovered PR to the terminal run that owns it (#245).
|
|
4068
|
-
*
|
|
4069
|
-
* A worker can fail before its completion report records `prUrl`, then have its
|
|
4070
|
-
* dirty tree committed and pushed by salvage. If that branch already has a PR,
|
|
4071
|
-
* the orchestrator otherwise has no policy-compliant path to inspect or merge
|
|
4072
|
-
* it: ownership is store-backed. Adoption is deliberately stricter than
|
|
4073
|
-
* admission. The tracker query proves the PR closes this run's issue; exact
|
|
4074
|
-
* branch and canonical repository matches prove it is this run's recovered
|
|
4075
|
-
* work, not an unrelated closer. Missing identity is refusal, never a guess.
|
|
4076
|
-
*
|
|
4077
|
-
* The newest run per issue is inspected, at most ten per tick and only inside
|
|
4078
|
-
* the same 30-day window as mediated PR verbs. The cursor advances through the
|
|
4079
|
-
* full eligible set so persistent non-matches cannot starve older recovered
|
|
4080
|
-
* work. Successful adoption is idempotent because the row gains `prUrl`;
|
|
4081
|
-
* non-matches are logged once per daemon process.
|
|
4082
|
-
*/
|
|
4083
|
-
const rejectedSalvagedPrRuns = new Set<string>();
|
|
4084
|
-
const salvagedPrCursor = new Map<string, string>();
|
|
4085
|
-
|
|
4086
|
-
export async function adoptSalvagedPrs(
|
|
4087
|
-
d: Pick<Deps, "project" | "tracker" | "store">,
|
|
4088
|
-
now = Date.now(),
|
|
4089
|
-
): Promise<void> {
|
|
4090
|
-
const { project, tracker, store } = d;
|
|
4091
|
-
const eligible = store
|
|
4092
|
-
.recentRuns(project.name, now - PR_LOOKUP_WINDOW_MS)
|
|
4093
|
-
.filter(
|
|
4094
|
-
(run) =>
|
|
4095
|
-
ADOPTABLE_PR_STATES[run.state] === true &&
|
|
4096
|
-
run.prUrl === undefined &&
|
|
4097
|
-
run.branch.trim() !== "",
|
|
4098
|
-
);
|
|
4099
|
-
const previous = salvagedPrCursor.get(project.name);
|
|
4100
|
-
const previousIndex =
|
|
4101
|
-
previous === undefined ? -1 : eligible.findIndex((run) => run.id === previous);
|
|
4102
|
-
const start = previousIndex === -1 ? 0 : (previousIndex + 1) % eligible.length;
|
|
4103
|
-
const candidates = Array.from(
|
|
4104
|
-
{ length: Math.min(SALVAGED_PR_ADOPTION_BATCH, eligible.length) },
|
|
4105
|
-
(_, offset) => eligible[(start + offset) % eligible.length]!,
|
|
4106
|
-
);
|
|
4107
|
-
const last = candidates.at(-1);
|
|
4108
|
-
if (last !== undefined) salvagedPrCursor.set(project.name, last.id);
|
|
4109
|
-
|
|
4110
|
-
for (const run of candidates) {
|
|
4111
|
-
const repo = project.routing.repos[run.repo];
|
|
4112
|
-
const repoIdentity = repo === undefined ? undefined : githubRepo(repo.cloneUrl);
|
|
4113
|
-
let closers: OpenCloser[];
|
|
4114
|
-
try {
|
|
4115
|
-
closers = await tracker.openClosersFor(run.issue);
|
|
4116
|
-
} catch (err) {
|
|
4117
|
-
log(`#${run.issue} PR adoption lookup failed (${errText(err)}) — retrying next tick`);
|
|
4118
|
-
continue;
|
|
4119
|
-
}
|
|
4120
|
-
|
|
4121
|
-
const closer = closers.find(
|
|
4122
|
-
(candidate) =>
|
|
4123
|
-
candidate.headRefName !== "" &&
|
|
4124
|
-
candidate.headRefName === run.branch &&
|
|
4125
|
-
repo !== undefined &&
|
|
4126
|
-
candidate.repo !== "" &&
|
|
4127
|
-
candidate.repo === repoIdentity,
|
|
4128
|
-
);
|
|
4129
|
-
if (closer === undefined) {
|
|
4130
|
-
if (!rejectedSalvagedPrRuns.has(run.id)) {
|
|
4131
|
-
const observed = closers[0];
|
|
4132
|
-
const reason =
|
|
4133
|
-
observed === undefined
|
|
4134
|
-
? "no open closing PR"
|
|
4135
|
-
: observed.headRefName === ""
|
|
4136
|
-
? "closer has no head branch identity"
|
|
4137
|
-
: observed.headRefName !== run.branch
|
|
4138
|
-
? `closer head ${observed.headRefName} does not match retained branch ${run.branch}`
|
|
4139
|
-
: observed.repo === ""
|
|
4140
|
-
? "closer has no repository identity"
|
|
4141
|
-
: `closer repository ${observed.repo} does not match routed repository`;
|
|
4142
|
-
log(`#${run.issue} PR not adopted onto attempt ${run.attempt}: ${reason}`);
|
|
4143
|
-
rejectedSalvagedPrRuns.add(run.id);
|
|
4144
|
-
}
|
|
4145
|
-
continue;
|
|
4146
|
-
}
|
|
4147
|
-
|
|
4148
|
-
const flag: SettlementFlag = {
|
|
4149
|
-
kind: "pr-adopted",
|
|
4150
|
-
file: "(recovery)",
|
|
4151
|
-
detail: `${closer.url} matched retained branch ${run.branch} in ${closer.repo}`,
|
|
4152
|
-
};
|
|
4153
|
-
store.updateRun(run.id, {
|
|
4154
|
-
prUrl: closer.url,
|
|
4155
|
-
settlementFlags: [...(run.settlementFlags ?? []), flag],
|
|
4156
|
-
});
|
|
4157
|
-
rejectedSalvagedPrRuns.delete(run.id);
|
|
4158
|
-
log(
|
|
4159
|
-
`#${run.issue} adopted PR ${closer.url} onto attempt ${run.attempt}` +
|
|
4160
|
-
(run.salvageSha === undefined ? "" : ` (salvaged head ${run.salvageSha})`),
|
|
4161
|
-
);
|
|
4162
|
-
}
|
|
4163
|
-
}
|
|
4164
|
-
|
|
4165
4136
|
/** Canonical `owner/repo` identity from a configured network clone URL. */
|
|
4166
4137
|
function githubRepo(cloneUrl: string): string | undefined {
|
|
4167
4138
|
const normalized = cloneUrl.replace(/\/$/, "").replace(/\.git$/, "");
|
|
@@ -4189,7 +4160,7 @@ type CleanupRetainedWorktree = (
|
|
|
4189
4160
|
* dirty or uniquely unpushed work.
|
|
4190
4161
|
*/
|
|
4191
4162
|
export async function cleanupRetainedRuns(
|
|
4192
|
-
d: Pick<Deps, "project" | "tracker" | "store">,
|
|
4163
|
+
d: Pick<Deps, "project" | "tracker" | "store" | "escalate">,
|
|
4193
4164
|
queuedIssues: ReadonlySet<number>,
|
|
4194
4165
|
cursor: RetainedCleanupCursor,
|
|
4195
4166
|
cleanup: CleanupRetainedWorktree = cleanupRetainedWorktree,
|
|
@@ -4254,9 +4225,35 @@ export async function cleanupRetainedRuns(
|
|
|
4254
4225
|
|
|
4255
4226
|
const outcome = await cleanup(mirrorPathFor(repo, project.mirrorRoot), run.worktree, run.branch);
|
|
4256
4227
|
if (outcome.kind === "removed") {
|
|
4257
|
-
store.updateRun(run.id, {
|
|
4228
|
+
store.updateRun(run.id, {
|
|
4229
|
+
worktree: "",
|
|
4230
|
+
...(run.quarantineDetail === undefined ? {} : { quarantineDetail: null }),
|
|
4231
|
+
});
|
|
4258
4232
|
log(`#${run.issue} retained worktree reaped: ${run.worktree} (${run.branch})`);
|
|
4233
|
+
} else if (outcome.reason === "quarantined") {
|
|
4234
|
+
// A tree whose object store cannot be made sound is potentially
|
|
4235
|
+
// stranded work: the daemon refuses to fetch into it, so its commits
|
|
4236
|
+
// cannot be verified against any remote. The row records the condition
|
|
4237
|
+
// so the status snapshot can name the tree, and the escalation ledger
|
|
4238
|
+
// dedupes on the stable summary below — a pass that keeps seeing the
|
|
4239
|
+
// same broken tree reports it once, never once per dispatch pass.
|
|
4240
|
+
store.updateRun(run.id, { quarantineDetail: outcome.detail });
|
|
4241
|
+
await safeEscalate(d, {
|
|
4242
|
+
tier: 1,
|
|
4243
|
+
project: project.name,
|
|
4244
|
+
issue: run.issue,
|
|
4245
|
+
summary: `#${run.issue} quarantined retained worktree — potentially stranded work`,
|
|
4246
|
+
detail: `${run.worktree} (${run.branch})\n${outcome.detail}`,
|
|
4247
|
+
});
|
|
4259
4248
|
} else {
|
|
4249
|
+
// Any other retained reason means the tree is back under ordinary
|
|
4250
|
+
// retention: its alternates were repaired (or never needed it), so the
|
|
4251
|
+
// quarantine — if one was marked — is over and the snapshot must not
|
|
4252
|
+
// keep naming it as quarantined.
|
|
4253
|
+
if (run.quarantineDetail !== undefined) {
|
|
4254
|
+
store.updateRun(run.id, { quarantineDetail: null });
|
|
4255
|
+
log(`#${run.issue} retained worktree no longer quarantined: ${outcome.detail}`);
|
|
4256
|
+
}
|
|
4260
4257
|
log(`#${run.issue} retained worktree kept (${outcome.reason}): ${outcome.detail}`);
|
|
4261
4258
|
}
|
|
4262
4259
|
}
|
|
@@ -4281,6 +4278,7 @@ export function summarizeDispatch(
|
|
|
4281
4278
|
holds: readonly AdmissionHold[],
|
|
4282
4279
|
completedAt = Date.now(),
|
|
4283
4280
|
settled = 0,
|
|
4281
|
+
parked = 0,
|
|
4284
4282
|
): DispatchSummary {
|
|
4285
4283
|
const groups = new Map<AdmissionHoldReason, { count: number; issues: number[]; details: string[] }>();
|
|
4286
4284
|
for (const hold of holds) {
|
|
@@ -4310,6 +4308,10 @@ export function summarizeDispatch(
|
|
|
4310
4308
|
...(group.details.length === 0 ? {} : { details: group.details }),
|
|
4311
4309
|
})),
|
|
4312
4310
|
settled,
|
|
4311
|
+
// Omitted at zero so a pass with nothing parked keeps the pre-#507 record
|
|
4312
|
+
// shape byte for byte — old persisted rows lack the key and readers use
|
|
4313
|
+
// `?? 0` either way.
|
|
4314
|
+
...(parked === 0 ? {} : { parked }),
|
|
4313
4315
|
};
|
|
4314
4316
|
}
|
|
4315
4317
|
|
|
@@ -4442,6 +4444,13 @@ export function upgradeVerifyDepsFor(d: Pick<Deps, "project" | "store">): Upgrad
|
|
|
4442
4444
|
);
|
|
4443
4445
|
return launched.ok === true ? { ok: true, unit: launched.unit } : { ok: false, stderr: launched.stderr };
|
|
4444
4446
|
},
|
|
4447
|
+
herdrSession: resolveHerdrSession(process.env),
|
|
4448
|
+
// The durable pane probe for the post-restart verifier (#832): every omp
|
|
4449
|
+
// process the fleet pane currently claims by herdr, resolved to its start
|
|
4450
|
+
// time — the evidence an external (pane-owned) orchestrator actually
|
|
4451
|
+
// restarted after the install began, through the same shared probe as the
|
|
4452
|
+
// in-process upgrade.
|
|
4453
|
+
probePaneOmp: (session) => herdrPaneOmpStarts(runCommand, session, processStartTimeMs),
|
|
4445
4454
|
log,
|
|
4446
4455
|
now: () => Date.now(),
|
|
4447
4456
|
};
|
|
@@ -4470,14 +4479,71 @@ async function verifyPendingUpgradeTick(
|
|
|
4470
4479
|
}
|
|
4471
4480
|
}
|
|
4472
4481
|
|
|
4482
|
+
/**
|
|
4483
|
+
* The daemon's conductor.db snapshot cadence (#289) — the ledger's only
|
|
4484
|
+
* durable copy, taken on the digest-aligned {@link dbSnapshotDue} window:
|
|
4485
|
+
* once per local day, at/after the digest's configured `at` when it has one.
|
|
4486
|
+
*
|
|
4487
|
+
* Host-global, like the store itself: one snapshot per day for every project,
|
|
4488
|
+
* so whichever project's daemon wins the day first publishes it and the
|
|
4489
|
+
* others no-op on the same durable marker. The marker is written through the
|
|
4490
|
+
* store's existing notification ledger (a one-row idempotence guard, the same
|
|
4491
|
+
* primitive escalations use not to act twice) *after* the snapshot is
|
|
4492
|
+
* published — a crash between publish and mark re-snapshots the next tick
|
|
4493
|
+
* instead of skipping the day, and a restored store that predates today's
|
|
4494
|
+
* marker takes a fresh snapshot on the next tick.
|
|
4495
|
+
*
|
|
4496
|
+
* Returns true when a snapshot was published. A store that does not exist
|
|
4497
|
+
* yet is a silent no-op: there is nothing to back up, and `doctor`'s
|
|
4498
|
+
* `db-backup` probe already treats that as a pass.
|
|
4499
|
+
*/
|
|
4500
|
+
export function runDbSnapshotCadence(args: {
|
|
4501
|
+
digestPolicy: ReportingPolicy["digest"];
|
|
4502
|
+
store: Pick<Store, "wasNotified" | "markNotified">;
|
|
4503
|
+
source: string;
|
|
4504
|
+
backupDir: string;
|
|
4505
|
+
now?: number;
|
|
4506
|
+
keep?: number;
|
|
4507
|
+
log?: (line: string) => void;
|
|
4508
|
+
}): boolean {
|
|
4509
|
+
const { digestPolicy, store, source, backupDir } = args;
|
|
4510
|
+
// No store yet (first boot before any run persisted) → nothing to back up;
|
|
4511
|
+
// `doctor`'s `db-backup` probe already treats that as a pass.
|
|
4512
|
+
if (!existsSync(source)) return false;
|
|
4513
|
+
const now = args.now ?? Date.now();
|
|
4514
|
+
const today = localDayKey(now, digestPolicy.timezone);
|
|
4515
|
+
const alreadyToday = store.wasNotified(dbSnapshotMarkerKey(today));
|
|
4516
|
+
if (!dbSnapshotDue({ digest: digestPolicy }, alreadyToday ? today : undefined, now)) {
|
|
4517
|
+
return false;
|
|
4518
|
+
}
|
|
4519
|
+
const published = snapshotDb(source, backupDir, now);
|
|
4520
|
+
(args.log ?? log)(`conductor.db snapshot published: ${published}`);
|
|
4521
|
+
store.markNotified(dbSnapshotMarkerKey(today));
|
|
4522
|
+
try {
|
|
4523
|
+
const removed = pruneDbSnapshots(backupDir, args.keep ?? DB_SNAPSHOT_RETENTION);
|
|
4524
|
+
if (removed > 0) {
|
|
4525
|
+
(args.log ?? log)(`db snapshot retention pruned ${removed} file(s) beyond the retained ${args.keep ?? DB_SNAPSHOT_RETENTION}`);
|
|
4526
|
+
}
|
|
4527
|
+
} catch (err) {
|
|
4528
|
+
// The snapshot is already published and marked; an over-bound directory
|
|
4529
|
+
// costs disk until the next due day retries the prune, never the backup.
|
|
4530
|
+
(args.log ?? log)(`db snapshot retention prune failed: ${errText(err)}`);
|
|
4531
|
+
}
|
|
4532
|
+
return true;
|
|
4533
|
+
}
|
|
4534
|
+
|
|
4473
4535
|
export async function tick(d: Deps, workers?: WorkerPool): Promise<void> {
|
|
4474
4536
|
// A config edit takes effect on the next tick, not the next daemon restart
|
|
4475
4537
|
// (#170). Re-resolve the project and its caps at the tick boundary so a tick
|
|
4476
4538
|
// and every run it admits see one consistent snapshot; a failed read keeps
|
|
4477
4539
|
// the boot values rather than wedging the tick, and the next tick tries
|
|
4478
|
-
// again.
|
|
4540
|
+
// again. The same reloaded config feeds the db-snapshot backup dir: when it
|
|
4541
|
+
// is unreadable the cadence falls back to the state-root default, matching
|
|
4542
|
+
// `doctor`'s probe of an absent field.
|
|
4543
|
+
let reloadedConfig: ConductorConfig | undefined;
|
|
4479
4544
|
try {
|
|
4480
4545
|
const cfg = loadConfig();
|
|
4546
|
+
reloadedConfig = cfg;
|
|
4481
4547
|
const fresh = findProject(cfg, d.project.name);
|
|
4482
4548
|
const freshCaps = resolveCaps(fresh, cfg.defaults);
|
|
4483
4549
|
if (JSON.stringify(fresh) !== JSON.stringify(d.project) || JSON.stringify(freshCaps) !== JSON.stringify(d.caps)) {
|
|
@@ -4485,6 +4551,7 @@ export async function tick(d: Deps, workers?: WorkerPool): Promise<void> {
|
|
|
4485
4551
|
}
|
|
4486
4552
|
d.project = fresh;
|
|
4487
4553
|
d.caps = freshCaps;
|
|
4554
|
+
d.host = cfg.host;
|
|
4488
4555
|
d.deliveryPolicyValid = true;
|
|
4489
4556
|
} catch (err) {
|
|
4490
4557
|
log(`config reload failed (${errText(err)}) — retaining boot values but blocking autonomous delivery`);
|
|
@@ -4559,6 +4626,25 @@ export async function tick(d: Deps, workers?: WorkerPool): Promise<void> {
|
|
|
4559
4626
|
log(`salvaged PR adoption sweep failed: ${errText(err)}`);
|
|
4560
4627
|
}
|
|
4561
4628
|
|
|
4629
|
+
// The conductor.db snapshot cadence (#289) is durability, not dispatch:
|
|
4630
|
+
// the verb ledger, the decision rows and the run history have no other
|
|
4631
|
+
// copy, so they are snapshotted once per day on the digest-aligned window.
|
|
4632
|
+
// Above the pause gate on purpose — a parked fleet still accumulates ledger
|
|
4633
|
+
// rows and still deserves a backup — and a failure costs the day's snapshot
|
|
4634
|
+
// placeholder, never the tick: failed cadence steps are logged and retried
|
|
4635
|
+
// by the five-minute loop, exactly like the sweeps above it.
|
|
4636
|
+
try {
|
|
4637
|
+
runDbSnapshotCadence({
|
|
4638
|
+
digestPolicy: (d.project.reporting ?? DEFAULT_REPORT_POLICY).digest,
|
|
4639
|
+
store: d.store,
|
|
4640
|
+
source: dbPath(),
|
|
4641
|
+
backupDir: dbBackupDirFor(reloadedConfig),
|
|
4642
|
+
log,
|
|
4643
|
+
});
|
|
4644
|
+
} catch (err) {
|
|
4645
|
+
log(`db snapshot cadence failed: ${errText(err)}`);
|
|
4646
|
+
}
|
|
4647
|
+
|
|
4562
4648
|
// Immediately after settlement and before any routing, so a class is on the
|
|
4563
4649
|
// row before the next dispatch decision reads its budgets (#132). Above the
|
|
4564
4650
|
// pause gate deliberately: classification and label reconciliation are
|
|
@@ -4568,7 +4654,15 @@ export async function tick(d: Deps, workers?: WorkerPool): Promise<void> {
|
|
|
4568
4654
|
// Each is guarded on its own: a tracker that fails mid-classification must not
|
|
4569
4655
|
// stop the label reconcile, and neither may stop the tick.
|
|
4570
4656
|
try {
|
|
4571
|
-
settled += await classifyAndRecover(
|
|
4657
|
+
settled += await classifyAndRecover({
|
|
4658
|
+
project: d.project,
|
|
4659
|
+
caps: d.caps,
|
|
4660
|
+
tracker: d.tracker,
|
|
4661
|
+
store: d.store,
|
|
4662
|
+
escalate: (e) => d.escalate(e),
|
|
4663
|
+
isPaused,
|
|
4664
|
+
setPaused,
|
|
4665
|
+
});
|
|
4572
4666
|
} catch (err) {
|
|
4573
4667
|
log(`classification sweep failed: ${errText(err)}`);
|
|
4574
4668
|
}
|
|
@@ -4618,6 +4712,37 @@ export async function tick(d: Deps, workers?: WorkerPool): Promise<void> {
|
|
|
4618
4712
|
log(`decision condition pass failed: ${errText(err)}`);
|
|
4619
4713
|
});
|
|
4620
4714
|
|
|
4715
|
+
// A spend-cap pause is self-expiring (#780). The spend gate below persists a
|
|
4716
|
+
// pause once today's spend reaches the cap, but once that latch is on disk
|
|
4717
|
+
// every later pass returns at the pause gate and never reaches the spend
|
|
4718
|
+
// check again — so the fleet stayed stopped after the rolling window reset
|
|
4719
|
+
// until an operator ran `resume`. The same measurement that closed the gate
|
|
4720
|
+
// reopens it: clear a per-project spend-cap pause when the current
|
|
4721
|
+
// rolling-window spend is below the cap (or no cap is configured at all —
|
|
4722
|
+
// the gate that justified the pause is gone). Two guards keep the clear from
|
|
4723
|
+
// ever overriding another stop: only the exact instance that was read is
|
|
4724
|
+
// removed (a hold that replaced the sentinel meanwhile is untouched, review
|
|
4725
|
+
// #780), and only the per-project sentinel is ever considered — the legacy
|
|
4726
|
+
// global sentinel, which may carry an operator/integrity/setup hold, is
|
|
4727
|
+
// never removed, so the pause gate below still observes it.
|
|
4728
|
+
const spendPausePath = pausedPath(d.project.name);
|
|
4729
|
+
const spendPause = pauseInstanceAt(spendPausePath);
|
|
4730
|
+
if (spendPause?.source === "spend-cap") {
|
|
4731
|
+
const spent = d.store.spendSince(d.project.name, startOfToday());
|
|
4732
|
+
if (
|
|
4733
|
+
(d.caps.dailySpendUsd === null || spent < d.caps.dailySpendUsd) &&
|
|
4734
|
+
clearPauseIfUnchanged(spendPausePath, spendPause)
|
|
4735
|
+
) {
|
|
4736
|
+
log(
|
|
4737
|
+
`spend-cap pause cleared: $${spent.toFixed(2)} ` +
|
|
4738
|
+
(d.caps.dailySpendUsd === null
|
|
4739
|
+
? "(no daily cap configured)"
|
|
4740
|
+
: `below the $${d.caps.dailySpendUsd.toFixed(2)} daily cap`) +
|
|
4741
|
+
" — dispatch resumes",
|
|
4742
|
+
);
|
|
4743
|
+
}
|
|
4744
|
+
}
|
|
4745
|
+
|
|
4621
4746
|
// A paused fleet claims nothing. Checked first so pausing takes effect on the
|
|
4622
4747
|
// next tick without signalling the process. But the pass still ran, and the
|
|
4623
4748
|
// operator has to be able to see it: record it as a held pass — the work the
|
|
@@ -4638,6 +4763,37 @@ export async function tick(d: Deps, workers?: WorkerPool): Promise<void> {
|
|
|
4638
4763
|
return;
|
|
4639
4764
|
}
|
|
4640
4765
|
|
|
4766
|
+
// The project drain is the self-expiring sibling of the pause fence (#484):
|
|
4767
|
+
// the same admission boundary — settlement above it, nothing claimed below —
|
|
4768
|
+
// but the intent is durable (it survives orchestrator loss) and bounded (the
|
|
4769
|
+
// record carries an absolute deadline, so a crash can never strand
|
|
4770
|
+
// admission). Three shapes, three behaviours:
|
|
4771
|
+
// - fresh drain with live runs → a held pass, exactly like a pause;
|
|
4772
|
+
// - fresh drain with nothing left to wait for → the drain is satisfied
|
|
4773
|
+
// and clears itself, so a completed drain never needs a second operator
|
|
4774
|
+
// action and this pass proceeds normally;
|
|
4775
|
+
// - malformed record → fails closed for THIS pass (it might be a fresh
|
|
4776
|
+
// fence we cannot read), and the same consume removed it, so it can
|
|
4777
|
+
// never become an unbounded permanent drain.
|
|
4778
|
+
const drain = consumeDrain(d.project.name);
|
|
4779
|
+
if (drain.kind === "active") {
|
|
4780
|
+
// Completion is the ACTIVE set, not the live-worker set: pushed-pending
|
|
4781
|
+
// and pushed-green PRs still make the `runs-settled` release gate fail, so
|
|
4782
|
+
// a drain that cleared while one remained would admit work on top of a
|
|
4783
|
+
// batch the releases still see as unfinished (#776 review #2).
|
|
4784
|
+
if (d.store.activeRuns(d.project.name).length === 0) {
|
|
4785
|
+
cancelDrain(d.project.name);
|
|
4786
|
+
log("project drain completed: no active runs remain — drain cleared");
|
|
4787
|
+
} else {
|
|
4788
|
+
d.store.recordDispatch(d.project.name, summarizeHeldPass(settled));
|
|
4789
|
+
return;
|
|
4790
|
+
}
|
|
4791
|
+
} else if (drain.kind === "error") {
|
|
4792
|
+
log(`project drain record invalid (${drain.problem}) — removed; this pass admits nothing`);
|
|
4793
|
+
d.store.recordDispatch(d.project.name, summarizeHeldPass(settled));
|
|
4794
|
+
return;
|
|
4795
|
+
}
|
|
4796
|
+
|
|
4641
4797
|
const { project, caps, store } = d;
|
|
4642
4798
|
|
|
4643
4799
|
// "Nobody patches the running conductor" is a hard boundary in both briefs —
|
|
@@ -4727,18 +4883,57 @@ export async function tick(d: Deps, workers?: WorkerPool): Promise<void> {
|
|
|
4727
4883
|
return pending.length === 0 ? issue : { ...issue, labels: effectiveLabels(issue.labels, pending) };
|
|
4728
4884
|
});
|
|
4729
4885
|
const { routed, unroutable } = route(effective, project);
|
|
4730
|
-
//
|
|
4731
|
-
//
|
|
4732
|
-
//
|
|
4733
|
-
|
|
4734
|
-
|
|
4735
|
-
|
|
4736
|
-
|
|
4737
|
-
|
|
4886
|
+
// route() drops a candidate for two reasons, only one of which is a claim
|
|
4887
|
+
// question. A lifecycle state label (agent:in-progress/blocked/failed)
|
|
4888
|
+
// marks a run-owned issue: it is genuinely in flight while its newest run
|
|
4889
|
+
// is live or settling, and a terminal newest run — or no run at all — means
|
|
4890
|
+
// the label is residual, the footprint of a settled run nobody cleared,
|
|
4891
|
+
// which is Duty 1 reconciliation work, not occupied capacity. A missing
|
|
4892
|
+
// queue label instead marks an issue the outbox is withdrawing (a pending
|
|
4893
|
+
// queue-label removal, e.g. releaseQueueLabel after a merge the PR did not
|
|
4894
|
+
// close), which belongs in none of the three populations. So `claimed` is
|
|
4895
|
+
// defined from actual ownership over genuinely lifecycle-labelled candidates
|
|
4896
|
+
// rather than as the residual of routing (#228, #611).
|
|
4897
|
+
const stateLabels = new Set(Object.values(project.stateLabels));
|
|
4898
|
+
const dropped = effective.filter(
|
|
4899
|
+
(issue) => !isEligible(issue, project) && issue.labels.some((l) => stateLabels.has(l)),
|
|
4900
|
+
);
|
|
4901
|
+
// The operator's park label is not a stale lifecycle label: the queue query
|
|
4902
|
+
// still returns a parked-and-queued issue, route() drops it as ineligible,
|
|
4903
|
+
// and the claim gate holds it as `issue-parked` when the park lands mid-pass
|
|
4904
|
+
// (#734). Counting it as stale-lifecycle below would summon the orchestrator
|
|
4905
|
+
// to reconcile a deliberate decision. Parked is its own population, derived
|
|
4906
|
+
// from the same isEligible read the gate uses, so the number status renders
|
|
4907
|
+
// cannot disagree with what admission would hold (#507).
|
|
4908
|
+
const parkedCandidates = effective.filter(
|
|
4909
|
+
(issue) => !isEligible(issue, project) && issue.labels.includes(project.stateLabels.backlog),
|
|
4910
|
+
);
|
|
4911
|
+
const parkedNumbers = new Set(parkedCandidates.map((issue) => issue.number));
|
|
4912
|
+
let claimed = 0;
|
|
4913
|
+
let parked = 0;
|
|
4914
|
+
const lifecycleHolds: AdmissionHold[] = [];
|
|
4915
|
+
for (const issue of dropped) {
|
|
4916
|
+
const newest = store.latestRun(project.name, issue.number);
|
|
4917
|
+
if (newest !== undefined && ACTIVE_STATES.includes(newest.state)) {
|
|
4918
|
+
claimed += 1;
|
|
4919
|
+
} else if (parkedNumbers.has(issue.number)) {
|
|
4920
|
+
// A park never kills a live run; a parked issue with no run is inventory
|
|
4921
|
+
// the operator is deliberately holding, not reconciliation work.
|
|
4922
|
+
parked += 1;
|
|
4923
|
+
} else {
|
|
4924
|
+
lifecycleHolds.push({ issue: issue.number, reason: "stale-lifecycle" });
|
|
4925
|
+
}
|
|
4926
|
+
}
|
|
4927
|
+
const routingHolds: AdmissionHold[] = [
|
|
4928
|
+
...unroutable.map(
|
|
4929
|
+
(u): AdmissionHold => ({ issue: u.issue.number, reason: `unroutable:${u.reason}` }),
|
|
4930
|
+
),
|
|
4931
|
+
...lifecycleHolds,
|
|
4932
|
+
];
|
|
4738
4933
|
const recordDispatch = (admitted: number, holds: readonly AdmissionHold[]): void => {
|
|
4739
4934
|
store.recordDispatch(
|
|
4740
4935
|
project.name,
|
|
4741
|
-
summarizeDispatch(ready.length, routed.length, claimed, admitted, holds, Date.now(), settled),
|
|
4936
|
+
summarizeDispatch(ready.length, routed.length, claimed, admitted, holds, Date.now(), settled, parked),
|
|
4742
4937
|
);
|
|
4743
4938
|
};
|
|
4744
4939
|
|
|
@@ -4783,7 +4978,8 @@ export async function tick(d: Deps, workers?: WorkerPool): Promise<void> {
|
|
|
4783
4978
|
summary: `Daily spend cap reached on ${new Date().toISOString().slice(0, 10)} — ${project.name} is paused`,
|
|
4784
4979
|
detail: [
|
|
4785
4980
|
`Spent $${spent.toFixed(2)} of the $${caps.dailySpendUsd.toFixed(2)} daily cap.`,
|
|
4786
|
-
"
|
|
4981
|
+
"Dispatch stays paused until today's spend falls below the cap, then resumes",
|
|
4982
|
+
"automatically on the next pass — `omp-conductor resume` only speeds that up.",
|
|
4787
4983
|
].join("\n"),
|
|
4788
4984
|
});
|
|
4789
4985
|
recordDispatch(0, [
|
|
@@ -4831,7 +5027,7 @@ export async function tick(d: Deps, workers?: WorkerPool): Promise<void> {
|
|
|
4831
5027
|
log(`dispatching ${pass.admitted.map((a) => `#${a.r.issue.number}`).join(" ")}`);
|
|
4832
5028
|
await dispatchAdmissions(
|
|
4833
5029
|
pass.admitted,
|
|
4834
|
-
(a) => handleIssue(d, a.r, a.attempt, a.lane),
|
|
5030
|
+
(a) => handleIssue(d, a.r, a.attempt, a.lane, a.model),
|
|
4835
5031
|
workers,
|
|
4836
5032
|
);
|
|
4837
5033
|
|
|
@@ -4859,6 +5055,23 @@ export interface DaemonHealthSnapshot {
|
|
|
4859
5055
|
codeGraph?: CodeGraphHealth;
|
|
4860
5056
|
/** Live workers in a non-running pause phase; absent/empty = nothing paused. */
|
|
4861
5057
|
workers?: { issue: number; runId: string; phase: WorkerPausePhase }[];
|
|
5058
|
+
/**
|
|
5059
|
+
* The live orchestrator surface, attested by the running daemon (#832):
|
|
5060
|
+
* which mode this project's fleet is in, and — when the daemon hosts the
|
|
5061
|
+
* orchestrator session itself — the extension version that session *loaded*
|
|
5062
|
+
* and the transcript it resumed. This is the read the upgrade's
|
|
5063
|
+
* session-reload verification checks: a restarted daemon whose orchestrator
|
|
5064
|
+
* child came up on the installed release answers `loaded` equal to that
|
|
5065
|
+
* release; one that never reloaded answers an older version; one that
|
|
5066
|
+
* failed to start answers `mode: "failed"`. `external` means the pane owns
|
|
5067
|
+
* the session and the daemon cannot attest it from here.
|
|
5068
|
+
*/
|
|
5069
|
+
orchestrator?: {
|
|
5070
|
+
mode: "embedded" | "external" | "failed";
|
|
5071
|
+
loaded?: string;
|
|
5072
|
+
sessionFile?: string;
|
|
5073
|
+
alive?: boolean;
|
|
5074
|
+
};
|
|
4862
5075
|
}
|
|
4863
5076
|
|
|
4864
5077
|
export interface DaemonHealth {
|
|
@@ -4874,6 +5087,7 @@ export function daemonHealthSnapshot(
|
|
|
4874
5087
|
paused = isPaused(project),
|
|
4875
5088
|
codeGraph?: CodeGraphHealth,
|
|
4876
5089
|
workerControls?: WorkerControlRegistry,
|
|
5090
|
+
orchestrator?: DaemonHealthSnapshot["orchestrator"],
|
|
4877
5091
|
): DaemonHealthSnapshot {
|
|
4878
5092
|
const dispatch = store.latestDispatch(project);
|
|
4879
5093
|
return {
|
|
@@ -4885,6 +5099,7 @@ export function daemonHealthSnapshot(
|
|
|
4885
5099
|
...(dispatch === undefined ? {} : { dispatch }),
|
|
4886
5100
|
...(codeGraph?.configured === true ? { codeGraph } : {}),
|
|
4887
5101
|
...(workerControls === undefined ? {} : { workers: workerControls.snapshot(project) }),
|
|
5102
|
+
...(orchestrator === undefined ? {} : { orchestrator }),
|
|
4888
5103
|
};
|
|
4889
5104
|
}
|
|
4890
5105
|
|
|
@@ -5202,6 +5417,55 @@ export async function daemonHttpResponse(req: Request, d: DaemonHttpDeps): Promi
|
|
|
5202
5417
|
return workerControl ?? new Response("not found\n", { status: 404 });
|
|
5203
5418
|
}
|
|
5204
5419
|
|
|
5420
|
+
/**
|
|
5421
|
+
* The effective reporting surface, derived from the configured policy — never
|
|
5422
|
+
* from the legacy preset name alone: an explicit policy without the
|
|
5423
|
+
* `scopePreset` back-annotation must present the same truth from `interruptOn`
|
|
5424
|
+
* and the digest cadence (#633).
|
|
5425
|
+
*/
|
|
5426
|
+
export interface ReportingSummary {
|
|
5427
|
+
/** The legacy preset this policy came from, when the config back-annotates one. */
|
|
5428
|
+
scopePreset?: ReportScope;
|
|
5429
|
+
/** Categories allowed to interrupt the operator's phone. */
|
|
5430
|
+
interruptOn: InterruptCategory[];
|
|
5431
|
+
/** Where non-interrupting outcomes accumulate: the effective digest config. */
|
|
5432
|
+
digest: ReportingPolicy["digest"];
|
|
5433
|
+
}
|
|
5434
|
+
|
|
5435
|
+
/**
|
|
5436
|
+
* The effective reporting summary. The loader always materialises the policy,
|
|
5437
|
+
* so `undefined` means the default (`DEFAULT_REPORT_POLICY`); the preset name
|
|
5438
|
+
* is carried only when the policy actually back-annotates one, so a rendered
|
|
5439
|
+
* status names the preset without ever letting a stale or absent preset
|
|
5440
|
+
* misstate what the policy does (#633).
|
|
5441
|
+
*/
|
|
5442
|
+
export function reportingSummary(policy: ReportingPolicy | undefined): ReportingSummary {
|
|
5443
|
+
const effective = policy ?? DEFAULT_REPORT_POLICY;
|
|
5444
|
+
return {
|
|
5445
|
+
...(effective.scopePreset === undefined ? {} : { scopePreset: effective.scopePreset }),
|
|
5446
|
+
interruptOn: [...effective.interruptOn],
|
|
5447
|
+
digest: { ...effective.digest },
|
|
5448
|
+
};
|
|
5449
|
+
}
|
|
5450
|
+
|
|
5451
|
+
/**
|
|
5452
|
+
* The active drain as structured status exposes it: the creation instant, the
|
|
5453
|
+
* absolute deadline, the purpose, and the live-run count the drain is waiting
|
|
5454
|
+
* to reach zero. Only present on the snapshot while the record is fresh.
|
|
5455
|
+
*/
|
|
5456
|
+
export interface DrainStatus {
|
|
5457
|
+
/** Epoch-ms instant the drain intent was recorded. */
|
|
5458
|
+
since: number;
|
|
5459
|
+
/** Absolute epoch-ms deadline — admission resumes automatically after it. */
|
|
5460
|
+
expiresAt: number;
|
|
5461
|
+
/** Purpose recorded at creation. */
|
|
5462
|
+
reason?: string;
|
|
5463
|
+
/** Runs still in the active set (live workers plus pushed-state PRs) — the
|
|
5464
|
+
* count a drain waits to reach zero, matching the `runs-settled` release
|
|
5465
|
+
* gate (#776 review #2). */
|
|
5466
|
+
remainingRuns: number;
|
|
5467
|
+
}
|
|
5468
|
+
|
|
5205
5469
|
export interface StatusSnapshot {
|
|
5206
5470
|
project: string;
|
|
5207
5471
|
configPath: string;
|
|
@@ -5213,10 +5477,27 @@ export interface StatusSnapshot {
|
|
|
5213
5477
|
* reading like a mistake (#220).
|
|
5214
5478
|
*/
|
|
5215
5479
|
pauseReason?: string;
|
|
5480
|
+
/**
|
|
5481
|
+
* The project's active self-expiring drain (#484): present only while a
|
|
5482
|
+
* fresh, valid drain record exists. Paused and drained are deliberately two
|
|
5483
|
+
* fields — a drain is bounded and self-clearing where a pause is not, and
|
|
5484
|
+
* the structured surface has to tell them apart without reading the pass
|
|
5485
|
+
* history. Human CLI wording over it is a later #484 child.
|
|
5486
|
+
*/
|
|
5487
|
+
drain?: DrainStatus;
|
|
5216
5488
|
/** Mechanical operator availability at the moment this snapshot was read. */
|
|
5217
5489
|
availability?: AvailabilityState;
|
|
5218
5490
|
/** Next digest opportunity under the same predicate that gates submission. */
|
|
5219
5491
|
digestSchedule?: DigestScheduleState;
|
|
5492
|
+
/**
|
|
5493
|
+
* The effective reporting policy: what interrupts, where everything else
|
|
5494
|
+
* goes, and the legacy preset name when one is back-annotated. On the
|
|
5495
|
+
* snapshot so the human `status`, the dashboard API and the CLI all read
|
|
5496
|
+
* the same truth about the reporting surface without opening config — a
|
|
5497
|
+
* routine outcome that reads as "Telegram is broken" unless the policy says
|
|
5498
|
+
* it is digest-only (#633).
|
|
5499
|
+
*/
|
|
5500
|
+
reporting?: ReportingSummary;
|
|
5220
5501
|
caps: Caps;
|
|
5221
5502
|
/**
|
|
5222
5503
|
* The effective per-shape release grants. On the snapshot rather than re-read
|
|
@@ -5225,6 +5506,14 @@ export interface StatusSnapshot {
|
|
|
5225
5506
|
* the config or the brief.
|
|
5226
5507
|
*/
|
|
5227
5508
|
releaseGrants: ResolvedGrants;
|
|
5509
|
+
/**
|
|
5510
|
+
* The effective review policy (#678). On the snapshot for the reason
|
|
5511
|
+
* `releaseGrants` is: the orchestrator's Duty 1 has to act on it every
|
|
5512
|
+
* tick, and a stale level or ceiling sitting only in a config file nobody
|
|
5513
|
+
* opens is exactly the drift this field exists to surface. The loader always
|
|
5514
|
+
* materialises it, so this is never absent on a real daemon.
|
|
5515
|
+
*/
|
|
5516
|
+
review: ReviewPolicy;
|
|
5228
5517
|
/** Occupied issues: live workers plus green PRs awaiting a human merge. */
|
|
5229
5518
|
activeRuns: RunRecord[];
|
|
5230
5519
|
/**
|
|
@@ -5239,6 +5528,11 @@ export interface StatusSnapshot {
|
|
|
5239
5528
|
/** Newest attempts holding a preserved WIP tip, or a tree that is still the
|
|
5240
5529
|
* only copy of work the daemon could not save. */
|
|
5241
5530
|
salvagedRuns: RunRecord[];
|
|
5531
|
+
/** Retained trees whose object store could not be made sound, so the daemon
|
|
5532
|
+
* refused to fetch into them and their commits cannot be verified against
|
|
5533
|
+
* any remote (#737). Distinguished from ordinary retention on purpose —
|
|
5534
|
+
* quarantine is "potentially stranded work", not routine housekeeping. */
|
|
5535
|
+
quarantinedRuns: RunRecord[];
|
|
5242
5536
|
/** One-shot issue ceilings waiting for the next claim. */
|
|
5243
5537
|
turnOverrides: TurnOverride[];
|
|
5244
5538
|
/** Reports the operator has not provably received: pending, in-flight with an
|
|
@@ -5325,6 +5619,19 @@ export function statusSnapshotFromStore(
|
|
|
5325
5619
|
// Read once: the provenance read touches the filesystem, and the renderer
|
|
5326
5620
|
// should never pay for it twice per status.
|
|
5327
5621
|
const reason = pauseProvenance(p.name)?.reason;
|
|
5622
|
+
// Same cost discipline as the pause read: the drain record is a file read,
|
|
5623
|
+
// and the active-drain view is only built when one is actually fresh. The
|
|
5624
|
+
// read is observational — it never mutates the record — so a status read
|
|
5625
|
+
// cannot consume a malformed marker the next dispatch pass still has to fail
|
|
5626
|
+
// closed on; the dispatch consume owns cleanup (#776 review #2).
|
|
5627
|
+
const drain = readDrain(p.name);
|
|
5628
|
+
// The live run list feeds the worker-capacity row; the drain's
|
|
5629
|
+
// remaining-runs count waits on the ACTIVE set (live workers plus
|
|
5630
|
+
// pushed-state PRs), the same population the `runs-settled` release gate
|
|
5631
|
+
// reads, so the two can never disagree about when a batch is finished
|
|
5632
|
+
// (#776 review #2).
|
|
5633
|
+
const live = store.liveRuns(p.name);
|
|
5634
|
+
const active = store.activeRuns(p.name);
|
|
5328
5635
|
// The live review-revision rounds, read from the same durable rows the
|
|
5329
5636
|
// restart recovery uses: a run whose revision is dispatched is read as
|
|
5330
5637
|
// `review-revision N` while its worker is live (#692).
|
|
@@ -5338,18 +5645,31 @@ export function statusSnapshotFromStore(
|
|
|
5338
5645
|
stateDir: stateDir(),
|
|
5339
5646
|
paused: isPaused(p.name),
|
|
5340
5647
|
...(reason === undefined ? {} : { pauseReason: reason }),
|
|
5648
|
+
...(drain.kind === "active"
|
|
5649
|
+
? {
|
|
5650
|
+
drain: {
|
|
5651
|
+
since: Date.parse(drain.drain.createdAt),
|
|
5652
|
+
expiresAt: Date.parse(drain.drain.expiresAt),
|
|
5653
|
+
...(drain.drain.reason === undefined ? {} : { reason: drain.drain.reason }),
|
|
5654
|
+
remainingRuns: active.length,
|
|
5655
|
+
} satisfies DrainStatus,
|
|
5656
|
+
}
|
|
5657
|
+
: {}),
|
|
5341
5658
|
availability: availabilityState(p.reporting, now),
|
|
5342
5659
|
digestSchedule: digestScheduleState(p.reporting ?? DEFAULT_REPORT_POLICY, lastDigestDay, now),
|
|
5660
|
+
reporting: reportingSummary(p.reporting ?? DEFAULT_REPORT_POLICY),
|
|
5343
5661
|
caps,
|
|
5344
5662
|
releaseGrants: resolveReleaseGrants(p),
|
|
5345
|
-
|
|
5663
|
+
review: resolveReview(p),
|
|
5664
|
+
activeRuns: active,
|
|
5346
5665
|
reviewRounds,
|
|
5347
5666
|
salvagedRuns: store.salvagedRuns(p.name),
|
|
5667
|
+
quarantinedRuns: store.quarantinedRuns(p.name),
|
|
5348
5668
|
turnOverrides: store.listTurnOverrides(p.name),
|
|
5349
5669
|
openReports: store.openReports(p.name),
|
|
5350
5670
|
digestBacklog: store.digestBacklog(p.name),
|
|
5351
5671
|
verbLedger: store.verbLedger(p.name, { limit: STATUS_LEDGER_SCAN }),
|
|
5352
|
-
liveWorkers:
|
|
5672
|
+
liveWorkers: live.length,
|
|
5353
5673
|
runsToday: store.runsStartedSince(p.name, since),
|
|
5354
5674
|
spendTodayUsd: store.spendSince(p.name, since),
|
|
5355
5675
|
...(dispatch === undefined ? {} : { dispatch }),
|
|
@@ -5396,6 +5716,14 @@ export function formatDispatchSummary(summary?: DispatchSummary): string {
|
|
|
5396
5716
|
` candidates ${summary.ready} ready / ${summary.claimed ?? 0} in flight / ${summary.routed} spare`,
|
|
5397
5717
|
` admitted ${summary.admitted}`,
|
|
5398
5718
|
);
|
|
5719
|
+
// The operator's park label, counted from the same eligibility read the
|
|
5720
|
+
// claim gate uses (#507): "0 claimable, 12 parked" and "0 claimable,
|
|
5721
|
+
// nothing to do" demand opposite orchestrator responses and must not
|
|
5722
|
+
// render alike. Omitted at zero so an empty queue stays the old shape.
|
|
5723
|
+
const parked = summary.parked ?? 0;
|
|
5724
|
+
if (parked > 0) {
|
|
5725
|
+
lines.push(` parked ${parked} — operator-held, never claimed`);
|
|
5726
|
+
}
|
|
5399
5727
|
if (summary.holds.length === 0) {
|
|
5400
5728
|
lines.push(" held 0");
|
|
5401
5729
|
} else {
|
|
@@ -5418,32 +5746,6 @@ export function formatDispatchSummary(summary?: DispatchSummary): string {
|
|
|
5418
5746
|
return lines.join("\n");
|
|
5419
5747
|
}
|
|
5420
5748
|
|
|
5421
|
-
/**
|
|
5422
|
-
* The WIP block: every issue whose newest attempt left work behind, and
|
|
5423
|
-
* whether that work is safe.
|
|
5424
|
-
*
|
|
5425
|
-
* Blocked runs used to be invisible here, which is exactly how #118 stayed
|
|
5426
|
-
* invisible for a full attempt cycle — the operator saw a blocked issue and had
|
|
5427
|
-
* no way to tell "stopped with 34 uncommitted files" from "stopped clean".
|
|
5428
|
-
* A preserved line is informational; an UNSALVAGED line is an alarm, and it
|
|
5429
|
-
* names the directory because that directory is the work.
|
|
5430
|
-
*/
|
|
5431
|
-
export function formatSalvagedRuns(runs: readonly RunRecord[]): string[] {
|
|
5432
|
-
if (runs.length === 0) return [];
|
|
5433
|
-
const lines = ["", "wip"];
|
|
5434
|
-
for (const r of runs) {
|
|
5435
|
-
lines.push(
|
|
5436
|
-
r.salvageError !== undefined && r.salvageAckAt === undefined
|
|
5437
|
-
? ` #${r.issue} UNSALVAGED ${r.worktree === "" ? "(path not recorded)" : r.worktree} — ` +
|
|
5438
|
-
`only copy, dispatch held (${r.salvageError})`
|
|
5439
|
-
: r.salvageError !== undefined
|
|
5440
|
-
? ` #${r.issue} accepted as lost attempt ${r.attempt} (${r.salvageError})`
|
|
5441
|
-
: ` #${r.issue} preserved ${r.salvageSha ?? "?"} on ${r.branch} (attempt ${r.attempt}, ${r.state})`,
|
|
5442
|
-
);
|
|
5443
|
-
}
|
|
5444
|
-
return lines;
|
|
5445
|
-
}
|
|
5446
|
-
|
|
5447
5749
|
/**
|
|
5448
5750
|
* The grant table, named shape by shape.
|
|
5449
5751
|
*
|
|
@@ -5461,6 +5763,7 @@ export function formatReleaseGrants(grants: ResolvedGrants): string[] {
|
|
|
5461
5763
|
...RELEASE_SHAPES.map((shape) => ` ${shape.padEnd(19)}${grants[shape]}`),
|
|
5462
5764
|
];
|
|
5463
5765
|
}
|
|
5766
|
+
|
|
5464
5767
|
export function formatBaseHealth(rows: readonly BaseHealth[]): string[] {
|
|
5465
5768
|
return rows.map((row) => {
|
|
5466
5769
|
const head = row.headSha.slice(0, 8);
|
|
@@ -5550,6 +5853,7 @@ export function formatStatus(s: StatusSnapshot): string {
|
|
|
5550
5853
|
lines.push(...formatBaseHealth(s.baseHealth));
|
|
5551
5854
|
lines.push(...formatFreezes(s.freezes));
|
|
5552
5855
|
lines.push(...formatSalvagedRuns(s.salvagedRuns));
|
|
5856
|
+
lines.push(...formatQuarantinedRuns(s.quarantinedRuns));
|
|
5553
5857
|
lines.push(...formatOpenReports(s.openReports));
|
|
5554
5858
|
lines.push(...formatVerbLedger(s.verbLedger));
|
|
5555
5859
|
// Deploy hint: a restart while workers are live orphans them (salvage runs
|
|
@@ -5623,61 +5927,6 @@ export function prepareConductor(project?: string): void {
|
|
|
5623
5927
|
setPaused(true, { source: "setup" }, project);
|
|
5624
5928
|
}
|
|
5625
5929
|
|
|
5626
|
-
/** Bounded per tick: each row costs tracker calls to gather facts for. */
|
|
5627
|
-
const CLASSIFY_BATCH = 20;
|
|
5628
|
-
|
|
5629
|
-
/** Tool calls quoted as evidence for a run that spun to its turn cap. */
|
|
5630
|
-
const SPIN_EVIDENCE_CALLS = 10;
|
|
5631
|
-
|
|
5632
|
-
/**
|
|
5633
|
-
* The last few tool names a transcript recorded, newest last.
|
|
5634
|
-
*
|
|
5635
|
-
* `turn-cap-spinning` escalates rather than requeueing, and the acceptance
|
|
5636
|
-
* criterion is that the escalation carries evidence of what the worker was doing
|
|
5637
|
-
* when it hit the cap — otherwise the orchestrator opens the transcript and
|
|
5638
|
-
* re-derives it, which is the manual triage this whole sweep removes.
|
|
5639
|
-
*/
|
|
5640
|
-
export function lastToolCalls(sessionFile: string | undefined, limit = SPIN_EVIDENCE_CALLS): string[] {
|
|
5641
|
-
if (sessionFile === undefined) return [];
|
|
5642
|
-
let text: string;
|
|
5643
|
-
try {
|
|
5644
|
-
text = readFileSync(sessionFile, "utf8");
|
|
5645
|
-
} catch {
|
|
5646
|
-
return [];
|
|
5647
|
-
}
|
|
5648
|
-
const names: string[] = [];
|
|
5649
|
-
for (const line of text.split("\n")) {
|
|
5650
|
-
if (line.length === 0) continue;
|
|
5651
|
-
let row: unknown;
|
|
5652
|
-
try {
|
|
5653
|
-
row = JSON.parse(line) as unknown;
|
|
5654
|
-
} catch {
|
|
5655
|
-
continue;
|
|
5656
|
-
}
|
|
5657
|
-
if (row === null || typeof row !== "object") continue;
|
|
5658
|
-
const rec = row as { readonly [key: string]: unknown };
|
|
5659
|
-
// Both shapes the harness has written: a top-level tool event, and a tool
|
|
5660
|
-
// block inside an assistant message.
|
|
5661
|
-
const direct = rec["toolName"];
|
|
5662
|
-
if (typeof direct === "string") {
|
|
5663
|
-
names.push(direct);
|
|
5664
|
-
continue;
|
|
5665
|
-
}
|
|
5666
|
-
const message = rec["message"];
|
|
5667
|
-
if (message === null || typeof message !== "object") continue;
|
|
5668
|
-
const content = (message as { readonly [key: string]: unknown })["content"];
|
|
5669
|
-
if (!Array.isArray(content)) continue;
|
|
5670
|
-
for (const part of content) {
|
|
5671
|
-
if (part === null || typeof part !== "object") continue;
|
|
5672
|
-
const p = part as { readonly [key: string]: unknown };
|
|
5673
|
-
if (p["type"] !== "tool_use") continue;
|
|
5674
|
-
const name = p["name"];
|
|
5675
|
-
if (typeof name === "string") names.push(name);
|
|
5676
|
-
}
|
|
5677
|
-
}
|
|
5678
|
-
return names.slice(-limit);
|
|
5679
|
-
}
|
|
5680
|
-
|
|
5681
5930
|
/** A provider refusal a session recorded before dying, or undefined for none. */
|
|
5682
5931
|
export interface SessionError {
|
|
5683
5932
|
status?: number;
|
|
@@ -5693,138 +5942,6 @@ export function completionLastError(
|
|
|
5693
5942
|
return providerCredit ?? providerTransient ?? verifiedReason ?? sessionErr?.message;
|
|
5694
5943
|
}
|
|
5695
5944
|
|
|
5696
|
-
/**
|
|
5697
|
-
* The last error a transcript recorded, or undefined when it recorded none.
|
|
5698
|
-
*
|
|
5699
|
-
* The harness writes `{"stopReason":"error","errorStatus":402,"errorId":402,
|
|
5700
|
-
* "errorMessage":"402 This request requires more credits, ..."}`. The daemon
|
|
5701
|
-
* read none of it, so three runs died `unknown` with an empty `lastError` and
|
|
5702
|
-
* charged an attempt each for a billing state (#220).
|
|
5703
|
-
*
|
|
5704
|
-
* Scanned newest-first: a session that recovered from an early error and then
|
|
5705
|
-
* died of something else must report the something else, and a session that
|
|
5706
|
-
* recovered from its only error and finished cleanly reports the error anyway
|
|
5707
|
-
* because there is no terminal verdict to outrank it (#220).
|
|
5708
|
-
*/
|
|
5709
|
-
export function readSessionError(sessionFile: string | undefined): SessionError | undefined {
|
|
5710
|
-
if (sessionFile === undefined) return undefined;
|
|
5711
|
-
let text: string;
|
|
5712
|
-
try {
|
|
5713
|
-
text = readFileSync(sessionFile, "utf8");
|
|
5714
|
-
} catch {
|
|
5715
|
-
return undefined;
|
|
5716
|
-
}
|
|
5717
|
-
const lines = text.split("\n");
|
|
5718
|
-
for (let i = lines.length - 1; i >= 0; i -= 1) {
|
|
5719
|
-
const line = lines[i];
|
|
5720
|
-
if (line === undefined || line.length === 0) continue;
|
|
5721
|
-
let row: unknown;
|
|
5722
|
-
try {
|
|
5723
|
-
row = JSON.parse(line) as unknown;
|
|
5724
|
-
} catch {
|
|
5725
|
-
continue;
|
|
5726
|
-
}
|
|
5727
|
-
if (row === null || typeof row !== "object") continue;
|
|
5728
|
-
const rec = row as { readonly [key: string]: unknown };
|
|
5729
|
-
if (rec["stopReason"] !== "error") continue;
|
|
5730
|
-
const message = rec["errorMessage"];
|
|
5731
|
-
if (typeof message !== "string" || message.trim() === "") continue;
|
|
5732
|
-
const status = rec["errorStatus"];
|
|
5733
|
-
return {
|
|
5734
|
-
...(typeof status === "number" && Number.isFinite(status) ? { status } : {}),
|
|
5735
|
-
message: message.trim(),
|
|
5736
|
-
};
|
|
5737
|
-
}
|
|
5738
|
-
return undefined;
|
|
5739
|
-
}
|
|
5740
|
-
|
|
5741
|
-
/**
|
|
5742
|
-
* Classify every unclassified terminal run, persist the verdict, and perform the
|
|
5743
|
-
* one recovery its class names (#132).
|
|
5744
|
-
*
|
|
5745
|
-
* Half this fleet's spend produced no merged PR, and every one of those runs
|
|
5746
|
-
* ended at a human who re-derived the same triage by hand and then threw the
|
|
5747
|
-
* conclusion away. The mechanical classes — a cancelled runner, a kill from a
|
|
5748
|
-
* daemon restart, a green PR whose base moved, a row whose PR had already merged
|
|
5749
|
-
* — need no judgement at all; the genuinely human ones are worth a person's
|
|
5750
|
-
* attention only if they arrive with their evidence already gathered.
|
|
5751
|
-
*
|
|
5752
|
-
* Facts are fetched per row and only the ones that row needs: a `killed` row
|
|
5753
|
-
* costs nothing, a `failed` row with a PR costs a state read and a check read.
|
|
5754
|
-
* A `pushed-green` row that classifies to nothing is left completely untouched —
|
|
5755
|
-
* it is healthy, and writing a class onto it would take it out of this sweep for
|
|
5756
|
-
* good.
|
|
5757
|
-
*/
|
|
5758
|
-
export async function classifyAndRecover(d: Deps): Promise<number> {
|
|
5759
|
-
const { project, caps, tracker, store } = d;
|
|
5760
|
-
// Settle recoveries, counted for the pass's dispatch record — a row whose PR
|
|
5761
|
-
// merged is settled here when the settle sweep could not establish identity
|
|
5762
|
-
// (#497). Every other exit returns 0.
|
|
5763
|
-
let settled = 0;
|
|
5764
|
-
for (const run of store.runsNeedingClassification(project.name, CLASSIFY_BATCH)) {
|
|
5765
|
-
const facts: ClassifyFacts = {};
|
|
5766
|
-
let classifiedRun = run;
|
|
5767
|
-
if (run.state === "failed" || run.state === "killed") {
|
|
5768
|
-
const sessionError = readSessionError(run.sessionFile);
|
|
5769
|
-
if (sessionError !== undefined) {
|
|
5770
|
-
if (run.lastError === undefined || run.lastError === sessionError.message) {
|
|
5771
|
-
facts.sessionError = sessionError;
|
|
5772
|
-
}
|
|
5773
|
-
if (run.lastError === undefined) {
|
|
5774
|
-
store.updateRun(run.id, { lastError: sessionError.message });
|
|
5775
|
-
classifiedRun = { ...run, lastError: sessionError.message };
|
|
5776
|
-
}
|
|
5777
|
-
}
|
|
5778
|
-
}
|
|
5779
|
-
try {
|
|
5780
|
-
if (run.prUrl !== undefined) {
|
|
5781
|
-
const pr = await tracker.prState(run.prUrl);
|
|
5782
|
-
if (pr !== undefined) facts.pr = pr;
|
|
5783
|
-
if (run.state === "pushed-green" && facts.pr === "open") {
|
|
5784
|
-
facts.mergeable = await tracker.mergeable(run.prUrl);
|
|
5785
|
-
}
|
|
5786
|
-
if (run.state === "failed" && facts.pr === "open") {
|
|
5787
|
-
facts.checks = await tracker.checkConclusions(run.prUrl);
|
|
5788
|
-
// When a check failed with a reachable log, pull its tail so the
|
|
5789
|
-
// table can tell an infra outage (#177) from a real test failure by
|
|
5790
|
-
// the log's own words. First failure wins; a log that cannot be
|
|
5791
|
-
// fetched is left undefined and classification stays conservative.
|
|
5792
|
-
// GitHub reports check states as `FAILURE` while the classifier reads
|
|
5793
|
-
// them lowercased — normalise so the live seam and the table agree on
|
|
5794
|
-
// which check is the failure whose log we pull (#177).
|
|
5795
|
-
const firstFailure = facts.checks.find((c) => normalise(c.state) === "failure" && c.link !== undefined);
|
|
5796
|
-
if (firstFailure?.link !== undefined) {
|
|
5797
|
-
facts.failingLog = await tracker.checkLog(firstFailure.link);
|
|
5798
|
-
}
|
|
5799
|
-
}
|
|
5800
|
-
}
|
|
5801
|
-
} catch (err) {
|
|
5802
|
-
// Per row, like every other sweep here: one unreachable PR must not stop
|
|
5803
|
-
// the rest from being classified. The next tick asks again for free.
|
|
5804
|
-
log(`#${run.issue} not classified: fact gathering failed (${errText(err)}) — retrying next tick`);
|
|
5805
|
-
continue;
|
|
5806
|
-
}
|
|
5807
|
-
|
|
5808
|
-
const { cls, recovery, evidence } = classifyRun(classifiedRun, facts, caps);
|
|
5809
|
-
|
|
5810
|
-
// A healthy green PR is not a failure of any class. Leaving the row
|
|
5811
|
-
// unclassified is what keeps it eligible for the sweep on the tick where its
|
|
5812
|
-
// base does move under it.
|
|
5813
|
-
if (run.state === "pushed-green" && cls === "unknown") continue;
|
|
5814
|
-
|
|
5815
|
-
const retry = run.failureClass !== undefined;
|
|
5816
|
-
store.updateRun(run.id, { failureClass: cls, recoveryAction: recovery });
|
|
5817
|
-
log(
|
|
5818
|
-
retry
|
|
5819
|
-
? `#${run.issue} retrying ${recovery} for ${cls}: ${evidence}`
|
|
5820
|
-
: `#${run.issue} classified ${cls} → ${recovery}: ${evidence}`,
|
|
5821
|
-
);
|
|
5822
|
-
if (recovery === "settle") settled += 1;
|
|
5823
|
-
await recoverRun(d, classifiedRun, cls, recovery, evidence);
|
|
5824
|
-
}
|
|
5825
|
-
return settled;
|
|
5826
|
-
}
|
|
5827
|
-
|
|
5828
5945
|
/**
|
|
5829
5946
|
* How many settled `ci-deterministic` rows one reconciliation pass may
|
|
5830
5947
|
* re-examine beyond the persisted review cursor. Each candidate costs GitHub
|
|
@@ -5973,479 +6090,6 @@ async function historicalInfraEvidence(d: Deps, run: RunRecord): Promise<string[
|
|
|
5973
6090
|
return chunks;
|
|
5974
6091
|
}
|
|
5975
6092
|
|
|
5976
|
-
/** Performs the one action a class names. Never chooses one of its own. */
|
|
5977
|
-
async function recoverRun(
|
|
5978
|
-
d: Deps,
|
|
5979
|
-
run: RunRecord,
|
|
5980
|
-
cls: FailureClass,
|
|
5981
|
-
recovery: RecoveryAction,
|
|
5982
|
-
evidence: string,
|
|
5983
|
-
): Promise<void> {
|
|
5984
|
-
const { project, caps, tracker, store } = d;
|
|
5985
|
-
const inProgress = project.stateLabels.inProgress;
|
|
5986
|
-
|
|
5987
|
-
if (recovery === "settle") {
|
|
5988
|
-
// Enqueue the release with the terminal write (see `settlePushedGreen`):
|
|
5989
|
-
// the outbox keeps the label and the row one fact, so a tracker refusal
|
|
5990
|
-
// can no longer strand `agent:in-progress` with nothing left to retry it
|
|
5991
|
-
// (#18, #201).
|
|
5992
|
-
releaseInProgress(d, run.issue, `PR merged: ${evidence}`);
|
|
5993
|
-
store.updateRun(run.id, { state: "merged", endedAt: Date.now(), recoveredAt: Date.now() });
|
|
5994
|
-
log(`#${run.issue} settled from ${cls}: ${evidence}`);
|
|
5995
|
-
return;
|
|
5996
|
-
}
|
|
5997
|
-
|
|
5998
|
-
if (recovery === "continue") {
|
|
5999
|
-
// Two classes recover by continuing, and only one of them has anything left
|
|
6000
|
-
// to do here.
|
|
6001
|
-
//
|
|
6002
|
-
// `turn-cap-progress` was already handed back by the completion path, which
|
|
6003
|
-
// swapped its labels and left the branch retained. There is nothing to
|
|
6004
|
-
// perform, and writing anything would be actively wrong: overwriting
|
|
6005
|
-
// `lastError` with a rebase brief tells the continuation worker to rebase a
|
|
6006
|
-
// run that simply ran out of turns, and re-swapping labels the completion
|
|
6007
|
-
// path already swapped is a pair of no-op `gh` calls. Record-only, so the
|
|
6008
|
-
// sweep stops re-offering it.
|
|
6009
|
-
if (cls === "turn-cap-progress") {
|
|
6010
|
-
store.updateRun(run.id, { recoveredAt: Date.now() });
|
|
6011
|
-
log(`#${run.issue} already continuing from ${cls}: ${evidence}`);
|
|
6012
|
-
return;
|
|
6013
|
-
}
|
|
6014
|
-
|
|
6015
|
-
// `wall-clock-cap-progress`: the completion path only auto-continues
|
|
6016
|
-
// turn-cap kills, so a wall-clock kill with work to show reaches this
|
|
6017
|
-
// sweep still holding the failed label. Swap it for the queue — the branch
|
|
6018
|
-
// is retained, so the next dispatch reattaches it and briefs a resume from
|
|
6019
|
-
// the recorded work. Killed rows gathered no tracker facts, so the
|
|
6020
|
-
// issue-open guard mirrors the requeue path's. The continuation gate is
|
|
6021
|
-
// the same one the turns path applies at kill time
|
|
6022
|
-
// (`shouldContinueAfterTurnsCap`): the row is already charged, and once
|
|
6023
|
-
// the ceiling is spent, handing back the queue label would offer a
|
|
6024
|
-
// candidate admission can never accept — only the failed label comes off,
|
|
6025
|
-
// and the exhaustion reaches a human (#490, #348).
|
|
6026
|
-
if (cls === "wall-clock-cap-progress") {
|
|
6027
|
-
const state = await tracker.issueState(run.issue).catch(() => undefined);
|
|
6028
|
-
if (state !== "open") {
|
|
6029
|
-
log(`#${run.issue} not continued from ${cls}: issue is ${state ?? "unreadable"}`);
|
|
6030
|
-
return;
|
|
6031
|
-
}
|
|
6032
|
-
const continuation = store.continuationsFor(project.name, run.issue);
|
|
6033
|
-
if (hasContinuationBudget(continuation, caps.maxContinuationsPerIssue)) {
|
|
6034
|
-
swapToQueue(d, run.issue, project.stateLabels.failed);
|
|
6035
|
-
store.updateRun(run.id, { recoveredAt: Date.now() });
|
|
6036
|
-
log(`#${run.issue} requeued for a wall-clock continuation: ${evidence}`);
|
|
6037
|
-
} else {
|
|
6038
|
-
store.enqueueLabelOps(project.name, [
|
|
6039
|
-
{ issue: run.issue, op: "remove", label: project.stateLabels.failed },
|
|
6040
|
-
]);
|
|
6041
|
-
store.updateRun(run.id, { recoveredAt: Date.now() });
|
|
6042
|
-
await safeEscalate(d, {
|
|
6043
|
-
tier: 1,
|
|
6044
|
-
project: project.name,
|
|
6045
|
-
issue: run.issue,
|
|
6046
|
-
summary: `#${run.issue} exhausted its continuation budget on wall-clock cap kills`,
|
|
6047
|
-
detail: [
|
|
6048
|
-
`Attempt ${run.attempt} hit the wall-clock cap with work to continue from, but ${continuation} continuation(s) are already charged against a cap of ${caps.maxContinuationsPerIssue}.`,
|
|
6049
|
-
evidence,
|
|
6050
|
-
`Work to continue: branch ${run.branch} at ${run.headSha ?? run.salvageSha}${run.prUrl === undefined ? "" : ` — ${run.prUrl}`}.`,
|
|
6051
|
-
"The issue cannot finish inside the wall-clock cap, so another run would burn a worker slot for the same outcome. Re-scope it, raise maxContinuationsPerIssue for it, or finish the remaining work by hand.",
|
|
6052
|
-
].join("\n"),
|
|
6053
|
-
});
|
|
6054
|
-
await postExhaustionPostmortem(
|
|
6055
|
-
d,
|
|
6056
|
-
run,
|
|
6057
|
-
`Attempt ${run.attempt} hit the wall-clock cap with work to continue from, but ${continuation} continuation(s) are already charged against a cap of ${caps.maxContinuationsPerIssue}.`,
|
|
6058
|
-
);
|
|
6059
|
-
log(`#${run.issue} not continued from ${cls}: continuation budget exhausted`);
|
|
6060
|
-
}
|
|
6061
|
-
return;
|
|
6062
|
-
}
|
|
6063
|
-
|
|
6064
|
-
// `merge-conflict`: the branch is retained and its PR is open, so #50's
|
|
6065
|
-
// continuation guard admits it and the next tick briefs a rebase.
|
|
6066
|
-
//
|
|
6067
|
-
// The outbox makes the retry contract one-sided: the swap is enqueued — a
|
|
6068
|
-
// durable local write that cannot fail on the tracker — before
|
|
6069
|
-
// `recoveredAt` is written, so the row can never again be taken out of
|
|
6070
|
-
// `runsNeedingClassification` with its label swap still owed. That was
|
|
6071
|
-
// the defect 0.4.4 claimed to have fixed and did not, for this one
|
|
6072
|
-
// recovery; the projector retries until the tracker takes the swap, and
|
|
6073
|
-
// while it is pending the eligibility overlay keeps the issue coherent
|
|
6074
|
-
// (#201).
|
|
6075
|
-
swapToQueue(d, run.issue, inProgress);
|
|
6076
|
-
store.updateRun(run.id, {
|
|
6077
|
-
state: "killed",
|
|
6078
|
-
lastError:
|
|
6079
|
-
"merge-conflict: base moved under a green PR — rebase, regenerate recorded artifacts, push without force",
|
|
6080
|
-
recoveredAt: Date.now(),
|
|
6081
|
-
});
|
|
6082
|
-
log(`#${run.issue} requeued for a rebase continuation: ${evidence}`);
|
|
6083
|
-
return;
|
|
6084
|
-
}
|
|
6085
|
-
|
|
6086
|
-
if (recovery === "requeue") {
|
|
6087
|
-
if (cls === "provider-credit") {
|
|
6088
|
-
await reactToProviderCredit(d, run.issue, evidence, run.sessionFile);
|
|
6089
|
-
}
|
|
6090
|
-
// A dispatch-infra requeue that keeps landing on the same issue means the
|
|
6091
|
-
// mirror for its repo is persistently broken — a ref-lock that retry already
|
|
6092
|
-
// exhausted, a dead remote. Requeueing forever spends a turn-0 run per tick
|
|
6093
|
-
// with no chance of success, so after a bounded number of strikes this
|
|
6094
|
-
// escalates to a human instead (#168, #177).
|
|
6095
|
-
if (cls === "dispatch-infra" && store.classCountFor(project.name, run.issue, "dispatch-infra") >= DISPATCH_INFRA_MAX_STRIKES) {
|
|
6096
|
-
await safeEscalate(d, {
|
|
6097
|
-
tier: 1,
|
|
6098
|
-
project: project.name,
|
|
6099
|
-
issue: run.issue,
|
|
6100
|
-
summary: `[dispatch-infra] #${run.issue}: the mirror for ${run.repo} is failing persistently — ${evidence}`,
|
|
6101
|
-
detail: [
|
|
6102
|
-
`The dispatcher could not provision a worktree for #${run.issue} ${DISPATCH_INFRA_MAX_STRIKES} times in a row, all before the worker's first turn.`,
|
|
6103
|
-
"The mirror on this host needs attention (check disk, SSH/HTTPS credentials, and the mirror root).",
|
|
6104
|
-
].join("\n"),
|
|
6105
|
-
});
|
|
6106
|
-
store.updateRun(run.id, { recoveredAt: Date.now() });
|
|
6107
|
-
log(`#${run.issue} escalated from persistent ${cls}: ${evidence}`);
|
|
6108
|
-
return;
|
|
6109
|
-
}
|
|
6110
|
-
// Same bound for provider-transient: an issue whose stream keeps stalling
|
|
6111
|
-
// mid-run is requeued free (no attempt, no continuation charged) — but a
|
|
6112
|
-
// provider that aborts three times for one issue is down, and a human has
|
|
6113
|
-
// to check its status before hand-requeueing (#220). The escalation names
|
|
6114
|
-
// every model the chain tried, so a merged branch built on a different
|
|
6115
|
-
// model is attributable (#286).
|
|
6116
|
-
if (
|
|
6117
|
-
cls === "provider-transient" &&
|
|
6118
|
-
store.classCountFor(project.name, run.issue, "provider-transient") >= PROVIDER_TRANSIENT_MAX_STRIKES
|
|
6119
|
-
) {
|
|
6120
|
-
const tried = formatModelsTried(modelsTried(store.runsForIssue(project.name, run.issue)));
|
|
6121
|
-
await safeEscalate(d, {
|
|
6122
|
-
tier: 1,
|
|
6123
|
-
project: project.name,
|
|
6124
|
-
issue: run.issue,
|
|
6125
|
-
summary: `[provider-transient] #${run.issue}: the provider keeps aborting mid-stream — ${evidence}`,
|
|
6126
|
-
detail: [
|
|
6127
|
-
`The provider aborted the stream for #${run.issue} ${PROVIDER_TRANSIENT_MAX_STRIKES} times without the run ever producing a verdict (0 tokens billed each time).`,
|
|
6128
|
-
...(tried === ""
|
|
6129
|
-
? []
|
|
6130
|
-
: [`Models tried: ${tried}.`]),
|
|
6131
|
-
"Check provider status before requeueing by hand.",
|
|
6132
|
-
].join("\n"),
|
|
6133
|
-
});
|
|
6134
|
-
store.updateRun(run.id, { recoveredAt: Date.now() });
|
|
6135
|
-
log(`#${run.issue} escalated from persistent ${cls}: ${evidence}`);
|
|
6136
|
-
return;
|
|
6137
|
-
}
|
|
6138
|
-
// Same bound for provider-capacity: a run the provider throttled into the
|
|
6139
|
-
// ground is requeued free (no attempt charged) — but a provider that
|
|
6140
|
-
// throttles the same issue three times is at capacity, and a human has to
|
|
6141
|
-
// check its status before hand-requeueing (#573). On a chain-configured
|
|
6142
|
-
// project each requeue already moved the next attempt to the next chain
|
|
6143
|
-
// model, so this escalation is what catches the no-chain case and the
|
|
6144
|
-
// exhausted chain; it names every model the chain tried.
|
|
6145
|
-
if (
|
|
6146
|
-
cls === "provider-capacity" &&
|
|
6147
|
-
store.classCountFor(project.name, run.issue, "provider-capacity") >= PROVIDER_CAPACITY_MAX_STRIKES
|
|
6148
|
-
) {
|
|
6149
|
-
const tried = formatModelsTried(modelsTried(store.runsForIssue(project.name, run.issue)));
|
|
6150
|
-
await safeEscalate(d, {
|
|
6151
|
-
tier: 1,
|
|
6152
|
-
project: project.name,
|
|
6153
|
-
issue: run.issue,
|
|
6154
|
-
summary: `[provider-capacity] #${run.issue}: the model provider is throttling this run into the ground — ${evidence}`,
|
|
6155
|
-
detail: [
|
|
6156
|
-
`The provider answered #${run.issue} with sustained in-session rate limits ${PROVIDER_CAPACITY_MAX_STRIKES} times in a row; the harness retried each and was exhausted.`,
|
|
6157
|
-
...(tried === ""
|
|
6158
|
-
? []
|
|
6159
|
-
: [`Models tried: ${tried}.`]),
|
|
6160
|
-
"Check the provider's rate-limit status (and its throughput-oriented routes) before requeueing by hand.",
|
|
6161
|
-
].join("\n"),
|
|
6162
|
-
});
|
|
6163
|
-
store.updateRun(run.id, { recoveredAt: Date.now() });
|
|
6164
|
-
log(`#${run.issue} escalated from persistent ${cls}: ${evidence}`);
|
|
6165
|
-
return;
|
|
6166
|
-
}
|
|
6167
|
-
// Only when the tracker still shows this issue as ours to hand back. An
|
|
6168
|
-
// issue that is closed, or has no state label, was resolved by another route
|
|
6169
|
-
// and requeueing it would dispatch work nobody asked for.
|
|
6170
|
-
const state = await tracker.issueState(run.issue).catch(() => undefined);
|
|
6171
|
-
if (state !== "open") {
|
|
6172
|
-
log(`#${run.issue} not requeued from ${cls}: issue is ${state ?? "unreadable"}`);
|
|
6173
|
-
return;
|
|
6174
|
-
}
|
|
6175
|
-
// A clean orphan whose queue label is already absent is a deliberate
|
|
6176
|
-
// withdrawal — the operator took the issue out of the queue (e.g. so a
|
|
6177
|
-
// daemon restart could not re-dispatch work known to be unsafe) — and
|
|
6178
|
-
// recovery must not recreate that intent (#423). Read the live label set;
|
|
6179
|
-
// when the queue label is gone, release the dispatcher-owned in-progress
|
|
6180
|
-
// label and stop, leaving the operator's withdrawal to survive recovery.
|
|
6181
|
-
if (cls === "orphan-clean") {
|
|
6182
|
-
const snapshot = await tracker.issueSnapshot(run.issue).catch(() => undefined);
|
|
6183
|
-
if (snapshot === undefined) {
|
|
6184
|
-
log(`#${run.issue} not requeued from orphan-clean: cannot confirm ${project.queueLabel} (unreadable, retrying)`);
|
|
6185
|
-
return;
|
|
6186
|
-
}
|
|
6187
|
-
if (!snapshot.labels.includes(project.queueLabel)) {
|
|
6188
|
-
store.enqueueLabelOps(project.name, [{ issue: run.issue, op: "remove", label: inProgress }]);
|
|
6189
|
-
store.updateRun(run.id, { recoveredAt: Date.now() });
|
|
6190
|
-
log(`#${run.issue} not requeued from orphan-clean: ${project.queueLabel} removed before recovery (operator withdrawal)`);
|
|
6191
|
-
return;
|
|
6192
|
-
}
|
|
6193
|
-
// #439: an orphan-clean row charges the continuation budget, so once
|
|
6194
|
-
// `hasContinuationBudget` is spent — exactly the predicate `admitCandidates`
|
|
6195
|
-
// holds the issue on — requeueing re-adds a queue label for a candidate the
|
|
6196
|
-
// dispatcher can never admit. Stop handing it back and hold it instead: the
|
|
6197
|
-
// queue label comes off (so admission never re-holds on every dispatch),
|
|
6198
|
-
// the in-progress label is released, and a single diagnosis escalates once.
|
|
6199
|
-
// `orphan-clean` is deliberately NOT a global exclusion from the budget (a
|
|
6200
|
-
// worker that genuinely keeps dying mid-work must still be bounded); this is
|
|
6201
|
-
// the missing ceiling check this path never had (#348's invariant).
|
|
6202
|
-
const continuations = store.continuationsFor(project.name, run.issue);
|
|
6203
|
-
if (!hasContinuationBudget(continuations, caps.maxContinuationsPerIssue)) {
|
|
6204
|
-
const runs = store.runsForIssue(project.name, run.issue);
|
|
6205
|
-
const breakdown = continuationBreakdown(runs);
|
|
6206
|
-
const artifact = newestContinuableRun(runs);
|
|
6207
|
-
const onlyDaemonStops = breakdown.size === 1 && breakdown.get("orphan-clean") === continuations;
|
|
6208
|
-
const classLine = Array.from(breakdown, ([cls, n]) => `${n} ${cls}`).join(", ");
|
|
6209
|
-
const artifactLine =
|
|
6210
|
-
artifact === undefined
|
|
6211
|
-
? "The attempts left no salvage commit, head SHA or PR — the branch is empty, so start clean from a re-scope rather than continuing from nothing."
|
|
6212
|
-
: `Work to continue: branch ${artifact.branch} at ${artifact.headSha ?? artifact.salvageSha}${artifact.prUrl === undefined ? "" : ` — ${artifact.prUrl}`}.`;
|
|
6213
|
-
store.enqueueLabelOps(project.name, [
|
|
6214
|
-
{ issue: run.issue, op: "remove", label: inProgress },
|
|
6215
|
-
{ issue: run.issue, op: "remove", label: project.queueLabel },
|
|
6216
|
-
]);
|
|
6217
|
-
store.updateRun(run.id, { recoveredAt: Date.now() });
|
|
6218
|
-
await safeEscalate(d, {
|
|
6219
|
-
tier: 1,
|
|
6220
|
-
project: project.name,
|
|
6221
|
-
issue: run.issue,
|
|
6222
|
-
summary: `#${run.issue} exhausted its ${caps.maxContinuationsPerIssue}-continuation budget on ${cls}`,
|
|
6223
|
-
detail: [
|
|
6224
|
-
`Attempt ${run.attempt} is not requeued: ${continuations} continuation(s) charged against a cap of ${caps.maxContinuationsPerIssue}.`,
|
|
6225
|
-
`Continuations by failure class: ${classLine}.`,
|
|
6226
|
-
onlyDaemonStops
|
|
6227
|
-
? "Every continuation was a daemon stop — the work never failed; the budget was spent by daemon deaths, not the issue."
|
|
6228
|
-
: "Continuations span real work — inspect what each attempt left behind before continuing.",
|
|
6229
|
-
artifactLine,
|
|
6230
|
-
"What you can do: raise maxContinuationsPerIssue for this issue, re-scope it, or continue from the preserved work (or start clean if none).",
|
|
6231
|
-
].join("\n"),
|
|
6232
|
-
});
|
|
6233
|
-
await postExhaustionPostmortem(
|
|
6234
|
-
d,
|
|
6235
|
-
run,
|
|
6236
|
-
`Attempt ${run.attempt} is not requeued: ${continuations} continuation(s) charged against a cap of ${caps.maxContinuationsPerIssue}.`,
|
|
6237
|
-
);
|
|
6238
|
-
log(`#${run.issue} not requeued from orphan-clean: continuation budget exhausted`);
|
|
6239
|
-
return;
|
|
6240
|
-
}
|
|
6241
|
-
}
|
|
6242
|
-
const label = cls === "orphan-clean" ? inProgress : project.stateLabels.failed;
|
|
6243
|
-
swapToQueue(d, run.issue, label);
|
|
6244
|
-
store.updateRun(run.id, { recoveredAt: Date.now() });
|
|
6245
|
-
log(`#${run.issue} requeued from ${cls}: ${evidence}`);
|
|
6246
|
-
return;
|
|
6247
|
-
}
|
|
6248
|
-
|
|
6249
|
-
if (recovery === "rerun-checks") {
|
|
6250
|
-
if (run.prUrl === undefined) return;
|
|
6251
|
-
try {
|
|
6252
|
-
await tracker.rerunFailedChecks(run.prUrl);
|
|
6253
|
-
} catch (err) {
|
|
6254
|
-
log(`#${run.issue} check re-run failed (${errText(err)}) — retrying next tick`);
|
|
6255
|
-
return;
|
|
6256
|
-
}
|
|
6257
|
-
// Back to pending rather than green: the existing settle sweep re-verifies
|
|
6258
|
-
// it against the recorded head on a later tick, so nothing here has to guess
|
|
6259
|
-
// whether the re-run passed.
|
|
6260
|
-
store.updateRun(run.id, { state: "pushed-pending", lastError: null, recoveredAt: Date.now() });
|
|
6261
|
-
log(`#${run.issue} re-ran infrastructure checks: ${evidence}`);
|
|
6262
|
-
return;
|
|
6263
|
-
}
|
|
6264
|
-
|
|
6265
|
-
if (recovery === "escalate") {
|
|
6266
|
-
const detail = [evidence];
|
|
6267
|
-
if (cls === "turn-cap-spinning" || cls === "wall-clock-cap-spinning") {
|
|
6268
|
-
const calls = lastToolCalls(run.sessionFile);
|
|
6269
|
-
detail.push(
|
|
6270
|
-
calls.length === 0
|
|
6271
|
-
? "transcript unreadable — no tool calls could be recovered"
|
|
6272
|
-
: `Last ${calls.length} tool calls: ${calls.join(" → ")}`,
|
|
6273
|
-
);
|
|
6274
|
-
}
|
|
6275
|
-
if (run.lastError !== undefined && cls !== "question") detail.push(run.lastError);
|
|
6276
|
-
// #172: an unwritten transcript is "the run died before it flushed", not a
|
|
6277
|
-
// link to a file the operator will open and find missing.
|
|
6278
|
-
detail.push(
|
|
6279
|
-
run.sessionFile === undefined
|
|
6280
|
-
? "Session: (no transcript)"
|
|
6281
|
-
: existsSync(run.sessionFile)
|
|
6282
|
-
? `Session: ${run.sessionFile}`
|
|
6283
|
-
: `Session: ${run.sessionFile} (file was never written — the run died before its transcript was flushed)`,
|
|
6284
|
-
);
|
|
6285
|
-
// The class and the run are in the summary, which is what the notifications
|
|
6286
|
-
// ledger dedupes on — so one class escalates once per run rather than every
|
|
6287
|
-
// five minutes.
|
|
6288
|
-
await safeEscalate(d, {
|
|
6289
|
-
tier: 1,
|
|
6290
|
-
project: project.name,
|
|
6291
|
-
issue: run.issue,
|
|
6292
|
-
runId: run.id,
|
|
6293
|
-
summary: `[${cls}] #${run.issue} attempt ${run.attempt}: ${evidence}`,
|
|
6294
|
-
detail: detail.join("\n"),
|
|
6295
|
-
});
|
|
6296
|
-
// The hand-off IS the recovery for these classes: there is nothing else this
|
|
6297
|
-
// package can do, and leaving the row unrecovered would re-escalate forever.
|
|
6298
|
-
store.updateRun(run.id, { recoveredAt: Date.now() });
|
|
6299
|
-
return;
|
|
6300
|
-
}
|
|
6301
|
-
|
|
6302
|
-
// `hold` (orphan-dirty) and `none`: recorded, nothing performed. The existing
|
|
6303
|
-
// unsalvaged-WIP admission hold already fails dispatch closed until an
|
|
6304
|
-
// operator acknowledges the tree, which is the only safe move when the
|
|
6305
|
-
// worktree holds the only copy of real work.
|
|
6306
|
-
}
|
|
6307
|
-
|
|
6308
|
-
/**
|
|
6309
|
-
* Enqueue a state-label → queue-label swap for projection (#201).
|
|
6310
|
-
*
|
|
6311
|
-
* The swap is two ops in id order — remove first, then add — which is the
|
|
6312
|
-
* atomicity the projector guarantees: the issue never sits newly eligible
|
|
6313
|
-
* without a queue label on its way back, and the add never lands before the
|
|
6314
|
-
* remove when GitHub fails between them. Enqueueing is a durable local write
|
|
6315
|
-
* that cannot fail on the tracker, so the caller records its recovery
|
|
6316
|
-
* immediately and the projector retries the swap until the tracker takes it —
|
|
6317
|
-
* that closes the 0.4.4 hole where a refused label swap stranded the row
|
|
6318
|
-
* permanently under a log line promising a retry.
|
|
6319
|
-
*/
|
|
6320
|
-
function swapToQueue(d: Pick<Deps, "project" | "store">, issue: number, label: string): void {
|
|
6321
|
-
d.store.enqueueLabelOps(d.project.name, [
|
|
6322
|
-
{ issue, op: "remove", label },
|
|
6323
|
-
{ issue, op: "add", label: d.project.queueLabel },
|
|
6324
|
-
]);
|
|
6325
|
-
}
|
|
6326
|
-
|
|
6327
|
-
/** Bounded listing per state label, so one reconcile cannot walk a whole repo. */
|
|
6328
|
-
const RECONCILE_LIMIT = 50;
|
|
6329
|
-
|
|
6330
|
-
/**
|
|
6331
|
-
* Clear state labels from issues that no longer need them (#132's `superseded`).
|
|
6332
|
-
*
|
|
6333
|
-
* Two structural signals, both cheap and both observed on this fleet: an issue
|
|
6334
|
-
* that is closed but still carries `agent:*`, and an open issue carrying
|
|
6335
|
-
* `failed` whose sub-issues have all closed. On 2026-08-09 four issues (#307,
|
|
6336
|
-
* #297, #140, #82) carried `agent:failed` while every one of them was already
|
|
6337
|
-
* complete — the label was residue of a turns-cap kill from two days earlier,
|
|
6338
|
-
* and nothing in the loop ever revisited it. The board counted four phantom
|
|
6339
|
-
* failures while the genuinely stuck issues were invisible.
|
|
6340
|
-
*
|
|
6341
|
-
* Positive evidence only. A tracker that cannot list answers empty, and an empty
|
|
6342
|
-
* answer removes nothing: a reconcile that guessed would strip the interlock
|
|
6343
|
-
* that keeps two workers off one issue.
|
|
6344
|
-
*/
|
|
6345
|
-
export async function reconcileStaleLabels(d: Deps): Promise<void> {
|
|
6346
|
-
const { project, tracker, store } = d;
|
|
6347
|
-
const labels = [project.stateLabels.failed, project.stateLabels.blocked, project.stateLabels.inProgress];
|
|
6348
|
-
|
|
6349
|
-
for (const label of labels) {
|
|
6350
|
-
const carrying = await tracker.listLabeled(label, RECONCILE_LIMIT).catch(() => []);
|
|
6351
|
-
for (const issue of carrying) {
|
|
6352
|
-
if (issue.state === "closed") {
|
|
6353
|
-
// Never retain an `agent:*` label on a closed issue: the work is done by
|
|
6354
|
-
// some route, and the label only makes the board lie about it. Enqueue
|
|
6355
|
-
// rather than call — a refused write must not lose the decision; the
|
|
6356
|
-
// projector retries the removal until the tracker takes it (#201).
|
|
6357
|
-
store.enqueueLabelOps(project.name, [{ issue: issue.number, op: "remove", label }]);
|
|
6358
|
-
log(`#${issue.number} reconciled: closed issue no longer carries ${label} (queued)`);
|
|
6359
|
-
continue;
|
|
6360
|
-
}
|
|
6361
|
-
|
|
6362
|
-
if (label !== project.stateLabels.failed) continue;
|
|
6363
|
-
const children = await tracker.childrenOf(issue.number).catch(() => []);
|
|
6364
|
-
if (children.length === 0 || children.some((c) => c.state !== "closed")) continue;
|
|
6365
|
-
|
|
6366
|
-
const key = `${project.name}:superseded:${issue.number}`;
|
|
6367
|
-
if (store.wasNotified(key)) continue;
|
|
6368
|
-
const list = children.map((c) => `#${c.number}`).join(", ");
|
|
6369
|
-
store.enqueueLabelOps(project.name, [{ issue: issue.number, op: "remove", label }]);
|
|
6370
|
-
try {
|
|
6371
|
-
await tracker.comment(
|
|
6372
|
-
issue.number,
|
|
6373
|
-
`superseded: all sub-issues closed (${list}) — this label was stale; propose closing if the ` +
|
|
6374
|
-
`acceptance criteria are met on the default branch.`,
|
|
6375
|
-
);
|
|
6376
|
-
store.markNotified(key);
|
|
6377
|
-
log(`#${issue.number} reconciled: superseded by ${list}`);
|
|
6378
|
-
} catch (err) {
|
|
6379
|
-
// The label removal is already queued and will land regardless; the
|
|
6380
|
-
// comment is the only half that can fail here (#201).
|
|
6381
|
-
log(`#${issue.number} could not comment the superseded note (${errText(err)}) — retrying next tick`);
|
|
6382
|
-
}
|
|
6383
|
-
}
|
|
6384
|
-
}
|
|
6385
|
-
}
|
|
6386
|
-
|
|
6387
|
-
/**
|
|
6388
|
-
* Settles `claimed`/`running` rows left by a dead daemon process and, before
|
|
6389
|
-
* marking each one `orphaned`, salvages any dirty worktree.
|
|
6390
|
-
*
|
|
6391
|
-
* Found live after a host restart killed two workers mid-run, and again on
|
|
6392
|
-
* every package deploy that restarted while workers were live (#35): without
|
|
6393
|
-
* the salvage call the next attempt's `worktree remove --force` destroyed
|
|
6394
|
-
* uncommitted edits that had no other copy. Cap-kills already salvaged (#27);
|
|
6395
|
-
* this is the same call site for the restart path.
|
|
6396
|
-
*
|
|
6397
|
-
* Only the rows change. The issue keeps its in-progress label — that label is
|
|
6398
|
-
* the crash guard against double-dispatch, and deciding what a dead worker's
|
|
6399
|
-
* remains are worth (an open PR? a salvaged sha? a clean tree?) is the
|
|
6400
|
-
* orchestrator's drain-duty judgement, not something to automate here. The
|
|
6401
|
-
* rows also keep counting toward `maxAttemptsPerIssue`, so a loop of deaths
|
|
6402
|
-
* still escalates instead of retrying forever.
|
|
6403
|
-
*
|
|
6404
|
-
* `pushed-green` rows are deliberately left alone: they hold no process — they
|
|
6405
|
-
* are finished work waiting on a human merge, and they must keep occupying the
|
|
6406
|
-
* issue so a second attempt cannot land on a live PR. What eventually settles
|
|
6407
|
-
* them is {@link settlePushedGreen}, on the tick, by asking the tracker what
|
|
6408
|
-
* became of the PR — the one question a restart cannot answer by inference.
|
|
6409
|
-
*/
|
|
6410
|
-
export async function reconcileOrphanedRuns(
|
|
6411
|
-
store: Store,
|
|
6412
|
-
project: string,
|
|
6413
|
-
/**
|
|
6414
|
-
* Resolves the privileged publisher for one orphaned run. Optional because a
|
|
6415
|
-
* test driving the row transitions has no repo to publish to; production
|
|
6416
|
-
* always passes it, and without it a salvaged WIP commit stays local — which
|
|
6417
|
-
* is the half of #121 that reaches a human.
|
|
6418
|
-
*/
|
|
6419
|
-
publish?: (run: RunRecord) => RunPublisher,
|
|
6420
|
-
): Promise<RunRecord[]> {
|
|
6421
|
-
// Live runs only: `pushed-green` holds no process, so it cannot be orphaned by
|
|
6422
|
-
// a process dying — it is finished work waiting on a human merge.
|
|
6423
|
-
const stale = store.liveRuns(project);
|
|
6424
|
-
const endedAt = Date.now();
|
|
6425
|
-
for (const r of stale) {
|
|
6426
|
-
// Salvage before the row flips: the worktree path is on the record, and
|
|
6427
|
-
// salvageWip is a no-op for a missing/clean tree. The clause matches the
|
|
6428
|
-
// cap-kill wording so triage reads the same either way, and the tree is
|
|
6429
|
-
// kept because an orphan's remains are the orchestrator's drain-duty call.
|
|
6430
|
-
const settlement =
|
|
6431
|
-
r.worktree === ""
|
|
6432
|
-
? undefined
|
|
6433
|
-
: await settleWorktree({
|
|
6434
|
-
issue: r.issue,
|
|
6435
|
-
attempt: r.attempt,
|
|
6436
|
-
ending: "killed by a daemon restart",
|
|
6437
|
-
worktree: r.worktree,
|
|
6438
|
-
branch: r.branch,
|
|
6439
|
-
// An orphan's tree is kept, so its commits are not about to be
|
|
6440
|
-
// deleted — but a WIP salvage still has to reach GitHub, which is
|
|
6441
|
-
// #121's whole point and is now the daemon's hop to make.
|
|
6442
|
-
publish: publish?.(r),
|
|
6443
|
-
tree: "keep",
|
|
6444
|
-
});
|
|
6445
|
-
store.updateRun(r.id, { state: "orphaned", endedAt, ...settlement?.patch });
|
|
6446
|
-
}
|
|
6447
|
-
return stale;
|
|
6448
|
-
}
|
|
6449
6093
|
|
|
6450
6094
|
// ------------------------------------------------------------------- the daemon
|
|
6451
6095
|
|
|
@@ -6606,6 +6250,25 @@ export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
|
|
|
6606
6250
|
const store = openStore(dbPath());
|
|
6607
6251
|
const verbPeerReader = peerCredentialReader();
|
|
6608
6252
|
const verbDir = ensureVerbSocketDir(stateDir());
|
|
6253
|
+
// The worker identity (#798). Probed once here for the startup banner only —
|
|
6254
|
+
// every dispatch re-resolves it through {@link launchIdentity}, because the
|
|
6255
|
+
// account, setpriv and the harness binding (#828) are all host state that can
|
|
6256
|
+
// arrive after this process did. A host that cannot establish it keeps its
|
|
6257
|
+
// control plane running and fails each worker launch closed with the reason,
|
|
6258
|
+
// so the operator hears a concrete host change instead of a fleet that
|
|
6259
|
+
// silently ran workers as root.
|
|
6260
|
+
const identityAtStartup = resolveWorkerIdentity();
|
|
6261
|
+
if (identityAtStartup.ok) {
|
|
6262
|
+
log(
|
|
6263
|
+
`worker identity: ${identityAtStartup.identity.account} uid=${identityAtStartup.identity.uid} ` +
|
|
6264
|
+
`gid=${identityAtStartup.identity.gid} home=${identityAtStartup.identity.home}`,
|
|
6265
|
+
);
|
|
6266
|
+
} else {
|
|
6267
|
+
log(
|
|
6268
|
+
"worker identity unavailable — worker dispatch fails closed until the host provides it " +
|
|
6269
|
+
`(re-checked at every launch, so no restart is needed once it does): ${identityAtStartup.reason}`,
|
|
6270
|
+
);
|
|
6271
|
+
}
|
|
6609
6272
|
const integrity: IntegrityGate = { baseline: packageManifest(), paged: false };
|
|
6610
6273
|
const usage = sharedUsageSource();
|
|
6611
6274
|
const turnLimits = createTurnLimitRegistry((runId, maxTurns) => {
|
|
@@ -6632,6 +6295,7 @@ export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
|
|
|
6632
6295
|
log(projects.length === 1 ? message : `[${project.name}] ${message}`);
|
|
6633
6296
|
};
|
|
6634
6297
|
const caps = resolveCaps(project, cfg.defaults);
|
|
6298
|
+
const host = cfg.host;
|
|
6635
6299
|
// One transient-server-error breaker per project (#642): admission's
|
|
6636
6300
|
// GraphQL checks and the orchestrator mutation commands share it, so a 503
|
|
6637
6301
|
// observed by either side gates both instead of one provider outage being
|
|
@@ -6710,7 +6374,11 @@ export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
|
|
|
6710
6374
|
recordReleaseBlock(project.name, "orchestrator", shape, context),
|
|
6711
6375
|
});
|
|
6712
6376
|
const transcript = orchestrator.sessionFile();
|
|
6713
|
-
|
|
6377
|
+
const loaded = orchestrator.extensionVersion();
|
|
6378
|
+
projectLog(
|
|
6379
|
+
`orchestrator session ready${transcript === undefined ? "" : ` · ${transcript}`}` +
|
|
6380
|
+
`${loaded === undefined ? "" : ` · loaded omp-conductor ${loaded}`}`,
|
|
6381
|
+
);
|
|
6714
6382
|
} catch (err) {
|
|
6715
6383
|
orchestratorStartError = errText(err);
|
|
6716
6384
|
projectLog(
|
|
@@ -6763,6 +6431,7 @@ export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
|
|
|
6763
6431
|
const d: Deps = {
|
|
6764
6432
|
project,
|
|
6765
6433
|
caps,
|
|
6434
|
+
host,
|
|
6766
6435
|
tracker,
|
|
6767
6436
|
store,
|
|
6768
6437
|
drain,
|
|
@@ -6778,6 +6447,16 @@ export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
|
|
|
6778
6447
|
probeCriticalBase: (repo, markers, branch) =>
|
|
6779
6448
|
probeCriticalBase(project, repo, branch, markers),
|
|
6780
6449
|
probeWorktreeLane: (input) => probeRunLane(input),
|
|
6450
|
+
// The live host, per launch — never the startup verdict above.
|
|
6451
|
+
workerIdentity: () => resolveWorkerIdentity(),
|
|
6452
|
+
// Grant each run's paths to the worker identity by ownership, before the
|
|
6453
|
+
// session that must edit them starts (#798). The recursive chown runs as
|
|
6454
|
+
// root from the dispatch path; the worker identity owns its checkout and
|
|
6455
|
+
// transcript and nothing else.
|
|
6456
|
+
grantWorkerPaths: (identity, worktreePath, sessionDir) => {
|
|
6457
|
+
chownRecursive(worktreePath, identity.uid, identity.gid);
|
|
6458
|
+
chownRecursive(sessionDir, identity.uid, identity.gid);
|
|
6459
|
+
},
|
|
6781
6460
|
// A cross-repo Depends-on prerequisite reads through the same GitHub
|
|
6782
6461
|
// credential/accounting seams as the project tracker — a fresh tracker
|
|
6783
6462
|
// scoped to the referenced repo, reusing the daemon's gh hooks so API
|
|
@@ -6787,6 +6466,14 @@ export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
|
|
|
6787
6466
|
onCall: () => store.bumpGhCalls?.(utcDay(), "daemon"),
|
|
6788
6467
|
onRefusal: (at) => store.recordGhRefusal?.(at),
|
|
6789
6468
|
}).issueSnapshot(issue),
|
|
6469
|
+
// The body twin of `probeIssueIn`, for the dependency-graph cycle pass
|
|
6470
|
+
// (#421): reads a reachable routed prerequisite's body through the same
|
|
6471
|
+
// per-repo tracker/credential/accounting seams.
|
|
6472
|
+
probeBodyIn: (ownerRepo, issue) =>
|
|
6473
|
+
makeTracker({ ...project, tracker: { kind: "github", repo: ownerRepo } }, undefined, {
|
|
6474
|
+
onCall: () => store.bumpGhCalls?.(utcDay(), "daemon"),
|
|
6475
|
+
onRefusal: (at) => store.recordGhRefusal?.(at),
|
|
6476
|
+
}).issueBody(issue),
|
|
6790
6477
|
...(verbPeerReader === undefined ? {} : { verbPeerReader }),
|
|
6791
6478
|
verbActions,
|
|
6792
6479
|
};
|
|
@@ -6944,15 +6631,31 @@ export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
|
|
|
6944
6631
|
},
|
|
6945
6632
|
health: () =>
|
|
6946
6633
|
daemonHealth(
|
|
6947
|
-
runtimes.map((runtime) =>
|
|
6948
|
-
|
|
6634
|
+
runtimes.map((runtime) => {
|
|
6635
|
+
const orch = runtime.orchestrator;
|
|
6636
|
+
const mode = runtime.d.project.escalation.orchestrator;
|
|
6637
|
+
return daemonHealthSnapshot(
|
|
6949
6638
|
store,
|
|
6950
6639
|
runtime.d.project.name,
|
|
6951
6640
|
isPaused(runtime.d.project.name),
|
|
6952
6641
|
runtime.codeGraph,
|
|
6953
6642
|
workerControls,
|
|
6954
|
-
|
|
6955
|
-
|
|
6643
|
+
orch === undefined
|
|
6644
|
+
? // External mode is the pane's session: the daemon hosts no
|
|
6645
|
+
// child to attest. An embedded orchestrator that failed to
|
|
6646
|
+
// start is a live outage of the surface the upgrade must
|
|
6647
|
+
// promise reloaded (#832) — say so out loud.
|
|
6648
|
+
{ mode: mode === "external" ? "external" : "failed" }
|
|
6649
|
+
: {
|
|
6650
|
+
mode: "embedded",
|
|
6651
|
+
...(orch.extensionVersion() === undefined
|
|
6652
|
+
? {}
|
|
6653
|
+
: { loaded: orch.extensionVersion() }),
|
|
6654
|
+
...(orch.sessionFile() === undefined ? {} : { sessionFile: orch.sessionFile() }),
|
|
6655
|
+
alive: orch.alive(),
|
|
6656
|
+
},
|
|
6657
|
+
);
|
|
6658
|
+
}),
|
|
6956
6659
|
),
|
|
6957
6660
|
}),
|
|
6958
6661
|
});
|