omp-conductor 0.17.0 → 0.18.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/REFERENCE.md +12 -8
- package/package.json +1 -1
- package/schema/config.schema.json +40 -1
- package/src/admission.ts +263 -44
- package/src/ask.ts +39 -3
- package/src/availability.ts +27 -1
- package/src/backups.ts +2 -2
- package/src/briefs/orchestrator.md +1 -0
- package/src/briefs/worker.md +38 -19
- package/src/command-help.ts +8 -1
- package/src/command-manifest.ts +5 -2
- package/src/commands/arm.ts +6 -3
- package/src/commands/message.ts +32 -4
- package/src/commands/watch.ts +62 -3
- package/src/config-schema.ts +53 -0
- package/src/config.ts +97 -1
- package/src/daemon.ts +1479 -1483
- package/src/decisions.ts +51 -6
- package/src/depends-on.ts +261 -1
- package/src/diff-flags.ts +350 -0
- package/src/digest-schedule.ts +37 -0
- package/src/doctor.ts +310 -22
- package/src/escalate.ts +560 -57
- package/src/failure-class.ts +71 -15
- package/src/fleet.ts +189 -34
- package/src/gitops.ts +103 -24
- package/src/graph-health.ts +20 -7
- package/src/graph.ts +313 -68
- package/src/lifecycle.ts +43 -7
- package/src/omp.ts +42 -0
- package/src/orchestrator-tick.ts +430 -162
- package/src/release-policy.ts +177 -5
- package/src/routing.ts +11 -3
- package/src/session-host.ts +16 -0
- package/src/settlement.ts +1728 -0
- package/src/setup-host.ts +193 -4
- package/src/setup-install.ts +91 -30
- package/src/setup-wizard.ts +1257 -78
- package/src/setup.ts +153 -6
- package/src/status-render.ts +36 -4
- package/src/store.ts +411 -17
- package/src/tracker/github.ts +607 -12
- package/src/types.ts +331 -5
- package/src/upgrade.ts +50 -19
- package/src/verbs/actions.ts +66 -18
- package/src/verbs/protocol.ts +45 -0
- package/src/verbs/server.ts +270 -13
- package/src/worker.ts +239 -6
- package/src/worktree.ts +115 -8
- package/systemd/omp-conductor-recover.sh +73 -0
- package/systemd/recover-unit-test.sh +61 -0
package/src/daemon.ts
CHANGED
|
@@ -8,30 +8,35 @@
|
|
|
8
8
|
* here and enforced before anything is claimed.
|
|
9
9
|
*/
|
|
10
10
|
import { createHash } from "node:crypto";
|
|
11
|
-
import { chmodSync, existsSync, mkdirSync, readdirSync, readFileSync, rmSync, statSync, writeFileSync } from "node:fs";
|
|
11
|
+
import { chmodSync, existsSync, mkdirSync, readdirSync, readFileSync, renameSync, rmSync, statSync, writeFileSync } from "node:fs";
|
|
12
12
|
import { dirname, join, relative } from "node:path";
|
|
13
13
|
import {
|
|
14
14
|
configPath,
|
|
15
|
+
dbBackupDirFor,
|
|
15
16
|
findProject,
|
|
16
17
|
loadConfig,
|
|
17
18
|
resolveCaps,
|
|
18
19
|
resolveReleaseGrants,
|
|
20
|
+
resolveReview,
|
|
19
21
|
stateDir,
|
|
20
22
|
} from "./config.ts";
|
|
21
23
|
import { availabilityState, type AvailabilityState } from "./availability.ts";
|
|
22
24
|
import {
|
|
23
|
-
UNREADABLE_TREE_FLAG,
|
|
24
|
-
analyseSettlement,
|
|
25
|
-
deriveChangedLine,
|
|
26
25
|
formatSettlementFlags,
|
|
27
26
|
settlementFlagSummary,
|
|
28
27
|
withDerivedChangedLine,
|
|
29
28
|
} from "./diff-flags.ts";
|
|
30
|
-
import {
|
|
29
|
+
import {
|
|
30
|
+
dbSnapshotDue,
|
|
31
|
+
dbSnapshotMarkerKey,
|
|
32
|
+
digestScheduleState,
|
|
33
|
+
localDayKey,
|
|
34
|
+
type DigestScheduleState,
|
|
35
|
+
} from "./digest-schedule.ts";
|
|
31
36
|
import { createEscalator, escalationIssueRef } from "./escalate.ts";
|
|
32
37
|
import { pendingCodeGraph, probeCodeGraph, type CodeGraphHealth } from "./graph-health.ts";
|
|
33
38
|
import { graphHint } from "./graph.ts";
|
|
34
|
-
import { acquireOnceLease, healthCheck, livingDaemon } from "./lifecycle.ts";
|
|
39
|
+
import { acquireOnceLease, healthCheck, livingDaemon, probeUnit, SYSTEMD_UNIT } from "./lifecycle.ts";
|
|
35
40
|
import { requestImmediateTick, resolveTickConfigCwd, STALL_MARKER_FILE } from "./orchestrator-tick.ts";
|
|
36
41
|
import { runDoctor } from "./doctor.ts";
|
|
37
42
|
import { appendJournal, launchTransientUnit, readUpgradeJournal } from "./upgrade-journal.ts";
|
|
@@ -52,34 +57,64 @@ import {
|
|
|
52
57
|
} from "./reports.ts";
|
|
53
58
|
import {
|
|
54
59
|
recordReleaseBlock,
|
|
60
|
+
sharedHostBriefNotice,
|
|
55
61
|
type GateShape,
|
|
56
62
|
type ReleaseBlockContext,
|
|
57
63
|
} from "./release-policy.ts";
|
|
58
|
-
import { branchName, effectiveLabels, route } from "./routing.ts";
|
|
64
|
+
import { branchName, effectiveLabels, isEligible, route } from "./routing.ts";
|
|
59
65
|
import type { Routed, UnroutableReason } from "./routing.ts";
|
|
60
|
-
import {
|
|
61
|
-
|
|
66
|
+
import {
|
|
67
|
+
admitCandidates,
|
|
68
|
+
effectiveLane,
|
|
69
|
+
hasContinuationBudget,
|
|
70
|
+
hasFailedAttemptBudget,
|
|
71
|
+
laneEcho,
|
|
72
|
+
} from "./admission.ts";
|
|
73
|
+
import type { Admission, AdmissionHold, FileLane } from "./admission.ts";
|
|
62
74
|
import { log, errText, safeEscalate } from "./log.ts";
|
|
75
|
+
import {
|
|
76
|
+
adoptSalvagedPrs,
|
|
77
|
+
classifyAndRecover,
|
|
78
|
+
collectSettlementFlags,
|
|
79
|
+
formatQuarantinedRuns,
|
|
80
|
+
formatSalvagedRuns,
|
|
81
|
+
reactToProviderCredit,
|
|
82
|
+
readSessionError,
|
|
83
|
+
reconcileOrphanedRuns,
|
|
84
|
+
reconcileStaleLabels,
|
|
85
|
+
recordOperatorStop,
|
|
86
|
+
settlePushedGreen,
|
|
87
|
+
settleWorktree,
|
|
88
|
+
swapToQueue,
|
|
89
|
+
} from "./settlement.ts";
|
|
63
90
|
import { materializeOmpSettings } from "./omp-settings.ts";
|
|
64
91
|
import { evaluateDecisionConditions, probeNpmVersion, probeRateLimitReset } from "./decisions.ts";
|
|
65
|
-
import {
|
|
92
|
+
import { infraLogSignature, infraSignatureVersion, providerCreditRefusal, providerTransientFault } from "./failure-class.ts";
|
|
66
93
|
import {
|
|
67
94
|
DEFAULT_MODEL_FALLBACK_THRESHOLD,
|
|
68
95
|
fallbackClause,
|
|
69
|
-
formatModelsTried,
|
|
70
|
-
modelsTried,
|
|
71
96
|
providerFailureFacts,
|
|
72
97
|
resolveDispatchModel,
|
|
73
98
|
} from "./model-fallback.ts";
|
|
74
99
|
import { projectLabels } from "./label-projection.ts";
|
|
75
|
-
import {
|
|
76
|
-
|
|
100
|
+
import {
|
|
101
|
+
DB_SNAPSHOT_RETENTION,
|
|
102
|
+
ACTIVE_STATES,
|
|
103
|
+
dbPath,
|
|
104
|
+
LIVE_STATES,
|
|
105
|
+
openStore,
|
|
106
|
+
pruneDbSnapshots,
|
|
107
|
+
snapshotDb,
|
|
108
|
+
utcDay,
|
|
109
|
+
} from "./store.ts";
|
|
110
|
+
import { GraphqlBreaker, makeTracker, type RateLimitStatus } from "./tracker/github.ts";
|
|
77
111
|
import { DEFAULT_REPORT_POLICY, RELEASE_SHAPES } from "./types.ts";
|
|
78
112
|
import type {
|
|
79
113
|
BaseFreeze,
|
|
80
114
|
BaseHealth,
|
|
81
115
|
AdmissionHoldReason,
|
|
82
116
|
Caps,
|
|
117
|
+
ConductorConfig,
|
|
83
118
|
DispatchSummary,
|
|
84
119
|
DigestBacklog,
|
|
85
120
|
Escalation,
|
|
@@ -92,11 +127,15 @@ import type {
|
|
|
92
127
|
PrState,
|
|
93
128
|
ProjectConfig,
|
|
94
129
|
ReadyIssue,
|
|
130
|
+
ReportingPolicy,
|
|
95
131
|
RepoTarget,
|
|
96
132
|
ReportRecord,
|
|
97
133
|
ResolvedGrants,
|
|
98
134
|
FailureClass,
|
|
99
135
|
RecoveryAction,
|
|
136
|
+
ReviewPolicy,
|
|
137
|
+
ReviewRevisionOutcome,
|
|
138
|
+
ReviewRevisionRecord,
|
|
100
139
|
RunRecord,
|
|
101
140
|
RunState,
|
|
102
141
|
SettlementFlag,
|
|
@@ -104,6 +143,7 @@ import type {
|
|
|
104
143
|
Tracker,
|
|
105
144
|
VerbLedgerEntry,
|
|
106
145
|
TurnOverride,
|
|
146
|
+
WorkflowRun,
|
|
107
147
|
} from "./types.ts";
|
|
108
148
|
import {
|
|
109
149
|
type KilledBy,
|
|
@@ -113,6 +153,7 @@ import {
|
|
|
113
153
|
type RunWorkerDeps,
|
|
114
154
|
ORPHAN_RESUME_PROMPT,
|
|
115
155
|
renderBrief,
|
|
156
|
+
renderReviewRevisionPrompt,
|
|
116
157
|
runWorker,
|
|
117
158
|
} from "./worker.ts";
|
|
118
159
|
import {
|
|
@@ -130,7 +171,6 @@ import { githubVerbActions } from "./verbs/actions.ts";
|
|
|
130
171
|
import { formatVerbLedger, STATUS_LEDGER_SCAN } from "./verbs/ledger.ts";
|
|
131
172
|
import {
|
|
132
173
|
listenVerbChannel,
|
|
133
|
-
PR_LOOKUP_WINDOW_MS,
|
|
134
174
|
type VerbActions,
|
|
135
175
|
type VerbDeps,
|
|
136
176
|
type VerbListener,
|
|
@@ -170,26 +210,7 @@ const GRAPH_HEALTH_INTERVAL_MS = 60_000;
|
|
|
170
210
|
* report an operator is waiting on must not sit in the outbox for the length of
|
|
171
211
|
* a poll interval, and delivery is owed even while claiming is paused (#123). */
|
|
172
212
|
const REPORT_DELIVERY_INTERVAL_MS = 30_000;
|
|
173
|
-
|
|
174
|
-
* retries — but only a bounded number of times. Three strikes for one issue
|
|
175
|
-
* means the mirror itself is broken, not unlucky, and the sweep escalates
|
|
176
|
-
* instead of burning a turn-0 run per tick forever (#168, #177). */
|
|
177
|
-
const DISPATCH_INFRA_MAX_STRIKES = 3;
|
|
178
|
-
/** A provider-transient requeue (stream stalled mid-run) is retried, but only a
|
|
179
|
-
* bounded number of times: three aborted streams for one issue means the
|
|
180
|
-
* provider itself is degraded, not unlucky, and the sweep escalates to a
|
|
181
|
-
* human instead of requeueing into a down provider forever (#220). */
|
|
182
|
-
const PROVIDER_TRANSIENT_MAX_STRIKES = 3;
|
|
183
|
-
/** A provider-capacity requeue (sustained in-session rate limiting) is retried,
|
|
184
|
-
* but only a bounded number of times: three throttled runs for one issue mean
|
|
185
|
-
* the provider is at capacity, not unlucky, and the sweep escalates to a human
|
|
186
|
-
* instead of requeueing into a throttled provider forever (#573). The issue's
|
|
187
|
-
* own chain moves onto its next model per strike (via {@link FAILOVER_CLASSES}),
|
|
188
|
-
* so a bounded chain is exhaustible; this caps the unbounded no-chain case. */
|
|
189
|
-
const PROVIDER_CAPACITY_MAX_STRIKES = 3;
|
|
190
|
-
/** Terminal rows inspected for a salvaged PR per tick. GitHub provenance reads
|
|
191
|
-
* are maintenance, but a backlog must not turn one tick into an API burst. */
|
|
192
|
-
const SALVAGED_PR_ADOPTION_BATCH = 10;
|
|
213
|
+
|
|
193
214
|
const DEFAULT_PORT = 8787;
|
|
194
215
|
const BRIEF_TEMPLATE_PATH = join(import.meta.dir, "briefs", "worker.md");
|
|
195
216
|
|
|
@@ -289,7 +310,9 @@ interface Deps {
|
|
|
289
310
|
/**
|
|
290
311
|
* Reads one active run's file lane for the admission file-lane interlock
|
|
291
312
|
* (#555): the union of its uncommitted worktree changes and its branch-vs-base
|
|
292
|
-
* diff.
|
|
313
|
+
* diff. Reconciliation is not occupancy: while the run merges its base, the
|
|
314
|
+
* probe reports only files it has actually diverged on, never the files the
|
|
315
|
+
* merge merely staged (#684). Wired by `runDaemon` to the mirror/worktree-backed
|
|
293
316
|
* {@link probeRunLane}; a test injects a fake. Absent, the interlock is inert
|
|
294
317
|
* (no lane is ever known occupied), which is the issue's "fail open": the
|
|
295
318
|
* gate adds holds, it never refuses a well-formed issue for lack of this
|
|
@@ -303,6 +326,13 @@ interface Deps {
|
|
|
303
326
|
* admission fails a routed cross-repo prerequisite closed.
|
|
304
327
|
*/
|
|
305
328
|
probeIssueIn?: (repo: string, issue: number) => Promise<IssueSnapshot | undefined>;
|
|
329
|
+
/**
|
|
330
|
+
* Reads one issue's BODY in a repository the admission tracker is not bound
|
|
331
|
+
* to — the dependency-graph cycle pass (#421). Wired by `runDaemon` to a
|
|
332
|
+
* repo-scoped tracker; a test injects a fake. Absent, a routed reachable
|
|
333
|
+
* body fails that branch closed rather than synthesising a cycle.
|
|
334
|
+
*/
|
|
335
|
+
probeBodyIn?: (repo: string, issue: number) => Promise<string | undefined>;
|
|
306
336
|
}
|
|
307
337
|
|
|
308
338
|
/**
|
|
@@ -610,6 +640,135 @@ export function setPaused(
|
|
|
610
640
|
}
|
|
611
641
|
}
|
|
612
642
|
|
|
643
|
+
// ----------------------------------------------------------------- admission
|
|
644
|
+
// acknowledgement (#651, review #3)
|
|
645
|
+
//
|
|
646
|
+
// The setup-fence's "acknowledgement" must come from the daemon itself, not a
|
|
647
|
+
// second synchronous worker count: a tick that passed its pause gate before
|
|
648
|
+
// the fence landed can sit in awaited tracker/routing/admission work and claim
|
|
649
|
+
// after the count and before setup mutates. Two places make that impossible
|
|
650
|
+
// and record it durably:
|
|
651
|
+
//
|
|
652
|
+
// - the tick's pause gate re-acknowledges every held pass (the daemon has
|
|
653
|
+
// reached its admission boundary and claims nothing);
|
|
654
|
+
// - the claim itself re-checks the pause immediately before creating the run
|
|
655
|
+
// row, so a tick already past its gate when the fence landed refuses at
|
|
656
|
+
// the claim — and writes the acknowledgement — instead of admitting.
|
|
657
|
+
//
|
|
658
|
+
// The acknowledgement file names the exact pause instance observed and the
|
|
659
|
+
// daemon generation that observed it, so the setup barrier can prove the
|
|
660
|
+
// acknowledged fence is the fence it froze and the acknowledgement belongs to
|
|
661
|
+
// the daemon it began with.
|
|
662
|
+
|
|
663
|
+
/** The durable admission acknowledgement a daemon writes when it observes a
|
|
664
|
+
* pause fence at an admission boundary. */
|
|
665
|
+
export interface AdmissionAckRecord {
|
|
666
|
+
/** The pause instance observed (source token, reason, creation instant). */
|
|
667
|
+
pause: { source: string; reason?: string; since: number };
|
|
668
|
+
/** The daemon generation that observed it — {@link daemonGeneration}. */
|
|
669
|
+
daemon: string;
|
|
670
|
+
/** When the daemon observed the fence. */
|
|
671
|
+
observedAt: number;
|
|
672
|
+
}
|
|
673
|
+
|
|
674
|
+
/** The acknowledgement file path. One per host: the fence is host-global. */
|
|
675
|
+
export function admissionAckPath(): string {
|
|
676
|
+
return join(stateDir(), "admission-ack.json");
|
|
677
|
+
}
|
|
678
|
+
|
|
679
|
+
/**
|
|
680
|
+
* The generation identity of the running daemon, or `undefined` when nothing
|
|
681
|
+
* provably runs. The live pidfile record wins (its pid and boot instant
|
|
682
|
+
* identify the exact instance, #377); a record-less ACTIVE unit is still a
|
|
683
|
+
* running daemon and its MainPID is the generation that lets a fence spot a
|
|
684
|
+
* supervisor restart (#651 review #2). Shared by the ack writer and the setup
|
|
685
|
+
* barrier's identity so both sides of the acknowledgement name the same
|
|
686
|
+
* instance by the same rule.
|
|
687
|
+
*/
|
|
688
|
+
export function daemonGeneration(): string | undefined {
|
|
689
|
+
const daemon = livingDaemon();
|
|
690
|
+
if (daemon !== undefined) return `${daemon.pid}@${daemon.startedAt}`;
|
|
691
|
+
const ownership = probeUnit(SYSTEMD_UNIT);
|
|
692
|
+
return ownership.kind === "active" ? `systemd:${ownership.pid}` : undefined;
|
|
693
|
+
}
|
|
694
|
+
|
|
695
|
+
/** Reads the current admission acknowledgement, if one is readable. A corrupt
|
|
696
|
+
* or unreadable file is absence — the barrier fails closed on the absence. */
|
|
697
|
+
export function readAdmissionAck(): AdmissionAckRecord | undefined {
|
|
698
|
+
let parsed: unknown;
|
|
699
|
+
try {
|
|
700
|
+
parsed = JSON.parse(readFileSync(admissionAckPath(), "utf8"));
|
|
701
|
+
} catch {
|
|
702
|
+
return undefined;
|
|
703
|
+
}
|
|
704
|
+
if (parsed === null || typeof parsed !== "object") return undefined;
|
|
705
|
+
const r = parsed as Record<string, unknown>;
|
|
706
|
+
const pause = r["pause"];
|
|
707
|
+
const daemon = r["daemon"];
|
|
708
|
+
const observedAt = r["observedAt"];
|
|
709
|
+
if (pause === null || typeof pause !== "object") return undefined;
|
|
710
|
+
const p = pause as Record<string, unknown>;
|
|
711
|
+
const source = p["source"];
|
|
712
|
+
const since = p["since"];
|
|
713
|
+
const reason = p["reason"];
|
|
714
|
+
if (typeof source !== "string" || source.length === 0) return undefined;
|
|
715
|
+
if (typeof since !== "number" || !Number.isFinite(since)) return undefined;
|
|
716
|
+
if (reason !== undefined && typeof reason !== "string") return undefined;
|
|
717
|
+
if (typeof daemon !== "string" || daemon.length === 0) return undefined;
|
|
718
|
+
if (typeof observedAt !== "number" || !Number.isFinite(observedAt)) return undefined;
|
|
719
|
+
return { pause: { source, since, ...(reason === undefined ? {} : { reason }) }, daemon, observedAt };
|
|
720
|
+
}
|
|
721
|
+
|
|
722
|
+
/**
|
|
723
|
+
* Writes the admission acknowledgement for the pause the daemon just observed.
|
|
724
|
+
* The observed instance is the most restrictive fence in force: a host-global
|
|
725
|
+
* sentinel gates every project and is the pause a host-wide transaction (setup,
|
|
726
|
+
* an all-projects hold) froze, so it wins over a project-scoped sentinel;
|
|
727
|
+
* otherwise the project's own sentinel is the effective one for the daemon's
|
|
728
|
+
* claims, matching {@link pauseInstance}. Recording the project sentinel while
|
|
729
|
+
* a global fence is also in force would name a narrower, older hold instead of
|
|
730
|
+
* the fence that actually gates the host — and a barrier that froze the global
|
|
731
|
+
* sentinel could then never match its own acknowledgement (#651 review #4). A
|
|
732
|
+
* no-op when no pause is readable: there is nothing to acknowledge. The
|
|
733
|
+
* generation is the writing daemon's own (the record it owns, or its
|
|
734
|
+
* supervised MainPID).
|
|
735
|
+
*/
|
|
736
|
+
export function writeAdmissionAck(project: string): void {
|
|
737
|
+
// `pauseInstance()` reads only the global sentinel; while it is in force it
|
|
738
|
+
// is the fence that gates every project, and the durable record must name it
|
|
739
|
+
// rather than a per-project hold that predates it.
|
|
740
|
+
const pause = pauseInstance() ?? pauseInstance(project);
|
|
741
|
+
if (pause === undefined) return;
|
|
742
|
+
const generation = daemonGeneration();
|
|
743
|
+
if (generation === undefined) return;
|
|
744
|
+
const path = admissionAckPath();
|
|
745
|
+
mkdirSync(dirname(path), { recursive: true });
|
|
746
|
+
const record: AdmissionAckRecord = { pause, daemon: generation, observedAt: Date.now() };
|
|
747
|
+
const tmp = `${path}.${process.pid.toString(36)}.${Date.now().toString(36)}.tmp`;
|
|
748
|
+
try {
|
|
749
|
+
writeFileSync(tmp, `${JSON.stringify(record, null, 2)}\n`, { mode: 0o600 });
|
|
750
|
+
renameSync(tmp, path);
|
|
751
|
+
} catch (err) {
|
|
752
|
+
rmSync(tmp, { force: true });
|
|
753
|
+
throw err;
|
|
754
|
+
}
|
|
755
|
+
}
|
|
756
|
+
|
|
757
|
+
/** Wake the daemon's dispatch loop to prompt a pass (best-effort; used by the
|
|
758
|
+
* setup barrier so a live daemon acknowledges the fence without waiting for
|
|
759
|
+
* its next five-minute tick). A refused/failed wake just lengthens the wait;
|
|
760
|
+
* the barrier's deadline is what bounds it. */
|
|
761
|
+
export async function wakeDaemon(port: number): Promise<void> {
|
|
762
|
+
try {
|
|
763
|
+
await fetch(`http://127.0.0.1:${port}/wake`, {
|
|
764
|
+
method: "POST",
|
|
765
|
+
signal: AbortSignal.timeout(1_500),
|
|
766
|
+
});
|
|
767
|
+
} catch {
|
|
768
|
+
// Best-effort by contract: the caller's bounded wait is the backstop.
|
|
769
|
+
}
|
|
770
|
+
}
|
|
771
|
+
|
|
613
772
|
// ----------------------------------------------------------- package integrity
|
|
614
773
|
|
|
615
774
|
/** Enough differing paths to tell a deploy from a tamper at a glance; the full
|
|
@@ -775,16 +934,29 @@ const DISCUSSION_CHARS_BUDGET = 8_000;
|
|
|
775
934
|
* the #517 failure. An empty comment list renders nothing, so a commentless
|
|
776
935
|
* issue's brief stays byte-identical to what this package has always shipped.
|
|
777
936
|
*/
|
|
778
|
-
function renderDiscussion(comments: IssueComment[] | "unread"): string {
|
|
937
|
+
function renderDiscussion(comments: IssueComment[] | "unread", lane?: FileLane): string {
|
|
779
938
|
if (comments === "unread") {
|
|
780
|
-
|
|
939
|
+
const lines = [
|
|
781
940
|
"## Discussion",
|
|
782
941
|
"",
|
|
783
942
|
"_The issue's comments could not be read at dispatch time. The live read below is_",
|
|
784
943
|
"_the only path to them; if it prints nothing, that is a failed read, not an_",
|
|
785
944
|
"_absence of discussion._",
|
|
786
945
|
"",
|
|
787
|
-
]
|
|
946
|
+
];
|
|
947
|
+
// The lane admission enforced is carried through dispatch, so even an
|
|
948
|
+
// unreadable thread cannot hide it from the worker (#608): the gate's
|
|
949
|
+
// effective declaration renders from the admission snapshot, not from the
|
|
950
|
+
// read that just failed. A body declaration needs no note — the body
|
|
951
|
+
// always renders.
|
|
952
|
+
if (lane !== undefined && lane.at !== "body") {
|
|
953
|
+
lines.push(
|
|
954
|
+
"_The effective file lane below was enforced at admission — it supersedes any earlier declaration._",
|
|
955
|
+
lane.source,
|
|
956
|
+
"",
|
|
957
|
+
);
|
|
958
|
+
}
|
|
959
|
+
return lines.join("\n");
|
|
788
960
|
}
|
|
789
961
|
if (comments.length === 0) return "";
|
|
790
962
|
const total = comments.length;
|
|
@@ -812,9 +984,67 @@ function renderDiscussion(comments: IssueComment[] | "unread"): string {
|
|
|
812
984
|
"",
|
|
813
985
|
);
|
|
814
986
|
}
|
|
987
|
+
// The file-lane gate resolves its effective lane across the whole thread,
|
|
988
|
+
// not just what this budget renders, so a winning declaration beyond the
|
|
989
|
+
// budget would otherwise control admission while the worker never sees it
|
|
990
|
+
// (#608). Reproduce the winning declaration verbatim here — `at` records
|
|
991
|
+
// the comment it came from — so the rendered brief and the gate agree on
|
|
992
|
+
// the same lane, and the note itself re-parses through the same
|
|
993
|
+
// `File lane:` grammar. The note is suppressed only when the winning
|
|
994
|
+
// declaration is provably visible in the rendered thread (the body always
|
|
995
|
+
// renders, so a body winner needs none); a carried admission lane whose
|
|
996
|
+
// comment the dispatch re-read shifted or dropped still renders here,
|
|
997
|
+
// whether or not the thread was truncated.
|
|
998
|
+
if (
|
|
999
|
+
lane !== undefined &&
|
|
1000
|
+
lane.at !== "body" &&
|
|
1001
|
+
!comments.slice(0, shown).some((c) => c.body.includes(lane.source))
|
|
1002
|
+
) {
|
|
1003
|
+
lines.push(
|
|
1004
|
+
`_The effective file lane was declared in comment ${lane.at + 1} — it supersedes any earlier declaration._`,
|
|
1005
|
+
lane.source,
|
|
1006
|
+
"",
|
|
1007
|
+
);
|
|
1008
|
+
}
|
|
815
1009
|
return lines.join("\n");
|
|
816
1010
|
}
|
|
817
1011
|
|
|
1012
|
+
/**
|
|
1013
|
+
* The parsed file lane as a brief section (#724): the file list the gate will
|
|
1014
|
+
* enforce, or the explicit fail-open note — the same `laneEcho` the promotion
|
|
1015
|
+
* verb prints, so the author and the worker read one parse. Deliberately only
|
|
1016
|
+
* the parse, never the source line: the prose already renders in the body or
|
|
1017
|
+
* the discussion, and in #720's case the prose is exactly what looked
|
|
1018
|
+
* reasonable to a human while parsed greedily.
|
|
1019
|
+
*/
|
|
1020
|
+
function laneBlock(lane: FileLane | undefined): string {
|
|
1021
|
+
const echo = laneEcho(lane);
|
|
1022
|
+
const body = lane === undefined ? `_${echo}_` : `\`${echo}\``;
|
|
1023
|
+
return ["## File lane (as parsed)", "", body, "", ""].join("\n");
|
|
1024
|
+
}
|
|
1025
|
+
|
|
1026
|
+
/**
|
|
1027
|
+
* What an orphan-resumed worker is told about the file lane on top of the
|
|
1028
|
+
* continuation notice (#608). The original brief already in the transcript
|
|
1029
|
+
* may show an earlier declaration, while admission enforces the current one —
|
|
1030
|
+
* replaying the whole brief would re-do the work, but continuing under a stale
|
|
1031
|
+
* lane is the collision the gate exists to stop. The declaration is rendered
|
|
1032
|
+
* verbatim, so it re-parses through the same `File lane:` grammar every
|
|
1033
|
+
* surfaced declaration uses.
|
|
1034
|
+
*/
|
|
1035
|
+
function resumeLaneBlock(lane: FileLane): string {
|
|
1036
|
+
return [
|
|
1037
|
+
"",
|
|
1038
|
+
"The file lane admission enforced for this continuation is below. It supersedes",
|
|
1039
|
+
"any lane declaration in the brief already in your transcript:",
|
|
1040
|
+
"",
|
|
1041
|
+
lane.source,
|
|
1042
|
+
"",
|
|
1043
|
+
`Parsed files: ${laneEcho(lane)}.`,
|
|
1044
|
+
"",
|
|
1045
|
+
].join("\n");
|
|
1046
|
+
}
|
|
1047
|
+
|
|
818
1048
|
/**
|
|
819
1049
|
* Record a state-label swap for projection (#201). The add enqueues before the
|
|
820
1050
|
* remove — the reverse order would leave a window where the issue carries no
|
|
@@ -828,211 +1058,6 @@ function swapLabel(store: Store, projectName: string, issue: number, from: strin
|
|
|
828
1058
|
{ issue, op: "remove", label: from },
|
|
829
1059
|
]);
|
|
830
1060
|
}
|
|
831
|
-
/**
|
|
832
|
-
* Persist the operator-stop transition before releasing its live controller.
|
|
833
|
-
* The row is terminal first, then its in-progress label is removed through the
|
|
834
|
-
* same durable projection outbox as every other lifecycle transition.
|
|
835
|
-
*/
|
|
836
|
-
export function recordOperatorStop(
|
|
837
|
-
store: Pick<Store, "updateRun" | "enqueueLabelOps">,
|
|
838
|
-
args: {
|
|
839
|
-
project: string;
|
|
840
|
-
issue: number;
|
|
841
|
-
runId: string;
|
|
842
|
-
inProgress: string;
|
|
843
|
-
reason: string;
|
|
844
|
-
patch: Partial<RunRecord>;
|
|
845
|
-
},
|
|
846
|
-
): void {
|
|
847
|
-
store.updateRun(args.runId, {
|
|
848
|
-
...args.patch,
|
|
849
|
-
state: "stopped",
|
|
850
|
-
lastError: `operator stopped: ${args.reason}`,
|
|
851
|
-
});
|
|
852
|
-
store.enqueueLabelOps(args.project, [
|
|
853
|
-
{ issue: args.issue, op: "remove", label: args.inProgress },
|
|
854
|
-
]);
|
|
855
|
-
}
|
|
856
|
-
|
|
857
|
-
|
|
858
|
-
/**
|
|
859
|
-
* The escalator throws when no transport is configured or Telegram rejects, and
|
|
860
|
-
* only records the dedup marker on success. A page that cannot be delivered
|
|
861
|
-
* must not take the tick down with it — log it and let the next tick retry.
|
|
862
|
-
*
|
|
863
|
-
* Returns whether it actually went out, because "page once" and "page once
|
|
864
|
-
* *successfully*" are different promises: a caller that latches a once-only
|
|
865
|
-
* gate on the attempt turns one failed delivery into permanent silence about a
|
|
866
|
-
* condition that is still true.
|
|
867
|
-
*/
|
|
868
|
-
|
|
869
|
-
async function reactToProviderCredit(
|
|
870
|
-
d: Deps,
|
|
871
|
-
issue: number,
|
|
872
|
-
message: string,
|
|
873
|
-
sessionFile: string | undefined,
|
|
874
|
-
): Promise<void> {
|
|
875
|
-
const { project } = d;
|
|
876
|
-
const alreadyPaused = isPaused(project.name);
|
|
877
|
-
if (!alreadyPaused) setPaused(true, { source: "provider-credit", reason: message }, project.name);
|
|
878
|
-
log(
|
|
879
|
-
`#${issue} provider refused for credit — dispatch ${alreadyPaused ? "remains paused" : "paused"}: ${message}`,
|
|
880
|
-
);
|
|
881
|
-
// Fleet-scoped and run-independent on purpose. The notification ledger
|
|
882
|
-
// dedupes on `project:issue:tier:summary`, so `NO_ISSUE` plus a summary
|
|
883
|
-
// carrying no run or attempt pages once for the fleet, not once per run.
|
|
884
|
-
await safeEscalate(d, {
|
|
885
|
-
tier: 2,
|
|
886
|
-
category: "fleet-stopped",
|
|
887
|
-
project: project.name,
|
|
888
|
-
issue: NO_ISSUE,
|
|
889
|
-
summary: `Model provider refused for credit — ${project.name} is paused`,
|
|
890
|
-
detail: [
|
|
891
|
-
message,
|
|
892
|
-
"",
|
|
893
|
-
"No implementation attempt was charged: this is a billing state, not a",
|
|
894
|
-
"failed implementation. Each affected issue keeps its queue label and",
|
|
895
|
-
"re-dispatches on `omp-conductor resume` once the provider has credit.",
|
|
896
|
-
`Session: ${sessionFile ?? "(no transcript)"}`,
|
|
897
|
-
].join("\n"),
|
|
898
|
-
});
|
|
899
|
-
}
|
|
900
|
-
|
|
901
|
-
/**
|
|
902
|
-
* What a salvage attempt contributes to the escalation: where the work went, or
|
|
903
|
-
* that it went nowhere. Split from the effects below for the same reason
|
|
904
|
-
* `checkIntegrity` is — this wording is the whole thing a human acts on, so it
|
|
905
|
-
* is worth a test holding it, and the sha in it is the only pointer to work
|
|
906
|
-
* that no longer has any other copy.
|
|
907
|
-
*
|
|
908
|
-
* `retained` is not cosmetic. These lines used to promise a tree "kept for
|
|
909
|
-
* inspection" unconditionally, which was true only because salvage ran only on
|
|
910
|
-
* the paths that keep one. A blocked run's tree is removed the moment its work
|
|
911
|
-
* is safely on the branch, and sending an operator to a path this process just
|
|
912
|
-
* deleted is the same class of mistake as #118 itself.
|
|
913
|
-
*/
|
|
914
|
-
export function salvageLines(
|
|
915
|
-
outcome: SalvageOutcome,
|
|
916
|
-
worktree: string,
|
|
917
|
-
retained: boolean,
|
|
918
|
-
): string[] {
|
|
919
|
-
const fate = retained
|
|
920
|
-
? `Worktree kept for inspection: ${worktree}`
|
|
921
|
-
: `Worktree removed: ${worktree}`;
|
|
922
|
-
|
|
923
|
-
if (outcome.kind === "nothing") return [`${fate} — nothing uncommitted to salvage`];
|
|
924
|
-
|
|
925
|
-
if (outcome.kind === "failed") {
|
|
926
|
-
return [
|
|
927
|
-
`WIP SALVAGE FAILED: ${outcome.error}`,
|
|
928
|
-
`Uncommitted work in ${worktree} is the only copy of it, so the tree was kept.`,
|
|
929
|
-
"This issue is held out of dispatch until the tree is recovered by hand and",
|
|
930
|
-
"`omp-conductor unblock <n> --force` records that you accepted it.",
|
|
931
|
-
];
|
|
932
|
-
}
|
|
933
|
-
|
|
934
|
-
const where =
|
|
935
|
-
`WIP committed to ${outcome.branch} @ ${outcome.sha}` +
|
|
936
|
-
(outcome.pushed
|
|
937
|
-
? " and pushed — the work outlives this worktree"
|
|
938
|
-
: ` but NOT pushed (${outcome.pushError ?? "no reason given"}) — it lives only in this host's mirror`);
|
|
939
|
-
// Manifest belongs in the escalation too: opening the commit is how the
|
|
940
|
-
// orchestrator talked itself into scrubbing a worker tree (#38).
|
|
941
|
-
const n = outcome.files.length;
|
|
942
|
-
const count = `${n} file${n === 1 ? "" : "s"}`;
|
|
943
|
-
const manifest =
|
|
944
|
-
outcome.newPaths.length === 0
|
|
945
|
-
? `${count} (all modifications to tracked paths)`
|
|
946
|
-
: `${count}; new: ${outcome.newPaths.slice(0, 12).join(", ")}${
|
|
947
|
-
outcome.newPaths.length > 12 ? `, … +${outcome.newPaths.length - 12} more` : ""
|
|
948
|
-
}`;
|
|
949
|
-
return [where, manifest, fate];
|
|
950
|
-
}
|
|
951
|
-
|
|
952
|
-
/** Everything a settled run has to record and say about its worktree. */
|
|
953
|
-
export interface WorktreeSettlement {
|
|
954
|
-
outcome: SalvageOutcome;
|
|
955
|
-
/** Whether the tree still exists now the run is over. */
|
|
956
|
-
retained: boolean;
|
|
957
|
-
/** Escalation lines naming where the work went. */
|
|
958
|
-
lines: string[];
|
|
959
|
-
/** Row fields recording the durable ref, or the failure that blocks a re-claim. */
|
|
960
|
-
patch: Pick<RunRecord, "salvageSha" | "salvageError">;
|
|
961
|
-
}
|
|
962
|
-
|
|
963
|
-
/**
|
|
964
|
-
* Decides what becomes of a finished run's worktree: save the work, then keep
|
|
965
|
-
* or remove the tree, then say which.
|
|
966
|
-
*
|
|
967
|
-
* One function because the two halves are one decision and splitting them is
|
|
968
|
-
* how #118 happened — the removal at the end of dispatch had no idea whether
|
|
969
|
-
* anything had been saved, and the salvage at the top of the failure branch had
|
|
970
|
-
* no idea the blocked branch fell through to a `--force` removal.
|
|
971
|
-
*
|
|
972
|
-
* A salvage that *fails* retains the tree whatever the caller asked for. There
|
|
973
|
-
* was real work, git refused to commit it, and the tree is now the only copy in
|
|
974
|
-
* existence: deleting it on schedule would be the data loss this whole path
|
|
975
|
-
* exists to prevent. The issue is held out of dispatch until an operator says
|
|
976
|
-
* otherwise, because the next attempt's `worktree remove --force` would finish
|
|
977
|
-
* the job (see `admitCandidates`).
|
|
978
|
-
*
|
|
979
|
-
* Exported so a test can drive the real decision against a real git tree.
|
|
980
|
-
*/
|
|
981
|
-
export async function settleWorktree(
|
|
982
|
-
args: {
|
|
983
|
-
issue: number;
|
|
984
|
-
attempt: number;
|
|
985
|
-
/** Clause for the commit subject: "killed by the turns cap", "blocked …". */
|
|
986
|
-
ending: string;
|
|
987
|
-
worktree: string;
|
|
988
|
-
/** The run's branch, so the pre-removal publish names the right ref. */
|
|
989
|
-
branch: string;
|
|
990
|
-
/**
|
|
991
|
-
* Publishes the run branch on the privileged side. Required rather than
|
|
992
|
-
* optional: the run's commits live in a repository of its own,
|
|
993
|
-
* so a removal that did not publish first would delete the only copy —
|
|
994
|
-
* which is #121's data loss with one extra step. `undefined` is a visible
|
|
995
|
-
* decision at the call site, never an omission.
|
|
996
|
-
*/
|
|
997
|
-
publish: RunPublisher | undefined;
|
|
998
|
-
} & (
|
|
999
|
-
| /** Terminal-failure and orphan trees are evidence, and are kept even when clean. */
|
|
1000
|
-
{ tree: "keep" }
|
|
1001
|
-
| { tree: "remove"; mirrorPath: string }
|
|
1002
|
-
),
|
|
1003
|
-
): Promise<WorktreeSettlement> {
|
|
1004
|
-
const { issue, attempt, ending, worktree, branch, publish } = args;
|
|
1005
|
-
const outcome = await salvageWip(worktree, issue, attempt, ending, publish);
|
|
1006
|
-
const retained = args.tree === "keep" || outcome.kind === "failed";
|
|
1007
|
-
if (!retained && args.tree === "remove") {
|
|
1008
|
-
// Before the removal, always — not only when salvage found something. A run
|
|
1009
|
-
// that *committed* and could not publish has its work in its own repository
|
|
1010
|
-
// and nowhere else, and salvage never sees a committed tree because it is
|
|
1011
|
-
// clean. The mirror fetch inside `publish` is what preserves it; the push
|
|
1012
|
-
// to GitHub can fail (no network, protected ref) and the work still lives.
|
|
1013
|
-
const published = await publish?.(branch);
|
|
1014
|
-
if (published !== undefined && !published.ok) {
|
|
1015
|
-
log(`#${issue} publish before removal failed, work is preserved in the mirror: ${published.stderr}`);
|
|
1016
|
-
}
|
|
1017
|
-
await removeWorktree(args.mirrorPath, worktree);
|
|
1018
|
-
}
|
|
1019
|
-
|
|
1020
|
-
const lines = salvageLines(outcome, worktree, retained);
|
|
1021
|
-
log(`#${issue} salvage: ${lines.join(" ")}`);
|
|
1022
|
-
return {
|
|
1023
|
-
outcome,
|
|
1024
|
-
retained,
|
|
1025
|
-
lines,
|
|
1026
|
-
patch:
|
|
1027
|
-
outcome.kind === "salvaged"
|
|
1028
|
-
? { salvageSha: outcome.sha }
|
|
1029
|
-
: outcome.kind === "failed"
|
|
1030
|
-
? { salvageError: outcome.error }
|
|
1031
|
-
: {},
|
|
1032
|
-
};
|
|
1033
|
-
}
|
|
1034
|
-
|
|
1035
|
-
|
|
1036
1061
|
/**
|
|
1037
1062
|
* How a run's end is named — in the salvage commit, and to whoever reads it.
|
|
1038
1063
|
* The whole clause, not a bare reason: a graceful block was not killed by
|
|
@@ -1066,6 +1091,15 @@ export async function buildBrief(
|
|
|
1066
1091
|
* when the tracker refused at dispatch — rare, but it must never read as
|
|
1067
1092
|
* "no comments" (the #517 failure mode). Absent means an empty list. */
|
|
1068
1093
|
comments?: IssueComment[] | "unread";
|
|
1094
|
+
/**
|
|
1095
|
+
* The effective file lane admission resolved for this candidate (#608).
|
|
1096
|
+
* When carried, the brief renders exactly this value — the gate's own
|
|
1097
|
+
* snapshot — instead of recomputing from the dispatch-time comment read,
|
|
1098
|
+
* so a changed or failed second read can neither hide nor reword the
|
|
1099
|
+
* lane admission enforced. Absent (unit-level callers), the lane is
|
|
1100
|
+
* resolved from the rendered thread itself.
|
|
1101
|
+
*/
|
|
1102
|
+
lane?: FileLane;
|
|
1069
1103
|
} = {},
|
|
1070
1104
|
): Promise<string> {
|
|
1071
1105
|
// Read per dispatch rather than caching: editing the brief then takes effect
|
|
@@ -1103,6 +1137,18 @@ export async function buildBrief(
|
|
|
1103
1137
|
"",
|
|
1104
1138
|
].join("\n")
|
|
1105
1139
|
: "";
|
|
1140
|
+
// The comments as rendered, and the effective file lane the brief must show
|
|
1141
|
+
// — the admission-carried value when dispatch has one, else the lane the
|
|
1142
|
+
// thread itself resolves to — passed to the renderer so a winning
|
|
1143
|
+
// declaration can be reproduced verbatim even when it sits beyond the
|
|
1144
|
+
// discussion budget, keeping the gate and the worker-visible brief on one
|
|
1145
|
+
// lane (#608).
|
|
1146
|
+
const comments = opts.comments ?? [];
|
|
1147
|
+
// The effective lane admission resolved (or resolves) for this candidate:
|
|
1148
|
+
// the carried admission snapshot when dispatch has one, else the thread
|
|
1149
|
+
// itself. Both the discussion renderer and the parsed-lane section draw on
|
|
1150
|
+
// the same value, so the brief shows one lane on every surface (#608, #724).
|
|
1151
|
+
const lane = opts.lane ?? (comments === "unread" ? undefined : effectiveLane(r.issue.body, comments));
|
|
1106
1152
|
return renderBrief(template, {
|
|
1107
1153
|
ISSUE_NUMBER: String(r.issue.number),
|
|
1108
1154
|
ISSUE_TITLE: r.issue.title,
|
|
@@ -1111,11 +1157,18 @@ export async function buildBrief(
|
|
|
1111
1157
|
BRANCH: branch,
|
|
1112
1158
|
WORKTREE: worktree,
|
|
1113
1159
|
ACCEPTANCE_CRITERIA: acceptanceCriteria(r.issue),
|
|
1114
|
-
ISSUE_COMMENTS: renderDiscussion(
|
|
1160
|
+
ISSUE_COMMENTS: renderDiscussion(comments, lane),
|
|
1161
|
+
FILE_LANE: laneBlock(lane),
|
|
1115
1162
|
GATES: gatesBlock(r.repo),
|
|
1116
|
-
//
|
|
1117
|
-
//
|
|
1118
|
-
//
|
|
1163
|
+
// The guarded shell suites and their sanctioned parse-only check, derived
|
|
1164
|
+
// from SHARED_HOST_SCRIPTS so the brief and the refusal share one list
|
|
1165
|
+
// (#687). Non-empty whenever the guard has paths, so the placeholder line
|
|
1166
|
+
// always renders to a line.
|
|
1167
|
+
SHARED_HOST_NOTICE: sharedHostBriefNotice(),
|
|
1168
|
+
// The brief's code-graph paragraph: the exact `project` key for a
|
|
1169
|
+
// configured repo, or an explicit "no graph" statement for an
|
|
1170
|
+
// unconfigured one — never silence, because a worker that knows there is
|
|
1171
|
+
// no graph stops looking for it.
|
|
1119
1172
|
GRAPH_HINT: graphHint(r.repo),
|
|
1120
1173
|
CONTINUATION: continuation,
|
|
1121
1174
|
});
|
|
@@ -1124,178 +1177,6 @@ export async function buildBrief(
|
|
|
1124
1177
|
// ------------------------------------------------------------------- one issue
|
|
1125
1178
|
|
|
1126
1179
|
|
|
1127
|
-
/** The failure classes `countContinuations` deliberately does not charge — the
|
|
1128
|
-
* inverted copy of its exclusions, kept beside the breakdown that consumes it
|
|
1129
|
-
* so the two can only drift together (#439). `orphan-clean` is absent on
|
|
1130
|
-
* purpose: daemon orphans consume the continuation budget, which is exactly
|
|
1131
|
-
* why the requeue side has to respect the ceiling instead of racing it. */
|
|
1132
|
-
const NON_CONTINUATION_CLASSES: Partial<Record<FailureClass, true>> = {
|
|
1133
|
-
"admin-kill": true,
|
|
1134
|
-
"settlement-stuck": true,
|
|
1135
|
-
"env-start-failure": true,
|
|
1136
|
-
"dispatch-infra": true,
|
|
1137
|
-
"provider-credit": true,
|
|
1138
|
-
"provider-transient": true,
|
|
1139
|
-
"provider-capacity": true,
|
|
1140
|
-
};
|
|
1141
|
-
|
|
1142
|
-
/** How one issue spent its continuation budget, grouped by failure class —
|
|
1143
|
-
* the exact rows `continuationsFor` charges, so an exhaustion escalation
|
|
1144
|
-
* reports the same budget it says is spent. `unclassified` groups rows that
|
|
1145
|
-
* charged before the class was written (a pre-upgrade NULL). */
|
|
1146
|
-
function continuationBreakdown(runs: readonly RunRecord[]): Map<string, number> {
|
|
1147
|
-
const perClass = new Map<string, number>();
|
|
1148
|
-
for (const r of runs) {
|
|
1149
|
-
const chargedAsKilledOrOrphaned =
|
|
1150
|
-
(r.state === "killed" || r.state === "orphaned" || r.state === "blocked") &&
|
|
1151
|
-
(r.failureClass === undefined || NON_CONTINUATION_CLASSES[r.failureClass] === undefined);
|
|
1152
|
-
const chargedAsReturned = r.state === "failed" && r.failureClass === "returned-for-revision";
|
|
1153
|
-
if (!chargedAsKilledOrOrphaned && !chargedAsReturned) continue;
|
|
1154
|
-
const cls = r.failureClass ?? "unclassified";
|
|
1155
|
-
perClass.set(cls, (perClass.get(cls) ?? 0) + 1);
|
|
1156
|
-
}
|
|
1157
|
-
return perClass;
|
|
1158
|
-
}
|
|
1159
|
-
|
|
1160
|
-
/** The newest attempt's preserved work, if any, so an exhaustion escalation can
|
|
1161
|
-
* say whether continuing is worthwhile: `salvageSha`/`headSha`/`prUrl` are the
|
|
1162
|
-
* three artifacts a run can leave, and all null on a branch nothing reached. */
|
|
1163
|
-
function newestContinuableRun(runs: readonly RunRecord[]): RunRecord | undefined {
|
|
1164
|
-
const newestFirst = [...runs].reverse();
|
|
1165
|
-
return newestFirst.find(
|
|
1166
|
-
(r) => r.salvageSha !== undefined || r.headSha !== undefined || r.prUrl !== undefined,
|
|
1167
|
-
);
|
|
1168
|
-
}
|
|
1169
|
-
|
|
1170
|
-
/** The fenced-block info string that marks an exhaustion postmortem comment, so
|
|
1171
|
-
* a grooming scout re-slicing the issue can find and parse the whole block by
|
|
1172
|
-
* grepping for it. */
|
|
1173
|
-
export const POSTMORTEM_MARKER = "conductor-postmortem";
|
|
1174
|
-
|
|
1175
|
-
/** How one issue's attempt chain failed, as "3× ci-deterministic, 1× …" — the
|
|
1176
|
-
* digest shape for naming what the exhaustion was. Groups every row by its
|
|
1177
|
-
* failure class, whether or not it charged the continuation budget, because
|
|
1178
|
-
* the postmortem tells the whole story and not just the budget half (#290). */
|
|
1179
|
-
export function attemptClassBreakdown(runs: readonly RunRecord[]): string {
|
|
1180
|
-
const perClass = new Map<string, number>();
|
|
1181
|
-
for (const r of runs) {
|
|
1182
|
-
const cls = r.failureClass ?? "unclassified";
|
|
1183
|
-
perClass.set(cls, (perClass.get(cls) ?? 0) + 1);
|
|
1184
|
-
}
|
|
1185
|
-
if (perClass.size === 0) return "unclassified";
|
|
1186
|
-
return Array.from(perClass, ([cls, n]) => `${n}× ${cls}`).join(", ");
|
|
1187
|
-
}
|
|
1188
|
-
|
|
1189
|
-
/** Wall-clock duration of one run as a compact human string ("45m", "1h30m"). */
|
|
1190
|
-
export function humanDuration(ms: number): string {
|
|
1191
|
-
const seconds = Math.max(0, Math.round(ms / 1_000));
|
|
1192
|
-
if (seconds < 60) return `${seconds}s`;
|
|
1193
|
-
const minutes = Math.round(seconds / 60);
|
|
1194
|
-
if (minutes < 60) return `${minutes}m`;
|
|
1195
|
-
const hours = Math.floor(minutes / 60);
|
|
1196
|
-
const rest = minutes % 60;
|
|
1197
|
-
return rest === 0 ? `${hours}h` : `${hours}h${rest}m`;
|
|
1198
|
-
}
|
|
1199
|
-
|
|
1200
|
-
/** Flatten and bound a run's last error to one greppable table line. */
|
|
1201
|
-
export function oneLineBrief(text: string | undefined): string | undefined {
|
|
1202
|
-
if (text === undefined || text.trim() === "") return undefined;
|
|
1203
|
-
const flat = text.replace(/\s+/g, " ").trim();
|
|
1204
|
-
return flat.length > 90 ? `${flat.slice(0, 89)}…` : flat;
|
|
1205
|
-
}
|
|
1206
|
-
|
|
1207
|
-
/**
|
|
1208
|
-
* The exhaustion postmortem block: one greppable fenced block covering every
|
|
1209
|
-
* attempt in the chain — continuation rows included — with per-attempt turns,
|
|
1210
|
-
* wall clock, failure class and a one-line last error, the explicit salvage
|
|
1211
|
-
* state, the spend total and the transcript paths for local inspection.
|
|
1212
|
-
*
|
|
1213
|
-
* Pure so the tests hold the shape, not the transport: the writer below owns
|
|
1214
|
-
* the once-only guarantee, this owns what "once" looks like.
|
|
1215
|
-
*/
|
|
1216
|
-
export function formatExhaustionPostmortem(args: {
|
|
1217
|
-
issue: number;
|
|
1218
|
-
runs: readonly RunRecord[];
|
|
1219
|
-
reason: string;
|
|
1220
|
-
}): string {
|
|
1221
|
-
const { issue, runs, reason } = args;
|
|
1222
|
-
const artifact = newestContinuableRun(runs);
|
|
1223
|
-
const totalSpend = runs.reduce((sum, r) => sum + r.spendUsd, 0);
|
|
1224
|
-
const spend = `$${totalSpend.toFixed(2)}`;
|
|
1225
|
-
const attemptLines = runs.map((r) => {
|
|
1226
|
-
const wall = r.endedAt === undefined ? "—" : humanDuration(r.endedAt - r.startedAt);
|
|
1227
|
-
const cls = r.failureClass ?? "unclassified";
|
|
1228
|
-
const error = oneLineBrief(r.lastError) ?? "—";
|
|
1229
|
-
return (
|
|
1230
|
-
` attempt ${r.attempt} ${r.state.padEnd(12)} turns ${r.turns}/${r.maxTurns} ` +
|
|
1231
|
-
`${cls.padEnd(24)} ${wall.padStart(4)} last error: ${error}`
|
|
1232
|
-
);
|
|
1233
|
-
});
|
|
1234
|
-
const salvage =
|
|
1235
|
-
artifact === undefined
|
|
1236
|
-
? "Salvaged WIP: absent — no attempt preserved a branch, head SHA or pull request."
|
|
1237
|
-
: `Salvaged WIP: present — branch ${artifact.branch} at ${artifact.headSha ?? artifact.salvageSha}` +
|
|
1238
|
-
`${artifact.prUrl === undefined ? "" : ` (PR ${artifact.prUrl})`}.`;
|
|
1239
|
-
return [
|
|
1240
|
-
`\`\`\`${POSTMORTEM_MARKER}`,
|
|
1241
|
-
`#${issue} exhausted: ${attemptClassBreakdown(runs)}`,
|
|
1242
|
-
reason,
|
|
1243
|
-
"",
|
|
1244
|
-
`Attempts (${runs.length} total, ${spend} spend):`,
|
|
1245
|
-
...attemptLines,
|
|
1246
|
-
salvage,
|
|
1247
|
-
`Spend total: ${spend} across ${runs.length} attempts.`,
|
|
1248
|
-
"Transcripts:",
|
|
1249
|
-
...runs.map((r) => (r.sessionFile === undefined ? " (none)" : ` ${r.sessionFile}`)),
|
|
1250
|
-
"```",
|
|
1251
|
-
].join("\n");
|
|
1252
|
-
}
|
|
1253
|
-
|
|
1254
|
-
/** Dedupe key prefix for the exhaustion postmortem comment, per issue. */
|
|
1255
|
-
function postmortemDedupeKey(project: string, issue: number): string {
|
|
1256
|
-
return `${project}:postmortem:${issue}`;
|
|
1257
|
-
}
|
|
1258
|
-
|
|
1259
|
-
/**
|
|
1260
|
-
* The exhaustion postmortem: written exactly once per issue, at the point the
|
|
1261
|
-
* continuation budget is spent and the issue is settled toward a human.
|
|
1262
|
-
*
|
|
1263
|
-
* The comment and the material event are both gated by the store's notification
|
|
1264
|
-
* ledger — the same idempotence guard the escalator uses — so re-settling an
|
|
1265
|
-
* already-postmortemed issue posts nothing and records nothing. A body-string
|
|
1266
|
-
* match on the issue would be the wrong guard: an issue re-scoped and re-run
|
|
1267
|
-
* would still carry the old block, and the guarantee asked of this is "decided
|
|
1268
|
-
* once", not "deduped against what is already written".
|
|
1269
|
-
*
|
|
1270
|
-
* The digest must name the exhaustion even if the comment write fails, so the
|
|
1271
|
-
* material event is recorded before the write and unconditionally (the ledger
|
|
1272
|
-
* is append-only, and the gate above already ran once). The comment failure is
|
|
1273
|
-
* logged rather than taking the sweep down with it.
|
|
1274
|
-
*/
|
|
1275
|
-
async function postExhaustionPostmortem(d: Deps, run: RunRecord, reason: string): Promise<void> {
|
|
1276
|
-
const { project, tracker, store } = d;
|
|
1277
|
-
const key = postmortemDedupeKey(project.name, run.issue);
|
|
1278
|
-
if (store.wasNotified(key)) return;
|
|
1279
|
-
const runs = store.runsForIssue(project.name, run.issue);
|
|
1280
|
-
const body = formatExhaustionPostmortem({ issue: run.issue, runs, reason });
|
|
1281
|
-
const occurredAt = Date.now();
|
|
1282
|
-
store.recordMaterialEvent({
|
|
1283
|
-
project: project.name,
|
|
1284
|
-
category: "exhaustion",
|
|
1285
|
-
summary: `#${run.issue} exhausted: ${attemptClassBreakdown(runs)}`,
|
|
1286
|
-
evidence: body,
|
|
1287
|
-
occurredAt,
|
|
1288
|
-
recordedAt: occurredAt,
|
|
1289
|
-
});
|
|
1290
|
-
try {
|
|
1291
|
-
await tracker.comment(run.issue, body);
|
|
1292
|
-
store.markNotified(key);
|
|
1293
|
-
log(`#${run.issue} posted exhaustion postmortem (${runs.length} attempts)`);
|
|
1294
|
-
} catch (err) {
|
|
1295
|
-
log(`#${run.issue} postmortem comment could not be posted (${errText(err)})`);
|
|
1296
|
-
}
|
|
1297
|
-
}
|
|
1298
|
-
|
|
1299
1180
|
export type ExtendTurnLimitResult =
|
|
1300
1181
|
| { kind: "extended"; runId: string; maxTurns: number }
|
|
1301
1182
|
| { kind: "not-increase"; runId: string; maxTurns: number }
|
|
@@ -1499,45 +1380,6 @@ export async function verifyPushedGreenClaim(
|
|
|
1499
1380
|
};
|
|
1500
1381
|
}
|
|
1501
1382
|
|
|
1502
|
-
/** What the settlement audit of one green run produced: the advisory flags
|
|
1503
|
-
* (test weakening only — the file-list disclosure is derived, not flagged),
|
|
1504
|
-
* whether the diff was cut short, and the `changed:` line composed from the
|
|
1505
|
-
* PR's own diff. */
|
|
1506
|
-
export interface SettlementAuditResult {
|
|
1507
|
-
flags: SettlementFlag[];
|
|
1508
|
-
truncated: boolean;
|
|
1509
|
-
/** The `changed:` file list derived from the PR's diff, present whenever the
|
|
1510
|
-
* diff could be read. Absent means the tree could not be read, and `flags`
|
|
1511
|
-
* then carries exactly {@link UNREADABLE_TREE_FLAG}. */
|
|
1512
|
-
changedLine?: string;
|
|
1513
|
-
}
|
|
1514
|
-
|
|
1515
|
-
/**
|
|
1516
|
-
* Audit a worker's own account of its work against the pull request it pushed.
|
|
1517
|
-
*
|
|
1518
|
-
* The thin half of the split #85 established: this fetches, {@link
|
|
1519
|
-
* analyseSettlement} decides. It runs beside {@link verifyPushedGreenClaim} and
|
|
1520
|
-
* shares none of its authority — that function decides a run's state, this one
|
|
1521
|
-
* cannot, by construction. It returns evidence and the caller appends it.
|
|
1522
|
-
*
|
|
1523
|
-
* A diff that cannot be read is a finding (`changed-line-missing`), never an
|
|
1524
|
-
* empty flag list: nothing was derived and nothing was checked, and that must
|
|
1525
|
-
* not read as a clean bill.
|
|
1526
|
-
*/
|
|
1527
|
-
export async function collectSettlementFlags(
|
|
1528
|
-
tracker: Pick<Tracker, "prDiff">,
|
|
1529
|
-
claim: { prUrl?: string; issueText: string },
|
|
1530
|
-
): Promise<SettlementAuditResult> {
|
|
1531
|
-
if (claim.prUrl === undefined) return { flags: [UNREADABLE_TREE_FLAG], truncated: false };
|
|
1532
|
-
const diff = await tracker.prDiff(claim.prUrl);
|
|
1533
|
-
if (diff === undefined) return { flags: [UNREADABLE_TREE_FLAG], truncated: false };
|
|
1534
|
-
return {
|
|
1535
|
-
flags: analyseSettlement({ issueText: claim.issueText, diff }),
|
|
1536
|
-
truncated: diff.truncated,
|
|
1537
|
-
changedLine: deriveChangedLine(diff),
|
|
1538
|
-
};
|
|
1539
|
-
}
|
|
1540
|
-
|
|
1541
1383
|
/**
|
|
1542
1384
|
* One attempt at one issue, from claim to terminal state. Everything is inside
|
|
1543
1385
|
* a single try/catch so that a bad issue costs its own run and nothing else.
|
|
@@ -1681,7 +1523,12 @@ function orphanResumeVerdict(
|
|
|
1681
1523
|
return { kind: "resume", prior };
|
|
1682
1524
|
}
|
|
1683
1525
|
|
|
1684
|
-
export async function handleIssue(
|
|
1526
|
+
export async function handleIssue(
|
|
1527
|
+
d: Deps,
|
|
1528
|
+
r: Routed,
|
|
1529
|
+
attempt: number,
|
|
1530
|
+
admittedLane?: FileLane,
|
|
1531
|
+
): Promise<void> {
|
|
1685
1532
|
const { project, caps, tracker, store } = d;
|
|
1686
1533
|
const issue = r.issue.number;
|
|
1687
1534
|
const branch = branchName(r.issue);
|
|
@@ -1824,6 +1671,24 @@ export async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<
|
|
|
1824
1671
|
return;
|
|
1825
1672
|
}
|
|
1826
1673
|
|
|
1674
|
+
// The claim-side of the pause fence (#651 review #3): the tick's pause
|
|
1675
|
+
// gate sits above routing, so a tick that passed that gate before an
|
|
1676
|
+
// operator wrote the freeze can still be mid-routing when the fence lands
|
|
1677
|
+
// — it would claim after the setup barrier's acknowledgement and before
|
|
1678
|
+
// its first mutation. The claim itself re-checks the pause and refuses,
|
|
1679
|
+
// writing the durable admission acknowledgement the barrier waits for.
|
|
1680
|
+
// A claim observed under the fence is the last admission boundary there
|
|
1681
|
+
// is: nothing may create a run row beside a held fleet.
|
|
1682
|
+
if (isPaused(d.project.name)) {
|
|
1683
|
+
try {
|
|
1684
|
+
writeAdmissionAck(d.project.name);
|
|
1685
|
+
} catch (err) {
|
|
1686
|
+
log(`admission acknowledgement write failed: ${errText(err)}`);
|
|
1687
|
+
}
|
|
1688
|
+
log(`#${issue} not claimed: dispatch is paused at claim time`);
|
|
1689
|
+
return;
|
|
1690
|
+
}
|
|
1691
|
+
|
|
1827
1692
|
// Claim on the STORE first, before anything that can fail. The run row —
|
|
1828
1693
|
// not the label — is the crash-safe guard against double dispatch: rows
|
|
1829
1694
|
// are local, written before any network call, and the startup orphan
|
|
@@ -1899,6 +1764,10 @@ export async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<
|
|
|
1899
1764
|
spendUsd: 0,
|
|
1900
1765
|
maxTurns: caps.workerMaxTurns,
|
|
1901
1766
|
startedAt: Date.now(),
|
|
1767
|
+
// #567: the orphan-clean attempt this claim continues, when the verdict
|
|
1768
|
+
// above fired. `undefined` for a fresh dispatch — the store maps that to
|
|
1769
|
+
// NULL, so a fresh row simply never carries the field.
|
|
1770
|
+
resumedFromRunId: resuming?.id,
|
|
1902
1771
|
});
|
|
1903
1772
|
// #434: carry a terminal predecessor's open PR onto the continuation row.
|
|
1904
1773
|
// The claim itself stays synchronous — `handleIssue` claims at the top of
|
|
@@ -2043,10 +1912,15 @@ export async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<
|
|
|
2043
1912
|
// The continuation notice replaces the brief for a resumed attempt (#536).
|
|
2044
1913
|
// The original brief is already in the resumed transcript; re-sending it is
|
|
2045
1914
|
// how a resumed worker ends up re-doing the work it just did. Everything
|
|
2046
|
-
// below the brief is fresh-dispatch-only, exactly as today
|
|
1915
|
+
// below the brief is fresh-dispatch-only, exactly as today — except the
|
|
1916
|
+
// file lane admission enforced for this continuation, which rides the
|
|
1917
|
+
// notice itself: the retained transcript's brief may show an earlier
|
|
1918
|
+
// declaration, and a worker continuing under a stale lane is the collision
|
|
1919
|
+
// the gate exists to stop (#608).
|
|
2047
1920
|
let brief: string;
|
|
2048
1921
|
if (resuming !== undefined) {
|
|
2049
|
-
brief =
|
|
1922
|
+
brief =
|
|
1923
|
+
ORPHAN_RESUME_PROMPT + (admittedLane === undefined ? "" : resumeLaneBlock(admittedLane));
|
|
2050
1924
|
} else {
|
|
2051
1925
|
// The discussion is rendered at dispatch so a worker never depends on a
|
|
2052
1926
|
// runtime `gh` read to see the orchestrator's grooming (#517). The read is
|
|
@@ -2068,6 +1942,7 @@ export async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<
|
|
|
2068
1942
|
? { salvagedSha: priorSalvage }
|
|
2069
1943
|
: {}),
|
|
2070
1944
|
comments,
|
|
1945
|
+
lane: admittedLane,
|
|
2071
1946
|
});
|
|
2072
1947
|
}
|
|
2073
1948
|
if (await settleStopBeforeSession()) return;
|
|
@@ -2183,6 +2058,9 @@ export async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<
|
|
|
2183
2058
|
? await collectSettlementFlags(tracker, {
|
|
2184
2059
|
prUrl: result.prUrl,
|
|
2185
2060
|
issueText: `${r.issue.title}\n${r.issue.body}`,
|
|
2061
|
+
// The claimed-proof check compares the PR's Verified commands
|
|
2062
|
+
// against what this run's session actually recorded.
|
|
2063
|
+
sessionFile: result.sessionFile,
|
|
2186
2064
|
})
|
|
2187
2065
|
: undefined;
|
|
2188
2066
|
if (result.state === "pushed-green" && audit?.truncated) {
|
|
@@ -2369,7 +2247,7 @@ export async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<
|
|
|
2369
2247
|
});
|
|
2370
2248
|
|
|
2371
2249
|
if (providerCredit !== undefined) {
|
|
2372
|
-
await reactToProviderCredit(d, issue, providerCredit, result.sessionFile);
|
|
2250
|
+
await reactToProviderCredit({ project: d.project, escalate: (e) => d.escalate(e), isPaused, setPaused }, issue, providerCredit, result.sessionFile);
|
|
2373
2251
|
swapToQueue(d, issue, inProgress);
|
|
2374
2252
|
} else if (continueTurns) {
|
|
2375
2253
|
// Requeue as one ordered pair: the in-progress removal before the
|
|
@@ -2535,141 +2413,771 @@ export async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<
|
|
|
2535
2413
|
}
|
|
2536
2414
|
}
|
|
2537
2415
|
|
|
2538
|
-
//
|
|
2539
|
-
|
|
2540
|
-
/** What a resolved PR turns its `pushed-green` row into. */
|
|
2541
|
-
export interface Settlement {
|
|
2542
|
-
state: "merged" | "failed";
|
|
2543
|
-
/** The log line after `#<n> settled: `, and — for a rejection — the row's own
|
|
2544
|
-
* `lastError`, because a `failed` row whose worker succeeded has to say so. */
|
|
2545
|
-
reason: string;
|
|
2546
|
-
}
|
|
2547
|
-
|
|
2548
|
-
/**
|
|
2549
|
-
* What one `pushed-green` row becomes now its PR has an answer, or undefined to
|
|
2550
|
-
* leave the row exactly as it is.
|
|
2551
|
-
*
|
|
2552
|
-
* A `pushed-green` row is the only one nothing ever revisited: the worker is
|
|
2553
|
-
* finished, `reconcileOrphanedRuns` only settles rows that held a process, and
|
|
2554
|
-
* `merged` went unwritten from day one. So they accumulated — three of them on
|
|
2555
|
-
* the reference fleet on 2026-08-07, every PR merged and every issue closed,
|
|
2556
|
-
* with `/healthz` still reporting three active runs and their issues
|
|
2557
|
-
* permanently unclaimable, because the busy set *is* the active set (#18).
|
|
2558
|
-
*
|
|
2559
|
-
* The mapping, and why each answer is the only honest one:
|
|
2560
|
-
*
|
|
2561
|
-
* - `merged` — the work landed. That is what `merged` was reserved for.
|
|
2562
|
-
* - `closed` — a human read the work and said no. Leaving it `pushed-green`
|
|
2563
|
-
* forever is a lie; `failed` records that it did not land and releases the
|
|
2564
|
-
* busy guard, so an issue a human re-queues can be attempted again. A row
|
|
2565
|
-
* that had reached `pushed-green` or `pushed-pending` is classified
|
|
2566
|
-
* `returned-for-revision` at settlement. A review decision asks for another
|
|
2567
|
-
* implementation pass, not a failure, so it consumes the continuation budget
|
|
2568
|
-
* instead of the failed-attempt budget.
|
|
2569
|
-
* - `open`, and undefined — nothing changes. Undefined is "could not tell": a
|
|
2570
|
-
* flaky network, a revoked token, a deleted PR. Settling on it would record a
|
|
2571
|
-
* merge that never happened, and the next tick asks again for free. An
|
|
2572
|
-
* ambiguous answer must never settle a row.
|
|
2573
|
-
*/
|
|
2574
|
-
export function settlementFor(pr: PrState | undefined, prUrl: string): Settlement | undefined {
|
|
2575
|
-
if (pr === "merged") return { state: "merged", reason: `${prUrl} merged` };
|
|
2576
|
-
if (pr === "closed") return { state: "failed", reason: `${prUrl} closed without merging` };
|
|
2577
|
-
return undefined;
|
|
2578
|
-
}
|
|
2416
|
+
// ------------------------------------------------------------ review revisions
|
|
2579
2417
|
|
|
2580
2418
|
/**
|
|
2581
|
-
*
|
|
2419
|
+
* The daemon half of the review-revision transport (#677): wake every pending
|
|
2420
|
+
* revision the orchestrator's verb recorded, on the tick, exactly like
|
|
2421
|
+
* admission — the verb has no handle on the worker machinery, and the CLI path
|
|
2422
|
+
* must behave like the embedded one. Bounded by the same worker slots, so a
|
|
2423
|
+
* queued revision can never push the fleet past `maxConcurrentWorkers`; the
|
|
2424
|
+
* remaining rows stay pending for the next tick.
|
|
2582
2425
|
*
|
|
2583
|
-
*
|
|
2584
|
-
* the
|
|
2585
|
-
*
|
|
2586
|
-
* settled
|
|
2587
|
-
* active set correctly; an authoritative `gh issue view` on each afterwards
|
|
2588
|
-
* still showed `agent:in-progress` — permanently unclaimable with no supported
|
|
2589
|
-
* way back (#18).
|
|
2590
|
-
*
|
|
2591
|
-
* The outbox makes the row transition and the label one fact again: the
|
|
2592
|
-
* removal is enqueued in the same breath as the row is terminalised, the
|
|
2593
|
-
* projector applies it with unbounded retry, and while it is pending the
|
|
2594
|
-
* eligibility overlay treats the label as already gone. A tracker that refuses
|
|
2595
|
-
* the write (403, rate limit) can no longer strand the row — that is `#184`
|
|
2596
|
-
* and `#198` closed. No `pushed-*` row is ever written terminal with its
|
|
2597
|
-
* label release owed but unrecorded, because enqueueing is a local store write
|
|
2598
|
-
* that cannot fail on the tracker.
|
|
2599
|
-
*
|
|
2600
|
-
* Synchronous. The op is durable the moment this returns.
|
|
2426
|
+
* The claim (`pushed-green` → `running`) is synchronous and atomic per row;
|
|
2427
|
+
* only the resumed worker runs in the pool. A row that moved since the verb
|
|
2428
|
+
* recorded it — merged after all, settled, re-claimed — fails the claim and is
|
|
2429
|
+
* settled `skipped` rather than woken on stale identity.
|
|
2601
2430
|
*/
|
|
2602
|
-
export function
|
|
2603
|
-
|
|
2604
|
-
|
|
2605
|
-
|
|
2606
|
-
|
|
2607
|
-
const
|
|
2608
|
-
|
|
2609
|
-
|
|
2431
|
+
export async function dispatchReviewRevisions(d: Deps, pool?: WorkerPool): Promise<void> {
|
|
2432
|
+
const pending = d.store.pendingReviewRevisions(d.project.name);
|
|
2433
|
+
if (pending.length === 0) return;
|
|
2434
|
+
const slots = Math.max(0, d.caps.maxConcurrentWorkers - d.store.liveRuns(d.project.name).length);
|
|
2435
|
+
const batch = pending.slice(0, slots);
|
|
2436
|
+
const launches: Promise<void>[] = [];
|
|
2437
|
+
for (const revision of batch) {
|
|
2438
|
+
if (d.drain?.draining === true) {
|
|
2439
|
+
log(
|
|
2440
|
+
`review revisions held: the daemon is draining (${batch.length - launches.length} of ${batch.length} not launched)`,
|
|
2441
|
+
);
|
|
2442
|
+
break;
|
|
2443
|
+
}
|
|
2444
|
+
if (!d.store.claimRunForReview(revision.runId)) {
|
|
2445
|
+
d.store.settleReviewRevision(revision.id, "skipped", Date.now());
|
|
2446
|
+
log(
|
|
2447
|
+
`#${revision.issue} review round ${revision.round} skipped: run ${revision.runId} is no longer pushed-green`,
|
|
2448
|
+
);
|
|
2449
|
+
continue;
|
|
2450
|
+
}
|
|
2451
|
+
d.store.markReviewRevisionDispatched(revision.id, Date.now());
|
|
2452
|
+
log(`#${revision.issue} review round ${revision.round} → resuming run ${revision.runId}`);
|
|
2453
|
+
launches.push(handleReviewRevision(d, revision));
|
|
2454
|
+
}
|
|
2455
|
+
if (pool !== undefined) {
|
|
2456
|
+
for (const launch of launches) pool.launch(launch);
|
|
2457
|
+
return;
|
|
2458
|
+
}
|
|
2459
|
+
// `--once`: like admissions, the tick waits for the workers it launched.
|
|
2460
|
+
await Promise.allSettled(launches);
|
|
2610
2461
|
}
|
|
2611
2462
|
|
|
2612
2463
|
/**
|
|
2613
|
-
*
|
|
2614
|
-
*
|
|
2464
|
+
* Resume one review-revision worker: the run row is already claimed
|
|
2465
|
+
* (`pushed-green` → `running` by the dispatch pass), so this resumes the SAME
|
|
2466
|
+
* OMP session — same session directory, `resume: true` at the harness, and the
|
|
2467
|
+
* `sessionFile` lineage compared at dispatch time — on the SAME branch and PR,
|
|
2468
|
+
* then settles the row back the way {@link handleIssue} settles any worker.
|
|
2615
2469
|
*
|
|
2616
|
-
*
|
|
2617
|
-
*
|
|
2618
|
-
*
|
|
2619
|
-
*
|
|
2620
|
-
*
|
|
2621
|
-
*
|
|
2622
|
-
* The label is released too, which reverses what this function first promised.
|
|
2623
|
-
* It used to leave tracker labels alone exactly as {@link reconcileOrphanedRuns}
|
|
2624
|
-
* does, reasoning that a merge closes the issue anyway and that deciding what an
|
|
2625
|
-
* issue's labels should say next is the orchestrator's drain duty. There turned
|
|
2626
|
-
* out to be no such path: on 2026-08-09 two settled rows left their issues
|
|
2627
|
-
* carrying `agent:in-progress` forever, with the brief forbidding the
|
|
2628
|
-
* orchestrator from touching it and `unblock` declining to (see
|
|
2629
|
-
* {@link releaseInProgress}). The row transition and the label are one fact, and
|
|
2630
|
-
* writing half of it is the whole of that bug.
|
|
2631
|
-
*
|
|
2632
|
-
* Releasing it is safe here specifically because of what these rows are. A
|
|
2633
|
-
* `pushed-green` or `pushed-pending` row has no process behind it — its worker
|
|
2634
|
-
* exited and its worktree is gone — so a terminal answer about its PR proves no
|
|
2635
|
-
* worker owns the issue, and the duplicate-dispatch interlock the label exists
|
|
2636
|
-
* for is spent. {@link reconcileOrphanedRuns} still leaves labels alone for the
|
|
2637
|
-
* opposite reason: an orphaned `running` row is work nobody has read yet. And
|
|
2638
|
-
* the brief's rule stays absolute, because this is a daemon-owned write through
|
|
2639
|
-
* the same Tracker port the dispatcher claimed the issue with — orphan detection
|
|
2640
|
-
* is only trustworthy while every state label on the tracker came from this
|
|
2641
|
-
* package.
|
|
2642
|
-
*
|
|
2643
|
-
* The two writes are ordered label-then-row, and the order is load-bearing. This
|
|
2644
|
-
* sweep is the only thing that revisits a `pushed-*` row, so the terminal state
|
|
2645
|
-
* is also the row's exit from it: written first, a tracker that then failed on
|
|
2646
|
-
* the label would leave `agent:in-progress` with nothing left to retry it — #18
|
|
2647
|
-
* exactly, in the last window able to reach it. Writing the label first makes
|
|
2648
|
-
* failure cost a repeated `gh` call on the next tick instead, and the row stays
|
|
2649
|
-
* in the busy set throughout, so no second worker can be sent at the issue while
|
|
2650
|
-
* it waits.
|
|
2470
|
+
* The run row is reused, never cloned: same attempt number, same prUrl, same
|
|
2471
|
+
* transcript. That is what keeps a revision round off both budgets — the round
|
|
2472
|
+
* is a state transition of one already-green row, green → running → green, and
|
|
2473
|
+
* no new row exists for either counter to count. Only a revision worker that
|
|
2474
|
+
* genuinely settles terminal falls through to the ordinary failure handling,
|
|
2475
|
+
* budgets and all, exactly as today.
|
|
2651
2476
|
*/
|
|
2652
|
-
|
|
2653
|
-
const
|
|
2654
|
-
const
|
|
2655
|
-
|
|
2656
|
-
|
|
2477
|
+
export async function handleReviewRevision(d: Deps, revision: ReviewRevisionRecord): Promise<void> {
|
|
2478
|
+
const { project, caps, tracker, store } = d;
|
|
2479
|
+
const run = store.getRun(revision.runId);
|
|
2480
|
+
if (run === undefined || run.state !== "running") {
|
|
2481
|
+
log(
|
|
2482
|
+
`#${revision.issue} review round ${revision.round} skipped: run ${revision.runId} is ${run?.state ?? "gone"} — ` +
|
|
2483
|
+
"the row moved between the dispatch claim and the resume.",
|
|
2484
|
+
);
|
|
2485
|
+
store.settleReviewRevision(revision.id, "skipped", Date.now());
|
|
2486
|
+
return;
|
|
2487
|
+
}
|
|
2488
|
+
const issue = run.issue;
|
|
2489
|
+
const branch = run.branch;
|
|
2490
|
+
const runId = run.id;
|
|
2491
|
+
const inProgress = project.stateLabels.inProgress;
|
|
2492
|
+
const repo = Object.values(project.routing.repos).find((candidate) => candidate.name === run.repo);
|
|
2493
|
+
if (repo === undefined) {
|
|
2494
|
+
log(`#${issue} review round ${revision.round} cannot dispatch: repo ${run.repo} is no longer routed`);
|
|
2495
|
+
store.updateRun(runId, { state: "pushed-green", endedAt: Date.now() });
|
|
2496
|
+
store.settleReviewRevision(revision.id, "skipped", Date.now());
|
|
2497
|
+
return;
|
|
2498
|
+
}
|
|
2499
|
+
const mirrorPath = mirrorPathFor(repo, project.mirrorRoot);
|
|
2500
|
+
|
|
2501
|
+
// The session being resumed is the one the pushed-green row recorded. A
|
|
2502
|
+
// revision without that transcript cannot continue the same session — the
|
|
2503
|
+
// exact fake this feature exists to prevent — so fail loudly instead of
|
|
2504
|
+
// silently starting a fresh one.
|
|
2505
|
+
const priorSessionFile = run.sessionFile;
|
|
2506
|
+
if (priorSessionFile === undefined || !existsSync(priorSessionFile)) {
|
|
2507
|
+
log(
|
|
2508
|
+
`#${issue} review round ${revision.round} cannot resume: the recorded transcript ` +
|
|
2509
|
+
`${priorSessionFile ?? "(none)"} is gone`,
|
|
2510
|
+
);
|
|
2511
|
+
store.updateRun(runId, {
|
|
2512
|
+
state: "pushed-green",
|
|
2513
|
+
endedAt: Date.now(),
|
|
2514
|
+
lastError: `review round ${revision.round} could not resume: the run's transcript is gone`,
|
|
2515
|
+
});
|
|
2516
|
+
store.settleReviewRevision(revision.id, "skipped", Date.now());
|
|
2517
|
+
await safeEscalate(d, {
|
|
2518
|
+
tier: 1,
|
|
2519
|
+
project: project.name,
|
|
2520
|
+
issue,
|
|
2521
|
+
runId,
|
|
2522
|
+
summary: `#${issue} review round ${revision.round} could not resume — the transcript is missing`,
|
|
2523
|
+
detail: [
|
|
2524
|
+
revision.prUrl,
|
|
2525
|
+
"",
|
|
2526
|
+
"The PR is still open and green; the run row was returned to pushed-green.",
|
|
2527
|
+
"Resume the manual path (close + unblock + continuation) or merge it as it stands.",
|
|
2528
|
+
].join("\n"),
|
|
2529
|
+
});
|
|
2530
|
+
return;
|
|
2531
|
+
}
|
|
2532
|
+
const sessionDir = dirname(priorSessionFile);
|
|
2657
2533
|
|
|
2658
|
-
|
|
2659
|
-
|
|
2660
|
-
|
|
2661
|
-
|
|
2662
|
-
|
|
2663
|
-
|
|
2664
|
-
|
|
2665
|
-
]);
|
|
2534
|
+
let worktreePath: string | undefined;
|
|
2535
|
+
let runRepo: RunRepoRef | undefined;
|
|
2536
|
+
let turnLimit: TurnLimitController | undefined;
|
|
2537
|
+
let workerControl: WorkerControlSlot | undefined;
|
|
2538
|
+
let workerSessionInstalled = false;
|
|
2539
|
+
let verbListener: VerbListener | undefined;
|
|
2540
|
+
const repoSlug = githubRepo(repo.cloneUrl);
|
|
2666
2541
|
|
|
2667
|
-
|
|
2668
|
-
|
|
2669
|
-
|
|
2670
|
-
|
|
2671
|
-
|
|
2672
|
-
|
|
2542
|
+
/**
|
|
2543
|
+
* Publishes the run's branch on the privileged side: run repo → mirror →
|
|
2544
|
+
* GitHub, fast-forward only — the same route `handleIssue` uses, so the
|
|
2545
|
+
* revised head the worker pushed through its own `conductor_push` is
|
|
2546
|
+
* re-verified against the remote before the tree dies.
|
|
2547
|
+
*/
|
|
2548
|
+
const publish: RunPublisher = async () => {
|
|
2549
|
+
if (runRepo === undefined) return { ok: false, stderr: "the run repository was never provisioned" };
|
|
2550
|
+
return pushRunBranch(project, runRepo);
|
|
2551
|
+
};
|
|
2552
|
+
|
|
2553
|
+
const settleStopBeforeSession = async (): Promise<boolean> => {
|
|
2554
|
+
const reason = workerControl?.requestedStop();
|
|
2555
|
+
if (reason === undefined || workerSessionInstalled) return false;
|
|
2556
|
+
turnLimit?.close();
|
|
2557
|
+
turnLimit = undefined;
|
|
2558
|
+
recordOperatorStop(store, {
|
|
2559
|
+
project: project.name,
|
|
2560
|
+
issue,
|
|
2561
|
+
runId,
|
|
2562
|
+
inProgress,
|
|
2563
|
+
reason,
|
|
2564
|
+
patch: {
|
|
2565
|
+
endedAt: Date.now(),
|
|
2566
|
+
report: ["Operator stopped the review revision before its session started.", `Reason: ${reason}`].join("\n"),
|
|
2567
|
+
},
|
|
2568
|
+
});
|
|
2569
|
+
store.settleReviewRevision(revision.id, "skipped", Date.now());
|
|
2570
|
+
log(`#${issue} review round ${revision.round} stopped by operator before its session started: ${reason}`);
|
|
2571
|
+
return true;
|
|
2572
|
+
};
|
|
2573
|
+
|
|
2574
|
+
const settleDrainBeforeSession = async (): Promise<boolean> => {
|
|
2575
|
+
if (d.drain?.draining !== true || workerSessionInstalled) return false;
|
|
2576
|
+
turnLimit?.close();
|
|
2577
|
+
turnLimit = undefined;
|
|
2578
|
+
// A shutdown that interrupts the wake is the same situation as the
|
|
2579
|
+
// provisioning-failure branch below, not an operator stop — nobody
|
|
2580
|
+
// stopped anything, the PR is still green and still open, and only the
|
|
2581
|
+
// wake failed; `settleStopBeforeSession` immediately above covers the
|
|
2582
|
+
// case where an operator genuinely did. So mirror that branch: restore
|
|
2583
|
+
// `pushed-green` naming the shutdown, settle the revision `skipped` so
|
|
2584
|
+
// the round is not double-counted, and leave the in-progress label alone
|
|
2585
|
+
// — the ordinary settle sweep releases it in the same breath it
|
|
2586
|
+
// terminalises the row when the PR resolves. Stopping the row here would
|
|
2587
|
+
// strand it: `settlePushedGreen` sweeps only pushed-* rows and
|
|
2588
|
+
// classification excludes `stopped`, so the issue would carry neither
|
|
2589
|
+
// label and nothing would ever revisit the still-green PR.
|
|
2590
|
+
store.updateRun(runId, {
|
|
2591
|
+
state: "pushed-green",
|
|
2592
|
+
endedAt: Date.now(),
|
|
2593
|
+
lastError: "daemon shutdown began after the review revision claim; the round was not launched",
|
|
2594
|
+
});
|
|
2595
|
+
store.settleReviewRevision(revision.id, "skipped", Date.now());
|
|
2596
|
+
log(`#${issue} review round ${revision.round} not launched: daemon shutdown began after the claim`);
|
|
2597
|
+
return true;
|
|
2598
|
+
};
|
|
2599
|
+
|
|
2600
|
+
// Live-run controls, opened before any provisioning so the operator stop
|
|
2601
|
+
// and shutdown fences cover the whole wake window, exactly as they do for a
|
|
2602
|
+
// fresh claim in `handleIssue` (#374). The original run's entries were
|
|
2603
|
+
// closed at its settle, so reopening by issue is safe.
|
|
2604
|
+
turnLimit = d.turnLimits.open(project.name, issue, runId, run.maxTurns);
|
|
2605
|
+
workerControl = d.workerControls.open(project.name, issue, runId);
|
|
2606
|
+
if (await settleStopBeforeSession()) return;
|
|
2607
|
+
if (await settleDrainBeforeSession()) return;
|
|
2608
|
+
|
|
2609
|
+
try {
|
|
2610
|
+
// Reattach the run's own branch at the same per-issue path the run used:
|
|
2611
|
+
// the worktree was removed at the pushed-green settle and the branch was
|
|
2612
|
+
// published, so provisioning is the same continuation reattach as a
|
|
2613
|
+
// normal re-claim. A provisioning failure restores the row — the PR is
|
|
2614
|
+
// still green and still open, and only the wake failed — and says so.
|
|
2615
|
+
try {
|
|
2616
|
+
const provisioned = await addRunRepo(repo, project.mirrorRoot, project.workspaceRoot, issue, branch);
|
|
2617
|
+
worktreePath = provisioned.path;
|
|
2618
|
+
runRepo = { repo, runRepoPath: worktreePath, branch };
|
|
2619
|
+
} catch (err) {
|
|
2620
|
+
const detail = errText(err);
|
|
2621
|
+
log(`#${issue} review round ${revision.round} cannot provision ${branch}: ${detail}`);
|
|
2622
|
+
// An operator stop that landed during provisioning wins over restoration —
|
|
2623
|
+
// it is a newer, explicit action, and the fences above already honoured
|
|
2624
|
+
// one that landed earlier.
|
|
2625
|
+
const pendingStop = workerControl?.requestedStop();
|
|
2626
|
+
if (pendingStop === undefined) {
|
|
2627
|
+
store.updateRun(runId, {
|
|
2628
|
+
state: "pushed-green",
|
|
2629
|
+
endedAt: Date.now(),
|
|
2630
|
+
lastError: `review round ${revision.round} could not provision the worktree: ${detail}`,
|
|
2631
|
+
});
|
|
2632
|
+
store.settleReviewRevision(revision.id, "skipped", Date.now());
|
|
2633
|
+
await safeEscalate(d, {
|
|
2634
|
+
tier: 1,
|
|
2635
|
+
project: project.name,
|
|
2636
|
+
issue,
|
|
2637
|
+
runId,
|
|
2638
|
+
summary: `#${issue} review round ${revision.round} could not be dispatched — worktree provisioning failed`,
|
|
2639
|
+
detail: [revision.prUrl, "", detail].join("\n"),
|
|
2640
|
+
});
|
|
2641
|
+
} else {
|
|
2642
|
+
recordOperatorStop(store, {
|
|
2643
|
+
project: project.name,
|
|
2644
|
+
issue,
|
|
2645
|
+
runId,
|
|
2646
|
+
inProgress,
|
|
2647
|
+
reason: pendingStop,
|
|
2648
|
+
patch: {
|
|
2649
|
+
endedAt: Date.now(),
|
|
2650
|
+
report: [
|
|
2651
|
+
`Operator stopped the review revision before its session started: ${pendingStop}`,
|
|
2652
|
+
`(worktree provisioning also failed: ${detail})`,
|
|
2653
|
+
].join("\n"),
|
|
2654
|
+
},
|
|
2655
|
+
});
|
|
2656
|
+
store.settleReviewRevision(revision.id, "skipped", Date.now());
|
|
2657
|
+
}
|
|
2658
|
+
return;
|
|
2659
|
+
}
|
|
2660
|
+
if (await settleStopBeforeSession()) return;
|
|
2661
|
+
if (await settleDrainBeforeSession()) return;
|
|
2662
|
+
|
|
2663
|
+
// The fleet-owned omp settings overlay, staged under the SAME session
|
|
2664
|
+
// directory the transcript lives in — a resumed session loads it exactly
|
|
2665
|
+
// as the original run did (#537).
|
|
2666
|
+
const ompSettingsFile = materializeOmpSettings(project, sessionDir);
|
|
2667
|
+
|
|
2668
|
+
// The run's mutation channel (#126): same run id, same issue, same repo —
|
|
2669
|
+
// so `conductor_push` publishes the run's own branch and nothing else.
|
|
2670
|
+
verbListener = await listenVerbChannel(
|
|
2671
|
+
verbDeps(d),
|
|
2672
|
+
{
|
|
2673
|
+
kind: "run",
|
|
2674
|
+
path: verbSocketPath(ensureVerbSocketDir(stateDir()), `run-${String(issue)}`),
|
|
2675
|
+
project: project.name,
|
|
2676
|
+
role: "worker",
|
|
2677
|
+
runId,
|
|
2678
|
+
issue,
|
|
2679
|
+
repo,
|
|
2680
|
+
runRepoPath: worktreePath,
|
|
2681
|
+
branch,
|
|
2682
|
+
},
|
|
2683
|
+
{ ...(d.verbPeerReader === undefined ? {} : { peerReader: d.verbPeerReader }) },
|
|
2684
|
+
);
|
|
2685
|
+
if (await settleStopBeforeSession()) return;
|
|
2686
|
+
if (await settleDrainBeforeSession()) return;
|
|
2687
|
+
|
|
2688
|
+
store.updateRun(runId, { worktree: worktreePath });
|
|
2689
|
+
|
|
2690
|
+
// The row's own counters are cumulative across the revision: the revision
|
|
2691
|
+
// worker meters its own session from zero, so its deltas are added to the
|
|
2692
|
+
// totals the run already recorded.
|
|
2693
|
+
const baseTurns = run.turns;
|
|
2694
|
+
const baseSpend = run.spendUsd;
|
|
2695
|
+
|
|
2696
|
+
log(`#${issue} review round ${revision.round} → resuming ${branch} (${priorSessionFile})`);
|
|
2697
|
+
|
|
2698
|
+
let result: WorkerResult;
|
|
2699
|
+
try {
|
|
2700
|
+
result = await runWorker({
|
|
2701
|
+
brief: renderReviewRevisionPrompt(revision.findings, revision.round),
|
|
2702
|
+
cwd: worktreePath,
|
|
2703
|
+
caps,
|
|
2704
|
+
...(repoSlug === undefined ? {} : { repoSlug }),
|
|
2705
|
+
maxTurns: () => turnLimit?.maxTurns() ?? run.maxTurns,
|
|
2706
|
+
onPauseControl: (control) => {
|
|
2707
|
+
workerSessionInstalled = true;
|
|
2708
|
+
workerControl?.install(control);
|
|
2709
|
+
},
|
|
2710
|
+
sessionDir,
|
|
2711
|
+
// The same-session half of the transport: `continueRecent` on the run's
|
|
2712
|
+
// own session directory, never a fresh `run-<uuid>` transcript.
|
|
2713
|
+
resume: true,
|
|
2714
|
+
socketPath: join(sessionDir, "ipc.sock"),
|
|
2715
|
+
verbSocketPath: verbListener.path,
|
|
2716
|
+
onSpawn: (pid) => {
|
|
2717
|
+
verbListener?.bindPid(pid);
|
|
2718
|
+
},
|
|
2719
|
+
onChildLog: (line) => {
|
|
2720
|
+
log(`#${issue} ${line}`);
|
|
2721
|
+
},
|
|
2722
|
+
// The continuation stays on the model the green run used (#286).
|
|
2723
|
+
...(run.model === undefined ? {} : { model: run.model }),
|
|
2724
|
+
...(ompSettingsFile === undefined ? {} : { ompSettingsFile }),
|
|
2725
|
+
releaseGrants: resolveReleaseGrants(project),
|
|
2726
|
+
onReleaseBlocked: workerReleaseBlockRecorder(project.name, issue, runId),
|
|
2727
|
+
onTurn: (n) => store.updateRun(runId, { turns: baseTurns + n }),
|
|
2728
|
+
onSpend: (usd) => store.updateRun(runId, { spendUsd: baseSpend + usd }),
|
|
2729
|
+
onKilled: () => {
|
|
2730
|
+
turnLimit?.close();
|
|
2731
|
+
turnLimit = undefined;
|
|
2732
|
+
},
|
|
2733
|
+
onSessionFile: (f) => {
|
|
2734
|
+
store.updateRun(runId, { sessionFile: f });
|
|
2735
|
+
// Same-file lineage is the proof the resume happened: a revision
|
|
2736
|
+
// that opened a different transcript fell back to a fresh session.
|
|
2737
|
+
if (f !== priorSessionFile) {
|
|
2738
|
+
log(
|
|
2739
|
+
`#${issue} review round ${revision.round} resume fell back to a fresh session: opened ${f} ` +
|
|
2740
|
+
`instead of ${priorSessionFile}`,
|
|
2741
|
+
);
|
|
2742
|
+
}
|
|
2743
|
+
},
|
|
2744
|
+
maySpawn: () => d.drain?.draining !== true,
|
|
2745
|
+
}, d.workerDeps);
|
|
2746
|
+
} finally {
|
|
2747
|
+
turnLimit?.close();
|
|
2748
|
+
turnLimit = undefined;
|
|
2749
|
+
}
|
|
2750
|
+
|
|
2751
|
+
if (result.modelFallbackMessage !== undefined) {
|
|
2752
|
+
log(`#${issue} review round ${revision.round} model fallback: ${result.modelFallbackMessage}`);
|
|
2753
|
+
}
|
|
2754
|
+
|
|
2755
|
+
const verified: { state: RunState; reason?: string } =
|
|
2756
|
+
result.state === "pushed-green"
|
|
2757
|
+
? await verifyPushedGreenClaim(tracker, result)
|
|
2758
|
+
: { state: result.state };
|
|
2759
|
+
const state = verified.state;
|
|
2760
|
+
|
|
2761
|
+
const settlement =
|
|
2762
|
+
state === "pushed-green" || state === "pushed-pending" || state === "merged"
|
|
2763
|
+
? undefined
|
|
2764
|
+
: await settleWorktree({
|
|
2765
|
+
issue,
|
|
2766
|
+
attempt: run.attempt,
|
|
2767
|
+
ending:
|
|
2768
|
+
state === "blocked"
|
|
2769
|
+
? "blocked for an operator decision"
|
|
2770
|
+
: state === "stopped"
|
|
2771
|
+
? `stopped by the operator: ${result.stoppedReason ?? "no reason recorded"}`
|
|
2772
|
+
: endedBy(result.killedBy),
|
|
2773
|
+
worktree: worktreePath,
|
|
2774
|
+
branch,
|
|
2775
|
+
publish,
|
|
2776
|
+
...(state === "failed" || state === "killed"
|
|
2777
|
+
? ({ tree: "keep" } as const)
|
|
2778
|
+
: ({ tree: "remove", mirrorPath } as const)),
|
|
2779
|
+
});
|
|
2780
|
+
if (settlement === undefined) {
|
|
2781
|
+
const published = await publish(branch);
|
|
2782
|
+
if (!published.ok) {
|
|
2783
|
+
log(`#${issue} publish before removal failed, work is preserved in the mirror: ${published.stderr}`);
|
|
2784
|
+
}
|
|
2785
|
+
await removeWorktree(mirrorPath, worktreePath);
|
|
2786
|
+
}
|
|
2787
|
+
|
|
2788
|
+
// The settlement report names the review round first (#692): a run that
|
|
2789
|
+
// reaches green again after a round must read as a revision outcome, not
|
|
2790
|
+
// as a fresh run's first push, and a round that failed must still say the
|
|
2791
|
+
// round — the state and classification below carry the failure itself.
|
|
2792
|
+
const finalReport = [
|
|
2793
|
+
`review round ${revision.round}: ${state}`,
|
|
2794
|
+
...(verified.reason === undefined ? [] : ["", verified.reason]),
|
|
2795
|
+
result.report,
|
|
2796
|
+
].join("\n");
|
|
2797
|
+
|
|
2798
|
+
const terminalPatch: Partial<RunRecord> = {
|
|
2799
|
+
endedAt: Date.now(),
|
|
2800
|
+
turns: baseTurns + result.turns,
|
|
2801
|
+
spendUsd: baseSpend + result.spendUsd,
|
|
2802
|
+
provider429Count: result.provider429Count,
|
|
2803
|
+
...(result.model === undefined ? {} : { resolvedModel: result.model }),
|
|
2804
|
+
...(result.provider === undefined ? {} : { resolvedProvider: result.provider }),
|
|
2805
|
+
retryFallbacks: result.retryFallbacks,
|
|
2806
|
+
retryFallbackSucceeded: result.retryFallbackSucceeded,
|
|
2807
|
+
modelRecoveries: result.modelRecoveries,
|
|
2808
|
+
autoRetryCount: result.autoRetryCount,
|
|
2809
|
+
autoCompactionCount: result.autoCompactionCount,
|
|
2810
|
+
// The worker only reports these when it actually established them; a
|
|
2811
|
+
// failure whose report named no PR must not wipe what the row already
|
|
2812
|
+
// owns (#468).
|
|
2813
|
+
...(result.prUrl === undefined ? {} : { prUrl: result.prUrl }),
|
|
2814
|
+
...(result.headSha === undefined ? {} : { headSha: result.headSha }),
|
|
2815
|
+
sessionFile: result.sessionFile,
|
|
2816
|
+
report: finalReport,
|
|
2817
|
+
...settlement?.patch,
|
|
2818
|
+
};
|
|
2819
|
+
if (state === "stopped") {
|
|
2820
|
+
recordOperatorStop(store, {
|
|
2821
|
+
project: project.name,
|
|
2822
|
+
issue,
|
|
2823
|
+
runId,
|
|
2824
|
+
inProgress,
|
|
2825
|
+
reason: result.stoppedReason ?? "no reason recorded",
|
|
2826
|
+
patch: terminalPatch,
|
|
2827
|
+
});
|
|
2828
|
+
} else {
|
|
2829
|
+
const sessionErr = state === "failed" || state === "killed" ? readSessionError(result.sessionFile) : undefined;
|
|
2830
|
+
const providerCredit = sessionErr === undefined ? undefined : providerCreditRefusal(sessionErr);
|
|
2831
|
+
const providerTransient =
|
|
2832
|
+
providerCredit !== undefined || sessionErr === undefined ? undefined : providerTransientFault(sessionErr);
|
|
2833
|
+
const lastError = completionLastError(providerCredit, providerTransient, verified.reason, sessionErr);
|
|
2834
|
+
store.updateRun(runId, {
|
|
2835
|
+
...terminalPatch,
|
|
2836
|
+
state,
|
|
2837
|
+
...(lastError === undefined ? {} : { lastError }),
|
|
2838
|
+
});
|
|
2839
|
+
}
|
|
2840
|
+
|
|
2841
|
+
const outcome: ReviewRevisionOutcome =
|
|
2842
|
+
state === "pushed-green" ? "revised" : state === "pushed-pending" ? "pending" : "failed";
|
|
2843
|
+
store.settleReviewRevision(revision.id, outcome, Date.now());
|
|
2844
|
+
log(`#${issue} review round ${revision.round} ${state}${result.prUrl ? ` ${result.prUrl}` : ""}`);
|
|
2845
|
+
|
|
2846
|
+
// Post-settle outcomes, mirroring handleIssue: a failed or killed revision
|
|
2847
|
+
// worker falls through to exactly the same label, requeue and escalation
|
|
2848
|
+
// handling as any failed run — budgets included — and a blocked one pages
|
|
2849
|
+
// the orchestrator for a decision.
|
|
2850
|
+
if (state === "stopped") {
|
|
2851
|
+
log(`#${issue} review round ${revision.round} stopped by operator: ${result.stoppedReason}`);
|
|
2852
|
+
} else if (state === "blocked") {
|
|
2853
|
+
swapLabel(store, project.name, issue, inProgress, project.stateLabels.blocked);
|
|
2854
|
+
await safeEscalate(d, {
|
|
2855
|
+
tier: 1,
|
|
2856
|
+
project: project.name,
|
|
2857
|
+
issue,
|
|
2858
|
+
runId,
|
|
2859
|
+
summary: `#${issue} is blocked on review round ${revision.round} and needs a decision`,
|
|
2860
|
+
detail: [`${result.prUrl ?? "(no PR URL)"}`, "", result.report].join("\n"),
|
|
2861
|
+
});
|
|
2862
|
+
} else if (state === "failed" || state === "killed") {
|
|
2863
|
+
const continuation = store.continuationsFor(project.name, issue);
|
|
2864
|
+
const continueTurns = shouldContinueAfterTurnsCap({
|
|
2865
|
+
killedBy: result.killedBy,
|
|
2866
|
+
prUrl: result.prUrl,
|
|
2867
|
+
headSha: result.headSha,
|
|
2868
|
+
salvageSha: settlement?.patch?.salvageSha,
|
|
2869
|
+
continuation,
|
|
2870
|
+
maxContinuations: caps.maxContinuationsPerIssue,
|
|
2871
|
+
});
|
|
2872
|
+
if (continueTurns) {
|
|
2873
|
+
store.enqueueLabelOps(project.name, [
|
|
2874
|
+
{ issue, op: "remove", label: inProgress },
|
|
2875
|
+
{ issue, op: "add", label: project.queueLabel },
|
|
2876
|
+
]);
|
|
2877
|
+
log(
|
|
2878
|
+
`#${issue} review round ${revision.round} turns-cap, continuation ` +
|
|
2879
|
+
`${continuation}/${caps.maxContinuationsPerIssue} — salvaged and re-queued`,
|
|
2880
|
+
);
|
|
2881
|
+
await safeEscalate(d, {
|
|
2882
|
+
tier: 1,
|
|
2883
|
+
project: project.name,
|
|
2884
|
+
issue,
|
|
2885
|
+
runId,
|
|
2886
|
+
summary: `#${issue} hit the turns cap on review round ${revision.round} — auto-requeued for continuation`,
|
|
2887
|
+
detail: [
|
|
2888
|
+
`${result.prUrl ?? "(no PR URL)"}`,
|
|
2889
|
+
"",
|
|
2890
|
+
"The queue label is back on; the next tick should reattach the branch",
|
|
2891
|
+
"and open a continuation brief. No failed label was applied.",
|
|
2892
|
+
"",
|
|
2893
|
+
result.report,
|
|
2894
|
+
].join("\n"),
|
|
2895
|
+
});
|
|
2896
|
+
} else {
|
|
2897
|
+
swapLabel(store, project.name, issue, inProgress, project.stateLabels.failed);
|
|
2898
|
+
await safeEscalate(d, {
|
|
2899
|
+
tier: 1,
|
|
2900
|
+
project: project.name,
|
|
2901
|
+
issue,
|
|
2902
|
+
runId,
|
|
2903
|
+
summary: result.killedBy
|
|
2904
|
+
? `#${issue} was killed on review round ${revision.round} by the ${result.killedBy} cap`
|
|
2905
|
+
: `#${issue} failed on review round ${revision.round}`,
|
|
2906
|
+
detail: [`${result.prUrl ?? "(no PR URL)"}`, "", result.report].join("\n"),
|
|
2907
|
+
});
|
|
2908
|
+
}
|
|
2909
|
+
}
|
|
2910
|
+
} catch (err) {
|
|
2911
|
+
// A crash anywhere after the worker ran settles like any run's dispatch
|
|
2912
|
+
// error: the row is terminal, the tree is kept, and the issue is relabelled
|
|
2913
|
+
// so nobody silently re-claims it.
|
|
2914
|
+
turnLimit?.close();
|
|
2915
|
+
turnLimit = undefined;
|
|
2916
|
+
const detail = errText(err);
|
|
2917
|
+
log(`#${issue} review round ${revision.round} errored: ${detail}`);
|
|
2918
|
+
const settlement =
|
|
2919
|
+
worktreePath === undefined
|
|
2920
|
+
? undefined
|
|
2921
|
+
: await settleWorktree({
|
|
2922
|
+
issue,
|
|
2923
|
+
attempt: run.attempt,
|
|
2924
|
+
ending: "killed by a dispatch error",
|
|
2925
|
+
worktree: worktreePath,
|
|
2926
|
+
branch,
|
|
2927
|
+
publish,
|
|
2928
|
+
tree: "keep",
|
|
2929
|
+
});
|
|
2930
|
+
store.updateRun(runId, {
|
|
2931
|
+
state: "failed",
|
|
2932
|
+
endedAt: Date.now(),
|
|
2933
|
+
lastError: detail,
|
|
2934
|
+
...settlement?.patch,
|
|
2935
|
+
});
|
|
2936
|
+
store.settleReviewRevision(revision.id, "failed", Date.now());
|
|
2937
|
+
swapLabel(store, project.name, issue, inProgress, project.stateLabels.failed);
|
|
2938
|
+
await safeEscalate(d, {
|
|
2939
|
+
tier: 1,
|
|
2940
|
+
project: project.name,
|
|
2941
|
+
issue,
|
|
2942
|
+
runId,
|
|
2943
|
+
summary: `#${issue} review round ${revision.round} could not be dispatched`,
|
|
2944
|
+
detail: [
|
|
2945
|
+
`${revision.prUrl}`,
|
|
2946
|
+
"",
|
|
2947
|
+
detail,
|
|
2948
|
+
...(settlement?.lines ?? []),
|
|
2949
|
+
].join("\n"),
|
|
2950
|
+
});
|
|
2951
|
+
} finally {
|
|
2952
|
+
turnLimit?.close();
|
|
2953
|
+
if (verbListener !== undefined) {
|
|
2954
|
+
try {
|
|
2955
|
+
await verbListener.close();
|
|
2956
|
+
} catch (err) {
|
|
2957
|
+
log(`#${issue} verb socket ${verbListener.path} did not close cleanly: ${errText(err)}`);
|
|
2958
|
+
}
|
|
2959
|
+
verbListener = undefined;
|
|
2960
|
+
}
|
|
2961
|
+
workerControl?.close();
|
|
2962
|
+
workerControl = undefined;
|
|
2963
|
+
}
|
|
2964
|
+
}
|
|
2965
|
+
|
|
2966
|
+
/**
|
|
2967
|
+
* Restart recovery for in-flight review revisions (#692): a revision the
|
|
2968
|
+
* previous daemon claimed (`pushed-green` → `running`) and lost when it died
|
|
2969
|
+
* is restored to the exact state the orchestrator's verb left it in — same
|
|
2970
|
+
* run, same PR, same findings, same round, same session file — so the next
|
|
2971
|
+
* dispatch pass resumes the SAME OMP session rather than leaving the run to
|
|
2972
|
+
* the failure classifier.
|
|
2973
|
+
*
|
|
2974
|
+
* Runs AFTER the startup orphan sweep, on purpose: the sweep has already
|
|
2975
|
+
* salvaged the crashed revision's worktree to the branch (so no work is lost)
|
|
2976
|
+
* and marked the row `orphaned` — and this reconcile then restores that row
|
|
2977
|
+
* to `pushed-green` and re-queues the round before the first tick, so the
|
|
2978
|
+
* failure classifier never sees it. That ordering matters twice over: an
|
|
2979
|
+
* orphaned row would charge the continuation budget for a round that must
|
|
2980
|
+
* stay off it (#677), and the `review_revisions` row is the only place the
|
|
2981
|
+
* findings and round live. A revision the previous daemon never claimed (its
|
|
2982
|
+
* run is still `pushed-green`, its row still pending) needs no restoration —
|
|
2983
|
+
* the ordinary dispatch pass wakes it on the next tick, exactly as #677
|
|
2984
|
+
* already guarantees.
|
|
2985
|
+
*
|
|
2986
|
+
* A round whose run cannot be restored — the run is gone, settled terminal,
|
|
2987
|
+
* or its repo is no longer routed — fails closed instead: the revision is
|
|
2988
|
+
* settled `skipped` with the findings still on the row, ONE escalation names
|
|
2989
|
+
* what could not be restored, and no worker is started against a PR that may
|
|
2990
|
+
* already have one.
|
|
2991
|
+
*/
|
|
2992
|
+
export async function reconcileCrashedReviewRevisions(d: Deps): Promise<ReviewRevisionRecord[]> {
|
|
2993
|
+
const { project, store } = d;
|
|
2994
|
+
const revisions = store.unsettledReviewRevisions(project.name);
|
|
2995
|
+
if (revisions.length === 0) return [];
|
|
2996
|
+
const recovered: ReviewRevisionRecord[] = [];
|
|
2997
|
+
for (const revision of revisions) {
|
|
2998
|
+
const run = store.getRun(revision.runId);
|
|
2999
|
+
if (run === undefined) {
|
|
3000
|
+
log(
|
|
3001
|
+
`#${revision.issue} review round ${revision.round} cannot be restored across the restart: ` +
|
|
3002
|
+
`run ${revision.runId} no longer exists`,
|
|
3003
|
+
);
|
|
3004
|
+
store.settleReviewRevision(revision.id, "skipped", Date.now());
|
|
3005
|
+
await safeEscalate(d, {
|
|
3006
|
+
tier: 1,
|
|
3007
|
+
project: project.name,
|
|
3008
|
+
issue: revision.issue,
|
|
3009
|
+
runId: revision.runId,
|
|
3010
|
+
summary: `#${revision.issue} review round ${revision.round} could not be restored across the restart`,
|
|
3011
|
+
detail: [
|
|
3012
|
+
revision.prUrl,
|
|
3013
|
+
"",
|
|
3014
|
+
"The daemon restarted while this review round was in flight and the round cannot be re-dispatched:",
|
|
3015
|
+
`the run row ${revision.runId} no longer exists.`,
|
|
3016
|
+
"The round was settled skipped; the findings stay on this revision's durable row.",
|
|
3017
|
+
"Re-review the PR (it is still open and green) or merge it as it stands.",
|
|
3018
|
+
].join("\n"),
|
|
3019
|
+
});
|
|
3020
|
+
continue;
|
|
3021
|
+
}
|
|
3022
|
+
// A revision the previous daemon never claimed: the run is still a
|
|
3023
|
+
// settled green row and the round is still pending, so the ordinary
|
|
3024
|
+
// dispatch pass wakes it on the next tick untouched.
|
|
3025
|
+
if (run.state === "pushed-green") continue;
|
|
3026
|
+
// The run a previous daemon claimed for this round and died on: the
|
|
3027
|
+
// orphan sweep just marked it `orphaned` (salvaging the tree to the
|
|
3028
|
+
// branch), so restore it to the reviewable state the verb recorded.
|
|
3029
|
+
const crashed =
|
|
3030
|
+
run.state === "claimed" || run.state === "running" || run.state === "orphaned";
|
|
3031
|
+
if (!crashed) {
|
|
3032
|
+
log(
|
|
3033
|
+
`#${revision.issue} review round ${revision.round} cannot be restored across the restart: run ` +
|
|
3034
|
+
`${revision.runId} is ${run.state}`,
|
|
3035
|
+
);
|
|
3036
|
+
store.settleReviewRevision(revision.id, "skipped", Date.now());
|
|
3037
|
+
await safeEscalate(d, {
|
|
3038
|
+
tier: 1,
|
|
3039
|
+
project: project.name,
|
|
3040
|
+
issue: revision.issue,
|
|
3041
|
+
runId: revision.runId,
|
|
3042
|
+
summary: `#${revision.issue} review round ${revision.round} could not be restored across the restart`,
|
|
3043
|
+
detail: [
|
|
3044
|
+
revision.prUrl,
|
|
3045
|
+
"",
|
|
3046
|
+
"The daemon restarted while this review round was in flight and the round cannot be re-dispatched:",
|
|
3047
|
+
`the run row is ${run.state}, not a state a review round resumes from.`,
|
|
3048
|
+
"The round was settled skipped; the findings stay on this revision's durable row.",
|
|
3049
|
+
"Re-review the PR (it is still open and green) or merge it as it stands.",
|
|
3050
|
+
].join("\n"),
|
|
3051
|
+
});
|
|
3052
|
+
continue;
|
|
3053
|
+
}
|
|
3054
|
+
try {
|
|
3055
|
+
// The crashed revision's worktree — salvaged and published by the orphan
|
|
3056
|
+
// sweep that just ran, so the branch in the mirror has every committed
|
|
3057
|
+
// byte — must be gone before the next dispatch, or `addRunRepo` refuses
|
|
3058
|
+
// the reattach as possibly holding a previous attempt's work (#692).
|
|
3059
|
+
// `settleWorktree` with `tree: "remove"` re-attempts the salvage instead
|
|
3060
|
+
// of trusting it (a tree whose salvage failed is the only copy of work
|
|
3061
|
+
// and is NEVER removed), and refuses the removal if that re-attempt
|
|
3062
|
+
// fails — the fail-closed half of this recovery.
|
|
3063
|
+
if (run.worktree !== "" && existsSync(run.worktree)) {
|
|
3064
|
+
const repo = Object.values(project.routing.repos).find((candidate) => candidate.name === run.repo);
|
|
3065
|
+
if (repo === undefined) {
|
|
3066
|
+
log(
|
|
3067
|
+
`#${revision.issue} review round ${revision.round} cannot clear worktree ${run.worktree}: ` +
|
|
3068
|
+
`repo ${run.repo} is no longer routed — the tree may hold the only copy of work`,
|
|
3069
|
+
);
|
|
3070
|
+
store.settleReviewRevision(revision.id, "skipped", Date.now());
|
|
3071
|
+
await safeEscalate(d, {
|
|
3072
|
+
tier: 1,
|
|
3073
|
+
project: project.name,
|
|
3074
|
+
issue: revision.issue,
|
|
3075
|
+
runId: revision.runId,
|
|
3076
|
+
summary: `#${revision.issue} review round ${revision.round} could not be restored across the restart`,
|
|
3077
|
+
detail: [
|
|
3078
|
+
revision.prUrl,
|
|
3079
|
+
"",
|
|
3080
|
+
"The daemon restarted while this review round was in flight and the round cannot be re-dispatched:",
|
|
3081
|
+
`repo ${run.repo} is no longer routed, so the crashed revision's worktree cannot be published and removed.`,
|
|
3082
|
+
`The worktree is retained for recovery: ${run.worktree}`,
|
|
3083
|
+
"Recover it by hand (it may hold the round's last work), then re-review or merge the PR.",
|
|
3084
|
+
].join("\n"),
|
|
3085
|
+
});
|
|
3086
|
+
continue;
|
|
3087
|
+
}
|
|
3088
|
+
const settlement = await settleWorktree({
|
|
3089
|
+
issue: run.issue,
|
|
3090
|
+
attempt: run.attempt,
|
|
3091
|
+
ending: "interrupted by a daemon restart during a review round",
|
|
3092
|
+
worktree: run.worktree,
|
|
3093
|
+
branch: run.branch,
|
|
3094
|
+
publish: (branch) => pushRunBranch(project, { repo, runRepoPath: run.worktree, branch }),
|
|
3095
|
+
tree: "remove",
|
|
3096
|
+
mirrorPath: mirrorPathFor(repo, project.mirrorRoot),
|
|
3097
|
+
});
|
|
3098
|
+
if (settlement.retained) {
|
|
3099
|
+
log(
|
|
3100
|
+
`#${revision.issue} review round ${revision.round} worktree ${run.worktree} retained: ` +
|
|
3101
|
+
"its salvage failed, so the tree is the only copy of work and will not be removed",
|
|
3102
|
+
);
|
|
3103
|
+
store.settleReviewRevision(revision.id, "skipped", Date.now());
|
|
3104
|
+
await safeEscalate(d, {
|
|
3105
|
+
tier: 1,
|
|
3106
|
+
project: project.name,
|
|
3107
|
+
issue: revision.issue,
|
|
3108
|
+
runId: revision.runId,
|
|
3109
|
+
summary: `#${revision.issue} review round ${revision.round} could not be restored across the restart`,
|
|
3110
|
+
detail: [
|
|
3111
|
+
revision.prUrl,
|
|
3112
|
+
"",
|
|
3113
|
+
"The daemon restarted while this review round was in flight and the round cannot be re-dispatched:",
|
|
3114
|
+
"the crashed revision's worktree could not be salvaged, so it is the only copy of work and was retained.",
|
|
3115
|
+
`Recover the tree by hand: ${run.worktree}`,
|
|
3116
|
+
"Then re-review the PR (still open and green) or merge it as it stands.",
|
|
3117
|
+
].join("\n"),
|
|
3118
|
+
});
|
|
3119
|
+
continue;
|
|
3120
|
+
}
|
|
3121
|
+
}
|
|
3122
|
+
} catch (err) {
|
|
3123
|
+
// A settleWorktree throw (mirror unreachable, git refused) must not take
|
|
3124
|
+
// the whole startup reconcile down with one run's tree: fail this round
|
|
3125
|
+
// closed and let the next one try.
|
|
3126
|
+
const detail = errText(err);
|
|
3127
|
+
log(`#${revision.issue} review round ${revision.round} restore errored: ${detail}`);
|
|
3128
|
+
store.settleReviewRevision(revision.id, "skipped", Date.now());
|
|
3129
|
+
await safeEscalate(d, {
|
|
3130
|
+
tier: 1,
|
|
3131
|
+
project: project.name,
|
|
3132
|
+
issue: revision.issue,
|
|
3133
|
+
runId: revision.runId,
|
|
3134
|
+
summary: `#${revision.issue} review round ${revision.round} could not be restored across the restart`,
|
|
3135
|
+
detail: [revision.prUrl, "", detail].join("\n"),
|
|
3136
|
+
});
|
|
3137
|
+
continue;
|
|
3138
|
+
}
|
|
3139
|
+
// The exact state the verb recorded: a pushed-green row plus a pending
|
|
3140
|
+
// revision whose durable row still carries the findings, reviewed head,
|
|
3141
|
+
// round and target session. The next tick's dispatch pass claims and
|
|
3142
|
+
// wakes it exactly like a revision that was never dispatched.
|
|
3143
|
+
store.updateRun(revision.runId, {
|
|
3144
|
+
state: "pushed-green",
|
|
3145
|
+
endedAt: Date.now(),
|
|
3146
|
+
worktree: "",
|
|
3147
|
+
lastError: `review round ${revision.round} was restored across a daemon restart; the round is re-queued`,
|
|
3148
|
+
});
|
|
3149
|
+
store.requeueReviewRevision(revision.id);
|
|
3150
|
+
log(
|
|
3151
|
+
`#${revision.issue} review round ${revision.round} restored across restart: run ${revision.runId} ` +
|
|
3152
|
+
"is pushed-green again and the round is re-queued — the next dispatch pass resumes the same session",
|
|
3153
|
+
);
|
|
3154
|
+
recovered.push(revision);
|
|
3155
|
+
}
|
|
3156
|
+
return recovered;
|
|
3157
|
+
}
|
|
3158
|
+
|
|
3159
|
+
// ------------------------------------------------------------------- settlement
|
|
3160
|
+
const BASE_CHECK_BATCH = 20;
|
|
3161
|
+
const BASE_CHECK_WINDOW_MS = 24 * 60 * 60 * 1_000;
|
|
3162
|
+
const BASE_STATUS_WINDOW_MS = 7 * 24 * 60 * 60 * 1_000;
|
|
3163
|
+
|
|
3164
|
+
const SUCCESSFUL_WORKFLOW_CONCLUSIONS = new Set(["success", "neutral", "skipped"]);
|
|
3165
|
+
|
|
3166
|
+
const FAILING_WORKFLOW_CONCLUSIONS = new Set([
|
|
3167
|
+
"failure",
|
|
3168
|
+
"cancelled",
|
|
3169
|
+
"timed_out",
|
|
3170
|
+
"action_required",
|
|
3171
|
+
"startup_failure",
|
|
3172
|
+
"stale",
|
|
3173
|
+
]);
|
|
3174
|
+
|
|
3175
|
+
function appendSettlementFlag(run: RunRecord, flag: SettlementFlag): SettlementFlag[] {
|
|
3176
|
+
const flags = run.settlementFlags ?? [];
|
|
3177
|
+
return flags.some((existing) => existing.kind === flag.kind && existing.detail === flag.detail)
|
|
3178
|
+
? flags
|
|
3179
|
+
: [...flags, flag];
|
|
3180
|
+
}
|
|
2673
3181
|
|
|
2674
3182
|
/**
|
|
2675
3183
|
* Observe Actions on exact merge commits for up to one day. A running workflow
|
|
@@ -2956,220 +3464,6 @@ export async function watchBaseHealth(
|
|
|
2956
3464
|
}
|
|
2957
3465
|
}
|
|
2958
3466
|
|
|
2959
|
-
export async function settlePushedGreen(
|
|
2960
|
-
d: Pick<Deps, "project" | "tracker" | "store">,
|
|
2961
|
-
): Promise<number> {
|
|
2962
|
-
const { project, tracker, store } = d;
|
|
2963
|
-
// Runs the sweep resolved by terminalising the row. Every terminal write
|
|
2964
|
-
// below increments it; the tick attributes it as `settled` on the pass's
|
|
2965
|
-
// dispatch record, so a held pass is visible as work done, not just as a
|
|
2966
|
-
// clock that moved (#497).
|
|
2967
|
-
let settled = 0;
|
|
2968
|
-
// Filtered from the active set rather than asked for with a new query: active
|
|
2969
|
-
// is live workers plus these, so the list is bounded by the worker cap plus
|
|
2970
|
-
// the number of PRs awaiting a merge — a handful, by construction. A fleet
|
|
2971
|
-
// where that is not a handful has a merge problem, not a dispatch one.
|
|
2972
|
-
const pending = store
|
|
2973
|
-
.activeRuns(project.name)
|
|
2974
|
-
.filter((r) => r.state === "pushed-green" || r.state === "pushed-pending");
|
|
2975
|
-
|
|
2976
|
-
for (const run of pending) {
|
|
2977
|
-
// Nothing to ask about. A pushed result requires a PR, so a malformed row
|
|
2978
|
-
// must not buy a `gh` call every five minutes forever.
|
|
2979
|
-
if (run.prUrl === undefined) continue;
|
|
2980
|
-
|
|
2981
|
-
let pr: PrState | undefined;
|
|
2982
|
-
try {
|
|
2983
|
-
pr = await tracker.prState(run.prUrl);
|
|
2984
|
-
} catch (err) {
|
|
2985
|
-
// Per row, like admission's held candidate. The GitHub adapter already
|
|
2986
|
-
// answers undefined instead of throwing, so this catch is the port's
|
|
2987
|
-
// contract rather than that adapter's behaviour — and a tracker that does
|
|
2988
|
-
// throw must cost its own row, not the whole sweep.
|
|
2989
|
-
log(`#${run.issue} not settled: PR state lookup failed (${errText(err)}) — retrying next tick`);
|
|
2990
|
-
continue;
|
|
2991
|
-
}
|
|
2992
|
-
|
|
2993
|
-
const settlement = settlementFor(pr, run.prUrl);
|
|
2994
|
-
if (settlement !== undefined) {
|
|
2995
|
-
// A mediated merge enters a second, bounded observation phase. Record the
|
|
2996
|
-
// exact merge commit before the row leaves the active set; if GitHub cannot
|
|
2997
|
-
// supply it yet, retry this settlement next tick rather than create a
|
|
2998
|
-
// merged row whose base result can never be attributed.
|
|
2999
|
-
let merged: MergedPrInfo | undefined;
|
|
3000
|
-
if (settlement.state === "merged") {
|
|
3001
|
-
try {
|
|
3002
|
-
merged = await tracker.mergedPrInfo(run.prUrl);
|
|
3003
|
-
} catch (err) {
|
|
3004
|
-
log(`#${run.issue} not settled: merge identity lookup failed (${errText(err)}) — retrying next tick`);
|
|
3005
|
-
continue;
|
|
3006
|
-
}
|
|
3007
|
-
if (merged === undefined) {
|
|
3008
|
-
log(`#${run.issue} not settled: merge identity unavailable — retrying next tick`);
|
|
3009
|
-
continue;
|
|
3010
|
-
}
|
|
3011
|
-
}
|
|
3012
|
-
|
|
3013
|
-
// The label removal and the terminal row are one fact again (#201): the
|
|
3014
|
-
// release is enqueued — a durable local write that cannot fail on the
|
|
3015
|
-
// tracker — in the same breath as the row is terminalised.
|
|
3016
|
-
releaseInProgress(d, run.issue, settlement.reason);
|
|
3017
|
-
const patch: Partial<RunRecord> = {
|
|
3018
|
-
state: settlement.state,
|
|
3019
|
-
endedAt: Date.now(),
|
|
3020
|
-
...(merged === undefined
|
|
3021
|
-
? {}
|
|
3022
|
-
: {
|
|
3023
|
-
mergeSha: merged.mergeSha,
|
|
3024
|
-
baseRef: merged.baseRef,
|
|
3025
|
-
baseCheck: "pending",
|
|
3026
|
-
}),
|
|
3027
|
-
};
|
|
3028
|
-
if (settlement.state === "failed") {
|
|
3029
|
-
patch.lastError = settlement.reason;
|
|
3030
|
-
patch.failureClass = "returned-for-revision";
|
|
3031
|
-
patch.recoveryAction = "none";
|
|
3032
|
-
}
|
|
3033
|
-
store.updateRun(run.id, patch);
|
|
3034
|
-
settled += 1;
|
|
3035
|
-
log(`#${run.issue} settled: ${settlement.reason}`);
|
|
3036
|
-
continue;
|
|
3037
|
-
}
|
|
3038
|
-
|
|
3039
|
-
if (run.state !== "pushed-pending" || pr !== "open" || run.headSha === undefined) continue;
|
|
3040
|
-
let verification;
|
|
3041
|
-
try {
|
|
3042
|
-
verification = await tracker.verifyPr(run.prUrl, run.headSha);
|
|
3043
|
-
} catch (err) {
|
|
3044
|
-
log(`#${run.issue} checks not settled (${errText(err)}) — retrying next tick`);
|
|
3045
|
-
continue;
|
|
3046
|
-
}
|
|
3047
|
-
if (verification === undefined) continue;
|
|
3048
|
-
if (verification.status === "green") {
|
|
3049
|
-
store.updateRun(run.id, { state: "pushed-green", lastError: null });
|
|
3050
|
-
log(`#${run.issue} checks settled: ${verification.reason}`);
|
|
3051
|
-
} else if (verification.status === "failed") {
|
|
3052
|
-
// Equally terminal, so the release is enqueued before the row writes,
|
|
3053
|
-
// for the same reason as the settlement branch above (see there). The
|
|
3054
|
-
// green branch releases nothing — that row is still awaiting a merge,
|
|
3055
|
-
// and its live PR is exactly the work the label must keep guarding.
|
|
3056
|
-
releaseInProgress(d, run.issue, verification.reason);
|
|
3057
|
-
store.updateRun(run.id, { state: "failed", lastError: verification.reason });
|
|
3058
|
-
settled += 1;
|
|
3059
|
-
log(`#${run.issue} checks failed: ${verification.reason}`);
|
|
3060
|
-
} else {
|
|
3061
|
-
store.updateRun(run.id, { lastError: verification.reason });
|
|
3062
|
-
}
|
|
3063
|
-
}
|
|
3064
|
-
return settled;
|
|
3065
|
-
}
|
|
3066
|
-
|
|
3067
|
-
const ADOPTABLE_PR_STATES: Partial<Record<RunState, true>> = {
|
|
3068
|
-
failed: true,
|
|
3069
|
-
killed: true,
|
|
3070
|
-
orphaned: true,
|
|
3071
|
-
blocked: true,
|
|
3072
|
-
};
|
|
3073
|
-
|
|
3074
|
-
/**
|
|
3075
|
-
* Reattaches a recovered PR to the terminal run that owns it (#245).
|
|
3076
|
-
*
|
|
3077
|
-
* A worker can fail before its completion report records `prUrl`, then have its
|
|
3078
|
-
* dirty tree committed and pushed by salvage. If that branch already has a PR,
|
|
3079
|
-
* the orchestrator otherwise has no policy-compliant path to inspect or merge
|
|
3080
|
-
* it: ownership is store-backed. Adoption is deliberately stricter than
|
|
3081
|
-
* admission. The tracker query proves the PR closes this run's issue; exact
|
|
3082
|
-
* branch and canonical repository matches prove it is this run's recovered
|
|
3083
|
-
* work, not an unrelated closer. Missing identity is refusal, never a guess.
|
|
3084
|
-
*
|
|
3085
|
-
* The newest run per issue is inspected, at most ten per tick and only inside
|
|
3086
|
-
* the same 30-day window as mediated PR verbs. The cursor advances through the
|
|
3087
|
-
* full eligible set so persistent non-matches cannot starve older recovered
|
|
3088
|
-
* work. Successful adoption is idempotent because the row gains `prUrl`;
|
|
3089
|
-
* non-matches are logged once per daemon process.
|
|
3090
|
-
*/
|
|
3091
|
-
const rejectedSalvagedPrRuns = new Set<string>();
|
|
3092
|
-
const salvagedPrCursor = new Map<string, string>();
|
|
3093
|
-
|
|
3094
|
-
export async function adoptSalvagedPrs(
|
|
3095
|
-
d: Pick<Deps, "project" | "tracker" | "store">,
|
|
3096
|
-
now = Date.now(),
|
|
3097
|
-
): Promise<void> {
|
|
3098
|
-
const { project, tracker, store } = d;
|
|
3099
|
-
const eligible = store
|
|
3100
|
-
.recentRuns(project.name, now - PR_LOOKUP_WINDOW_MS)
|
|
3101
|
-
.filter(
|
|
3102
|
-
(run) =>
|
|
3103
|
-
ADOPTABLE_PR_STATES[run.state] === true &&
|
|
3104
|
-
run.prUrl === undefined &&
|
|
3105
|
-
run.branch.trim() !== "",
|
|
3106
|
-
);
|
|
3107
|
-
const previous = salvagedPrCursor.get(project.name);
|
|
3108
|
-
const previousIndex =
|
|
3109
|
-
previous === undefined ? -1 : eligible.findIndex((run) => run.id === previous);
|
|
3110
|
-
const start = previousIndex === -1 ? 0 : (previousIndex + 1) % eligible.length;
|
|
3111
|
-
const candidates = Array.from(
|
|
3112
|
-
{ length: Math.min(SALVAGED_PR_ADOPTION_BATCH, eligible.length) },
|
|
3113
|
-
(_, offset) => eligible[(start + offset) % eligible.length]!,
|
|
3114
|
-
);
|
|
3115
|
-
const last = candidates.at(-1);
|
|
3116
|
-
if (last !== undefined) salvagedPrCursor.set(project.name, last.id);
|
|
3117
|
-
|
|
3118
|
-
for (const run of candidates) {
|
|
3119
|
-
const repo = project.routing.repos[run.repo];
|
|
3120
|
-
const repoIdentity = repo === undefined ? undefined : githubRepo(repo.cloneUrl);
|
|
3121
|
-
let closers: OpenCloser[];
|
|
3122
|
-
try {
|
|
3123
|
-
closers = await tracker.openClosersFor(run.issue);
|
|
3124
|
-
} catch (err) {
|
|
3125
|
-
log(`#${run.issue} PR adoption lookup failed (${errText(err)}) — retrying next tick`);
|
|
3126
|
-
continue;
|
|
3127
|
-
}
|
|
3128
|
-
|
|
3129
|
-
const closer = closers.find(
|
|
3130
|
-
(candidate) =>
|
|
3131
|
-
candidate.headRefName !== "" &&
|
|
3132
|
-
candidate.headRefName === run.branch &&
|
|
3133
|
-
repo !== undefined &&
|
|
3134
|
-
candidate.repo !== "" &&
|
|
3135
|
-
candidate.repo === repoIdentity,
|
|
3136
|
-
);
|
|
3137
|
-
if (closer === undefined) {
|
|
3138
|
-
if (!rejectedSalvagedPrRuns.has(run.id)) {
|
|
3139
|
-
const observed = closers[0];
|
|
3140
|
-
const reason =
|
|
3141
|
-
observed === undefined
|
|
3142
|
-
? "no open closing PR"
|
|
3143
|
-
: observed.headRefName === ""
|
|
3144
|
-
? "closer has no head branch identity"
|
|
3145
|
-
: observed.headRefName !== run.branch
|
|
3146
|
-
? `closer head ${observed.headRefName} does not match retained branch ${run.branch}`
|
|
3147
|
-
: observed.repo === ""
|
|
3148
|
-
? "closer has no repository identity"
|
|
3149
|
-
: `closer repository ${observed.repo} does not match routed repository`;
|
|
3150
|
-
log(`#${run.issue} PR not adopted onto attempt ${run.attempt}: ${reason}`);
|
|
3151
|
-
rejectedSalvagedPrRuns.add(run.id);
|
|
3152
|
-
}
|
|
3153
|
-
continue;
|
|
3154
|
-
}
|
|
3155
|
-
|
|
3156
|
-
const flag: SettlementFlag = {
|
|
3157
|
-
kind: "pr-adopted",
|
|
3158
|
-
file: "(recovery)",
|
|
3159
|
-
detail: `${closer.url} matched retained branch ${run.branch} in ${closer.repo}`,
|
|
3160
|
-
};
|
|
3161
|
-
store.updateRun(run.id, {
|
|
3162
|
-
prUrl: closer.url,
|
|
3163
|
-
settlementFlags: [...(run.settlementFlags ?? []), flag],
|
|
3164
|
-
});
|
|
3165
|
-
rejectedSalvagedPrRuns.delete(run.id);
|
|
3166
|
-
log(
|
|
3167
|
-
`#${run.issue} adopted PR ${closer.url} onto attempt ${run.attempt}` +
|
|
3168
|
-
(run.salvageSha === undefined ? "" : ` (salvaged head ${run.salvageSha})`),
|
|
3169
|
-
);
|
|
3170
|
-
}
|
|
3171
|
-
}
|
|
3172
|
-
|
|
3173
3467
|
/** Canonical `owner/repo` identity from a configured network clone URL. */
|
|
3174
3468
|
function githubRepo(cloneUrl: string): string | undefined {
|
|
3175
3469
|
const normalized = cloneUrl.replace(/\/$/, "").replace(/\.git$/, "");
|
|
@@ -3197,7 +3491,7 @@ type CleanupRetainedWorktree = (
|
|
|
3197
3491
|
* dirty or uniquely unpushed work.
|
|
3198
3492
|
*/
|
|
3199
3493
|
export async function cleanupRetainedRuns(
|
|
3200
|
-
d: Pick<Deps, "project" | "tracker" | "store">,
|
|
3494
|
+
d: Pick<Deps, "project" | "tracker" | "store" | "escalate">,
|
|
3201
3495
|
queuedIssues: ReadonlySet<number>,
|
|
3202
3496
|
cursor: RetainedCleanupCursor,
|
|
3203
3497
|
cleanup: CleanupRetainedWorktree = cleanupRetainedWorktree,
|
|
@@ -3262,9 +3556,35 @@ export async function cleanupRetainedRuns(
|
|
|
3262
3556
|
|
|
3263
3557
|
const outcome = await cleanup(mirrorPathFor(repo, project.mirrorRoot), run.worktree, run.branch);
|
|
3264
3558
|
if (outcome.kind === "removed") {
|
|
3265
|
-
store.updateRun(run.id, {
|
|
3559
|
+
store.updateRun(run.id, {
|
|
3560
|
+
worktree: "",
|
|
3561
|
+
...(run.quarantineDetail === undefined ? {} : { quarantineDetail: null }),
|
|
3562
|
+
});
|
|
3266
3563
|
log(`#${run.issue} retained worktree reaped: ${run.worktree} (${run.branch})`);
|
|
3564
|
+
} else if (outcome.reason === "quarantined") {
|
|
3565
|
+
// A tree whose object store cannot be made sound is potentially
|
|
3566
|
+
// stranded work: the daemon refuses to fetch into it, so its commits
|
|
3567
|
+
// cannot be verified against any remote. The row records the condition
|
|
3568
|
+
// so the status snapshot can name the tree, and the escalation ledger
|
|
3569
|
+
// dedupes on the stable summary below — a pass that keeps seeing the
|
|
3570
|
+
// same broken tree reports it once, never once per dispatch pass.
|
|
3571
|
+
store.updateRun(run.id, { quarantineDetail: outcome.detail });
|
|
3572
|
+
await safeEscalate(d, {
|
|
3573
|
+
tier: 1,
|
|
3574
|
+
project: project.name,
|
|
3575
|
+
issue: run.issue,
|
|
3576
|
+
summary: `#${run.issue} quarantined retained worktree — potentially stranded work`,
|
|
3577
|
+
detail: `${run.worktree} (${run.branch})\n${outcome.detail}`,
|
|
3578
|
+
});
|
|
3267
3579
|
} else {
|
|
3580
|
+
// Any other retained reason means the tree is back under ordinary
|
|
3581
|
+
// retention: its alternates were repaired (or never needed it), so the
|
|
3582
|
+
// quarantine — if one was marked — is over and the snapshot must not
|
|
3583
|
+
// keep naming it as quarantined.
|
|
3584
|
+
if (run.quarantineDetail !== undefined) {
|
|
3585
|
+
store.updateRun(run.id, { quarantineDetail: null });
|
|
3586
|
+
log(`#${run.issue} retained worktree no longer quarantined: ${outcome.detail}`);
|
|
3587
|
+
}
|
|
3268
3588
|
log(`#${run.issue} retained worktree kept (${outcome.reason}): ${outcome.detail}`);
|
|
3269
3589
|
}
|
|
3270
3590
|
}
|
|
@@ -3277,6 +3597,7 @@ const DEGRADED_HOLDS: ReadonlySet<AdmissionHoldReason> = new Set([
|
|
|
3277
3597
|
"parent-lookup-error",
|
|
3278
3598
|
"open-pr-lookup-error",
|
|
3279
3599
|
"issue-state-lookup-error",
|
|
3600
|
+
"critical-base-verify-error",
|
|
3280
3601
|
]);
|
|
3281
3602
|
|
|
3282
3603
|
/** Groups transient decisions into the bounded record exposed by status. */
|
|
@@ -3471,10 +3792,63 @@ async function verifyPendingUpgradeTick(
|
|
|
3471
3792
|
);
|
|
3472
3793
|
}
|
|
3473
3794
|
} catch (err) {
|
|
3474
|
-
// A torn journal must not take the tick down: the operator still reads it
|
|
3475
|
-
// through `status`, and the next tick retries the same pass.
|
|
3476
|
-
log(`upgrade verification pass failed: ${errText(err)}`);
|
|
3795
|
+
// A torn journal must not take the tick down: the operator still reads it
|
|
3796
|
+
// through `status`, and the next tick retries the same pass.
|
|
3797
|
+
log(`upgrade verification pass failed: ${errText(err)}`);
|
|
3798
|
+
}
|
|
3799
|
+
}
|
|
3800
|
+
|
|
3801
|
+
/**
|
|
3802
|
+
* The daemon's conductor.db snapshot cadence (#289) — the ledger's only
|
|
3803
|
+
* durable copy, taken on the digest-aligned {@link dbSnapshotDue} window:
|
|
3804
|
+
* once per local day, at/after the digest's configured `at` when it has one.
|
|
3805
|
+
*
|
|
3806
|
+
* Host-global, like the store itself: one snapshot per day for every project,
|
|
3807
|
+
* so whichever project's daemon wins the day first publishes it and the
|
|
3808
|
+
* others no-op on the same durable marker. The marker is written through the
|
|
3809
|
+
* store's existing notification ledger (a one-row idempotence guard, the same
|
|
3810
|
+
* primitive escalations use not to act twice) *after* the snapshot is
|
|
3811
|
+
* published — a crash between publish and mark re-snapshots the next tick
|
|
3812
|
+
* instead of skipping the day, and a restored store that predates today's
|
|
3813
|
+
* marker takes a fresh snapshot on the next tick.
|
|
3814
|
+
*
|
|
3815
|
+
* Returns true when a snapshot was published. A store that does not exist
|
|
3816
|
+
* yet is a silent no-op: there is nothing to back up, and `doctor`'s
|
|
3817
|
+
* `db-backup` probe already treats that as a pass.
|
|
3818
|
+
*/
|
|
3819
|
+
export function runDbSnapshotCadence(args: {
|
|
3820
|
+
digestPolicy: ReportingPolicy["digest"];
|
|
3821
|
+
store: Pick<Store, "wasNotified" | "markNotified">;
|
|
3822
|
+
source: string;
|
|
3823
|
+
backupDir: string;
|
|
3824
|
+
now?: number;
|
|
3825
|
+
keep?: number;
|
|
3826
|
+
log?: (line: string) => void;
|
|
3827
|
+
}): boolean {
|
|
3828
|
+
const { digestPolicy, store, source, backupDir } = args;
|
|
3829
|
+
// No store yet (first boot before any run persisted) → nothing to back up;
|
|
3830
|
+
// `doctor`'s `db-backup` probe already treats that as a pass.
|
|
3831
|
+
if (!existsSync(source)) return false;
|
|
3832
|
+
const now = args.now ?? Date.now();
|
|
3833
|
+
const today = localDayKey(now, digestPolicy.timezone);
|
|
3834
|
+
const alreadyToday = store.wasNotified(dbSnapshotMarkerKey(today));
|
|
3835
|
+
if (!dbSnapshotDue({ digest: digestPolicy }, alreadyToday ? today : undefined, now)) {
|
|
3836
|
+
return false;
|
|
3837
|
+
}
|
|
3838
|
+
const published = snapshotDb(source, backupDir, now);
|
|
3839
|
+
(args.log ?? log)(`conductor.db snapshot published: ${published}`);
|
|
3840
|
+
store.markNotified(dbSnapshotMarkerKey(today));
|
|
3841
|
+
try {
|
|
3842
|
+
const removed = pruneDbSnapshots(backupDir, args.keep ?? DB_SNAPSHOT_RETENTION);
|
|
3843
|
+
if (removed > 0) {
|
|
3844
|
+
(args.log ?? log)(`db snapshot retention pruned ${removed} file(s) beyond the retained ${args.keep ?? DB_SNAPSHOT_RETENTION}`);
|
|
3845
|
+
}
|
|
3846
|
+
} catch (err) {
|
|
3847
|
+
// The snapshot is already published and marked; an over-bound directory
|
|
3848
|
+
// costs disk until the next due day retries the prune, never the backup.
|
|
3849
|
+
(args.log ?? log)(`db snapshot retention prune failed: ${errText(err)}`);
|
|
3477
3850
|
}
|
|
3851
|
+
return true;
|
|
3478
3852
|
}
|
|
3479
3853
|
|
|
3480
3854
|
export async function tick(d: Deps, workers?: WorkerPool): Promise<void> {
|
|
@@ -3482,9 +3856,13 @@ export async function tick(d: Deps, workers?: WorkerPool): Promise<void> {
|
|
|
3482
3856
|
// (#170). Re-resolve the project and its caps at the tick boundary so a tick
|
|
3483
3857
|
// and every run it admits see one consistent snapshot; a failed read keeps
|
|
3484
3858
|
// the boot values rather than wedging the tick, and the next tick tries
|
|
3485
|
-
// again.
|
|
3859
|
+
// again. The same reloaded config feeds the db-snapshot backup dir: when it
|
|
3860
|
+
// is unreadable the cadence falls back to the state-root default, matching
|
|
3861
|
+
// `doctor`'s probe of an absent field.
|
|
3862
|
+
let reloadedConfig: ConductorConfig | undefined;
|
|
3486
3863
|
try {
|
|
3487
3864
|
const cfg = loadConfig();
|
|
3865
|
+
reloadedConfig = cfg;
|
|
3488
3866
|
const fresh = findProject(cfg, d.project.name);
|
|
3489
3867
|
const freshCaps = resolveCaps(fresh, cfg.defaults);
|
|
3490
3868
|
if (JSON.stringify(fresh) !== JSON.stringify(d.project) || JSON.stringify(freshCaps) !== JSON.stringify(d.caps)) {
|
|
@@ -3566,6 +3944,25 @@ export async function tick(d: Deps, workers?: WorkerPool): Promise<void> {
|
|
|
3566
3944
|
log(`salvaged PR adoption sweep failed: ${errText(err)}`);
|
|
3567
3945
|
}
|
|
3568
3946
|
|
|
3947
|
+
// The conductor.db snapshot cadence (#289) is durability, not dispatch:
|
|
3948
|
+
// the verb ledger, the decision rows and the run history have no other
|
|
3949
|
+
// copy, so they are snapshotted once per day on the digest-aligned window.
|
|
3950
|
+
// Above the pause gate on purpose — a parked fleet still accumulates ledger
|
|
3951
|
+
// rows and still deserves a backup — and a failure costs the day's snapshot
|
|
3952
|
+
// placeholder, never the tick: failed cadence steps are logged and retried
|
|
3953
|
+
// by the five-minute loop, exactly like the sweeps above it.
|
|
3954
|
+
try {
|
|
3955
|
+
runDbSnapshotCadence({
|
|
3956
|
+
digestPolicy: (d.project.reporting ?? DEFAULT_REPORT_POLICY).digest,
|
|
3957
|
+
store: d.store,
|
|
3958
|
+
source: dbPath(),
|
|
3959
|
+
backupDir: dbBackupDirFor(reloadedConfig),
|
|
3960
|
+
log,
|
|
3961
|
+
});
|
|
3962
|
+
} catch (err) {
|
|
3963
|
+
log(`db snapshot cadence failed: ${errText(err)}`);
|
|
3964
|
+
}
|
|
3965
|
+
|
|
3569
3966
|
// Immediately after settlement and before any routing, so a class is on the
|
|
3570
3967
|
// row before the next dispatch decision reads its budgets (#132). Above the
|
|
3571
3968
|
// pause gate deliberately: classification and label reconciliation are
|
|
@@ -3575,7 +3972,15 @@ export async function tick(d: Deps, workers?: WorkerPool): Promise<void> {
|
|
|
3575
3972
|
// Each is guarded on its own: a tracker that fails mid-classification must not
|
|
3576
3973
|
// stop the label reconcile, and neither may stop the tick.
|
|
3577
3974
|
try {
|
|
3578
|
-
settled += await classifyAndRecover(
|
|
3975
|
+
settled += await classifyAndRecover({
|
|
3976
|
+
project: d.project,
|
|
3977
|
+
caps: d.caps,
|
|
3978
|
+
tracker: d.tracker,
|
|
3979
|
+
store: d.store,
|
|
3980
|
+
escalate: (e) => d.escalate(e),
|
|
3981
|
+
isPaused,
|
|
3982
|
+
setPaused,
|
|
3983
|
+
});
|
|
3579
3984
|
} catch (err) {
|
|
3580
3985
|
log(`classification sweep failed: ${errText(err)}`);
|
|
3581
3986
|
}
|
|
@@ -3632,6 +4037,15 @@ export async function tick(d: Deps, workers?: WorkerPool): Promise<void> {
|
|
|
3632
4037
|
// so `last dispatch` keeps moving while the fleet is deliberately parked. A
|
|
3633
4038
|
// frozen clock is then a stall report, not a hold (#497).
|
|
3634
4039
|
if (isPaused(d.project.name)) {
|
|
4040
|
+
// The held pass is the daemon-side admission acknowledgement: the daemon
|
|
4041
|
+
// has reached its admission boundary under the fence and claims nothing.
|
|
4042
|
+
// Written durably so the setup barrier can prove the fence was observed
|
|
4043
|
+
// (#651 review #3) — a pause cannot acknowledge itself.
|
|
4044
|
+
try {
|
|
4045
|
+
writeAdmissionAck(d.project.name);
|
|
4046
|
+
} catch (err) {
|
|
4047
|
+
log(`admission acknowledgement write failed: ${errText(err)}`);
|
|
4048
|
+
}
|
|
3635
4049
|
d.store.recordDispatch(d.project.name, summarizeHeldPass(settled));
|
|
3636
4050
|
return;
|
|
3637
4051
|
}
|
|
@@ -3694,6 +4108,18 @@ export async function tick(d: Deps, workers?: WorkerPool): Promise<void> {
|
|
|
3694
4108
|
return;
|
|
3695
4109
|
}
|
|
3696
4110
|
|
|
4111
|
+
// Review revisions are dispatch, not admission (#677): the orchestrator's
|
|
4112
|
+
// verb recorded them durably and the run row already occupies its issue, so
|
|
4113
|
+
// they bypass the ready queue and the claim path entirely — this pass wakes
|
|
4114
|
+
// them (bounded by the same worker slots) BEFORE the queue's capacity gate
|
|
4115
|
+
// reads live runs, so a claimed revision counts against capacity exactly like
|
|
4116
|
+
// the worker it is about to spawn.
|
|
4117
|
+
try {
|
|
4118
|
+
await dispatchReviewRevisions(d, workers);
|
|
4119
|
+
} catch (err) {
|
|
4120
|
+
log(`review revision dispatch failed: ${errText(err)}`);
|
|
4121
|
+
}
|
|
4122
|
+
|
|
3697
4123
|
// route() filters the queue through isEligible() itself, so anything already
|
|
3698
4124
|
// carrying a state label is gone before it gets here.
|
|
3699
4125
|
const ready = await d.tracker.listReady();
|
|
@@ -3713,14 +4139,37 @@ export async function tick(d: Deps, workers?: WorkerPool): Promise<void> {
|
|
|
3713
4139
|
return pending.length === 0 ? issue : { ...issue, labels: effectiveLabels(issue.labels, pending) };
|
|
3714
4140
|
});
|
|
3715
4141
|
const { routed, unroutable } = route(effective, project);
|
|
3716
|
-
//
|
|
3717
|
-
//
|
|
3718
|
-
//
|
|
3719
|
-
|
|
3720
|
-
|
|
3721
|
-
|
|
3722
|
-
|
|
3723
|
-
|
|
4142
|
+
// route() drops a candidate for two reasons, only one of which is a claim
|
|
4143
|
+
// question. A lifecycle state label (agent:in-progress/blocked/failed)
|
|
4144
|
+
// marks a run-owned issue: it is genuinely in flight while its newest run
|
|
4145
|
+
// is live or settling, and a terminal newest run — or no run at all — means
|
|
4146
|
+
// the label is residual, the footprint of a settled run nobody cleared,
|
|
4147
|
+
// which is Duty 1 reconciliation work, not occupied capacity. A missing
|
|
4148
|
+
// queue label instead marks an issue the outbox is withdrawing (a pending
|
|
4149
|
+
// queue-label removal, e.g. releaseQueueLabel after a merge the PR did not
|
|
4150
|
+
// close), which belongs in none of the three populations. So `claimed` is
|
|
4151
|
+
// defined from actual ownership over genuinely lifecycle-labelled candidates
|
|
4152
|
+
// rather than as the residual of routing (#228, #611).
|
|
4153
|
+
const stateLabels = new Set(Object.values(project.stateLabels));
|
|
4154
|
+
const dropped = effective.filter(
|
|
4155
|
+
(issue) => !isEligible(issue, project) && issue.labels.some((l) => stateLabels.has(l)),
|
|
4156
|
+
);
|
|
4157
|
+
let claimed = 0;
|
|
4158
|
+
const lifecycleHolds: AdmissionHold[] = [];
|
|
4159
|
+
for (const issue of dropped) {
|
|
4160
|
+
const newest = store.latestRun(project.name, issue.number);
|
|
4161
|
+
if (newest !== undefined && ACTIVE_STATES.includes(newest.state)) {
|
|
4162
|
+
claimed += 1;
|
|
4163
|
+
} else {
|
|
4164
|
+
lifecycleHolds.push({ issue: issue.number, reason: "stale-lifecycle" });
|
|
4165
|
+
}
|
|
4166
|
+
}
|
|
4167
|
+
const routingHolds: AdmissionHold[] = [
|
|
4168
|
+
...unroutable.map(
|
|
4169
|
+
(u): AdmissionHold => ({ issue: u.issue.number, reason: `unroutable:${u.reason}` }),
|
|
4170
|
+
),
|
|
4171
|
+
...lifecycleHolds,
|
|
4172
|
+
];
|
|
3724
4173
|
const recordDispatch = (admitted: number, holds: readonly AdmissionHold[]): void => {
|
|
3725
4174
|
store.recordDispatch(
|
|
3726
4175
|
project.name,
|
|
@@ -3817,7 +4266,7 @@ export async function tick(d: Deps, workers?: WorkerPool): Promise<void> {
|
|
|
3817
4266
|
log(`dispatching ${pass.admitted.map((a) => `#${a.r.issue.number}`).join(" ")}`);
|
|
3818
4267
|
await dispatchAdmissions(
|
|
3819
4268
|
pass.admitted,
|
|
3820
|
-
(a) => handleIssue(d, a.r, a.attempt),
|
|
4269
|
+
(a) => handleIssue(d, a.r, a.attempt, a.lane),
|
|
3821
4270
|
workers,
|
|
3822
4271
|
);
|
|
3823
4272
|
|
|
@@ -4211,11 +4660,33 @@ export interface StatusSnapshot {
|
|
|
4211
4660
|
* the config or the brief.
|
|
4212
4661
|
*/
|
|
4213
4662
|
releaseGrants: ResolvedGrants;
|
|
4663
|
+
/**
|
|
4664
|
+
* The effective review policy (#678). On the snapshot for the reason
|
|
4665
|
+
* `releaseGrants` is: the orchestrator's Duty 1 has to act on it every
|
|
4666
|
+
* tick, and a stale level or ceiling sitting only in a config file nobody
|
|
4667
|
+
* opens is exactly the drift this field exists to surface. The loader always
|
|
4668
|
+
* materialises it, so this is never absent on a real daemon.
|
|
4669
|
+
*/
|
|
4670
|
+
review: ReviewPolicy;
|
|
4214
4671
|
/** Occupied issues: live workers plus green PRs awaiting a human merge. */
|
|
4215
4672
|
activeRuns: RunRecord[];
|
|
4673
|
+
/**
|
|
4674
|
+
* runId → live review-revision round, for runs whose revision worker is
|
|
4675
|
+
* currently dispatched (#692). Derived from the durable `review_revisions`
|
|
4676
|
+
* rows, so the rendered `review-revision N` state rests on the same facts
|
|
4677
|
+
* the restart recovery reads — never on a label or a guess. A plain object
|
|
4678
|
+
* (not a Map) so the dashboard's JSON round-trip of the snapshot preserves
|
|
4679
|
+
* it byte for byte.
|
|
4680
|
+
*/
|
|
4681
|
+
reviewRounds?: Readonly<Record<string, number>>;
|
|
4216
4682
|
/** Newest attempts holding a preserved WIP tip, or a tree that is still the
|
|
4217
4683
|
* only copy of work the daemon could not save. */
|
|
4218
4684
|
salvagedRuns: RunRecord[];
|
|
4685
|
+
/** Retained trees whose object store could not be made sound, so the daemon
|
|
4686
|
+
* refused to fetch into them and their commits cannot be verified against
|
|
4687
|
+
* any remote (#737). Distinguished from ordinary retention on purpose —
|
|
4688
|
+
* quarantine is "potentially stranded work", not routine housekeeping. */
|
|
4689
|
+
quarantinedRuns: RunRecord[];
|
|
4219
4690
|
/** One-shot issue ceilings waiting for the next claim. */
|
|
4220
4691
|
turnOverrides: TurnOverride[];
|
|
4221
4692
|
/** Reports the operator has not provably received: pending, in-flight with an
|
|
@@ -4302,6 +4773,13 @@ export function statusSnapshotFromStore(
|
|
|
4302
4773
|
// Read once: the provenance read touches the filesystem, and the renderer
|
|
4303
4774
|
// should never pay for it twice per status.
|
|
4304
4775
|
const reason = pauseProvenance(p.name)?.reason;
|
|
4776
|
+
// The live review-revision rounds, read from the same durable rows the
|
|
4777
|
+
// restart recovery uses: a run whose revision is dispatched is read as
|
|
4778
|
+
// `review-revision N` while its worker is live (#692).
|
|
4779
|
+
const reviewRounds: Record<string, number> = {};
|
|
4780
|
+
for (const revision of store.unsettledReviewRevisions(p.name)) {
|
|
4781
|
+
if (revision.dispatchedAt !== undefined) reviewRounds[revision.runId] = revision.round;
|
|
4782
|
+
}
|
|
4305
4783
|
return {
|
|
4306
4784
|
project: p.name,
|
|
4307
4785
|
configPath: configPath(),
|
|
@@ -4312,8 +4790,11 @@ export function statusSnapshotFromStore(
|
|
|
4312
4790
|
digestSchedule: digestScheduleState(p.reporting ?? DEFAULT_REPORT_POLICY, lastDigestDay, now),
|
|
4313
4791
|
caps,
|
|
4314
4792
|
releaseGrants: resolveReleaseGrants(p),
|
|
4793
|
+
review: resolveReview(p),
|
|
4315
4794
|
activeRuns: store.activeRuns(p.name),
|
|
4795
|
+
reviewRounds,
|
|
4316
4796
|
salvagedRuns: store.salvagedRuns(p.name),
|
|
4797
|
+
quarantinedRuns: store.quarantinedRuns(p.name),
|
|
4317
4798
|
turnOverrides: store.listTurnOverrides(p.name),
|
|
4318
4799
|
openReports: store.openReports(p.name),
|
|
4319
4800
|
digestBacklog: store.digestBacklog(p.name),
|
|
@@ -4387,32 +4868,6 @@ export function formatDispatchSummary(summary?: DispatchSummary): string {
|
|
|
4387
4868
|
return lines.join("\n");
|
|
4388
4869
|
}
|
|
4389
4870
|
|
|
4390
|
-
/**
|
|
4391
|
-
* The WIP block: every issue whose newest attempt left work behind, and
|
|
4392
|
-
* whether that work is safe.
|
|
4393
|
-
*
|
|
4394
|
-
* Blocked runs used to be invisible here, which is exactly how #118 stayed
|
|
4395
|
-
* invisible for a full attempt cycle — the operator saw a blocked issue and had
|
|
4396
|
-
* no way to tell "stopped with 34 uncommitted files" from "stopped clean".
|
|
4397
|
-
* A preserved line is informational; an UNSALVAGED line is an alarm, and it
|
|
4398
|
-
* names the directory because that directory is the work.
|
|
4399
|
-
*/
|
|
4400
|
-
export function formatSalvagedRuns(runs: readonly RunRecord[]): string[] {
|
|
4401
|
-
if (runs.length === 0) return [];
|
|
4402
|
-
const lines = ["", "wip"];
|
|
4403
|
-
for (const r of runs) {
|
|
4404
|
-
lines.push(
|
|
4405
|
-
r.salvageError !== undefined && r.salvageAckAt === undefined
|
|
4406
|
-
? ` #${r.issue} UNSALVAGED ${r.worktree === "" ? "(path not recorded)" : r.worktree} — ` +
|
|
4407
|
-
`only copy, dispatch held (${r.salvageError})`
|
|
4408
|
-
: r.salvageError !== undefined
|
|
4409
|
-
? ` #${r.issue} accepted as lost attempt ${r.attempt} (${r.salvageError})`
|
|
4410
|
-
: ` #${r.issue} preserved ${r.salvageSha ?? "?"} on ${r.branch} (attempt ${r.attempt}, ${r.state})`,
|
|
4411
|
-
);
|
|
4412
|
-
}
|
|
4413
|
-
return lines;
|
|
4414
|
-
}
|
|
4415
|
-
|
|
4416
4871
|
/**
|
|
4417
4872
|
* The grant table, named shape by shape.
|
|
4418
4873
|
*
|
|
@@ -4430,6 +4885,7 @@ export function formatReleaseGrants(grants: ResolvedGrants): string[] {
|
|
|
4430
4885
|
...RELEASE_SHAPES.map((shape) => ` ${shape.padEnd(19)}${grants[shape]}`),
|
|
4431
4886
|
];
|
|
4432
4887
|
}
|
|
4888
|
+
|
|
4433
4889
|
export function formatBaseHealth(rows: readonly BaseHealth[]): string[] {
|
|
4434
4890
|
return rows.map((row) => {
|
|
4435
4891
|
const head = row.headSha.slice(0, 8);
|
|
@@ -4519,6 +4975,7 @@ export function formatStatus(s: StatusSnapshot): string {
|
|
|
4519
4975
|
lines.push(...formatBaseHealth(s.baseHealth));
|
|
4520
4976
|
lines.push(...formatFreezes(s.freezes));
|
|
4521
4977
|
lines.push(...formatSalvagedRuns(s.salvagedRuns));
|
|
4978
|
+
lines.push(...formatQuarantinedRuns(s.quarantinedRuns));
|
|
4522
4979
|
lines.push(...formatOpenReports(s.openReports));
|
|
4523
4980
|
lines.push(...formatVerbLedger(s.verbLedger));
|
|
4524
4981
|
// Deploy hint: a restart while workers are live orphans them (salvage runs
|
|
@@ -4592,61 +5049,6 @@ export function prepareConductor(project?: string): void {
|
|
|
4592
5049
|
setPaused(true, { source: "setup" }, project);
|
|
4593
5050
|
}
|
|
4594
5051
|
|
|
4595
|
-
/** Bounded per tick: each row costs tracker calls to gather facts for. */
|
|
4596
|
-
const CLASSIFY_BATCH = 20;
|
|
4597
|
-
|
|
4598
|
-
/** Tool calls quoted as evidence for a run that spun to its turn cap. */
|
|
4599
|
-
const SPIN_EVIDENCE_CALLS = 10;
|
|
4600
|
-
|
|
4601
|
-
/**
|
|
4602
|
-
* The last few tool names a transcript recorded, newest last.
|
|
4603
|
-
*
|
|
4604
|
-
* `turn-cap-spinning` escalates rather than requeueing, and the acceptance
|
|
4605
|
-
* criterion is that the escalation carries evidence of what the worker was doing
|
|
4606
|
-
* when it hit the cap — otherwise the orchestrator opens the transcript and
|
|
4607
|
-
* re-derives it, which is the manual triage this whole sweep removes.
|
|
4608
|
-
*/
|
|
4609
|
-
export function lastToolCalls(sessionFile: string | undefined, limit = SPIN_EVIDENCE_CALLS): string[] {
|
|
4610
|
-
if (sessionFile === undefined) return [];
|
|
4611
|
-
let text: string;
|
|
4612
|
-
try {
|
|
4613
|
-
text = readFileSync(sessionFile, "utf8");
|
|
4614
|
-
} catch {
|
|
4615
|
-
return [];
|
|
4616
|
-
}
|
|
4617
|
-
const names: string[] = [];
|
|
4618
|
-
for (const line of text.split("\n")) {
|
|
4619
|
-
if (line.length === 0) continue;
|
|
4620
|
-
let row: unknown;
|
|
4621
|
-
try {
|
|
4622
|
-
row = JSON.parse(line) as unknown;
|
|
4623
|
-
} catch {
|
|
4624
|
-
continue;
|
|
4625
|
-
}
|
|
4626
|
-
if (row === null || typeof row !== "object") continue;
|
|
4627
|
-
const rec = row as { readonly [key: string]: unknown };
|
|
4628
|
-
// Both shapes the harness has written: a top-level tool event, and a tool
|
|
4629
|
-
// block inside an assistant message.
|
|
4630
|
-
const direct = rec["toolName"];
|
|
4631
|
-
if (typeof direct === "string") {
|
|
4632
|
-
names.push(direct);
|
|
4633
|
-
continue;
|
|
4634
|
-
}
|
|
4635
|
-
const message = rec["message"];
|
|
4636
|
-
if (message === null || typeof message !== "object") continue;
|
|
4637
|
-
const content = (message as { readonly [key: string]: unknown })["content"];
|
|
4638
|
-
if (!Array.isArray(content)) continue;
|
|
4639
|
-
for (const part of content) {
|
|
4640
|
-
if (part === null || typeof part !== "object") continue;
|
|
4641
|
-
const p = part as { readonly [key: string]: unknown };
|
|
4642
|
-
if (p["type"] !== "tool_use") continue;
|
|
4643
|
-
const name = p["name"];
|
|
4644
|
-
if (typeof name === "string") names.push(name);
|
|
4645
|
-
}
|
|
4646
|
-
}
|
|
4647
|
-
return names.slice(-limit);
|
|
4648
|
-
}
|
|
4649
|
-
|
|
4650
5052
|
/** A provider refusal a session recorded before dying, or undefined for none. */
|
|
4651
5053
|
export interface SessionError {
|
|
4652
5054
|
status?: number;
|
|
@@ -4663,608 +5065,154 @@ export function completionLastError(
|
|
|
4663
5065
|
}
|
|
4664
5066
|
|
|
4665
5067
|
/**
|
|
4666
|
-
*
|
|
4667
|
-
*
|
|
4668
|
-
*
|
|
4669
|
-
*
|
|
4670
|
-
*
|
|
4671
|
-
* charged an attempt each for a billing state (#220).
|
|
4672
|
-
*
|
|
4673
|
-
* Scanned newest-first: a session that recovered from an early error and then
|
|
4674
|
-
* died of something else must report the something else, and a session that
|
|
4675
|
-
* recovered from its only error and finished cleanly reports the error anyway
|
|
4676
|
-
* because there is no terminal verdict to outrank it (#220).
|
|
5068
|
+
* How many settled `ci-deterministic` rows one reconciliation pass may
|
|
5069
|
+
* re-examine beyond the persisted review cursor. Each candidate costs GitHub
|
|
5070
|
+
* calls to re-fetch the evidence its check log carried, so a fleet with a
|
|
5071
|
+
* long misclassified history works through it over several daemon starts
|
|
5072
|
+
* rather than spending one boot's budget on all of it (#638).
|
|
4677
5073
|
*/
|
|
4678
|
-
|
|
4679
|
-
|
|
4680
|
-
|
|
4681
|
-
|
|
4682
|
-
|
|
4683
|
-
|
|
4684
|
-
|
|
4685
|
-
|
|
4686
|
-
|
|
4687
|
-
|
|
4688
|
-
|
|
4689
|
-
|
|
4690
|
-
|
|
4691
|
-
try {
|
|
4692
|
-
row = JSON.parse(line) as unknown;
|
|
4693
|
-
} catch {
|
|
4694
|
-
continue;
|
|
4695
|
-
}
|
|
4696
|
-
if (row === null || typeof row !== "object") continue;
|
|
4697
|
-
const rec = row as { readonly [key: string]: unknown };
|
|
4698
|
-
if (rec["stopReason"] !== "error") continue;
|
|
4699
|
-
const message = rec["errorMessage"];
|
|
4700
|
-
if (typeof message !== "string" || message.trim() === "") continue;
|
|
4701
|
-
const status = rec["errorStatus"];
|
|
4702
|
-
return {
|
|
4703
|
-
...(typeof status === "number" && Number.isFinite(status) ? { status } : {}),
|
|
4704
|
-
message: message.trim(),
|
|
4705
|
-
};
|
|
4706
|
-
}
|
|
4707
|
-
return undefined;
|
|
4708
|
-
}
|
|
5074
|
+
const HISTORICAL_INFRA_BATCH = 20;
|
|
5075
|
+
|
|
5076
|
+
/** How many workflow runs for the head commit one row's evidence pass reads —
|
|
5077
|
+
* and the ceiling past which the row is refused as undecided rather than
|
|
5078
|
+
* decided from a prefix of its head-pinned runs (review #654). */
|
|
5079
|
+
const HISTORICAL_INFRA_RUNS = 3;
|
|
5080
|
+
|
|
5081
|
+
/** How many attempts of one workflow run may be read for the failed log — and
|
|
5082
|
+
* the ceiling past which the run is refused as undecided rather than read as
|
|
5083
|
+
* a prefix that could hide a later attempt's real failure (review #654). A
|
|
5084
|
+
* failed job rerun to green leaves the failure in an earlier attempt; the
|
|
5085
|
+
* bound keeps one pathological run from costing the whole pass. */
|
|
5086
|
+
const HISTORICAL_INFRA_ATTEMPTS = 5;
|
|
4709
5087
|
|
|
4710
5088
|
/**
|
|
4711
|
-
*
|
|
4712
|
-
*
|
|
5089
|
+
* Repair settled `ci-deterministic` rows whose re-fetched check log carries a
|
|
5090
|
+
* closed infrastructure signature (#638). The forward classifier now names
|
|
5091
|
+
* the codeload setup 429 of #177 `ci-infra`, but a row already classified
|
|
5092
|
+
* `ci-deterministic`/`escalate` never re-enters the classification sweep, so
|
|
5093
|
+
* the old verdict charges an implementation attempt forever. Re-fetching the
|
|
5094
|
+
* head-pinned workflow-run attempt logs through the tracker, matching the
|
|
5095
|
+
* classifier's own closed signature list, and reclassifying `failureClass`
|
|
5096
|
+
* alone returns the attempt without re-animating a months-old run into
|
|
5097
|
+
* recovery.
|
|
4713
5098
|
*
|
|
4714
|
-
*
|
|
4715
|
-
*
|
|
4716
|
-
*
|
|
4717
|
-
*
|
|
4718
|
-
*
|
|
4719
|
-
*
|
|
5099
|
+
* Bounded, idempotent and resumable: one batch per call, reclassifications
|
|
5100
|
+
* only, and the store's update is guarded by the row still reading
|
|
5101
|
+
* `ci-deterministic`, so a second pass touches nothing it already repaired.
|
|
5102
|
+
* The batch resumes below a persisted review cursor, so each row is evaluated
|
|
5103
|
+
* once rather than rescanning the newest non-matches forever and starving
|
|
5104
|
+
* older repairable rows. A row whose evidence could not be read is a
|
|
5105
|
+
* no-mutation refusal: the cursor never advances past it, so the next pass
|
|
5106
|
+
* asks again; a row that was read and shown *not* to be infrastructure
|
|
5107
|
+
* advances the cursor. Every repair requires *all* gathered failed logs to
|
|
5108
|
+
* carry an infra signature — a setup 429 in one attempt must not waive a
|
|
5109
|
+
* compile/test failure in a sibling attempt.
|
|
4720
5110
|
*
|
|
4721
|
-
*
|
|
4722
|
-
* costs nothing, a `failed` row with a PR costs a state read and a check read.
|
|
4723
|
-
* A `pushed-green` row that classifies to nothing is left completely untouched —
|
|
4724
|
-
* it is healthy, and writing a class onto it would take it out of this sweep for
|
|
4725
|
-
* good.
|
|
5111
|
+
* Returns how many rows it repaired, for the boot log.
|
|
4726
5112
|
*/
|
|
4727
|
-
export async function
|
|
4728
|
-
const { project,
|
|
4729
|
-
|
|
4730
|
-
|
|
4731
|
-
//
|
|
4732
|
-
|
|
4733
|
-
|
|
4734
|
-
|
|
4735
|
-
|
|
4736
|
-
|
|
4737
|
-
|
|
4738
|
-
|
|
4739
|
-
|
|
4740
|
-
|
|
4741
|
-
|
|
4742
|
-
|
|
4743
|
-
|
|
4744
|
-
|
|
4745
|
-
|
|
4746
|
-
|
|
4747
|
-
}
|
|
4748
|
-
|
|
4749
|
-
|
|
4750
|
-
|
|
4751
|
-
|
|
4752
|
-
|
|
4753
|
-
|
|
4754
|
-
|
|
4755
|
-
|
|
4756
|
-
|
|
4757
|
-
|
|
4758
|
-
|
|
4759
|
-
|
|
4760
|
-
|
|
4761
|
-
|
|
4762
|
-
|
|
4763
|
-
facts.failingLog = await tracker.checkLog(firstFailure.link);
|
|
4764
|
-
}
|
|
4765
|
-
}
|
|
4766
|
-
}
|
|
4767
|
-
} catch (err) {
|
|
4768
|
-
// Per row, like every other sweep here: one unreachable PR must not stop
|
|
4769
|
-
// the rest from being classified. The next tick asks again for free.
|
|
4770
|
-
log(`#${run.issue} not classified: fact gathering failed (${errText(err)}) — retrying next tick`);
|
|
4771
|
-
continue;
|
|
4772
|
-
}
|
|
4773
|
-
|
|
4774
|
-
const { cls, recovery, evidence } = classifyRun(classifiedRun, facts, caps);
|
|
4775
|
-
|
|
4776
|
-
// A healthy green PR is not a failure of any class. Leaving the row
|
|
4777
|
-
// unclassified is what keeps it eligible for the sweep on the tick where its
|
|
4778
|
-
// base does move under it.
|
|
4779
|
-
if (run.state === "pushed-green" && cls === "unknown") continue;
|
|
4780
|
-
|
|
4781
|
-
const retry = run.failureClass !== undefined;
|
|
4782
|
-
store.updateRun(run.id, { failureClass: cls, recoveryAction: recovery });
|
|
4783
|
-
log(
|
|
4784
|
-
retry
|
|
4785
|
-
? `#${run.issue} retrying ${recovery} for ${cls}: ${evidence}`
|
|
4786
|
-
: `#${run.issue} classified ${cls} → ${recovery}: ${evidence}`,
|
|
4787
|
-
);
|
|
4788
|
-
if (recovery === "settle") settled += 1;
|
|
4789
|
-
await recoverRun(d, classifiedRun, cls, recovery, evidence);
|
|
4790
|
-
}
|
|
4791
|
-
return settled;
|
|
4792
|
-
}
|
|
4793
|
-
|
|
4794
|
-
/** Performs the one action a class names. Never chooses one of its own. */
|
|
4795
|
-
async function recoverRun(
|
|
4796
|
-
d: Deps,
|
|
4797
|
-
run: RunRecord,
|
|
4798
|
-
cls: FailureClass,
|
|
4799
|
-
recovery: RecoveryAction,
|
|
4800
|
-
evidence: string,
|
|
4801
|
-
): Promise<void> {
|
|
4802
|
-
const { project, caps, tracker, store } = d;
|
|
4803
|
-
const inProgress = project.stateLabels.inProgress;
|
|
4804
|
-
|
|
4805
|
-
if (recovery === "settle") {
|
|
4806
|
-
// Enqueue the release with the terminal write (see `settlePushedGreen`):
|
|
4807
|
-
// the outbox keeps the label and the row one fact, so a tracker refusal
|
|
4808
|
-
// can no longer strand `agent:in-progress` with nothing left to retry it
|
|
4809
|
-
// (#18, #201).
|
|
4810
|
-
releaseInProgress(d, run.issue, `PR merged: ${evidence}`);
|
|
4811
|
-
store.updateRun(run.id, { state: "merged", endedAt: Date.now(), recoveredAt: Date.now() });
|
|
4812
|
-
log(`#${run.issue} settled from ${cls}: ${evidence}`);
|
|
4813
|
-
return;
|
|
4814
|
-
}
|
|
4815
|
-
|
|
4816
|
-
if (recovery === "continue") {
|
|
4817
|
-
// Two classes recover by continuing, and only one of them has anything left
|
|
4818
|
-
// to do here.
|
|
4819
|
-
//
|
|
4820
|
-
// `turn-cap-progress` was already handed back by the completion path, which
|
|
4821
|
-
// swapped its labels and left the branch retained. There is nothing to
|
|
4822
|
-
// perform, and writing anything would be actively wrong: overwriting
|
|
4823
|
-
// `lastError` with a rebase brief tells the continuation worker to rebase a
|
|
4824
|
-
// run that simply ran out of turns, and re-swapping labels the completion
|
|
4825
|
-
// path already swapped is a pair of no-op `gh` calls. Record-only, so the
|
|
4826
|
-
// sweep stops re-offering it.
|
|
4827
|
-
if (cls === "turn-cap-progress") {
|
|
4828
|
-
store.updateRun(run.id, { recoveredAt: Date.now() });
|
|
4829
|
-
log(`#${run.issue} already continuing from ${cls}: ${evidence}`);
|
|
4830
|
-
return;
|
|
4831
|
-
}
|
|
4832
|
-
|
|
4833
|
-
// `wall-clock-cap-progress`: the completion path only auto-continues
|
|
4834
|
-
// turn-cap kills, so a wall-clock kill with work to show reaches this
|
|
4835
|
-
// sweep still holding the failed label. Swap it for the queue — the branch
|
|
4836
|
-
// is retained, so the next dispatch reattaches it and briefs a resume from
|
|
4837
|
-
// the recorded work. Killed rows gathered no tracker facts, so the
|
|
4838
|
-
// issue-open guard mirrors the requeue path's. The continuation gate is
|
|
4839
|
-
// the same one the turns path applies at kill time
|
|
4840
|
-
// (`shouldContinueAfterTurnsCap`): the row is already charged, and once
|
|
4841
|
-
// the ceiling is spent, handing back the queue label would offer a
|
|
4842
|
-
// candidate admission can never accept — only the failed label comes off,
|
|
4843
|
-
// and the exhaustion reaches a human (#490, #348).
|
|
4844
|
-
if (cls === "wall-clock-cap-progress") {
|
|
4845
|
-
const state = await tracker.issueState(run.issue).catch(() => undefined);
|
|
4846
|
-
if (state !== "open") {
|
|
4847
|
-
log(`#${run.issue} not continued from ${cls}: issue is ${state ?? "unreadable"}`);
|
|
4848
|
-
return;
|
|
4849
|
-
}
|
|
4850
|
-
const continuation = store.continuationsFor(project.name, run.issue);
|
|
4851
|
-
if (hasContinuationBudget(continuation, caps.maxContinuationsPerIssue)) {
|
|
4852
|
-
swapToQueue(d, run.issue, project.stateLabels.failed);
|
|
4853
|
-
store.updateRun(run.id, { recoveredAt: Date.now() });
|
|
4854
|
-
log(`#${run.issue} requeued for a wall-clock continuation: ${evidence}`);
|
|
4855
|
-
} else {
|
|
4856
|
-
store.enqueueLabelOps(project.name, [
|
|
4857
|
-
{ issue: run.issue, op: "remove", label: project.stateLabels.failed },
|
|
4858
|
-
]);
|
|
4859
|
-
store.updateRun(run.id, { recoveredAt: Date.now() });
|
|
4860
|
-
await safeEscalate(d, {
|
|
4861
|
-
tier: 1,
|
|
4862
|
-
project: project.name,
|
|
4863
|
-
issue: run.issue,
|
|
4864
|
-
summary: `#${run.issue} exhausted its continuation budget on wall-clock cap kills`,
|
|
4865
|
-
detail: [
|
|
4866
|
-
`Attempt ${run.attempt} hit the wall-clock cap with work to continue from, but ${continuation} continuation(s) are already charged against a cap of ${caps.maxContinuationsPerIssue}.`,
|
|
4867
|
-
evidence,
|
|
4868
|
-
`Work to continue: branch ${run.branch} at ${run.headSha ?? run.salvageSha}${run.prUrl === undefined ? "" : ` — ${run.prUrl}`}.`,
|
|
4869
|
-
"The issue cannot finish inside the wall-clock cap, so another run would burn a worker slot for the same outcome. Re-scope it, raise maxContinuationsPerIssue for it, or finish the remaining work by hand.",
|
|
4870
|
-
].join("\n"),
|
|
4871
|
-
});
|
|
4872
|
-
await postExhaustionPostmortem(
|
|
4873
|
-
d,
|
|
4874
|
-
run,
|
|
4875
|
-
`Attempt ${run.attempt} hit the wall-clock cap with work to continue from, but ${continuation} continuation(s) are already charged against a cap of ${caps.maxContinuationsPerIssue}.`,
|
|
4876
|
-
);
|
|
4877
|
-
log(`#${run.issue} not continued from ${cls}: continuation budget exhausted`);
|
|
4878
|
-
}
|
|
4879
|
-
return;
|
|
4880
|
-
}
|
|
4881
|
-
|
|
4882
|
-
// `merge-conflict`: the branch is retained and its PR is open, so #50's
|
|
4883
|
-
// continuation guard admits it and the next tick briefs a rebase.
|
|
4884
|
-
//
|
|
4885
|
-
// The outbox makes the retry contract one-sided: the swap is enqueued — a
|
|
4886
|
-
// durable local write that cannot fail on the tracker — before
|
|
4887
|
-
// `recoveredAt` is written, so the row can never again be taken out of
|
|
4888
|
-
// `runsNeedingClassification` with its label swap still owed. That was
|
|
4889
|
-
// the defect 0.4.4 claimed to have fixed and did not, for this one
|
|
4890
|
-
// recovery; the projector retries until the tracker takes the swap, and
|
|
4891
|
-
// while it is pending the eligibility overlay keeps the issue coherent
|
|
4892
|
-
// (#201).
|
|
4893
|
-
swapToQueue(d, run.issue, inProgress);
|
|
4894
|
-
store.updateRun(run.id, {
|
|
4895
|
-
state: "killed",
|
|
4896
|
-
lastError:
|
|
4897
|
-
"merge-conflict: base moved under a green PR — rebase, regenerate recorded artifacts, push without force",
|
|
4898
|
-
recoveredAt: Date.now(),
|
|
4899
|
-
});
|
|
4900
|
-
log(`#${run.issue} requeued for a rebase continuation: ${evidence}`);
|
|
4901
|
-
return;
|
|
4902
|
-
}
|
|
4903
|
-
|
|
4904
|
-
if (recovery === "requeue") {
|
|
4905
|
-
if (cls === "provider-credit") {
|
|
4906
|
-
await reactToProviderCredit(d, run.issue, evidence, run.sessionFile);
|
|
4907
|
-
}
|
|
4908
|
-
// A dispatch-infra requeue that keeps landing on the same issue means the
|
|
4909
|
-
// mirror for its repo is persistently broken — a ref-lock that retry already
|
|
4910
|
-
// exhausted, a dead remote. Requeueing forever spends a turn-0 run per tick
|
|
4911
|
-
// with no chance of success, so after a bounded number of strikes this
|
|
4912
|
-
// escalates to a human instead (#168, #177).
|
|
4913
|
-
if (cls === "dispatch-infra" && store.classCountFor(project.name, run.issue, "dispatch-infra") >= DISPATCH_INFRA_MAX_STRIKES) {
|
|
4914
|
-
await safeEscalate(d, {
|
|
4915
|
-
tier: 1,
|
|
4916
|
-
project: project.name,
|
|
4917
|
-
issue: run.issue,
|
|
4918
|
-
summary: `[dispatch-infra] #${run.issue}: the mirror for ${run.repo} is failing persistently — ${evidence}`,
|
|
4919
|
-
detail: [
|
|
4920
|
-
`The dispatcher could not provision a worktree for #${run.issue} ${DISPATCH_INFRA_MAX_STRIKES} times in a row, all before the worker's first turn.`,
|
|
4921
|
-
"The mirror on this host needs attention (check disk, SSH/HTTPS credentials, and the mirror root).",
|
|
4922
|
-
].join("\n"),
|
|
4923
|
-
});
|
|
4924
|
-
store.updateRun(run.id, { recoveredAt: Date.now() });
|
|
4925
|
-
log(`#${run.issue} escalated from persistent ${cls}: ${evidence}`);
|
|
4926
|
-
return;
|
|
4927
|
-
}
|
|
4928
|
-
// Same bound for provider-transient: an issue whose stream keeps stalling
|
|
4929
|
-
// mid-run is requeued free (no attempt, no continuation charged) — but a
|
|
4930
|
-
// provider that aborts three times for one issue is down, and a human has
|
|
4931
|
-
// to check its status before hand-requeueing (#220). The escalation names
|
|
4932
|
-
// every model the chain tried, so a merged branch built on a different
|
|
4933
|
-
// model is attributable (#286).
|
|
4934
|
-
if (
|
|
4935
|
-
cls === "provider-transient" &&
|
|
4936
|
-
store.classCountFor(project.name, run.issue, "provider-transient") >= PROVIDER_TRANSIENT_MAX_STRIKES
|
|
4937
|
-
) {
|
|
4938
|
-
const tried = formatModelsTried(modelsTried(store.runsForIssue(project.name, run.issue)));
|
|
4939
|
-
await safeEscalate(d, {
|
|
4940
|
-
tier: 1,
|
|
4941
|
-
project: project.name,
|
|
4942
|
-
issue: run.issue,
|
|
4943
|
-
summary: `[provider-transient] #${run.issue}: the provider keeps aborting mid-stream — ${evidence}`,
|
|
4944
|
-
detail: [
|
|
4945
|
-
`The provider aborted the stream for #${run.issue} ${PROVIDER_TRANSIENT_MAX_STRIKES} times without the run ever producing a verdict (0 tokens billed each time).`,
|
|
4946
|
-
...(tried === ""
|
|
4947
|
-
? []
|
|
4948
|
-
: [`Models tried: ${tried}.`]),
|
|
4949
|
-
"Check provider status before requeueing by hand.",
|
|
4950
|
-
].join("\n"),
|
|
4951
|
-
});
|
|
4952
|
-
store.updateRun(run.id, { recoveredAt: Date.now() });
|
|
4953
|
-
log(`#${run.issue} escalated from persistent ${cls}: ${evidence}`);
|
|
4954
|
-
return;
|
|
4955
|
-
}
|
|
4956
|
-
// Same bound for provider-capacity: a run the provider throttled into the
|
|
4957
|
-
// ground is requeued free (no attempt charged) — but a provider that
|
|
4958
|
-
// throttles the same issue three times is at capacity, and a human has to
|
|
4959
|
-
// check its status before hand-requeueing (#573). On a chain-configured
|
|
4960
|
-
// project each requeue already moved the next attempt to the next chain
|
|
4961
|
-
// model, so this escalation is what catches the no-chain case and the
|
|
4962
|
-
// exhausted chain; it names every model the chain tried.
|
|
4963
|
-
if (
|
|
4964
|
-
cls === "provider-capacity" &&
|
|
4965
|
-
store.classCountFor(project.name, run.issue, "provider-capacity") >= PROVIDER_CAPACITY_MAX_STRIKES
|
|
4966
|
-
) {
|
|
4967
|
-
const tried = formatModelsTried(modelsTried(store.runsForIssue(project.name, run.issue)));
|
|
4968
|
-
await safeEscalate(d, {
|
|
4969
|
-
tier: 1,
|
|
4970
|
-
project: project.name,
|
|
4971
|
-
issue: run.issue,
|
|
4972
|
-
summary: `[provider-capacity] #${run.issue}: the model provider is throttling this run into the ground — ${evidence}`,
|
|
4973
|
-
detail: [
|
|
4974
|
-
`The provider answered #${run.issue} with sustained in-session rate limits ${PROVIDER_CAPACITY_MAX_STRIKES} times in a row; the harness retried each and was exhausted.`,
|
|
4975
|
-
...(tried === ""
|
|
4976
|
-
? []
|
|
4977
|
-
: [`Models tried: ${tried}.`]),
|
|
4978
|
-
"Check the provider's rate-limit status (and its throughput-oriented routes) before requeueing by hand.",
|
|
4979
|
-
].join("\n"),
|
|
4980
|
-
});
|
|
4981
|
-
store.updateRun(run.id, { recoveredAt: Date.now() });
|
|
4982
|
-
log(`#${run.issue} escalated from persistent ${cls}: ${evidence}`);
|
|
4983
|
-
return;
|
|
4984
|
-
}
|
|
4985
|
-
// Only when the tracker still shows this issue as ours to hand back. An
|
|
4986
|
-
// issue that is closed, or has no state label, was resolved by another route
|
|
4987
|
-
// and requeueing it would dispatch work nobody asked for.
|
|
4988
|
-
const state = await tracker.issueState(run.issue).catch(() => undefined);
|
|
4989
|
-
if (state !== "open") {
|
|
4990
|
-
log(`#${run.issue} not requeued from ${cls}: issue is ${state ?? "unreadable"}`);
|
|
4991
|
-
return;
|
|
4992
|
-
}
|
|
4993
|
-
// A clean orphan whose queue label is already absent is a deliberate
|
|
4994
|
-
// withdrawal — the operator took the issue out of the queue (e.g. so a
|
|
4995
|
-
// daemon restart could not re-dispatch work known to be unsafe) — and
|
|
4996
|
-
// recovery must not recreate that intent (#423). Read the live label set;
|
|
4997
|
-
// when the queue label is gone, release the dispatcher-owned in-progress
|
|
4998
|
-
// label and stop, leaving the operator's withdrawal to survive recovery.
|
|
4999
|
-
if (cls === "orphan-clean") {
|
|
5000
|
-
const snapshot = await tracker.issueSnapshot(run.issue).catch(() => undefined);
|
|
5001
|
-
if (snapshot === undefined) {
|
|
5002
|
-
log(`#${run.issue} not requeued from orphan-clean: cannot confirm ${project.queueLabel} (unreadable, retrying)`);
|
|
5003
|
-
return;
|
|
5004
|
-
}
|
|
5005
|
-
if (!snapshot.labels.includes(project.queueLabel)) {
|
|
5006
|
-
store.enqueueLabelOps(project.name, [{ issue: run.issue, op: "remove", label: inProgress }]);
|
|
5007
|
-
store.updateRun(run.id, { recoveredAt: Date.now() });
|
|
5008
|
-
log(`#${run.issue} not requeued from orphan-clean: ${project.queueLabel} removed before recovery (operator withdrawal)`);
|
|
5009
|
-
return;
|
|
5010
|
-
}
|
|
5011
|
-
// #439: an orphan-clean row charges the continuation budget, so once
|
|
5012
|
-
// `hasContinuationBudget` is spent — exactly the predicate `admitCandidates`
|
|
5013
|
-
// holds the issue on — requeueing re-adds a queue label for a candidate the
|
|
5014
|
-
// dispatcher can never admit. Stop handing it back and hold it instead: the
|
|
5015
|
-
// queue label comes off (so admission never re-holds on every dispatch),
|
|
5016
|
-
// the in-progress label is released, and a single diagnosis escalates once.
|
|
5017
|
-
// `orphan-clean` is deliberately NOT a global exclusion from the budget (a
|
|
5018
|
-
// worker that genuinely keeps dying mid-work must still be bounded); this is
|
|
5019
|
-
// the missing ceiling check this path never had (#348's invariant).
|
|
5020
|
-
const continuations = store.continuationsFor(project.name, run.issue);
|
|
5021
|
-
if (!hasContinuationBudget(continuations, caps.maxContinuationsPerIssue)) {
|
|
5022
|
-
const runs = store.runsForIssue(project.name, run.issue);
|
|
5023
|
-
const breakdown = continuationBreakdown(runs);
|
|
5024
|
-
const artifact = newestContinuableRun(runs);
|
|
5025
|
-
const onlyDaemonStops = breakdown.size === 1 && breakdown.get("orphan-clean") === continuations;
|
|
5026
|
-
const classLine = Array.from(breakdown, ([cls, n]) => `${n} ${cls}`).join(", ");
|
|
5027
|
-
const artifactLine =
|
|
5028
|
-
artifact === undefined
|
|
5029
|
-
? "The attempts left no salvage commit, head SHA or PR — the branch is empty, so start clean from a re-scope rather than continuing from nothing."
|
|
5030
|
-
: `Work to continue: branch ${artifact.branch} at ${artifact.headSha ?? artifact.salvageSha}${artifact.prUrl === undefined ? "" : ` — ${artifact.prUrl}`}.`;
|
|
5031
|
-
store.enqueueLabelOps(project.name, [
|
|
5032
|
-
{ issue: run.issue, op: "remove", label: inProgress },
|
|
5033
|
-
{ issue: run.issue, op: "remove", label: project.queueLabel },
|
|
5034
|
-
]);
|
|
5035
|
-
store.updateRun(run.id, { recoveredAt: Date.now() });
|
|
5036
|
-
await safeEscalate(d, {
|
|
5037
|
-
tier: 1,
|
|
5038
|
-
project: project.name,
|
|
5039
|
-
issue: run.issue,
|
|
5040
|
-
summary: `#${run.issue} exhausted its ${caps.maxContinuationsPerIssue}-continuation budget on ${cls}`,
|
|
5041
|
-
detail: [
|
|
5042
|
-
`Attempt ${run.attempt} is not requeued: ${continuations} continuation(s) charged against a cap of ${caps.maxContinuationsPerIssue}.`,
|
|
5043
|
-
`Continuations by failure class: ${classLine}.`,
|
|
5044
|
-
onlyDaemonStops
|
|
5045
|
-
? "Every continuation was a daemon stop — the work never failed; the budget was spent by daemon deaths, not the issue."
|
|
5046
|
-
: "Continuations span real work — inspect what each attempt left behind before continuing.",
|
|
5047
|
-
artifactLine,
|
|
5048
|
-
"What you can do: raise maxContinuationsPerIssue for this issue, re-scope it, or continue from the preserved work (or start clean if none).",
|
|
5049
|
-
].join("\n"),
|
|
5050
|
-
});
|
|
5051
|
-
await postExhaustionPostmortem(
|
|
5052
|
-
d,
|
|
5053
|
-
run,
|
|
5054
|
-
`Attempt ${run.attempt} is not requeued: ${continuations} continuation(s) charged against a cap of ${caps.maxContinuationsPerIssue}.`,
|
|
5055
|
-
);
|
|
5056
|
-
log(`#${run.issue} not requeued from orphan-clean: continuation budget exhausted`);
|
|
5057
|
-
return;
|
|
5058
|
-
}
|
|
5059
|
-
}
|
|
5060
|
-
const label = cls === "orphan-clean" ? inProgress : project.stateLabels.failed;
|
|
5061
|
-
swapToQueue(d, run.issue, label);
|
|
5062
|
-
store.updateRun(run.id, { recoveredAt: Date.now() });
|
|
5063
|
-
log(`#${run.issue} requeued from ${cls}: ${evidence}`);
|
|
5064
|
-
return;
|
|
5065
|
-
}
|
|
5066
|
-
|
|
5067
|
-
if (recovery === "rerun-checks") {
|
|
5068
|
-
if (run.prUrl === undefined) return;
|
|
5069
|
-
try {
|
|
5070
|
-
await tracker.rerunFailedChecks(run.prUrl);
|
|
5071
|
-
} catch (err) {
|
|
5072
|
-
log(`#${run.issue} check re-run failed (${errText(err)}) — retrying next tick`);
|
|
5073
|
-
return;
|
|
5074
|
-
}
|
|
5075
|
-
// Back to pending rather than green: the existing settle sweep re-verifies
|
|
5076
|
-
// it against the recorded head on a later tick, so nothing here has to guess
|
|
5077
|
-
// whether the re-run passed.
|
|
5078
|
-
store.updateRun(run.id, { state: "pushed-pending", lastError: null, recoveredAt: Date.now() });
|
|
5079
|
-
log(`#${run.issue} re-ran infrastructure checks: ${evidence}`);
|
|
5080
|
-
return;
|
|
5081
|
-
}
|
|
5082
|
-
|
|
5083
|
-
if (recovery === "escalate") {
|
|
5084
|
-
const detail = [evidence];
|
|
5085
|
-
if (cls === "turn-cap-spinning" || cls === "wall-clock-cap-spinning") {
|
|
5086
|
-
const calls = lastToolCalls(run.sessionFile);
|
|
5087
|
-
detail.push(
|
|
5088
|
-
calls.length === 0
|
|
5089
|
-
? "transcript unreadable — no tool calls could be recovered"
|
|
5090
|
-
: `Last ${calls.length} tool calls: ${calls.join(" → ")}`,
|
|
5113
|
+
export async function reconcileHistoricalInfra(d: Deps): Promise<number> {
|
|
5114
|
+
const { project, store } = d;
|
|
5115
|
+
const version = infraSignatureVersion();
|
|
5116
|
+
const persisted = store.historicalInfraCursor(project.name);
|
|
5117
|
+
// A cursor stamped by an older signature list is stale: the classifier now
|
|
5118
|
+
// recognises more evidence, so the pass restarts from the newest row rather
|
|
5119
|
+
// than skipping past newly repairable history (#638).
|
|
5120
|
+
const cursor =
|
|
5121
|
+
persisted !== undefined && persisted.classifierVersion === version
|
|
5122
|
+
? { startedAt: persisted.startedAt, rowid: persisted.rowid }
|
|
5123
|
+
: undefined;
|
|
5124
|
+
const candidates = store.historicalInfraCandidates(project.name, HISTORICAL_INFRA_BATCH, cursor);
|
|
5125
|
+
let repaired = 0;
|
|
5126
|
+
let lastDecided: { startedAt: number; rowid: number } | undefined;
|
|
5127
|
+
for (const run of candidates) {
|
|
5128
|
+
const chunks = await historicalInfraEvidence(d, run);
|
|
5129
|
+
// Unreachable evidence is undecided exactly like a fresh classifier is: a
|
|
5130
|
+
// no-mutation refusal the next pass asks again, and the cursor stops here
|
|
5131
|
+
// so the row is re-offered rather than being skipped past.
|
|
5132
|
+
if (chunks === undefined) break;
|
|
5133
|
+
lastDecided = { startedAt: run.startedAt, rowid: run.rowid };
|
|
5134
|
+
// The head's runs were read and none holds a failing log — determinately
|
|
5135
|
+
// not infrastructure, so an old misclassified verdict stays charged.
|
|
5136
|
+
if (chunks.length === 0) continue;
|
|
5137
|
+
// Mixed guard: anything gathered that is not itself a closed infra
|
|
5138
|
+
// signature (a compile/test failure in a sibling attempt, a product 429)
|
|
5139
|
+
// refuses the whole row. One setup 429 is not permission to waive a real
|
|
5140
|
+
// implementation failure.
|
|
5141
|
+
if (!chunks.every((chunk) => infraLogSignature(chunk) !== undefined)) continue;
|
|
5142
|
+
if (store.reclassifyInfra(run.id)) {
|
|
5143
|
+
repaired += 1;
|
|
5144
|
+
// The every-guard above guarantees a signature and a first chunk; the
|
|
5145
|
+
// non-null assertions make the same fact readable to the type checker.
|
|
5146
|
+
log(
|
|
5147
|
+
`#${run.issue} repaired historical ${run.failureClass ?? "ci-deterministic"} → ci-infra (run ${run.id}): ` +
|
|
5148
|
+
`"${infraLogSignature(chunks[0]!)}" in the failed check log`,
|
|
5091
5149
|
);
|
|
5092
5150
|
}
|
|
5093
|
-
|
|
5094
|
-
|
|
5095
|
-
|
|
5096
|
-
|
|
5097
|
-
|
|
5098
|
-
|
|
5099
|
-
|
|
5100
|
-
? `Session: ${run.sessionFile}`
|
|
5101
|
-
: `Session: ${run.sessionFile} (file was never written — the run died before its transcript was flushed)`,
|
|
5151
|
+
}
|
|
5152
|
+
if (lastDecided !== undefined) {
|
|
5153
|
+
store.setHistoricalInfraCursor(
|
|
5154
|
+
project.name,
|
|
5155
|
+
lastDecided.startedAt,
|
|
5156
|
+
lastDecided.rowid,
|
|
5157
|
+
version,
|
|
5102
5158
|
);
|
|
5103
|
-
// The class and the run are in the summary, which is what the notifications
|
|
5104
|
-
// ledger dedupes on — so one class escalates once per run rather than every
|
|
5105
|
-
// five minutes.
|
|
5106
|
-
await safeEscalate(d, {
|
|
5107
|
-
tier: 1,
|
|
5108
|
-
project: project.name,
|
|
5109
|
-
issue: run.issue,
|
|
5110
|
-
runId: run.id,
|
|
5111
|
-
summary: `[${cls}] #${run.issue} attempt ${run.attempt}: ${evidence}`,
|
|
5112
|
-
detail: detail.join("\n"),
|
|
5113
|
-
});
|
|
5114
|
-
// The hand-off IS the recovery for these classes: there is nothing else this
|
|
5115
|
-
// package can do, and leaving the row unrecovered would re-escalate forever.
|
|
5116
|
-
store.updateRun(run.id, { recoveredAt: Date.now() });
|
|
5117
|
-
return;
|
|
5118
5159
|
}
|
|
5119
|
-
|
|
5120
|
-
// `hold` (orphan-dirty) and `none`: recorded, nothing performed. The existing
|
|
5121
|
-
// unsalvaged-WIP admission hold already fails dispatch closed until an
|
|
5122
|
-
// operator acknowledges the tree, which is the only safe move when the
|
|
5123
|
-
// worktree holds the only copy of real work.
|
|
5124
|
-
}
|
|
5125
|
-
|
|
5126
|
-
/**
|
|
5127
|
-
* Enqueue a state-label → queue-label swap for projection (#201).
|
|
5128
|
-
*
|
|
5129
|
-
* The swap is two ops in id order — remove first, then add — which is the
|
|
5130
|
-
* atomicity the projector guarantees: the issue never sits newly eligible
|
|
5131
|
-
* without a queue label on its way back, and the add never lands before the
|
|
5132
|
-
* remove when GitHub fails between them. Enqueueing is a durable local write
|
|
5133
|
-
* that cannot fail on the tracker, so the caller records its recovery
|
|
5134
|
-
* immediately and the projector retries the swap until the tracker takes it —
|
|
5135
|
-
* that closes the 0.4.4 hole where a refused label swap stranded the row
|
|
5136
|
-
* permanently under a log line promising a retry.
|
|
5137
|
-
*/
|
|
5138
|
-
function swapToQueue(d: Pick<Deps, "project" | "store">, issue: number, label: string): void {
|
|
5139
|
-
d.store.enqueueLabelOps(d.project.name, [
|
|
5140
|
-
{ issue, op: "remove", label },
|
|
5141
|
-
{ issue, op: "add", label: d.project.queueLabel },
|
|
5142
|
-
]);
|
|
5160
|
+
return repaired;
|
|
5143
5161
|
}
|
|
5144
5162
|
|
|
5145
|
-
/** Bounded listing per state label, so one reconcile cannot walk a whole repo. */
|
|
5146
|
-
const RECONCILE_LIMIT = 50;
|
|
5147
|
-
|
|
5148
5163
|
/**
|
|
5149
|
-
*
|
|
5150
|
-
*
|
|
5151
|
-
*
|
|
5152
|
-
*
|
|
5153
|
-
*
|
|
5154
|
-
*
|
|
5155
|
-
*
|
|
5156
|
-
* and nothing in the loop ever revisited it. The board counted four phantom
|
|
5157
|
-
* failures while the genuinely stuck issues were invisible.
|
|
5164
|
+
* The failed check logs of a settled `ci-deterministic` row, head-pinned to
|
|
5165
|
+
* the exact commit the row ran against, or `undefined` when the evidence is
|
|
5166
|
+
* unreachable (#638). Only the workflow-run history at `run.headSha` is
|
|
5167
|
+
* evidence: the PR's *current* check rollup is not, because a later push's
|
|
5168
|
+
* checks must never classify an earlier run's row. `workflowRunsAt` is itself
|
|
5169
|
+
* head-scoped, and each run's failed attempts are read through the tracker's
|
|
5170
|
+
* guarded runner (call/refusal accounting and the rate-limit breaker apply).
|
|
5158
5171
|
*
|
|
5159
|
-
*
|
|
5160
|
-
*
|
|
5161
|
-
*
|
|
5172
|
+
* Returns the per-failed-job failed logs gathered across the head's runs (one
|
|
5173
|
+
* chunk per failed job, full and untruncated), or `undefined` when any read
|
|
5174
|
+
* could not be made — the run register, an attempt's job register, a failed
|
|
5175
|
+
* job's log, or the head-run list itself — or when a bounded evidence set
|
|
5176
|
+
* exceeded its limit (more head-pinned runs than `HISTORICAL_INFRA_RUNS`, or
|
|
5177
|
+
* more attempts than `HISTORICAL_INFRA_ATTEMPTS`). Undefined is a no-mutation
|
|
5178
|
+
* refusal the next pass asks again; the cursor never advances past it. `[]`
|
|
5179
|
+
* means the head's runs were read and none holds a failing log — a
|
|
5180
|
+
* determinately non-infrastructure answer.
|
|
5162
5181
|
*/
|
|
5163
|
-
|
|
5164
|
-
|
|
5165
|
-
|
|
5166
|
-
|
|
5167
|
-
|
|
5168
|
-
|
|
5169
|
-
|
|
5170
|
-
|
|
5171
|
-
|
|
5172
|
-
|
|
5173
|
-
|
|
5174
|
-
|
|
5175
|
-
|
|
5176
|
-
log(`#${issue.number} reconciled: closed issue no longer carries ${label} (queued)`);
|
|
5177
|
-
continue;
|
|
5178
|
-
}
|
|
5179
|
-
|
|
5180
|
-
if (label !== project.stateLabels.failed) continue;
|
|
5181
|
-
const children = await tracker.childrenOf(issue.number).catch(() => []);
|
|
5182
|
-
if (children.length === 0 || children.some((c) => c.state !== "closed")) continue;
|
|
5183
|
-
|
|
5184
|
-
const key = `${project.name}:superseded:${issue.number}`;
|
|
5185
|
-
if (store.wasNotified(key)) continue;
|
|
5186
|
-
const list = children.map((c) => `#${c.number}`).join(", ");
|
|
5187
|
-
store.enqueueLabelOps(project.name, [{ issue: issue.number, op: "remove", label }]);
|
|
5188
|
-
try {
|
|
5189
|
-
await tracker.comment(
|
|
5190
|
-
issue.number,
|
|
5191
|
-
`superseded: all sub-issues closed (${list}) — this label was stale; propose closing if the ` +
|
|
5192
|
-
`acceptance criteria are met on the default branch.`,
|
|
5193
|
-
);
|
|
5194
|
-
store.markNotified(key);
|
|
5195
|
-
log(`#${issue.number} reconciled: superseded by ${list}`);
|
|
5196
|
-
} catch (err) {
|
|
5197
|
-
// The label removal is already queued and will land regardless; the
|
|
5198
|
-
// comment is the only half that can fail here (#201).
|
|
5199
|
-
log(`#${issue.number} could not comment the superseded note (${errText(err)}) — retrying next tick`);
|
|
5200
|
-
}
|
|
5201
|
-
}
|
|
5182
|
+
async function historicalInfraEvidence(d: Deps, run: RunRecord): Promise<string[] | undefined> {
|
|
5183
|
+
// Without a head SHA there is no safe way to pin evidence to this row; a
|
|
5184
|
+
// head-less row is determinately not a repair candidate, so it advances the
|
|
5185
|
+
// cursor rather than stalling the pass.
|
|
5186
|
+
if (run.headSha === undefined) return [];
|
|
5187
|
+
const target = d.project.routing.repos[run.repo];
|
|
5188
|
+
const repoIdentity = target === undefined ? undefined : githubRepo(target.cloneUrl);
|
|
5189
|
+
if (repoIdentity === undefined) return [];
|
|
5190
|
+
let runs: WorkflowRun[] | undefined;
|
|
5191
|
+
try {
|
|
5192
|
+
runs = await d.tracker.workflowRunsAt(repoIdentity, run.headSha);
|
|
5193
|
+
} catch {
|
|
5194
|
+
return undefined;
|
|
5202
5195
|
}
|
|
5203
|
-
|
|
5204
|
-
|
|
5205
|
-
|
|
5206
|
-
|
|
5207
|
-
|
|
5208
|
-
|
|
5209
|
-
|
|
5210
|
-
|
|
5211
|
-
|
|
5212
|
-
|
|
5213
|
-
|
|
5214
|
-
|
|
5215
|
-
|
|
5216
|
-
|
|
5217
|
-
|
|
5218
|
-
* orchestrator's drain-duty judgement, not something to automate here. The
|
|
5219
|
-
* rows also keep counting toward `maxAttemptsPerIssue`, so a loop of deaths
|
|
5220
|
-
* still escalates instead of retrying forever.
|
|
5221
|
-
*
|
|
5222
|
-
* `pushed-green` rows are deliberately left alone: they hold no process — they
|
|
5223
|
-
* are finished work waiting on a human merge, and they must keep occupying the
|
|
5224
|
-
* issue so a second attempt cannot land on a live PR. What eventually settles
|
|
5225
|
-
* them is {@link settlePushedGreen}, on the tick, by asking the tracker what
|
|
5226
|
-
* became of the PR — the one question a restart cannot answer by inference.
|
|
5227
|
-
*/
|
|
5228
|
-
export async function reconcileOrphanedRuns(
|
|
5229
|
-
store: Store,
|
|
5230
|
-
project: string,
|
|
5231
|
-
/**
|
|
5232
|
-
* Resolves the privileged publisher for one orphaned run. Optional because a
|
|
5233
|
-
* test driving the row transitions has no repo to publish to; production
|
|
5234
|
-
* always passes it, and without it a salvaged WIP commit stays local — which
|
|
5235
|
-
* is the half of #121 that reaches a human.
|
|
5236
|
-
*/
|
|
5237
|
-
publish?: (run: RunRecord) => RunPublisher,
|
|
5238
|
-
): Promise<RunRecord[]> {
|
|
5239
|
-
// Live runs only: `pushed-green` holds no process, so it cannot be orphaned by
|
|
5240
|
-
// a process dying — it is finished work waiting on a human merge.
|
|
5241
|
-
const stale = store.liveRuns(project);
|
|
5242
|
-
const endedAt = Date.now();
|
|
5243
|
-
for (const r of stale) {
|
|
5244
|
-
// Salvage before the row flips: the worktree path is on the record, and
|
|
5245
|
-
// salvageWip is a no-op for a missing/clean tree. The clause matches the
|
|
5246
|
-
// cap-kill wording so triage reads the same either way, and the tree is
|
|
5247
|
-
// kept because an orphan's remains are the orchestrator's drain-duty call.
|
|
5248
|
-
const settlement =
|
|
5249
|
-
r.worktree === ""
|
|
5250
|
-
? undefined
|
|
5251
|
-
: await settleWorktree({
|
|
5252
|
-
issue: r.issue,
|
|
5253
|
-
attempt: r.attempt,
|
|
5254
|
-
ending: "killed by a daemon restart",
|
|
5255
|
-
worktree: r.worktree,
|
|
5256
|
-
branch: r.branch,
|
|
5257
|
-
// An orphan's tree is kept, so its commits are not about to be
|
|
5258
|
-
// deleted — but a WIP salvage still has to reach GitHub, which is
|
|
5259
|
-
// #121's whole point and is now the daemon's hop to make.
|
|
5260
|
-
publish: publish?.(r),
|
|
5261
|
-
tree: "keep",
|
|
5262
|
-
});
|
|
5263
|
-
store.updateRun(r.id, { state: "orphaned", endedAt, ...settlement?.patch });
|
|
5196
|
+
if (runs === undefined) return undefined;
|
|
5197
|
+
// Refuse rather than decide from the first `HISTORICAL_INFRA_RUNS`: three
|
|
5198
|
+
// setup-429 runs must not waive a row whose fourth head-pinned run carries
|
|
5199
|
+
// the real compile/test failure (review #654). Undecided means the cursor
|
|
5200
|
+
// never advances past this row, so the next pass asks again.
|
|
5201
|
+
if (runs.length > HISTORICAL_INFRA_RUNS) return undefined;
|
|
5202
|
+
const chunks: string[] = [];
|
|
5203
|
+
for (const wf of runs) {
|
|
5204
|
+
const logs = await d.tracker.runFailedAttemptLogs(
|
|
5205
|
+
repoIdentity,
|
|
5206
|
+
wf.url,
|
|
5207
|
+
HISTORICAL_INFRA_ATTEMPTS,
|
|
5208
|
+
);
|
|
5209
|
+
if (logs === undefined) return undefined;
|
|
5210
|
+
chunks.push(...logs);
|
|
5264
5211
|
}
|
|
5265
|
-
return
|
|
5212
|
+
return chunks;
|
|
5266
5213
|
}
|
|
5267
5214
|
|
|
5215
|
+
|
|
5268
5216
|
// ------------------------------------------------------------------- the daemon
|
|
5269
5217
|
|
|
5270
5218
|
interface ProjectRuntime {
|
|
@@ -5450,12 +5398,17 @@ export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
|
|
|
5450
5398
|
log(projects.length === 1 ? message : `[${project.name}] ${message}`);
|
|
5451
5399
|
};
|
|
5452
5400
|
const caps = resolveCaps(project, cfg.defaults);
|
|
5401
|
+
// One transient-server-error breaker per project (#642): admission's
|
|
5402
|
+
// GraphQL checks and the orchestrator mutation commands share it, so a 503
|
|
5403
|
+
// observed by either side gates both instead of one provider outage being
|
|
5404
|
+
// re-asked by every candidate AND every mutation.
|
|
5405
|
+
const graphqlBreaker = new GraphqlBreaker();
|
|
5453
5406
|
const tracker = makeTracker(project, undefined, {
|
|
5454
5407
|
onCall: () => store.bumpGhCalls?.(utcDay(), "daemon"),
|
|
5455
5408
|
onRefusal: (at) => store.recordGhRefusal?.(at),
|
|
5456
5409
|
onNotModified: () => store.bumpGhCalls?.(utcDay(), "daemon-304"),
|
|
5457
|
-
});
|
|
5458
|
-
const verbActions = githubVerbActions(project);
|
|
5410
|
+
}, { graphqlBreaker });
|
|
5411
|
+
const verbActions = githubVerbActions(project, undefined, undefined, graphqlBreaker);
|
|
5459
5412
|
|
|
5460
5413
|
if (alive === undefined || alive.pid === process.pid) {
|
|
5461
5414
|
const orphanPublisher = (run: RunRecord): RunPublisher => {
|
|
@@ -5600,10 +5553,38 @@ export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
|
|
|
5600
5553
|
onCall: () => store.bumpGhCalls?.(utcDay(), "daemon"),
|
|
5601
5554
|
onRefusal: (at) => store.recordGhRefusal?.(at),
|
|
5602
5555
|
}).issueSnapshot(issue),
|
|
5556
|
+
// The body twin of `probeIssueIn`, for the dependency-graph cycle pass
|
|
5557
|
+
// (#421): reads a reachable routed prerequisite's body through the same
|
|
5558
|
+
// per-repo tracker/credential/accounting seams.
|
|
5559
|
+
probeBodyIn: (ownerRepo, issue) =>
|
|
5560
|
+
makeTracker({ ...project, tracker: { kind: "github", repo: ownerRepo } }, undefined, {
|
|
5561
|
+
onCall: () => store.bumpGhCalls?.(utcDay(), "daemon"),
|
|
5562
|
+
onRefusal: (at) => store.recordGhRefusal?.(at),
|
|
5563
|
+
}).issueBody(issue),
|
|
5603
5564
|
...(verbPeerReader === undefined ? {} : { verbPeerReader }),
|
|
5604
5565
|
verbActions,
|
|
5605
5566
|
};
|
|
5606
5567
|
runtimeDeps = d;
|
|
5568
|
+
// Review-revision restart recovery (#692): a revision the previous daemon
|
|
5569
|
+
// claimed and lost — its run is `running`/`orphaned` by the orphan sweep
|
|
5570
|
+
// above — is restored to `pushed-green` and re-queued so this process's
|
|
5571
|
+
// first dispatch pass resumes the exact session. Runs under the same
|
|
5572
|
+
// dead-daemon guard as `reconcileOrphanedRuns`, and after it, so the
|
|
5573
|
+
// revision's worktree has already been salvaged to the branch before it is
|
|
5574
|
+
// cleared for a fresh reattach. A round that cannot be restored was
|
|
5575
|
+
// settled and escalated by the reconcile itself.
|
|
5576
|
+
if (alive === undefined || alive.pid === process.pid) {
|
|
5577
|
+
try {
|
|
5578
|
+
for (const revision of await reconcileCrashedReviewRevisions(d)) {
|
|
5579
|
+
projectLog(
|
|
5580
|
+
`#${revision.issue} review round ${revision.round} recovered across restart — ` +
|
|
5581
|
+
`run ${revision.runId} is pushed-green again and awaits its next dispatch pass`,
|
|
5582
|
+
);
|
|
5583
|
+
}
|
|
5584
|
+
} catch (err) {
|
|
5585
|
+
projectLog(`review revision restart recovery failed: ${errText(err)}`);
|
|
5586
|
+
}
|
|
5587
|
+
}
|
|
5607
5588
|
// Startup reconciliation: close an incident carried over from a previous
|
|
5608
5589
|
// process when the orchestrator is up (one recovery notice), or open one
|
|
5609
5590
|
// when it failed to start (one down page). A daemon restarted while still
|
|
@@ -5625,6 +5606,21 @@ export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
|
|
|
5625
5606
|
});
|
|
5626
5607
|
}
|
|
5627
5608
|
|
|
5609
|
+
// Historical budget repair (#638): one bounded pass per daemon process,
|
|
5610
|
+
// before the first tick, so admission and `unblock` read the repaired counts
|
|
5611
|
+
// from the moment the daemon starts. A classifier fix changes future
|
|
5612
|
+
// verdicts but not rows already settled under the old one; re-fetching their
|
|
5613
|
+
// failed check logs and reclassifying the exact closed infrastructure
|
|
5614
|
+
// signature returns those attempts without re-animating the runs. Best-effort
|
|
5615
|
+
// per project — one unreachable project must not stop the rest from booting.
|
|
5616
|
+
for (const runtime of runtimes) {
|
|
5617
|
+
try {
|
|
5618
|
+
await reconcileHistoricalInfra(runtime.d);
|
|
5619
|
+
} catch (err) {
|
|
5620
|
+
log(`historical infra reconciliation failed: ${errText(err)}`);
|
|
5621
|
+
}
|
|
5622
|
+
}
|
|
5623
|
+
|
|
5628
5624
|
if (o.once) {
|
|
5629
5625
|
try {
|
|
5630
5626
|
for (const runtime of runtimes) await tick(runtime.d, workers);
|