omp-conductor 0.16.2 → 0.17.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +38 -4
- package/REFERENCE.md +18 -12
- package/package.json +2 -1
- package/schema/config.schema.json +16 -0
- package/src/admission.ts +159 -43
- package/src/availability.ts +27 -1
- package/src/briefs/worker.md +2 -0
- package/src/clack-ui.ts +83 -0
- package/src/command-manifest.ts +16 -7
- package/src/commands/arm.ts +11 -3
- package/src/commands/decision.ts +17 -7
- package/src/commands/doctor.ts +18 -1
- package/src/commands/hold.ts +9 -7
- package/src/commands/ledger.ts +25 -4
- package/src/commands/message.ts +32 -4
- package/src/commands/setup.ts +61 -10
- package/src/commands/stats.ts +9 -5
- package/src/commands/status.ts +32 -5
- package/src/commands/tail.ts +13 -1
- package/src/commands/watch.ts +16 -7
- package/src/config-schema.ts +20 -0
- package/src/config.ts +37 -0
- package/src/daemon.ts +1240 -18
- package/src/doctor.ts +310 -22
- package/src/escalate.ts +560 -57
- package/src/failure-class.ts +56 -13
- package/src/fleet.ts +224 -47
- package/src/gitops.ts +103 -24
- package/src/lifecycle.ts +7 -2
- package/src/orchestrator-tick.ts +372 -157
- package/src/privileged.ts +3 -0
- package/src/release-policy.ts +177 -5
- package/src/setup-answers.ts +135 -0
- package/src/setup-host.ts +193 -4
- package/src/setup-install.ts +2 -0
- package/src/setup-probe.ts +1 -0
- package/src/setup-wizard.ts +1296 -101
- package/src/setup.ts +60 -3
- package/src/status-render.ts +11 -1
- package/src/store.ts +333 -12
- package/src/tracker/github.ts +562 -13
- package/src/types.ts +204 -2
- package/src/ui/progress.ts +32 -0
- package/src/ui/style.ts +11 -0
- package/src/upgrade.ts +50 -19
- package/src/verbs/actions.ts +66 -18
- package/src/verbs/protocol.ts +45 -0
- package/src/verbs/server.ts +212 -11
- package/src/wizard-ui.ts +14 -5
- package/src/worker.ts +26 -0
- package/systemd/omp-conductor-recover.sh +73 -0
- package/systemd/recover-unit-test.sh +61 -0
package/src/daemon.ts
CHANGED
|
@@ -8,7 +8,7 @@
|
|
|
8
8
|
* here and enforced before anything is claimed.
|
|
9
9
|
*/
|
|
10
10
|
import { createHash } from "node:crypto";
|
|
11
|
-
import { chmodSync, existsSync, mkdirSync, readdirSync, readFileSync, rmSync, statSync, writeFileSync } from "node:fs";
|
|
11
|
+
import { chmodSync, existsSync, mkdirSync, readdirSync, readFileSync, renameSync, rmSync, statSync, writeFileSync } from "node:fs";
|
|
12
12
|
import { dirname, join, relative } from "node:path";
|
|
13
13
|
import {
|
|
14
14
|
configPath,
|
|
@@ -31,7 +31,7 @@ import { digestScheduleState, type DigestScheduleState } from "./digest-schedule
|
|
|
31
31
|
import { createEscalator, escalationIssueRef } from "./escalate.ts";
|
|
32
32
|
import { pendingCodeGraph, probeCodeGraph, type CodeGraphHealth } from "./graph-health.ts";
|
|
33
33
|
import { graphHint } from "./graph.ts";
|
|
34
|
-
import { acquireOnceLease, healthCheck, livingDaemon } from "./lifecycle.ts";
|
|
34
|
+
import { acquireOnceLease, healthCheck, livingDaemon, probeUnit, SYSTEMD_UNIT } from "./lifecycle.ts";
|
|
35
35
|
import { requestImmediateTick, resolveTickConfigCwd, STALL_MARKER_FILE } from "./orchestrator-tick.ts";
|
|
36
36
|
import { runDoctor } from "./doctor.ts";
|
|
37
37
|
import { appendJournal, launchTransientUnit, readUpgradeJournal } from "./upgrade-journal.ts";
|
|
@@ -52,17 +52,23 @@ import {
|
|
|
52
52
|
} from "./reports.ts";
|
|
53
53
|
import {
|
|
54
54
|
recordReleaseBlock,
|
|
55
|
+
sharedHostBriefNotice,
|
|
55
56
|
type GateShape,
|
|
56
57
|
type ReleaseBlockContext,
|
|
57
58
|
} from "./release-policy.ts";
|
|
58
59
|
import { branchName, effectiveLabels, route } from "./routing.ts";
|
|
59
60
|
import type { Routed, UnroutableReason } from "./routing.ts";
|
|
60
|
-
import {
|
|
61
|
-
|
|
61
|
+
import {
|
|
62
|
+
admitCandidates,
|
|
63
|
+
effectiveLane,
|
|
64
|
+
hasContinuationBudget,
|
|
65
|
+
hasFailedAttemptBudget,
|
|
66
|
+
} from "./admission.ts";
|
|
67
|
+
import type { Admission, AdmissionHold, FileLane } from "./admission.ts";
|
|
62
68
|
import { log, errText, safeEscalate } from "./log.ts";
|
|
63
69
|
import { materializeOmpSettings } from "./omp-settings.ts";
|
|
64
70
|
import { evaluateDecisionConditions, probeNpmVersion, probeRateLimitReset } from "./decisions.ts";
|
|
65
|
-
import { classifyRun, providerCreditRefusal, providerTransientFault, type ClassifyFacts } from "./failure-class.ts";
|
|
71
|
+
import { classifyRun, infraLogSignature, infraSignatureVersion, normalise, providerCreditRefusal, providerTransientFault, type ClassifyFacts } from "./failure-class.ts";
|
|
66
72
|
import {
|
|
67
73
|
DEFAULT_MODEL_FALLBACK_THRESHOLD,
|
|
68
74
|
fallbackClause,
|
|
@@ -73,7 +79,7 @@ import {
|
|
|
73
79
|
} from "./model-fallback.ts";
|
|
74
80
|
import { projectLabels } from "./label-projection.ts";
|
|
75
81
|
import { dbPath, LIVE_STATES, openStore, utcDay } from "./store.ts";
|
|
76
|
-
import { makeTracker, type RateLimitStatus } from "./tracker/github.ts";
|
|
82
|
+
import { GraphqlBreaker, makeTracker, type RateLimitStatus } from "./tracker/github.ts";
|
|
77
83
|
import { DEFAULT_REPORT_POLICY, RELEASE_SHAPES } from "./types.ts";
|
|
78
84
|
import type {
|
|
79
85
|
BaseFreeze,
|
|
@@ -97,6 +103,8 @@ import type {
|
|
|
97
103
|
ResolvedGrants,
|
|
98
104
|
FailureClass,
|
|
99
105
|
RecoveryAction,
|
|
106
|
+
ReviewRevisionOutcome,
|
|
107
|
+
ReviewRevisionRecord,
|
|
100
108
|
RunRecord,
|
|
101
109
|
RunState,
|
|
102
110
|
SettlementFlag,
|
|
@@ -104,6 +112,7 @@ import type {
|
|
|
104
112
|
Tracker,
|
|
105
113
|
VerbLedgerEntry,
|
|
106
114
|
TurnOverride,
|
|
115
|
+
WorkflowRun,
|
|
107
116
|
} from "./types.ts";
|
|
108
117
|
import {
|
|
109
118
|
type KilledBy,
|
|
@@ -113,6 +122,7 @@ import {
|
|
|
113
122
|
type RunWorkerDeps,
|
|
114
123
|
ORPHAN_RESUME_PROMPT,
|
|
115
124
|
renderBrief,
|
|
125
|
+
renderReviewRevisionPrompt,
|
|
116
126
|
runWorker,
|
|
117
127
|
} from "./worker.ts";
|
|
118
128
|
import {
|
|
@@ -289,7 +299,9 @@ interface Deps {
|
|
|
289
299
|
/**
|
|
290
300
|
* Reads one active run's file lane for the admission file-lane interlock
|
|
291
301
|
* (#555): the union of its uncommitted worktree changes and its branch-vs-base
|
|
292
|
-
* diff.
|
|
302
|
+
* diff. Reconciliation is not occupancy: while the run merges its base, the
|
|
303
|
+
* probe reports only files it has actually diverged on, never the files the
|
|
304
|
+
* merge merely staged (#684). Wired by `runDaemon` to the mirror/worktree-backed
|
|
293
305
|
* {@link probeRunLane}; a test injects a fake. Absent, the interlock is inert
|
|
294
306
|
* (no lane is ever known occupied), which is the issue's "fail open": the
|
|
295
307
|
* gate adds holds, it never refuses a well-formed issue for lack of this
|
|
@@ -610,6 +622,135 @@ export function setPaused(
|
|
|
610
622
|
}
|
|
611
623
|
}
|
|
612
624
|
|
|
625
|
+
// ----------------------------------------------------------------- admission
|
|
626
|
+
// acknowledgement (#651, review #3)
|
|
627
|
+
//
|
|
628
|
+
// The setup-fence's "acknowledgement" must come from the daemon itself, not a
|
|
629
|
+
// second synchronous worker count: a tick that passed its pause gate before
|
|
630
|
+
// the fence landed can sit in awaited tracker/routing/admission work and claim
|
|
631
|
+
// after the count and before setup mutates. Two places make that impossible
|
|
632
|
+
// and record it durably:
|
|
633
|
+
//
|
|
634
|
+
// - the tick's pause gate re-acknowledges every held pass (the daemon has
|
|
635
|
+
// reached its admission boundary and claims nothing);
|
|
636
|
+
// - the claim itself re-checks the pause immediately before creating the run
|
|
637
|
+
// row, so a tick already past its gate when the fence landed refuses at
|
|
638
|
+
// the claim — and writes the acknowledgement — instead of admitting.
|
|
639
|
+
//
|
|
640
|
+
// The acknowledgement file names the exact pause instance observed and the
|
|
641
|
+
// daemon generation that observed it, so the setup barrier can prove the
|
|
642
|
+
// acknowledged fence is the fence it froze and the acknowledgement belongs to
|
|
643
|
+
// the daemon it began with.
|
|
644
|
+
|
|
645
|
+
/** The durable admission acknowledgement a daemon writes when it observes a
|
|
646
|
+
* pause fence at an admission boundary. */
|
|
647
|
+
export interface AdmissionAckRecord {
|
|
648
|
+
/** The pause instance observed (source token, reason, creation instant). */
|
|
649
|
+
pause: { source: string; reason?: string; since: number };
|
|
650
|
+
/** The daemon generation that observed it — {@link daemonGeneration}. */
|
|
651
|
+
daemon: string;
|
|
652
|
+
/** When the daemon observed the fence. */
|
|
653
|
+
observedAt: number;
|
|
654
|
+
}
|
|
655
|
+
|
|
656
|
+
/** The acknowledgement file path. One per host: the fence is host-global. */
|
|
657
|
+
export function admissionAckPath(): string {
|
|
658
|
+
return join(stateDir(), "admission-ack.json");
|
|
659
|
+
}
|
|
660
|
+
|
|
661
|
+
/**
|
|
662
|
+
* The generation identity of the running daemon, or `undefined` when nothing
|
|
663
|
+
* provably runs. The live pidfile record wins (its pid and boot instant
|
|
664
|
+
* identify the exact instance, #377); a record-less ACTIVE unit is still a
|
|
665
|
+
* running daemon and its MainPID is the generation that lets a fence spot a
|
|
666
|
+
* supervisor restart (#651 review #2). Shared by the ack writer and the setup
|
|
667
|
+
* barrier's identity so both sides of the acknowledgement name the same
|
|
668
|
+
* instance by the same rule.
|
|
669
|
+
*/
|
|
670
|
+
export function daemonGeneration(): string | undefined {
|
|
671
|
+
const daemon = livingDaemon();
|
|
672
|
+
if (daemon !== undefined) return `${daemon.pid}@${daemon.startedAt}`;
|
|
673
|
+
const ownership = probeUnit(SYSTEMD_UNIT);
|
|
674
|
+
return ownership.kind === "active" ? `systemd:${ownership.pid}` : undefined;
|
|
675
|
+
}
|
|
676
|
+
|
|
677
|
+
/** Reads the current admission acknowledgement, if one is readable. A corrupt
|
|
678
|
+
* or unreadable file is absence — the barrier fails closed on the absence. */
|
|
679
|
+
export function readAdmissionAck(): AdmissionAckRecord | undefined {
|
|
680
|
+
let parsed: unknown;
|
|
681
|
+
try {
|
|
682
|
+
parsed = JSON.parse(readFileSync(admissionAckPath(), "utf8"));
|
|
683
|
+
} catch {
|
|
684
|
+
return undefined;
|
|
685
|
+
}
|
|
686
|
+
if (parsed === null || typeof parsed !== "object") return undefined;
|
|
687
|
+
const r = parsed as Record<string, unknown>;
|
|
688
|
+
const pause = r["pause"];
|
|
689
|
+
const daemon = r["daemon"];
|
|
690
|
+
const observedAt = r["observedAt"];
|
|
691
|
+
if (pause === null || typeof pause !== "object") return undefined;
|
|
692
|
+
const p = pause as Record<string, unknown>;
|
|
693
|
+
const source = p["source"];
|
|
694
|
+
const since = p["since"];
|
|
695
|
+
const reason = p["reason"];
|
|
696
|
+
if (typeof source !== "string" || source.length === 0) return undefined;
|
|
697
|
+
if (typeof since !== "number" || !Number.isFinite(since)) return undefined;
|
|
698
|
+
if (reason !== undefined && typeof reason !== "string") return undefined;
|
|
699
|
+
if (typeof daemon !== "string" || daemon.length === 0) return undefined;
|
|
700
|
+
if (typeof observedAt !== "number" || !Number.isFinite(observedAt)) return undefined;
|
|
701
|
+
return { pause: { source, since, ...(reason === undefined ? {} : { reason }) }, daemon, observedAt };
|
|
702
|
+
}
|
|
703
|
+
|
|
704
|
+
/**
|
|
705
|
+
* Writes the admission acknowledgement for the pause the daemon just observed.
|
|
706
|
+
* The observed instance is the most restrictive fence in force: a host-global
|
|
707
|
+
* sentinel gates every project and is the pause a host-wide transaction (setup,
|
|
708
|
+
* an all-projects hold) froze, so it wins over a project-scoped sentinel;
|
|
709
|
+
* otherwise the project's own sentinel is the effective one for the daemon's
|
|
710
|
+
* claims, matching {@link pauseInstance}. Recording the project sentinel while
|
|
711
|
+
* a global fence is also in force would name a narrower, older hold instead of
|
|
712
|
+
* the fence that actually gates the host — and a barrier that froze the global
|
|
713
|
+
* sentinel could then never match its own acknowledgement (#651 review #4). A
|
|
714
|
+
* no-op when no pause is readable: there is nothing to acknowledge. The
|
|
715
|
+
* generation is the writing daemon's own (the record it owns, or its
|
|
716
|
+
* supervised MainPID).
|
|
717
|
+
*/
|
|
718
|
+
export function writeAdmissionAck(project: string): void {
|
|
719
|
+
// `pauseInstance()` reads only the global sentinel; while it is in force it
|
|
720
|
+
// is the fence that gates every project, and the durable record must name it
|
|
721
|
+
// rather than a per-project hold that predates it.
|
|
722
|
+
const pause = pauseInstance() ?? pauseInstance(project);
|
|
723
|
+
if (pause === undefined) return;
|
|
724
|
+
const generation = daemonGeneration();
|
|
725
|
+
if (generation === undefined) return;
|
|
726
|
+
const path = admissionAckPath();
|
|
727
|
+
mkdirSync(dirname(path), { recursive: true });
|
|
728
|
+
const record: AdmissionAckRecord = { pause, daemon: generation, observedAt: Date.now() };
|
|
729
|
+
const tmp = `${path}.${process.pid.toString(36)}.${Date.now().toString(36)}.tmp`;
|
|
730
|
+
try {
|
|
731
|
+
writeFileSync(tmp, `${JSON.stringify(record, null, 2)}\n`, { mode: 0o600 });
|
|
732
|
+
renameSync(tmp, path);
|
|
733
|
+
} catch (err) {
|
|
734
|
+
rmSync(tmp, { force: true });
|
|
735
|
+
throw err;
|
|
736
|
+
}
|
|
737
|
+
}
|
|
738
|
+
|
|
739
|
+
/** Wake the daemon's dispatch loop to prompt a pass (best-effort; used by the
|
|
740
|
+
* setup barrier so a live daemon acknowledges the fence without waiting for
|
|
741
|
+
* its next five-minute tick). A refused/failed wake just lengthens the wait;
|
|
742
|
+
* the barrier's deadline is what bounds it. */
|
|
743
|
+
export async function wakeDaemon(port: number): Promise<void> {
|
|
744
|
+
try {
|
|
745
|
+
await fetch(`http://127.0.0.1:${port}/wake`, {
|
|
746
|
+
method: "POST",
|
|
747
|
+
signal: AbortSignal.timeout(1_500),
|
|
748
|
+
});
|
|
749
|
+
} catch {
|
|
750
|
+
// Best-effort by contract: the caller's bounded wait is the backstop.
|
|
751
|
+
}
|
|
752
|
+
}
|
|
753
|
+
|
|
613
754
|
// ----------------------------------------------------------- package integrity
|
|
614
755
|
|
|
615
756
|
/** Enough differing paths to tell a deploy from a tamper at a glance; the full
|
|
@@ -775,16 +916,29 @@ const DISCUSSION_CHARS_BUDGET = 8_000;
|
|
|
775
916
|
* the #517 failure. An empty comment list renders nothing, so a commentless
|
|
776
917
|
* issue's brief stays byte-identical to what this package has always shipped.
|
|
777
918
|
*/
|
|
778
|
-
function renderDiscussion(comments: IssueComment[] | "unread"): string {
|
|
919
|
+
function renderDiscussion(comments: IssueComment[] | "unread", lane?: FileLane): string {
|
|
779
920
|
if (comments === "unread") {
|
|
780
|
-
|
|
921
|
+
const lines = [
|
|
781
922
|
"## Discussion",
|
|
782
923
|
"",
|
|
783
924
|
"_The issue's comments could not be read at dispatch time. The live read below is_",
|
|
784
925
|
"_the only path to them; if it prints nothing, that is a failed read, not an_",
|
|
785
926
|
"_absence of discussion._",
|
|
786
927
|
"",
|
|
787
|
-
]
|
|
928
|
+
];
|
|
929
|
+
// The lane admission enforced is carried through dispatch, so even an
|
|
930
|
+
// unreadable thread cannot hide it from the worker (#608): the gate's
|
|
931
|
+
// effective declaration renders from the admission snapshot, not from the
|
|
932
|
+
// read that just failed. A body declaration needs no note — the body
|
|
933
|
+
// always renders.
|
|
934
|
+
if (lane !== undefined && lane.at !== "body") {
|
|
935
|
+
lines.push(
|
|
936
|
+
"_The effective file lane below was enforced at admission — it supersedes any earlier declaration._",
|
|
937
|
+
lane.source,
|
|
938
|
+
"",
|
|
939
|
+
);
|
|
940
|
+
}
|
|
941
|
+
return lines.join("\n");
|
|
788
942
|
}
|
|
789
943
|
if (comments.length === 0) return "";
|
|
790
944
|
const total = comments.length;
|
|
@@ -812,9 +966,51 @@ function renderDiscussion(comments: IssueComment[] | "unread"): string {
|
|
|
812
966
|
"",
|
|
813
967
|
);
|
|
814
968
|
}
|
|
969
|
+
// The file-lane gate resolves its effective lane across the whole thread,
|
|
970
|
+
// not just what this budget renders, so a winning declaration beyond the
|
|
971
|
+
// budget would otherwise control admission while the worker never sees it
|
|
972
|
+
// (#608). Reproduce the winning declaration verbatim here — `at` records
|
|
973
|
+
// the comment it came from — so the rendered brief and the gate agree on
|
|
974
|
+
// the same lane, and the note itself re-parses through the same
|
|
975
|
+
// `File lane:` grammar. The note is suppressed only when the winning
|
|
976
|
+
// declaration is provably visible in the rendered thread (the body always
|
|
977
|
+
// renders, so a body winner needs none); a carried admission lane whose
|
|
978
|
+
// comment the dispatch re-read shifted or dropped still renders here,
|
|
979
|
+
// whether or not the thread was truncated.
|
|
980
|
+
if (
|
|
981
|
+
lane !== undefined &&
|
|
982
|
+
lane.at !== "body" &&
|
|
983
|
+
!comments.slice(0, shown).some((c) => c.body.includes(lane.source))
|
|
984
|
+
) {
|
|
985
|
+
lines.push(
|
|
986
|
+
`_The effective file lane was declared in comment ${lane.at + 1} — it supersedes any earlier declaration._`,
|
|
987
|
+
lane.source,
|
|
988
|
+
"",
|
|
989
|
+
);
|
|
990
|
+
}
|
|
815
991
|
return lines.join("\n");
|
|
816
992
|
}
|
|
817
993
|
|
|
994
|
+
/**
|
|
995
|
+
* What an orphan-resumed worker is told about the file lane on top of the
|
|
996
|
+
* continuation notice (#608). The original brief already in the transcript
|
|
997
|
+
* may show an earlier declaration, while admission enforces the current one —
|
|
998
|
+
* replaying the whole brief would re-do the work, but continuing under a stale
|
|
999
|
+
* lane is the collision the gate exists to stop. The declaration is rendered
|
|
1000
|
+
* verbatim, so it re-parses through the same `File lane:` grammar every
|
|
1001
|
+
* surfaced declaration uses.
|
|
1002
|
+
*/
|
|
1003
|
+
function resumeLaneBlock(lane: FileLane): string {
|
|
1004
|
+
return [
|
|
1005
|
+
"",
|
|
1006
|
+
"The file lane admission enforced for this continuation is below. It supersedes",
|
|
1007
|
+
"any lane declaration in the brief already in your transcript:",
|
|
1008
|
+
"",
|
|
1009
|
+
lane.source,
|
|
1010
|
+
"",
|
|
1011
|
+
].join("\n");
|
|
1012
|
+
}
|
|
1013
|
+
|
|
818
1014
|
/**
|
|
819
1015
|
* Record a state-label swap for projection (#201). The add enqueues before the
|
|
820
1016
|
* remove — the reverse order would leave a window where the issue carries no
|
|
@@ -1066,6 +1262,15 @@ export async function buildBrief(
|
|
|
1066
1262
|
* when the tracker refused at dispatch — rare, but it must never read as
|
|
1067
1263
|
* "no comments" (the #517 failure mode). Absent means an empty list. */
|
|
1068
1264
|
comments?: IssueComment[] | "unread";
|
|
1265
|
+
/**
|
|
1266
|
+
* The effective file lane admission resolved for this candidate (#608).
|
|
1267
|
+
* When carried, the brief renders exactly this value — the gate's own
|
|
1268
|
+
* snapshot — instead of recomputing from the dispatch-time comment read,
|
|
1269
|
+
* so a changed or failed second read can neither hide nor reword the
|
|
1270
|
+
* lane admission enforced. Absent (unit-level callers), the lane is
|
|
1271
|
+
* resolved from the rendered thread itself.
|
|
1272
|
+
*/
|
|
1273
|
+
lane?: FileLane;
|
|
1069
1274
|
} = {},
|
|
1070
1275
|
): Promise<string> {
|
|
1071
1276
|
// Read per dispatch rather than caching: editing the brief then takes effect
|
|
@@ -1103,6 +1308,13 @@ export async function buildBrief(
|
|
|
1103
1308
|
"",
|
|
1104
1309
|
].join("\n")
|
|
1105
1310
|
: "";
|
|
1311
|
+
// The comments as rendered, and the effective file lane the brief must show
|
|
1312
|
+
// — the admission-carried value when dispatch has one, else the lane the
|
|
1313
|
+
// thread itself resolves to — passed to the renderer so a winning
|
|
1314
|
+
// declaration can be reproduced verbatim even when it sits beyond the
|
|
1315
|
+
// discussion budget, keeping the gate and the worker-visible brief on one
|
|
1316
|
+
// lane (#608).
|
|
1317
|
+
const comments = opts.comments ?? [];
|
|
1106
1318
|
return renderBrief(template, {
|
|
1107
1319
|
ISSUE_NUMBER: String(r.issue.number),
|
|
1108
1320
|
ISSUE_TITLE: r.issue.title,
|
|
@@ -1111,8 +1323,16 @@ export async function buildBrief(
|
|
|
1111
1323
|
BRANCH: branch,
|
|
1112
1324
|
WORKTREE: worktree,
|
|
1113
1325
|
ACCEPTANCE_CRITERIA: acceptanceCriteria(r.issue),
|
|
1114
|
-
ISSUE_COMMENTS: renderDiscussion(
|
|
1326
|
+
ISSUE_COMMENTS: renderDiscussion(
|
|
1327
|
+
comments,
|
|
1328
|
+
opts.lane ?? (comments === "unread" ? undefined : effectiveLane(r.issue.body, comments)),
|
|
1329
|
+
),
|
|
1115
1330
|
GATES: gatesBlock(r.repo),
|
|
1331
|
+
// The guarded shell suites and their sanctioned parse-only check, derived
|
|
1332
|
+
// from SHARED_HOST_SCRIPTS so the brief and the refusal share one list
|
|
1333
|
+
// (#687). Non-empty whenever the guard has paths, so the placeholder line
|
|
1334
|
+
// always renders to a line.
|
|
1335
|
+
SHARED_HOST_NOTICE: sharedHostBriefNotice(),
|
|
1116
1336
|
// Empty for a repo with no `graphProject`, and empty means *nothing*: the
|
|
1117
1337
|
// placeholder sits flush against the next list item in the template, so an
|
|
1118
1338
|
// unconfigured render leaves no blank line where a hint would have gone.
|
|
@@ -1681,7 +1901,12 @@ function orphanResumeVerdict(
|
|
|
1681
1901
|
return { kind: "resume", prior };
|
|
1682
1902
|
}
|
|
1683
1903
|
|
|
1684
|
-
export async function handleIssue(
|
|
1904
|
+
export async function handleIssue(
|
|
1905
|
+
d: Deps,
|
|
1906
|
+
r: Routed,
|
|
1907
|
+
attempt: number,
|
|
1908
|
+
admittedLane?: FileLane,
|
|
1909
|
+
): Promise<void> {
|
|
1685
1910
|
const { project, caps, tracker, store } = d;
|
|
1686
1911
|
const issue = r.issue.number;
|
|
1687
1912
|
const branch = branchName(r.issue);
|
|
@@ -1824,6 +2049,24 @@ export async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<
|
|
|
1824
2049
|
return;
|
|
1825
2050
|
}
|
|
1826
2051
|
|
|
2052
|
+
// The claim-side of the pause fence (#651 review #3): the tick's pause
|
|
2053
|
+
// gate sits above routing, so a tick that passed that gate before an
|
|
2054
|
+
// operator wrote the freeze can still be mid-routing when the fence lands
|
|
2055
|
+
// — it would claim after the setup barrier's acknowledgement and before
|
|
2056
|
+
// its first mutation. The claim itself re-checks the pause and refuses,
|
|
2057
|
+
// writing the durable admission acknowledgement the barrier waits for.
|
|
2058
|
+
// A claim observed under the fence is the last admission boundary there
|
|
2059
|
+
// is: nothing may create a run row beside a held fleet.
|
|
2060
|
+
if (isPaused(d.project.name)) {
|
|
2061
|
+
try {
|
|
2062
|
+
writeAdmissionAck(d.project.name);
|
|
2063
|
+
} catch (err) {
|
|
2064
|
+
log(`admission acknowledgement write failed: ${errText(err)}`);
|
|
2065
|
+
}
|
|
2066
|
+
log(`#${issue} not claimed: dispatch is paused at claim time`);
|
|
2067
|
+
return;
|
|
2068
|
+
}
|
|
2069
|
+
|
|
1827
2070
|
// Claim on the STORE first, before anything that can fail. The run row —
|
|
1828
2071
|
// not the label — is the crash-safe guard against double dispatch: rows
|
|
1829
2072
|
// are local, written before any network call, and the startup orphan
|
|
@@ -2043,10 +2286,15 @@ export async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<
|
|
|
2043
2286
|
// The continuation notice replaces the brief for a resumed attempt (#536).
|
|
2044
2287
|
// The original brief is already in the resumed transcript; re-sending it is
|
|
2045
2288
|
// how a resumed worker ends up re-doing the work it just did. Everything
|
|
2046
|
-
// below the brief is fresh-dispatch-only, exactly as today
|
|
2289
|
+
// below the brief is fresh-dispatch-only, exactly as today — except the
|
|
2290
|
+
// file lane admission enforced for this continuation, which rides the
|
|
2291
|
+
// notice itself: the retained transcript's brief may show an earlier
|
|
2292
|
+
// declaration, and a worker continuing under a stale lane is the collision
|
|
2293
|
+
// the gate exists to stop (#608).
|
|
2047
2294
|
let brief: string;
|
|
2048
2295
|
if (resuming !== undefined) {
|
|
2049
|
-
brief =
|
|
2296
|
+
brief =
|
|
2297
|
+
ORPHAN_RESUME_PROMPT + (admittedLane === undefined ? "" : resumeLaneBlock(admittedLane));
|
|
2050
2298
|
} else {
|
|
2051
2299
|
// The discussion is rendered at dispatch so a worker never depends on a
|
|
2052
2300
|
// runtime `gh` read to see the orchestrator's grooming (#517). The read is
|
|
@@ -2068,6 +2316,7 @@ export async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<
|
|
|
2068
2316
|
? { salvagedSha: priorSalvage }
|
|
2069
2317
|
: {}),
|
|
2070
2318
|
comments,
|
|
2319
|
+
lane: admittedLane,
|
|
2071
2320
|
});
|
|
2072
2321
|
}
|
|
2073
2322
|
if (await settleStopBeforeSession()) return;
|
|
@@ -2535,6 +2784,749 @@ export async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<
|
|
|
2535
2784
|
}
|
|
2536
2785
|
}
|
|
2537
2786
|
|
|
2787
|
+
// ------------------------------------------------------------ review revisions
|
|
2788
|
+
|
|
2789
|
+
/**
|
|
2790
|
+
* The daemon half of the review-revision transport (#677): wake every pending
|
|
2791
|
+
* revision the orchestrator's verb recorded, on the tick, exactly like
|
|
2792
|
+
* admission — the verb has no handle on the worker machinery, and the CLI path
|
|
2793
|
+
* must behave like the embedded one. Bounded by the same worker slots, so a
|
|
2794
|
+
* queued revision can never push the fleet past `maxConcurrentWorkers`; the
|
|
2795
|
+
* remaining rows stay pending for the next tick.
|
|
2796
|
+
*
|
|
2797
|
+
* The claim (`pushed-green` → `running`) is synchronous and atomic per row;
|
|
2798
|
+
* only the resumed worker runs in the pool. A row that moved since the verb
|
|
2799
|
+
* recorded it — merged after all, settled, re-claimed — fails the claim and is
|
|
2800
|
+
* settled `skipped` rather than woken on stale identity.
|
|
2801
|
+
*/
|
|
2802
|
+
export async function dispatchReviewRevisions(d: Deps, pool?: WorkerPool): Promise<void> {
|
|
2803
|
+
const pending = d.store.pendingReviewRevisions(d.project.name);
|
|
2804
|
+
if (pending.length === 0) return;
|
|
2805
|
+
const slots = Math.max(0, d.caps.maxConcurrentWorkers - d.store.liveRuns(d.project.name).length);
|
|
2806
|
+
const batch = pending.slice(0, slots);
|
|
2807
|
+
const launches: Promise<void>[] = [];
|
|
2808
|
+
for (const revision of batch) {
|
|
2809
|
+
if (d.drain?.draining === true) {
|
|
2810
|
+
log(
|
|
2811
|
+
`review revisions held: the daemon is draining (${batch.length - launches.length} of ${batch.length} not launched)`,
|
|
2812
|
+
);
|
|
2813
|
+
break;
|
|
2814
|
+
}
|
|
2815
|
+
if (!d.store.claimRunForReview(revision.runId)) {
|
|
2816
|
+
d.store.settleReviewRevision(revision.id, "skipped", Date.now());
|
|
2817
|
+
log(
|
|
2818
|
+
`#${revision.issue} review round ${revision.round} skipped: run ${revision.runId} is no longer pushed-green`,
|
|
2819
|
+
);
|
|
2820
|
+
continue;
|
|
2821
|
+
}
|
|
2822
|
+
d.store.markReviewRevisionDispatched(revision.id, Date.now());
|
|
2823
|
+
log(`#${revision.issue} review round ${revision.round} → resuming run ${revision.runId}`);
|
|
2824
|
+
launches.push(handleReviewRevision(d, revision));
|
|
2825
|
+
}
|
|
2826
|
+
if (pool !== undefined) {
|
|
2827
|
+
for (const launch of launches) pool.launch(launch);
|
|
2828
|
+
return;
|
|
2829
|
+
}
|
|
2830
|
+
// `--once`: like admissions, the tick waits for the workers it launched.
|
|
2831
|
+
await Promise.allSettled(launches);
|
|
2832
|
+
}
|
|
2833
|
+
|
|
2834
|
+
/**
|
|
2835
|
+
* Resume one review-revision worker: the run row is already claimed
|
|
2836
|
+
* (`pushed-green` → `running` by the dispatch pass), so this resumes the SAME
|
|
2837
|
+
* OMP session — same session directory, `resume: true` at the harness, and the
|
|
2838
|
+
* `sessionFile` lineage compared at dispatch time — on the SAME branch and PR,
|
|
2839
|
+
* then settles the row back the way {@link handleIssue} settles any worker.
|
|
2840
|
+
*
|
|
2841
|
+
* The run row is reused, never cloned: same attempt number, same prUrl, same
|
|
2842
|
+
* transcript. That is what keeps a revision round off both budgets — the round
|
|
2843
|
+
* is a state transition of one already-green row, green → running → green, and
|
|
2844
|
+
* no new row exists for either counter to count. Only a revision worker that
|
|
2845
|
+
* genuinely settles terminal falls through to the ordinary failure handling,
|
|
2846
|
+
* budgets and all, exactly as today.
|
|
2847
|
+
*/
|
|
2848
|
+
export async function handleReviewRevision(d: Deps, revision: ReviewRevisionRecord): Promise<void> {
|
|
2849
|
+
const { project, caps, tracker, store } = d;
|
|
2850
|
+
const run = store.getRun(revision.runId);
|
|
2851
|
+
if (run === undefined || run.state !== "running") {
|
|
2852
|
+
log(
|
|
2853
|
+
`#${revision.issue} review round ${revision.round} skipped: run ${revision.runId} is ${run?.state ?? "gone"} — ` +
|
|
2854
|
+
"the row moved between the dispatch claim and the resume.",
|
|
2855
|
+
);
|
|
2856
|
+
store.settleReviewRevision(revision.id, "skipped", Date.now());
|
|
2857
|
+
return;
|
|
2858
|
+
}
|
|
2859
|
+
const issue = run.issue;
|
|
2860
|
+
const branch = run.branch;
|
|
2861
|
+
const runId = run.id;
|
|
2862
|
+
const inProgress = project.stateLabels.inProgress;
|
|
2863
|
+
const repo = Object.values(project.routing.repos).find((candidate) => candidate.name === run.repo);
|
|
2864
|
+
if (repo === undefined) {
|
|
2865
|
+
log(`#${issue} review round ${revision.round} cannot dispatch: repo ${run.repo} is no longer routed`);
|
|
2866
|
+
store.updateRun(runId, { state: "pushed-green", endedAt: Date.now() });
|
|
2867
|
+
store.settleReviewRevision(revision.id, "skipped", Date.now());
|
|
2868
|
+
return;
|
|
2869
|
+
}
|
|
2870
|
+
const mirrorPath = mirrorPathFor(repo, project.mirrorRoot);
|
|
2871
|
+
|
|
2872
|
+
// The session being resumed is the one the pushed-green row recorded. A
|
|
2873
|
+
// revision without that transcript cannot continue the same session — the
|
|
2874
|
+
// exact fake this feature exists to prevent — so fail loudly instead of
|
|
2875
|
+
// silently starting a fresh one.
|
|
2876
|
+
const priorSessionFile = run.sessionFile;
|
|
2877
|
+
if (priorSessionFile === undefined || !existsSync(priorSessionFile)) {
|
|
2878
|
+
log(
|
|
2879
|
+
`#${issue} review round ${revision.round} cannot resume: the recorded transcript ` +
|
|
2880
|
+
`${priorSessionFile ?? "(none)"} is gone`,
|
|
2881
|
+
);
|
|
2882
|
+
store.updateRun(runId, {
|
|
2883
|
+
state: "pushed-green",
|
|
2884
|
+
endedAt: Date.now(),
|
|
2885
|
+
lastError: `review round ${revision.round} could not resume: the run's transcript is gone`,
|
|
2886
|
+
});
|
|
2887
|
+
store.settleReviewRevision(revision.id, "skipped", Date.now());
|
|
2888
|
+
await safeEscalate(d, {
|
|
2889
|
+
tier: 1,
|
|
2890
|
+
project: project.name,
|
|
2891
|
+
issue,
|
|
2892
|
+
runId,
|
|
2893
|
+
summary: `#${issue} review round ${revision.round} could not resume — the transcript is missing`,
|
|
2894
|
+
detail: [
|
|
2895
|
+
revision.prUrl,
|
|
2896
|
+
"",
|
|
2897
|
+
"The PR is still open and green; the run row was returned to pushed-green.",
|
|
2898
|
+
"Resume the manual path (close + unblock + continuation) or merge it as it stands.",
|
|
2899
|
+
].join("\n"),
|
|
2900
|
+
});
|
|
2901
|
+
return;
|
|
2902
|
+
}
|
|
2903
|
+
const sessionDir = dirname(priorSessionFile);
|
|
2904
|
+
|
|
2905
|
+
let worktreePath: string | undefined;
|
|
2906
|
+
let runRepo: RunRepoRef | undefined;
|
|
2907
|
+
let turnLimit: TurnLimitController | undefined;
|
|
2908
|
+
let workerControl: WorkerControlSlot | undefined;
|
|
2909
|
+
let workerSessionInstalled = false;
|
|
2910
|
+
let verbListener: VerbListener | undefined;
|
|
2911
|
+
const repoSlug = githubRepo(repo.cloneUrl);
|
|
2912
|
+
|
|
2913
|
+
/**
|
|
2914
|
+
* Publishes the run's branch on the privileged side: run repo → mirror →
|
|
2915
|
+
* GitHub, fast-forward only — the same route `handleIssue` uses, so the
|
|
2916
|
+
* revised head the worker pushed through its own `conductor_push` is
|
|
2917
|
+
* re-verified against the remote before the tree dies.
|
|
2918
|
+
*/
|
|
2919
|
+
const publish: RunPublisher = async () => {
|
|
2920
|
+
if (runRepo === undefined) return { ok: false, stderr: "the run repository was never provisioned" };
|
|
2921
|
+
return pushRunBranch(project, runRepo);
|
|
2922
|
+
};
|
|
2923
|
+
|
|
2924
|
+
const settleStopBeforeSession = async (): Promise<boolean> => {
|
|
2925
|
+
const reason = workerControl?.requestedStop();
|
|
2926
|
+
if (reason === undefined || workerSessionInstalled) return false;
|
|
2927
|
+
turnLimit?.close();
|
|
2928
|
+
turnLimit = undefined;
|
|
2929
|
+
recordOperatorStop(store, {
|
|
2930
|
+
project: project.name,
|
|
2931
|
+
issue,
|
|
2932
|
+
runId,
|
|
2933
|
+
inProgress,
|
|
2934
|
+
reason,
|
|
2935
|
+
patch: {
|
|
2936
|
+
endedAt: Date.now(),
|
|
2937
|
+
report: ["Operator stopped the review revision before its session started.", `Reason: ${reason}`].join("\n"),
|
|
2938
|
+
},
|
|
2939
|
+
});
|
|
2940
|
+
store.settleReviewRevision(revision.id, "skipped", Date.now());
|
|
2941
|
+
log(`#${issue} review round ${revision.round} stopped by operator before its session started: ${reason}`);
|
|
2942
|
+
return true;
|
|
2943
|
+
};
|
|
2944
|
+
|
|
2945
|
+
const settleDrainBeforeSession = async (): Promise<boolean> => {
|
|
2946
|
+
if (d.drain?.draining !== true || workerSessionInstalled) return false;
|
|
2947
|
+
turnLimit?.close();
|
|
2948
|
+
turnLimit = undefined;
|
|
2949
|
+
// A shutdown that interrupts the wake is the same situation as the
|
|
2950
|
+
// provisioning-failure branch below, not an operator stop — nobody
|
|
2951
|
+
// stopped anything, the PR is still green and still open, and only the
|
|
2952
|
+
// wake failed; `settleStopBeforeSession` immediately above covers the
|
|
2953
|
+
// case where an operator genuinely did. So mirror that branch: restore
|
|
2954
|
+
// `pushed-green` naming the shutdown, settle the revision `skipped` so
|
|
2955
|
+
// the round is not double-counted, and leave the in-progress label alone
|
|
2956
|
+
// — the ordinary settle sweep releases it in the same breath it
|
|
2957
|
+
// terminalises the row when the PR resolves. Stopping the row here would
|
|
2958
|
+
// strand it: `settlePushedGreen` sweeps only pushed-* rows and
|
|
2959
|
+
// classification excludes `stopped`, so the issue would carry neither
|
|
2960
|
+
// label and nothing would ever revisit the still-green PR.
|
|
2961
|
+
store.updateRun(runId, {
|
|
2962
|
+
state: "pushed-green",
|
|
2963
|
+
endedAt: Date.now(),
|
|
2964
|
+
lastError: "daemon shutdown began after the review revision claim; the round was not launched",
|
|
2965
|
+
});
|
|
2966
|
+
store.settleReviewRevision(revision.id, "skipped", Date.now());
|
|
2967
|
+
log(`#${issue} review round ${revision.round} not launched: daemon shutdown began after the claim`);
|
|
2968
|
+
return true;
|
|
2969
|
+
};
|
|
2970
|
+
|
|
2971
|
+
// Live-run controls, opened before any provisioning so the operator stop
|
|
2972
|
+
// and shutdown fences cover the whole wake window, exactly as they do for a
|
|
2973
|
+
// fresh claim in `handleIssue` (#374). The original run's entries were
|
|
2974
|
+
// closed at its settle, so reopening by issue is safe.
|
|
2975
|
+
turnLimit = d.turnLimits.open(project.name, issue, runId, run.maxTurns);
|
|
2976
|
+
workerControl = d.workerControls.open(project.name, issue, runId);
|
|
2977
|
+
if (await settleStopBeforeSession()) return;
|
|
2978
|
+
if (await settleDrainBeforeSession()) return;
|
|
2979
|
+
|
|
2980
|
+
try {
|
|
2981
|
+
// Reattach the run's own branch at the same per-issue path the run used:
|
|
2982
|
+
// the worktree was removed at the pushed-green settle and the branch was
|
|
2983
|
+
// published, so provisioning is the same continuation reattach as a
|
|
2984
|
+
// normal re-claim. A provisioning failure restores the row — the PR is
|
|
2985
|
+
// still green and still open, and only the wake failed — and says so.
|
|
2986
|
+
try {
|
|
2987
|
+
const provisioned = await addRunRepo(repo, project.mirrorRoot, project.workspaceRoot, issue, branch);
|
|
2988
|
+
worktreePath = provisioned.path;
|
|
2989
|
+
runRepo = { repo, runRepoPath: worktreePath, branch };
|
|
2990
|
+
} catch (err) {
|
|
2991
|
+
const detail = errText(err);
|
|
2992
|
+
log(`#${issue} review round ${revision.round} cannot provision ${branch}: ${detail}`);
|
|
2993
|
+
// An operator stop that landed during provisioning wins over restoration —
|
|
2994
|
+
// it is a newer, explicit action, and the fences above already honoured
|
|
2995
|
+
// one that landed earlier.
|
|
2996
|
+
const pendingStop = workerControl?.requestedStop();
|
|
2997
|
+
if (pendingStop === undefined) {
|
|
2998
|
+
store.updateRun(runId, {
|
|
2999
|
+
state: "pushed-green",
|
|
3000
|
+
endedAt: Date.now(),
|
|
3001
|
+
lastError: `review round ${revision.round} could not provision the worktree: ${detail}`,
|
|
3002
|
+
});
|
|
3003
|
+
store.settleReviewRevision(revision.id, "skipped", Date.now());
|
|
3004
|
+
await safeEscalate(d, {
|
|
3005
|
+
tier: 1,
|
|
3006
|
+
project: project.name,
|
|
3007
|
+
issue,
|
|
3008
|
+
runId,
|
|
3009
|
+
summary: `#${issue} review round ${revision.round} could not be dispatched — worktree provisioning failed`,
|
|
3010
|
+
detail: [revision.prUrl, "", detail].join("\n"),
|
|
3011
|
+
});
|
|
3012
|
+
} else {
|
|
3013
|
+
recordOperatorStop(store, {
|
|
3014
|
+
project: project.name,
|
|
3015
|
+
issue,
|
|
3016
|
+
runId,
|
|
3017
|
+
inProgress,
|
|
3018
|
+
reason: pendingStop,
|
|
3019
|
+
patch: {
|
|
3020
|
+
endedAt: Date.now(),
|
|
3021
|
+
report: [
|
|
3022
|
+
`Operator stopped the review revision before its session started: ${pendingStop}`,
|
|
3023
|
+
`(worktree provisioning also failed: ${detail})`,
|
|
3024
|
+
].join("\n"),
|
|
3025
|
+
},
|
|
3026
|
+
});
|
|
3027
|
+
store.settleReviewRevision(revision.id, "skipped", Date.now());
|
|
3028
|
+
}
|
|
3029
|
+
return;
|
|
3030
|
+
}
|
|
3031
|
+
if (await settleStopBeforeSession()) return;
|
|
3032
|
+
if (await settleDrainBeforeSession()) return;
|
|
3033
|
+
|
|
3034
|
+
// The fleet-owned omp settings overlay, staged under the SAME session
|
|
3035
|
+
// directory the transcript lives in — a resumed session loads it exactly
|
|
3036
|
+
// as the original run did (#537).
|
|
3037
|
+
const ompSettingsFile = materializeOmpSettings(project, sessionDir);
|
|
3038
|
+
|
|
3039
|
+
// The run's mutation channel (#126): same run id, same issue, same repo —
|
|
3040
|
+
// so `conductor_push` publishes the run's own branch and nothing else.
|
|
3041
|
+
verbListener = await listenVerbChannel(
|
|
3042
|
+
verbDeps(d),
|
|
3043
|
+
{
|
|
3044
|
+
kind: "run",
|
|
3045
|
+
path: verbSocketPath(ensureVerbSocketDir(stateDir()), `run-${String(issue)}`),
|
|
3046
|
+
project: project.name,
|
|
3047
|
+
role: "worker",
|
|
3048
|
+
runId,
|
|
3049
|
+
issue,
|
|
3050
|
+
repo,
|
|
3051
|
+
runRepoPath: worktreePath,
|
|
3052
|
+
branch,
|
|
3053
|
+
},
|
|
3054
|
+
{ ...(d.verbPeerReader === undefined ? {} : { peerReader: d.verbPeerReader }) },
|
|
3055
|
+
);
|
|
3056
|
+
if (await settleStopBeforeSession()) return;
|
|
3057
|
+
if (await settleDrainBeforeSession()) return;
|
|
3058
|
+
|
|
3059
|
+
store.updateRun(runId, { worktree: worktreePath });
|
|
3060
|
+
|
|
3061
|
+
// The row's own counters are cumulative across the revision: the revision
|
|
3062
|
+
// worker meters its own session from zero, so its deltas are added to the
|
|
3063
|
+
// totals the run already recorded.
|
|
3064
|
+
const baseTurns = run.turns;
|
|
3065
|
+
const baseSpend = run.spendUsd;
|
|
3066
|
+
|
|
3067
|
+
log(`#${issue} review round ${revision.round} → resuming ${branch} (${priorSessionFile})`);
|
|
3068
|
+
|
|
3069
|
+
let result: WorkerResult;
|
|
3070
|
+
try {
|
|
3071
|
+
result = await runWorker({
|
|
3072
|
+
brief: renderReviewRevisionPrompt(revision.findings, revision.round),
|
|
3073
|
+
cwd: worktreePath,
|
|
3074
|
+
caps,
|
|
3075
|
+
...(repoSlug === undefined ? {} : { repoSlug }),
|
|
3076
|
+
maxTurns: () => turnLimit?.maxTurns() ?? run.maxTurns,
|
|
3077
|
+
onPauseControl: (control) => {
|
|
3078
|
+
workerSessionInstalled = true;
|
|
3079
|
+
workerControl?.install(control);
|
|
3080
|
+
},
|
|
3081
|
+
sessionDir,
|
|
3082
|
+
// The same-session half of the transport: `continueRecent` on the run's
|
|
3083
|
+
// own session directory, never a fresh `run-<uuid>` transcript.
|
|
3084
|
+
resume: true,
|
|
3085
|
+
socketPath: join(sessionDir, "ipc.sock"),
|
|
3086
|
+
verbSocketPath: verbListener.path,
|
|
3087
|
+
onSpawn: (pid) => {
|
|
3088
|
+
verbListener?.bindPid(pid);
|
|
3089
|
+
},
|
|
3090
|
+
onChildLog: (line) => {
|
|
3091
|
+
log(`#${issue} ${line}`);
|
|
3092
|
+
},
|
|
3093
|
+
// The continuation stays on the model the green run used (#286).
|
|
3094
|
+
...(run.model === undefined ? {} : { model: run.model }),
|
|
3095
|
+
...(ompSettingsFile === undefined ? {} : { ompSettingsFile }),
|
|
3096
|
+
releaseGrants: resolveReleaseGrants(project),
|
|
3097
|
+
onReleaseBlocked: workerReleaseBlockRecorder(project.name, issue, runId),
|
|
3098
|
+
onTurn: (n) => store.updateRun(runId, { turns: baseTurns + n }),
|
|
3099
|
+
onSpend: (usd) => store.updateRun(runId, { spendUsd: baseSpend + usd }),
|
|
3100
|
+
onKilled: () => {
|
|
3101
|
+
turnLimit?.close();
|
|
3102
|
+
turnLimit = undefined;
|
|
3103
|
+
},
|
|
3104
|
+
onSessionFile: (f) => {
|
|
3105
|
+
store.updateRun(runId, { sessionFile: f });
|
|
3106
|
+
// Same-file lineage is the proof the resume happened: a revision
|
|
3107
|
+
// that opened a different transcript fell back to a fresh session.
|
|
3108
|
+
if (f !== priorSessionFile) {
|
|
3109
|
+
log(
|
|
3110
|
+
`#${issue} review round ${revision.round} resume fell back to a fresh session: opened ${f} ` +
|
|
3111
|
+
`instead of ${priorSessionFile}`,
|
|
3112
|
+
);
|
|
3113
|
+
}
|
|
3114
|
+
},
|
|
3115
|
+
maySpawn: () => d.drain?.draining !== true,
|
|
3116
|
+
}, d.workerDeps);
|
|
3117
|
+
} finally {
|
|
3118
|
+
turnLimit?.close();
|
|
3119
|
+
turnLimit = undefined;
|
|
3120
|
+
}
|
|
3121
|
+
|
|
3122
|
+
if (result.modelFallbackMessage !== undefined) {
|
|
3123
|
+
log(`#${issue} review round ${revision.round} model fallback: ${result.modelFallbackMessage}`);
|
|
3124
|
+
}
|
|
3125
|
+
|
|
3126
|
+
const verified: { state: RunState; reason?: string } =
|
|
3127
|
+
result.state === "pushed-green"
|
|
3128
|
+
? await verifyPushedGreenClaim(tracker, result)
|
|
3129
|
+
: { state: result.state };
|
|
3130
|
+
const state = verified.state;
|
|
3131
|
+
|
|
3132
|
+
const settlement =
|
|
3133
|
+
state === "pushed-green" || state === "pushed-pending" || state === "merged"
|
|
3134
|
+
? undefined
|
|
3135
|
+
: await settleWorktree({
|
|
3136
|
+
issue,
|
|
3137
|
+
attempt: run.attempt,
|
|
3138
|
+
ending:
|
|
3139
|
+
state === "blocked"
|
|
3140
|
+
? "blocked for an operator decision"
|
|
3141
|
+
: state === "stopped"
|
|
3142
|
+
? `stopped by the operator: ${result.stoppedReason ?? "no reason recorded"}`
|
|
3143
|
+
: endedBy(result.killedBy),
|
|
3144
|
+
worktree: worktreePath,
|
|
3145
|
+
branch,
|
|
3146
|
+
publish,
|
|
3147
|
+
...(state === "failed" || state === "killed"
|
|
3148
|
+
? ({ tree: "keep" } as const)
|
|
3149
|
+
: ({ tree: "remove", mirrorPath } as const)),
|
|
3150
|
+
});
|
|
3151
|
+
if (settlement === undefined) {
|
|
3152
|
+
const published = await publish(branch);
|
|
3153
|
+
if (!published.ok) {
|
|
3154
|
+
log(`#${issue} publish before removal failed, work is preserved in the mirror: ${published.stderr}`);
|
|
3155
|
+
}
|
|
3156
|
+
await removeWorktree(mirrorPath, worktreePath);
|
|
3157
|
+
}
|
|
3158
|
+
|
|
3159
|
+
// The settlement report names the review round first (#692): a run that
|
|
3160
|
+
// reaches green again after a round must read as a revision outcome, not
|
|
3161
|
+
// as a fresh run's first push, and a round that failed must still say the
|
|
3162
|
+
// round — the state and classification below carry the failure itself.
|
|
3163
|
+
const finalReport = [
|
|
3164
|
+
`review round ${revision.round}: ${state}`,
|
|
3165
|
+
...(verified.reason === undefined ? [] : ["", verified.reason]),
|
|
3166
|
+
result.report,
|
|
3167
|
+
].join("\n");
|
|
3168
|
+
|
|
3169
|
+
const terminalPatch: Partial<RunRecord> = {
|
|
3170
|
+
endedAt: Date.now(),
|
|
3171
|
+
turns: baseTurns + result.turns,
|
|
3172
|
+
spendUsd: baseSpend + result.spendUsd,
|
|
3173
|
+
provider429Count: result.provider429Count,
|
|
3174
|
+
...(result.model === undefined ? {} : { resolvedModel: result.model }),
|
|
3175
|
+
...(result.provider === undefined ? {} : { resolvedProvider: result.provider }),
|
|
3176
|
+
retryFallbacks: result.retryFallbacks,
|
|
3177
|
+
retryFallbackSucceeded: result.retryFallbackSucceeded,
|
|
3178
|
+
modelRecoveries: result.modelRecoveries,
|
|
3179
|
+
autoRetryCount: result.autoRetryCount,
|
|
3180
|
+
autoCompactionCount: result.autoCompactionCount,
|
|
3181
|
+
// The worker only reports these when it actually established them; a
|
|
3182
|
+
// failure whose report named no PR must not wipe what the row already
|
|
3183
|
+
// owns (#468).
|
|
3184
|
+
...(result.prUrl === undefined ? {} : { prUrl: result.prUrl }),
|
|
3185
|
+
...(result.headSha === undefined ? {} : { headSha: result.headSha }),
|
|
3186
|
+
sessionFile: result.sessionFile,
|
|
3187
|
+
report: finalReport,
|
|
3188
|
+
...settlement?.patch,
|
|
3189
|
+
};
|
|
3190
|
+
if (state === "stopped") {
|
|
3191
|
+
recordOperatorStop(store, {
|
|
3192
|
+
project: project.name,
|
|
3193
|
+
issue,
|
|
3194
|
+
runId,
|
|
3195
|
+
inProgress,
|
|
3196
|
+
reason: result.stoppedReason ?? "no reason recorded",
|
|
3197
|
+
patch: terminalPatch,
|
|
3198
|
+
});
|
|
3199
|
+
} else {
|
|
3200
|
+
const sessionErr = state === "failed" || state === "killed" ? readSessionError(result.sessionFile) : undefined;
|
|
3201
|
+
const providerCredit = sessionErr === undefined ? undefined : providerCreditRefusal(sessionErr);
|
|
3202
|
+
const providerTransient =
|
|
3203
|
+
providerCredit !== undefined || sessionErr === undefined ? undefined : providerTransientFault(sessionErr);
|
|
3204
|
+
const lastError = completionLastError(providerCredit, providerTransient, verified.reason, sessionErr);
|
|
3205
|
+
store.updateRun(runId, {
|
|
3206
|
+
...terminalPatch,
|
|
3207
|
+
state,
|
|
3208
|
+
...(lastError === undefined ? {} : { lastError }),
|
|
3209
|
+
});
|
|
3210
|
+
}
|
|
3211
|
+
|
|
3212
|
+
const outcome: ReviewRevisionOutcome =
|
|
3213
|
+
state === "pushed-green" ? "revised" : state === "pushed-pending" ? "pending" : "failed";
|
|
3214
|
+
store.settleReviewRevision(revision.id, outcome, Date.now());
|
|
3215
|
+
log(`#${issue} review round ${revision.round} ${state}${result.prUrl ? ` ${result.prUrl}` : ""}`);
|
|
3216
|
+
|
|
3217
|
+
// Post-settle outcomes, mirroring handleIssue: a failed or killed revision
|
|
3218
|
+
// worker falls through to exactly the same label, requeue and escalation
|
|
3219
|
+
// handling as any failed run — budgets included — and a blocked one pages
|
|
3220
|
+
// the orchestrator for a decision.
|
|
3221
|
+
if (state === "stopped") {
|
|
3222
|
+
log(`#${issue} review round ${revision.round} stopped by operator: ${result.stoppedReason}`);
|
|
3223
|
+
} else if (state === "blocked") {
|
|
3224
|
+
swapLabel(store, project.name, issue, inProgress, project.stateLabels.blocked);
|
|
3225
|
+
await safeEscalate(d, {
|
|
3226
|
+
tier: 1,
|
|
3227
|
+
project: project.name,
|
|
3228
|
+
issue,
|
|
3229
|
+
runId,
|
|
3230
|
+
summary: `#${issue} is blocked on review round ${revision.round} and needs a decision`,
|
|
3231
|
+
detail: [`${result.prUrl ?? "(no PR URL)"}`, "", result.report].join("\n"),
|
|
3232
|
+
});
|
|
3233
|
+
} else if (state === "failed" || state === "killed") {
|
|
3234
|
+
const continuation = store.continuationsFor(project.name, issue);
|
|
3235
|
+
const continueTurns = shouldContinueAfterTurnsCap({
|
|
3236
|
+
killedBy: result.killedBy,
|
|
3237
|
+
prUrl: result.prUrl,
|
|
3238
|
+
headSha: result.headSha,
|
|
3239
|
+
salvageSha: settlement?.patch?.salvageSha,
|
|
3240
|
+
continuation,
|
|
3241
|
+
maxContinuations: caps.maxContinuationsPerIssue,
|
|
3242
|
+
});
|
|
3243
|
+
if (continueTurns) {
|
|
3244
|
+
store.enqueueLabelOps(project.name, [
|
|
3245
|
+
{ issue, op: "remove", label: inProgress },
|
|
3246
|
+
{ issue, op: "add", label: project.queueLabel },
|
|
3247
|
+
]);
|
|
3248
|
+
log(
|
|
3249
|
+
`#${issue} review round ${revision.round} turns-cap, continuation ` +
|
|
3250
|
+
`${continuation}/${caps.maxContinuationsPerIssue} — salvaged and re-queued`,
|
|
3251
|
+
);
|
|
3252
|
+
await safeEscalate(d, {
|
|
3253
|
+
tier: 1,
|
|
3254
|
+
project: project.name,
|
|
3255
|
+
issue,
|
|
3256
|
+
runId,
|
|
3257
|
+
summary: `#${issue} hit the turns cap on review round ${revision.round} — auto-requeued for continuation`,
|
|
3258
|
+
detail: [
|
|
3259
|
+
`${result.prUrl ?? "(no PR URL)"}`,
|
|
3260
|
+
"",
|
|
3261
|
+
"The queue label is back on; the next tick should reattach the branch",
|
|
3262
|
+
"and open a continuation brief. No failed label was applied.",
|
|
3263
|
+
"",
|
|
3264
|
+
result.report,
|
|
3265
|
+
].join("\n"),
|
|
3266
|
+
});
|
|
3267
|
+
} else {
|
|
3268
|
+
swapLabel(store, project.name, issue, inProgress, project.stateLabels.failed);
|
|
3269
|
+
await safeEscalate(d, {
|
|
3270
|
+
tier: 1,
|
|
3271
|
+
project: project.name,
|
|
3272
|
+
issue,
|
|
3273
|
+
runId,
|
|
3274
|
+
summary: result.killedBy
|
|
3275
|
+
? `#${issue} was killed on review round ${revision.round} by the ${result.killedBy} cap`
|
|
3276
|
+
: `#${issue} failed on review round ${revision.round}`,
|
|
3277
|
+
detail: [`${result.prUrl ?? "(no PR URL)"}`, "", result.report].join("\n"),
|
|
3278
|
+
});
|
|
3279
|
+
}
|
|
3280
|
+
}
|
|
3281
|
+
} catch (err) {
|
|
3282
|
+
// A crash anywhere after the worker ran settles like any run's dispatch
|
|
3283
|
+
// error: the row is terminal, the tree is kept, and the issue is relabelled
|
|
3284
|
+
// so nobody silently re-claims it.
|
|
3285
|
+
turnLimit?.close();
|
|
3286
|
+
turnLimit = undefined;
|
|
3287
|
+
const detail = errText(err);
|
|
3288
|
+
log(`#${issue} review round ${revision.round} errored: ${detail}`);
|
|
3289
|
+
const settlement =
|
|
3290
|
+
worktreePath === undefined
|
|
3291
|
+
? undefined
|
|
3292
|
+
: await settleWorktree({
|
|
3293
|
+
issue,
|
|
3294
|
+
attempt: run.attempt,
|
|
3295
|
+
ending: "killed by a dispatch error",
|
|
3296
|
+
worktree: worktreePath,
|
|
3297
|
+
branch,
|
|
3298
|
+
publish,
|
|
3299
|
+
tree: "keep",
|
|
3300
|
+
});
|
|
3301
|
+
store.updateRun(runId, {
|
|
3302
|
+
state: "failed",
|
|
3303
|
+
endedAt: Date.now(),
|
|
3304
|
+
lastError: detail,
|
|
3305
|
+
...settlement?.patch,
|
|
3306
|
+
});
|
|
3307
|
+
store.settleReviewRevision(revision.id, "failed", Date.now());
|
|
3308
|
+
swapLabel(store, project.name, issue, inProgress, project.stateLabels.failed);
|
|
3309
|
+
await safeEscalate(d, {
|
|
3310
|
+
tier: 1,
|
|
3311
|
+
project: project.name,
|
|
3312
|
+
issue,
|
|
3313
|
+
runId,
|
|
3314
|
+
summary: `#${issue} review round ${revision.round} could not be dispatched`,
|
|
3315
|
+
detail: [
|
|
3316
|
+
`${revision.prUrl}`,
|
|
3317
|
+
"",
|
|
3318
|
+
detail,
|
|
3319
|
+
...(settlement?.lines ?? []),
|
|
3320
|
+
].join("\n"),
|
|
3321
|
+
});
|
|
3322
|
+
} finally {
|
|
3323
|
+
turnLimit?.close();
|
|
3324
|
+
if (verbListener !== undefined) {
|
|
3325
|
+
try {
|
|
3326
|
+
await verbListener.close();
|
|
3327
|
+
} catch (err) {
|
|
3328
|
+
log(`#${issue} verb socket ${verbListener.path} did not close cleanly: ${errText(err)}`);
|
|
3329
|
+
}
|
|
3330
|
+
verbListener = undefined;
|
|
3331
|
+
}
|
|
3332
|
+
workerControl?.close();
|
|
3333
|
+
workerControl = undefined;
|
|
3334
|
+
}
|
|
3335
|
+
}
|
|
3336
|
+
|
|
3337
|
+
/**
|
|
3338
|
+
* Restart recovery for in-flight review revisions (#692): a revision the
|
|
3339
|
+
* previous daemon claimed (`pushed-green` → `running`) and lost when it died
|
|
3340
|
+
* is restored to the exact state the orchestrator's verb left it in — same
|
|
3341
|
+
* run, same PR, same findings, same round, same session file — so the next
|
|
3342
|
+
* dispatch pass resumes the SAME OMP session rather than leaving the run to
|
|
3343
|
+
* the failure classifier.
|
|
3344
|
+
*
|
|
3345
|
+
* Runs AFTER the startup orphan sweep, on purpose: the sweep has already
|
|
3346
|
+
* salvaged the crashed revision's worktree to the branch (so no work is lost)
|
|
3347
|
+
* and marked the row `orphaned` — and this reconcile then restores that row
|
|
3348
|
+
* to `pushed-green` and re-queues the round before the first tick, so the
|
|
3349
|
+
* failure classifier never sees it. That ordering matters twice over: an
|
|
3350
|
+
* orphaned row would charge the continuation budget for a round that must
|
|
3351
|
+
* stay off it (#677), and the `review_revisions` row is the only place the
|
|
3352
|
+
* findings and round live. A revision the previous daemon never claimed (its
|
|
3353
|
+
* run is still `pushed-green`, its row still pending) needs no restoration —
|
|
3354
|
+
* the ordinary dispatch pass wakes it on the next tick, exactly as #677
|
|
3355
|
+
* already guarantees.
|
|
3356
|
+
*
|
|
3357
|
+
* A round whose run cannot be restored — the run is gone, settled terminal,
|
|
3358
|
+
* or its repo is no longer routed — fails closed instead: the revision is
|
|
3359
|
+
* settled `skipped` with the findings still on the row, ONE escalation names
|
|
3360
|
+
* what could not be restored, and no worker is started against a PR that may
|
|
3361
|
+
* already have one.
|
|
3362
|
+
*/
|
|
3363
|
+
export async function reconcileCrashedReviewRevisions(d: Deps): Promise<ReviewRevisionRecord[]> {
|
|
3364
|
+
const { project, store } = d;
|
|
3365
|
+
const revisions = store.unsettledReviewRevisions(project.name);
|
|
3366
|
+
if (revisions.length === 0) return [];
|
|
3367
|
+
const recovered: ReviewRevisionRecord[] = [];
|
|
3368
|
+
for (const revision of revisions) {
|
|
3369
|
+
const run = store.getRun(revision.runId);
|
|
3370
|
+
if (run === undefined) {
|
|
3371
|
+
log(
|
|
3372
|
+
`#${revision.issue} review round ${revision.round} cannot be restored across the restart: ` +
|
|
3373
|
+
`run ${revision.runId} no longer exists`,
|
|
3374
|
+
);
|
|
3375
|
+
store.settleReviewRevision(revision.id, "skipped", Date.now());
|
|
3376
|
+
await safeEscalate(d, {
|
|
3377
|
+
tier: 1,
|
|
3378
|
+
project: project.name,
|
|
3379
|
+
issue: revision.issue,
|
|
3380
|
+
runId: revision.runId,
|
|
3381
|
+
summary: `#${revision.issue} review round ${revision.round} could not be restored across the restart`,
|
|
3382
|
+
detail: [
|
|
3383
|
+
revision.prUrl,
|
|
3384
|
+
"",
|
|
3385
|
+
"The daemon restarted while this review round was in flight and the round cannot be re-dispatched:",
|
|
3386
|
+
`the run row ${revision.runId} no longer exists.`,
|
|
3387
|
+
"The round was settled skipped; the findings stay on this revision's durable row.",
|
|
3388
|
+
"Re-review the PR (it is still open and green) or merge it as it stands.",
|
|
3389
|
+
].join("\n"),
|
|
3390
|
+
});
|
|
3391
|
+
continue;
|
|
3392
|
+
}
|
|
3393
|
+
// A revision the previous daemon never claimed: the run is still a
|
|
3394
|
+
// settled green row and the round is still pending, so the ordinary
|
|
3395
|
+
// dispatch pass wakes it on the next tick untouched.
|
|
3396
|
+
if (run.state === "pushed-green") continue;
|
|
3397
|
+
// The run a previous daemon claimed for this round and died on: the
|
|
3398
|
+
// orphan sweep just marked it `orphaned` (salvaging the tree to the
|
|
3399
|
+
// branch), so restore it to the reviewable state the verb recorded.
|
|
3400
|
+
const crashed =
|
|
3401
|
+
run.state === "claimed" || run.state === "running" || run.state === "orphaned";
|
|
3402
|
+
if (!crashed) {
|
|
3403
|
+
log(
|
|
3404
|
+
`#${revision.issue} review round ${revision.round} cannot be restored across the restart: run ` +
|
|
3405
|
+
`${revision.runId} is ${run.state}`,
|
|
3406
|
+
);
|
|
3407
|
+
store.settleReviewRevision(revision.id, "skipped", Date.now());
|
|
3408
|
+
await safeEscalate(d, {
|
|
3409
|
+
tier: 1,
|
|
3410
|
+
project: project.name,
|
|
3411
|
+
issue: revision.issue,
|
|
3412
|
+
runId: revision.runId,
|
|
3413
|
+
summary: `#${revision.issue} review round ${revision.round} could not be restored across the restart`,
|
|
3414
|
+
detail: [
|
|
3415
|
+
revision.prUrl,
|
|
3416
|
+
"",
|
|
3417
|
+
"The daemon restarted while this review round was in flight and the round cannot be re-dispatched:",
|
|
3418
|
+
`the run row is ${run.state}, not a state a review round resumes from.`,
|
|
3419
|
+
"The round was settled skipped; the findings stay on this revision's durable row.",
|
|
3420
|
+
"Re-review the PR (it is still open and green) or merge it as it stands.",
|
|
3421
|
+
].join("\n"),
|
|
3422
|
+
});
|
|
3423
|
+
continue;
|
|
3424
|
+
}
|
|
3425
|
+
try {
|
|
3426
|
+
// The crashed revision's worktree — salvaged and published by the orphan
|
|
3427
|
+
// sweep that just ran, so the branch in the mirror has every committed
|
|
3428
|
+
// byte — must be gone before the next dispatch, or `addRunRepo` refuses
|
|
3429
|
+
// the reattach as possibly holding a previous attempt's work (#692).
|
|
3430
|
+
// `settleWorktree` with `tree: "remove"` re-attempts the salvage instead
|
|
3431
|
+
// of trusting it (a tree whose salvage failed is the only copy of work
|
|
3432
|
+
// and is NEVER removed), and refuses the removal if that re-attempt
|
|
3433
|
+
// fails — the fail-closed half of this recovery.
|
|
3434
|
+
if (run.worktree !== "" && existsSync(run.worktree)) {
|
|
3435
|
+
const repo = Object.values(project.routing.repos).find((candidate) => candidate.name === run.repo);
|
|
3436
|
+
if (repo === undefined) {
|
|
3437
|
+
log(
|
|
3438
|
+
`#${revision.issue} review round ${revision.round} cannot clear worktree ${run.worktree}: ` +
|
|
3439
|
+
`repo ${run.repo} is no longer routed — the tree may hold the only copy of work`,
|
|
3440
|
+
);
|
|
3441
|
+
store.settleReviewRevision(revision.id, "skipped", Date.now());
|
|
3442
|
+
await safeEscalate(d, {
|
|
3443
|
+
tier: 1,
|
|
3444
|
+
project: project.name,
|
|
3445
|
+
issue: revision.issue,
|
|
3446
|
+
runId: revision.runId,
|
|
3447
|
+
summary: `#${revision.issue} review round ${revision.round} could not be restored across the restart`,
|
|
3448
|
+
detail: [
|
|
3449
|
+
revision.prUrl,
|
|
3450
|
+
"",
|
|
3451
|
+
"The daemon restarted while this review round was in flight and the round cannot be re-dispatched:",
|
|
3452
|
+
`repo ${run.repo} is no longer routed, so the crashed revision's worktree cannot be published and removed.`,
|
|
3453
|
+
`The worktree is retained for recovery: ${run.worktree}`,
|
|
3454
|
+
"Recover it by hand (it may hold the round's last work), then re-review or merge the PR.",
|
|
3455
|
+
].join("\n"),
|
|
3456
|
+
});
|
|
3457
|
+
continue;
|
|
3458
|
+
}
|
|
3459
|
+
const settlement = await settleWorktree({
|
|
3460
|
+
issue: run.issue,
|
|
3461
|
+
attempt: run.attempt,
|
|
3462
|
+
ending: "interrupted by a daemon restart during a review round",
|
|
3463
|
+
worktree: run.worktree,
|
|
3464
|
+
branch: run.branch,
|
|
3465
|
+
publish: (branch) => pushRunBranch(project, { repo, runRepoPath: run.worktree, branch }),
|
|
3466
|
+
tree: "remove",
|
|
3467
|
+
mirrorPath: mirrorPathFor(repo, project.mirrorRoot),
|
|
3468
|
+
});
|
|
3469
|
+
if (settlement.retained) {
|
|
3470
|
+
log(
|
|
3471
|
+
`#${revision.issue} review round ${revision.round} worktree ${run.worktree} retained: ` +
|
|
3472
|
+
"its salvage failed, so the tree is the only copy of work and will not be removed",
|
|
3473
|
+
);
|
|
3474
|
+
store.settleReviewRevision(revision.id, "skipped", Date.now());
|
|
3475
|
+
await safeEscalate(d, {
|
|
3476
|
+
tier: 1,
|
|
3477
|
+
project: project.name,
|
|
3478
|
+
issue: revision.issue,
|
|
3479
|
+
runId: revision.runId,
|
|
3480
|
+
summary: `#${revision.issue} review round ${revision.round} could not be restored across the restart`,
|
|
3481
|
+
detail: [
|
|
3482
|
+
revision.prUrl,
|
|
3483
|
+
"",
|
|
3484
|
+
"The daemon restarted while this review round was in flight and the round cannot be re-dispatched:",
|
|
3485
|
+
"the crashed revision's worktree could not be salvaged, so it is the only copy of work and was retained.",
|
|
3486
|
+
`Recover the tree by hand: ${run.worktree}`,
|
|
3487
|
+
"Then re-review the PR (still open and green) or merge it as it stands.",
|
|
3488
|
+
].join("\n"),
|
|
3489
|
+
});
|
|
3490
|
+
continue;
|
|
3491
|
+
}
|
|
3492
|
+
}
|
|
3493
|
+
} catch (err) {
|
|
3494
|
+
// A settleWorktree throw (mirror unreachable, git refused) must not take
|
|
3495
|
+
// the whole startup reconcile down with one run's tree: fail this round
|
|
3496
|
+
// closed and let the next one try.
|
|
3497
|
+
const detail = errText(err);
|
|
3498
|
+
log(`#${revision.issue} review round ${revision.round} restore errored: ${detail}`);
|
|
3499
|
+
store.settleReviewRevision(revision.id, "skipped", Date.now());
|
|
3500
|
+
await safeEscalate(d, {
|
|
3501
|
+
tier: 1,
|
|
3502
|
+
project: project.name,
|
|
3503
|
+
issue: revision.issue,
|
|
3504
|
+
runId: revision.runId,
|
|
3505
|
+
summary: `#${revision.issue} review round ${revision.round} could not be restored across the restart`,
|
|
3506
|
+
detail: [revision.prUrl, "", detail].join("\n"),
|
|
3507
|
+
});
|
|
3508
|
+
continue;
|
|
3509
|
+
}
|
|
3510
|
+
// The exact state the verb recorded: a pushed-green row plus a pending
|
|
3511
|
+
// revision whose durable row still carries the findings, reviewed head,
|
|
3512
|
+
// round and target session. The next tick's dispatch pass claims and
|
|
3513
|
+
// wakes it exactly like a revision that was never dispatched.
|
|
3514
|
+
store.updateRun(revision.runId, {
|
|
3515
|
+
state: "pushed-green",
|
|
3516
|
+
endedAt: Date.now(),
|
|
3517
|
+
worktree: "",
|
|
3518
|
+
lastError: `review round ${revision.round} was restored across a daemon restart; the round is re-queued`,
|
|
3519
|
+
});
|
|
3520
|
+
store.requeueReviewRevision(revision.id);
|
|
3521
|
+
log(
|
|
3522
|
+
`#${revision.issue} review round ${revision.round} restored across restart: run ${revision.runId} ` +
|
|
3523
|
+
"is pushed-green again and the round is re-queued — the next dispatch pass resumes the same session",
|
|
3524
|
+
);
|
|
3525
|
+
recovered.push(revision);
|
|
3526
|
+
}
|
|
3527
|
+
return recovered;
|
|
3528
|
+
}
|
|
3529
|
+
|
|
2538
3530
|
// ------------------------------------------------------------------- settlement
|
|
2539
3531
|
|
|
2540
3532
|
/** What a resolved PR turns its `pushed-green` row into. */
|
|
@@ -3277,6 +4269,7 @@ const DEGRADED_HOLDS: ReadonlySet<AdmissionHoldReason> = new Set([
|
|
|
3277
4269
|
"parent-lookup-error",
|
|
3278
4270
|
"open-pr-lookup-error",
|
|
3279
4271
|
"issue-state-lookup-error",
|
|
4272
|
+
"critical-base-verify-error",
|
|
3280
4273
|
]);
|
|
3281
4274
|
|
|
3282
4275
|
/** Groups transient decisions into the bounded record exposed by status. */
|
|
@@ -3632,6 +4625,15 @@ export async function tick(d: Deps, workers?: WorkerPool): Promise<void> {
|
|
|
3632
4625
|
// so `last dispatch` keeps moving while the fleet is deliberately parked. A
|
|
3633
4626
|
// frozen clock is then a stall report, not a hold (#497).
|
|
3634
4627
|
if (isPaused(d.project.name)) {
|
|
4628
|
+
// The held pass is the daemon-side admission acknowledgement: the daemon
|
|
4629
|
+
// has reached its admission boundary under the fence and claims nothing.
|
|
4630
|
+
// Written durably so the setup barrier can prove the fence was observed
|
|
4631
|
+
// (#651 review #3) — a pause cannot acknowledge itself.
|
|
4632
|
+
try {
|
|
4633
|
+
writeAdmissionAck(d.project.name);
|
|
4634
|
+
} catch (err) {
|
|
4635
|
+
log(`admission acknowledgement write failed: ${errText(err)}`);
|
|
4636
|
+
}
|
|
3635
4637
|
d.store.recordDispatch(d.project.name, summarizeHeldPass(settled));
|
|
3636
4638
|
return;
|
|
3637
4639
|
}
|
|
@@ -3694,6 +4696,18 @@ export async function tick(d: Deps, workers?: WorkerPool): Promise<void> {
|
|
|
3694
4696
|
return;
|
|
3695
4697
|
}
|
|
3696
4698
|
|
|
4699
|
+
// Review revisions are dispatch, not admission (#677): the orchestrator's
|
|
4700
|
+
// verb recorded them durably and the run row already occupies its issue, so
|
|
4701
|
+
// they bypass the ready queue and the claim path entirely — this pass wakes
|
|
4702
|
+
// them (bounded by the same worker slots) BEFORE the queue's capacity gate
|
|
4703
|
+
// reads live runs, so a claimed revision counts against capacity exactly like
|
|
4704
|
+
// the worker it is about to spawn.
|
|
4705
|
+
try {
|
|
4706
|
+
await dispatchReviewRevisions(d, workers);
|
|
4707
|
+
} catch (err) {
|
|
4708
|
+
log(`review revision dispatch failed: ${errText(err)}`);
|
|
4709
|
+
}
|
|
4710
|
+
|
|
3697
4711
|
// route() filters the queue through isEligible() itself, so anything already
|
|
3698
4712
|
// carrying a state label is gone before it gets here.
|
|
3699
4713
|
const ready = await d.tracker.listReady();
|
|
@@ -3817,7 +4831,7 @@ export async function tick(d: Deps, workers?: WorkerPool): Promise<void> {
|
|
|
3817
4831
|
log(`dispatching ${pass.admitted.map((a) => `#${a.r.issue.number}`).join(" ")}`);
|
|
3818
4832
|
await dispatchAdmissions(
|
|
3819
4833
|
pass.admitted,
|
|
3820
|
-
(a) => handleIssue(d, a.r, a.attempt),
|
|
4834
|
+
(a) => handleIssue(d, a.r, a.attempt, a.lane),
|
|
3821
4835
|
workers,
|
|
3822
4836
|
);
|
|
3823
4837
|
|
|
@@ -4213,6 +5227,15 @@ export interface StatusSnapshot {
|
|
|
4213
5227
|
releaseGrants: ResolvedGrants;
|
|
4214
5228
|
/** Occupied issues: live workers plus green PRs awaiting a human merge. */
|
|
4215
5229
|
activeRuns: RunRecord[];
|
|
5230
|
+
/**
|
|
5231
|
+
* runId → live review-revision round, for runs whose revision worker is
|
|
5232
|
+
* currently dispatched (#692). Derived from the durable `review_revisions`
|
|
5233
|
+
* rows, so the rendered `review-revision N` state rests on the same facts
|
|
5234
|
+
* the restart recovery reads — never on a label or a guess. A plain object
|
|
5235
|
+
* (not a Map) so the dashboard's JSON round-trip of the snapshot preserves
|
|
5236
|
+
* it byte for byte.
|
|
5237
|
+
*/
|
|
5238
|
+
reviewRounds?: Readonly<Record<string, number>>;
|
|
4216
5239
|
/** Newest attempts holding a preserved WIP tip, or a tree that is still the
|
|
4217
5240
|
* only copy of work the daemon could not save. */
|
|
4218
5241
|
salvagedRuns: RunRecord[];
|
|
@@ -4302,6 +5325,13 @@ export function statusSnapshotFromStore(
|
|
|
4302
5325
|
// Read once: the provenance read touches the filesystem, and the renderer
|
|
4303
5326
|
// should never pay for it twice per status.
|
|
4304
5327
|
const reason = pauseProvenance(p.name)?.reason;
|
|
5328
|
+
// The live review-revision rounds, read from the same durable rows the
|
|
5329
|
+
// restart recovery uses: a run whose revision is dispatched is read as
|
|
5330
|
+
// `review-revision N` while its worker is live (#692).
|
|
5331
|
+
const reviewRounds: Record<string, number> = {};
|
|
5332
|
+
for (const revision of store.unsettledReviewRevisions(p.name)) {
|
|
5333
|
+
if (revision.dispatchedAt !== undefined) reviewRounds[revision.runId] = revision.round;
|
|
5334
|
+
}
|
|
4305
5335
|
return {
|
|
4306
5336
|
project: p.name,
|
|
4307
5337
|
configPath: configPath(),
|
|
@@ -4313,6 +5343,7 @@ export function statusSnapshotFromStore(
|
|
|
4313
5343
|
caps,
|
|
4314
5344
|
releaseGrants: resolveReleaseGrants(p),
|
|
4315
5345
|
activeRuns: store.activeRuns(p.name),
|
|
5346
|
+
reviewRounds,
|
|
4316
5347
|
salvagedRuns: store.salvagedRuns(p.name),
|
|
4317
5348
|
turnOverrides: store.listTurnOverrides(p.name),
|
|
4318
5349
|
openReports: store.openReports(p.name),
|
|
@@ -4758,7 +5789,10 @@ export async function classifyAndRecover(d: Deps): Promise<number> {
|
|
|
4758
5789
|
// table can tell an infra outage (#177) from a real test failure by
|
|
4759
5790
|
// the log's own words. First failure wins; a log that cannot be
|
|
4760
5791
|
// fetched is left undefined and classification stays conservative.
|
|
4761
|
-
|
|
5792
|
+
// GitHub reports check states as `FAILURE` while the classifier reads
|
|
5793
|
+
// them lowercased — normalise so the live seam and the table agree on
|
|
5794
|
+
// which check is the failure whose log we pull (#177).
|
|
5795
|
+
const firstFailure = facts.checks.find((c) => normalise(c.state) === "failure" && c.link !== undefined);
|
|
4762
5796
|
if (firstFailure?.link !== undefined) {
|
|
4763
5797
|
facts.failingLog = await tracker.checkLog(firstFailure.link);
|
|
4764
5798
|
}
|
|
@@ -4791,6 +5825,154 @@ export async function classifyAndRecover(d: Deps): Promise<number> {
|
|
|
4791
5825
|
return settled;
|
|
4792
5826
|
}
|
|
4793
5827
|
|
|
5828
|
+
/**
|
|
5829
|
+
* How many settled `ci-deterministic` rows one reconciliation pass may
|
|
5830
|
+
* re-examine beyond the persisted review cursor. Each candidate costs GitHub
|
|
5831
|
+
* calls to re-fetch the evidence its check log carried, so a fleet with a
|
|
5832
|
+
* long misclassified history works through it over several daemon starts
|
|
5833
|
+
* rather than spending one boot's budget on all of it (#638).
|
|
5834
|
+
*/
|
|
5835
|
+
const HISTORICAL_INFRA_BATCH = 20;
|
|
5836
|
+
|
|
5837
|
+
/** How many workflow runs for the head commit one row's evidence pass reads —
|
|
5838
|
+
* and the ceiling past which the row is refused as undecided rather than
|
|
5839
|
+
* decided from a prefix of its head-pinned runs (review #654). */
|
|
5840
|
+
const HISTORICAL_INFRA_RUNS = 3;
|
|
5841
|
+
|
|
5842
|
+
/** How many attempts of one workflow run may be read for the failed log — and
|
|
5843
|
+
* the ceiling past which the run is refused as undecided rather than read as
|
|
5844
|
+
* a prefix that could hide a later attempt's real failure (review #654). A
|
|
5845
|
+
* failed job rerun to green leaves the failure in an earlier attempt; the
|
|
5846
|
+
* bound keeps one pathological run from costing the whole pass. */
|
|
5847
|
+
const HISTORICAL_INFRA_ATTEMPTS = 5;
|
|
5848
|
+
|
|
5849
|
+
/**
|
|
5850
|
+
* Repair settled `ci-deterministic` rows whose re-fetched check log carries a
|
|
5851
|
+
* closed infrastructure signature (#638). The forward classifier now names
|
|
5852
|
+
* the codeload setup 429 of #177 `ci-infra`, but a row already classified
|
|
5853
|
+
* `ci-deterministic`/`escalate` never re-enters the classification sweep, so
|
|
5854
|
+
* the old verdict charges an implementation attempt forever. Re-fetching the
|
|
5855
|
+
* head-pinned workflow-run attempt logs through the tracker, matching the
|
|
5856
|
+
* classifier's own closed signature list, and reclassifying `failureClass`
|
|
5857
|
+
* alone returns the attempt without re-animating a months-old run into
|
|
5858
|
+
* recovery.
|
|
5859
|
+
*
|
|
5860
|
+
* Bounded, idempotent and resumable: one batch per call, reclassifications
|
|
5861
|
+
* only, and the store's update is guarded by the row still reading
|
|
5862
|
+
* `ci-deterministic`, so a second pass touches nothing it already repaired.
|
|
5863
|
+
* The batch resumes below a persisted review cursor, so each row is evaluated
|
|
5864
|
+
* once rather than rescanning the newest non-matches forever and starving
|
|
5865
|
+
* older repairable rows. A row whose evidence could not be read is a
|
|
5866
|
+
* no-mutation refusal: the cursor never advances past it, so the next pass
|
|
5867
|
+
* asks again; a row that was read and shown *not* to be infrastructure
|
|
5868
|
+
* advances the cursor. Every repair requires *all* gathered failed logs to
|
|
5869
|
+
* carry an infra signature — a setup 429 in one attempt must not waive a
|
|
5870
|
+
* compile/test failure in a sibling attempt.
|
|
5871
|
+
*
|
|
5872
|
+
* Returns how many rows it repaired, for the boot log.
|
|
5873
|
+
*/
|
|
5874
|
+
export async function reconcileHistoricalInfra(d: Deps): Promise<number> {
|
|
5875
|
+
const { project, store } = d;
|
|
5876
|
+
const version = infraSignatureVersion();
|
|
5877
|
+
const persisted = store.historicalInfraCursor(project.name);
|
|
5878
|
+
// A cursor stamped by an older signature list is stale: the classifier now
|
|
5879
|
+
// recognises more evidence, so the pass restarts from the newest row rather
|
|
5880
|
+
// than skipping past newly repairable history (#638).
|
|
5881
|
+
const cursor =
|
|
5882
|
+
persisted !== undefined && persisted.classifierVersion === version
|
|
5883
|
+
? { startedAt: persisted.startedAt, rowid: persisted.rowid }
|
|
5884
|
+
: undefined;
|
|
5885
|
+
const candidates = store.historicalInfraCandidates(project.name, HISTORICAL_INFRA_BATCH, cursor);
|
|
5886
|
+
let repaired = 0;
|
|
5887
|
+
let lastDecided: { startedAt: number; rowid: number } | undefined;
|
|
5888
|
+
for (const run of candidates) {
|
|
5889
|
+
const chunks = await historicalInfraEvidence(d, run);
|
|
5890
|
+
// Unreachable evidence is undecided exactly like a fresh classifier is: a
|
|
5891
|
+
// no-mutation refusal the next pass asks again, and the cursor stops here
|
|
5892
|
+
// so the row is re-offered rather than being skipped past.
|
|
5893
|
+
if (chunks === undefined) break;
|
|
5894
|
+
lastDecided = { startedAt: run.startedAt, rowid: run.rowid };
|
|
5895
|
+
// The head's runs were read and none holds a failing log — determinately
|
|
5896
|
+
// not infrastructure, so an old misclassified verdict stays charged.
|
|
5897
|
+
if (chunks.length === 0) continue;
|
|
5898
|
+
// Mixed guard: anything gathered that is not itself a closed infra
|
|
5899
|
+
// signature (a compile/test failure in a sibling attempt, a product 429)
|
|
5900
|
+
// refuses the whole row. One setup 429 is not permission to waive a real
|
|
5901
|
+
// implementation failure.
|
|
5902
|
+
if (!chunks.every((chunk) => infraLogSignature(chunk) !== undefined)) continue;
|
|
5903
|
+
if (store.reclassifyInfra(run.id)) {
|
|
5904
|
+
repaired += 1;
|
|
5905
|
+
// The every-guard above guarantees a signature and a first chunk; the
|
|
5906
|
+
// non-null assertions make the same fact readable to the type checker.
|
|
5907
|
+
log(
|
|
5908
|
+
`#${run.issue} repaired historical ${run.failureClass ?? "ci-deterministic"} → ci-infra (run ${run.id}): ` +
|
|
5909
|
+
`"${infraLogSignature(chunks[0]!)}" in the failed check log`,
|
|
5910
|
+
);
|
|
5911
|
+
}
|
|
5912
|
+
}
|
|
5913
|
+
if (lastDecided !== undefined) {
|
|
5914
|
+
store.setHistoricalInfraCursor(
|
|
5915
|
+
project.name,
|
|
5916
|
+
lastDecided.startedAt,
|
|
5917
|
+
lastDecided.rowid,
|
|
5918
|
+
version,
|
|
5919
|
+
);
|
|
5920
|
+
}
|
|
5921
|
+
return repaired;
|
|
5922
|
+
}
|
|
5923
|
+
|
|
5924
|
+
/**
|
|
5925
|
+
* The failed check logs of a settled `ci-deterministic` row, head-pinned to
|
|
5926
|
+
* the exact commit the row ran against, or `undefined` when the evidence is
|
|
5927
|
+
* unreachable (#638). Only the workflow-run history at `run.headSha` is
|
|
5928
|
+
* evidence: the PR's *current* check rollup is not, because a later push's
|
|
5929
|
+
* checks must never classify an earlier run's row. `workflowRunsAt` is itself
|
|
5930
|
+
* head-scoped, and each run's failed attempts are read through the tracker's
|
|
5931
|
+
* guarded runner (call/refusal accounting and the rate-limit breaker apply).
|
|
5932
|
+
*
|
|
5933
|
+
* Returns the per-failed-job failed logs gathered across the head's runs (one
|
|
5934
|
+
* chunk per failed job, full and untruncated), or `undefined` when any read
|
|
5935
|
+
* could not be made — the run register, an attempt's job register, a failed
|
|
5936
|
+
* job's log, or the head-run list itself — or when a bounded evidence set
|
|
5937
|
+
* exceeded its limit (more head-pinned runs than `HISTORICAL_INFRA_RUNS`, or
|
|
5938
|
+
* more attempts than `HISTORICAL_INFRA_ATTEMPTS`). Undefined is a no-mutation
|
|
5939
|
+
* refusal the next pass asks again; the cursor never advances past it. `[]`
|
|
5940
|
+
* means the head's runs were read and none holds a failing log — a
|
|
5941
|
+
* determinately non-infrastructure answer.
|
|
5942
|
+
*/
|
|
5943
|
+
async function historicalInfraEvidence(d: Deps, run: RunRecord): Promise<string[] | undefined> {
|
|
5944
|
+
// Without a head SHA there is no safe way to pin evidence to this row; a
|
|
5945
|
+
// head-less row is determinately not a repair candidate, so it advances the
|
|
5946
|
+
// cursor rather than stalling the pass.
|
|
5947
|
+
if (run.headSha === undefined) return [];
|
|
5948
|
+
const target = d.project.routing.repos[run.repo];
|
|
5949
|
+
const repoIdentity = target === undefined ? undefined : githubRepo(target.cloneUrl);
|
|
5950
|
+
if (repoIdentity === undefined) return [];
|
|
5951
|
+
let runs: WorkflowRun[] | undefined;
|
|
5952
|
+
try {
|
|
5953
|
+
runs = await d.tracker.workflowRunsAt(repoIdentity, run.headSha);
|
|
5954
|
+
} catch {
|
|
5955
|
+
return undefined;
|
|
5956
|
+
}
|
|
5957
|
+
if (runs === undefined) return undefined;
|
|
5958
|
+
// Refuse rather than decide from the first `HISTORICAL_INFRA_RUNS`: three
|
|
5959
|
+
// setup-429 runs must not waive a row whose fourth head-pinned run carries
|
|
5960
|
+
// the real compile/test failure (review #654). Undecided means the cursor
|
|
5961
|
+
// never advances past this row, so the next pass asks again.
|
|
5962
|
+
if (runs.length > HISTORICAL_INFRA_RUNS) return undefined;
|
|
5963
|
+
const chunks: string[] = [];
|
|
5964
|
+
for (const wf of runs) {
|
|
5965
|
+
const logs = await d.tracker.runFailedAttemptLogs(
|
|
5966
|
+
repoIdentity,
|
|
5967
|
+
wf.url,
|
|
5968
|
+
HISTORICAL_INFRA_ATTEMPTS,
|
|
5969
|
+
);
|
|
5970
|
+
if (logs === undefined) return undefined;
|
|
5971
|
+
chunks.push(...logs);
|
|
5972
|
+
}
|
|
5973
|
+
return chunks;
|
|
5974
|
+
}
|
|
5975
|
+
|
|
4794
5976
|
/** Performs the one action a class names. Never chooses one of its own. */
|
|
4795
5977
|
async function recoverRun(
|
|
4796
5978
|
d: Deps,
|
|
@@ -5450,12 +6632,17 @@ export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
|
|
|
5450
6632
|
log(projects.length === 1 ? message : `[${project.name}] ${message}`);
|
|
5451
6633
|
};
|
|
5452
6634
|
const caps = resolveCaps(project, cfg.defaults);
|
|
6635
|
+
// One transient-server-error breaker per project (#642): admission's
|
|
6636
|
+
// GraphQL checks and the orchestrator mutation commands share it, so a 503
|
|
6637
|
+
// observed by either side gates both instead of one provider outage being
|
|
6638
|
+
// re-asked by every candidate AND every mutation.
|
|
6639
|
+
const graphqlBreaker = new GraphqlBreaker();
|
|
5453
6640
|
const tracker = makeTracker(project, undefined, {
|
|
5454
6641
|
onCall: () => store.bumpGhCalls?.(utcDay(), "daemon"),
|
|
5455
6642
|
onRefusal: (at) => store.recordGhRefusal?.(at),
|
|
5456
6643
|
onNotModified: () => store.bumpGhCalls?.(utcDay(), "daemon-304"),
|
|
5457
|
-
});
|
|
5458
|
-
const verbActions = githubVerbActions(project);
|
|
6644
|
+
}, { graphqlBreaker });
|
|
6645
|
+
const verbActions = githubVerbActions(project, undefined, undefined, graphqlBreaker);
|
|
5459
6646
|
|
|
5460
6647
|
if (alive === undefined || alive.pid === process.pid) {
|
|
5461
6648
|
const orphanPublisher = (run: RunRecord): RunPublisher => {
|
|
@@ -5604,6 +6791,26 @@ export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
|
|
|
5604
6791
|
verbActions,
|
|
5605
6792
|
};
|
|
5606
6793
|
runtimeDeps = d;
|
|
6794
|
+
// Review-revision restart recovery (#692): a revision the previous daemon
|
|
6795
|
+
// claimed and lost — its run is `running`/`orphaned` by the orphan sweep
|
|
6796
|
+
// above — is restored to `pushed-green` and re-queued so this process's
|
|
6797
|
+
// first dispatch pass resumes the exact session. Runs under the same
|
|
6798
|
+
// dead-daemon guard as `reconcileOrphanedRuns`, and after it, so the
|
|
6799
|
+
// revision's worktree has already been salvaged to the branch before it is
|
|
6800
|
+
// cleared for a fresh reattach. A round that cannot be restored was
|
|
6801
|
+
// settled and escalated by the reconcile itself.
|
|
6802
|
+
if (alive === undefined || alive.pid === process.pid) {
|
|
6803
|
+
try {
|
|
6804
|
+
for (const revision of await reconcileCrashedReviewRevisions(d)) {
|
|
6805
|
+
projectLog(
|
|
6806
|
+
`#${revision.issue} review round ${revision.round} recovered across restart — ` +
|
|
6807
|
+
`run ${revision.runId} is pushed-green again and awaits its next dispatch pass`,
|
|
6808
|
+
);
|
|
6809
|
+
}
|
|
6810
|
+
} catch (err) {
|
|
6811
|
+
projectLog(`review revision restart recovery failed: ${errText(err)}`);
|
|
6812
|
+
}
|
|
6813
|
+
}
|
|
5607
6814
|
// Startup reconciliation: close an incident carried over from a previous
|
|
5608
6815
|
// process when the orchestrator is up (one recovery notice), or open one
|
|
5609
6816
|
// when it failed to start (one down page). A daemon restarted while still
|
|
@@ -5625,6 +6832,21 @@ export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
|
|
|
5625
6832
|
});
|
|
5626
6833
|
}
|
|
5627
6834
|
|
|
6835
|
+
// Historical budget repair (#638): one bounded pass per daemon process,
|
|
6836
|
+
// before the first tick, so admission and `unblock` read the repaired counts
|
|
6837
|
+
// from the moment the daemon starts. A classifier fix changes future
|
|
6838
|
+
// verdicts but not rows already settled under the old one; re-fetching their
|
|
6839
|
+
// failed check logs and reclassifying the exact closed infrastructure
|
|
6840
|
+
// signature returns those attempts without re-animating the runs. Best-effort
|
|
6841
|
+
// per project — one unreachable project must not stop the rest from booting.
|
|
6842
|
+
for (const runtime of runtimes) {
|
|
6843
|
+
try {
|
|
6844
|
+
await reconcileHistoricalInfra(runtime.d);
|
|
6845
|
+
} catch (err) {
|
|
6846
|
+
log(`historical infra reconciliation failed: ${errText(err)}`);
|
|
6847
|
+
}
|
|
6848
|
+
}
|
|
6849
|
+
|
|
5628
6850
|
if (o.once) {
|
|
5629
6851
|
try {
|
|
5630
6852
|
for (const runtime of runtimes) await tick(runtime.d, workers);
|