omp-conductor 0.18.2 → 0.19.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +105 -40
- package/REFERENCE.md +865 -30
- package/package.json +1 -1
- package/schema/config.schema.json +26 -0
- package/src/admission.ts +212 -26
- package/src/ask.ts +288 -1
- package/src/briefs/orchestrator.md +6 -5
- package/src/cli.ts +5 -1
- package/src/command-help.ts +9 -1
- package/src/command-manifest.ts +36 -3
- package/src/commands/arm.ts +5 -1
- package/src/commands/context.ts +2 -0
- package/src/commands/message.ts +26 -2
- package/src/commands/reconcile-units.ts +104 -0
- package/src/commands/release-composition.ts +232 -0
- package/src/commands/resume.ts +2 -27
- package/src/commands/setup.ts +101 -16
- package/src/commands/stats.ts +11 -30
- package/src/commands/tail.ts +31 -1
- package/src/commands/upgrade.ts +20 -3
- package/src/commands/verb.ts +2 -1
- package/src/config-schema.ts +19 -0
- package/src/config.ts +80 -0
- package/src/credential-class.ts +366 -0
- package/src/daemon.ts +1218 -288
- package/src/dashboard/app.js +504 -2
- package/src/dashboard/controls.ts +336 -0
- package/src/dashboard/index.html +30 -0
- package/src/dashboard/server.ts +271 -30
- package/src/dashboard/style.css +116 -0
- package/src/dashboard/transcript.ts +173 -0
- package/src/doctor.ts +379 -22
- package/src/failure-class.ts +59 -0
- package/src/fleet.ts +511 -101
- package/src/host.ts +6 -130
- package/src/omp.ts +29 -0
- package/src/orchestrator-tick.ts +343 -88
- package/src/pause.ts +233 -0
- package/src/settlement.ts +159 -2
- package/src/setup-answers.ts +97 -0
- package/src/setup-host.ts +325 -1159
- package/src/setup-install.ts +204 -27
- package/src/setup-wizard.ts +111 -50
- package/src/setup.ts +33 -0
- package/src/spend-telemetry.ts +117 -0
- package/src/stats.ts +35 -0
- package/src/status-render.ts +348 -19
- package/src/store.ts +1229 -55
- package/src/telegram-freshness.ts +269 -0
- package/src/to-spec.ts +27 -0
- package/src/types.ts +697 -4
- package/src/unblock.ts +22 -0
- package/src/unit-reconcile.ts +303 -0
- package/src/upgrade-verify.ts +8 -1
- package/src/upgrade.ts +326 -47
- package/src/verbs/actions.ts +124 -10
- package/src/verbs/protocol.ts +70 -2
- package/src/verbs/server.ts +447 -8
- package/src/wake.ts +48 -0
- package/src/worker.ts +403 -3
package/src/verbs/server.ts
CHANGED
|
@@ -40,15 +40,23 @@
|
|
|
40
40
|
|
|
41
41
|
import { randomUUID } from "node:crypto";
|
|
42
42
|
import { createServer, type Server, type Socket } from "node:net";
|
|
43
|
+
import { join } from "node:path";
|
|
43
44
|
|
|
44
45
|
import { resolvePolicy, resolveReleaseGrants, resolveReview } from "../config.ts";
|
|
46
|
+
import { wakeDispatch } from "../wake.ts";
|
|
45
47
|
// The mediated release is the drain's closing gesture (#791): a successful
|
|
46
48
|
// release ends this project's bounded release window, so the privileged half
|
|
47
49
|
// clears the record through the same operator surface the CLI uses.
|
|
48
50
|
import { cancelDrain } from "../fleet.ts";
|
|
49
51
|
import { effectiveLane, laneEcho, writeLaneSectionHeading } from "../admission.ts";
|
|
50
52
|
import { chainEntriesFromDiff, chainViolations } from "../chain-check.ts";
|
|
51
|
-
import {
|
|
53
|
+
import {
|
|
54
|
+
repoSlugFor,
|
|
55
|
+
type LaneFile,
|
|
56
|
+
type LaneSource,
|
|
57
|
+
type RunLaneProbe,
|
|
58
|
+
type readBaseChain as readBaseChainType,
|
|
59
|
+
} from "../gitops.ts";
|
|
52
60
|
import { releaseRefusal } from "../release-policy.ts";
|
|
53
61
|
import { PR_LOOKUP_WINDOW_MS, REVISABLE_RUN_STATES, prReviewReadiness } from "../decisions.ts";
|
|
54
62
|
import { LIVE_STATES } from "../store.ts";
|
|
@@ -146,6 +154,16 @@ export interface ReleaseExecution {
|
|
|
146
154
|
shape: ReleaseShape;
|
|
147
155
|
repo: RepoTarget;
|
|
148
156
|
tag?: string;
|
|
157
|
+
/**
|
|
158
|
+
* The exact commit a tag shape must point at (#695). Absent means "whatever
|
|
159
|
+
* the released repo's default branch is at cut time", which is the behaviour
|
|
160
|
+
* that stranded a tag every time the release window needed a merge: cutting
|
|
161
|
+
* required a quiet window, reaching that window required merging the open PR,
|
|
162
|
+
* and merging moved the branch past the tag that had just been cut. A pinned
|
|
163
|
+
* commit removes the dependence on *when* the tag is cut; the privileged half
|
|
164
|
+
* still refuses one that is not reachable from that branch.
|
|
165
|
+
*/
|
|
166
|
+
sha?: string;
|
|
149
167
|
artefact?: string;
|
|
150
168
|
environment?: string;
|
|
151
169
|
}
|
|
@@ -178,6 +196,14 @@ export interface VerbActions {
|
|
|
178
196
|
fields: { title?: string; body?: string },
|
|
179
197
|
): Promise<{ ok: true } | { ok: false; stderr: string }>;
|
|
180
198
|
mergePr(prUrl: string, headSha: string): Promise<ActionOutcome>;
|
|
199
|
+
/**
|
|
200
|
+
* Close one pull request, having first posted `comment` on it (#876).
|
|
201
|
+
*
|
|
202
|
+
* Both halves in one action because the ordering is the contract: a close that
|
|
203
|
+
* lands while its explanation fails leaves a PR nobody can account for, so the
|
|
204
|
+
* comment goes first and a failed comment means no close.
|
|
205
|
+
*/
|
|
206
|
+
closePr(prUrl: string, comment: string): Promise<ActionOutcome>;
|
|
181
207
|
release(execution: ReleaseExecution): Promise<ActionOutcome>;
|
|
182
208
|
/**
|
|
183
209
|
* Request the fleet upgrade itself to `version`: refuse when npm does not
|
|
@@ -221,6 +247,17 @@ export interface VerbDeps {
|
|
|
221
247
|
* whether the merge would fork the chain, it never changes it (#227).
|
|
222
248
|
*/
|
|
223
249
|
chain: { readBaseChain: typeof readBaseChainType };
|
|
250
|
+
/**
|
|
251
|
+
* Read-only access to one run's actual file lane (#925), kept off
|
|
252
|
+
* {@link VerbDeps.actions} for the same reason {@link VerbDeps.chain} is:
|
|
253
|
+
* `conductor_pr_recover` consults what a branch carries, it never changes it.
|
|
254
|
+
*
|
|
255
|
+
* Required rather than optional on purpose. This seam decides a fail-CLOSED
|
|
256
|
+
* refusal, so "the field was not wired" must be a compile error rather than a
|
|
257
|
+
* silently skipped check — an unwired guard that reports success is worse
|
|
258
|
+
* than no guard, because nothing anywhere says the lane went unproven.
|
|
259
|
+
*/
|
|
260
|
+
lane: { probeRunLane: RunLaneProbe };
|
|
224
261
|
}
|
|
225
262
|
|
|
226
263
|
/**
|
|
@@ -553,6 +590,12 @@ async function decide(deps: VerbDeps, channel: VerbChannel, raw: unknown): Promi
|
|
|
553
590
|
verb === "conductor_pr_update_branch" ||
|
|
554
591
|
verb === "conductor_pr_update" ||
|
|
555
592
|
verb === "conductor_pr_review" ||
|
|
593
|
+
// A clearance disposes of evidence against a PR a pre-pause run already
|
|
594
|
+
// pushed (#913). It starts no work, and a pause that refused it would
|
|
595
|
+
// simply relocate the deadlock it exists to break: `conductor_pr_merge`
|
|
596
|
+
// stays available under a pause, so the one thing that unblocks it must
|
|
597
|
+
// too.
|
|
598
|
+
verb === "conductor_pr_review_clear" ||
|
|
556
599
|
verb === "conductor_pr_recover" ||
|
|
557
600
|
verb === "conductor_label");
|
|
558
601
|
const stopped =
|
|
@@ -567,7 +610,8 @@ async function decide(deps: VerbDeps, channel: VerbChannel, raw: unknown): Promi
|
|
|
567
610
|
"fleet-paused",
|
|
568
611
|
`refused: ${stopped}. The pause refuses new claims and work-starting mutations. ` +
|
|
569
612
|
"Completion verbs for runs admitted before the pause (conductor_pr_merge, conductor_pr_update_branch, " +
|
|
570
|
-
"conductor_pr_update,
|
|
613
|
+
"conductor_pr_update, conductor_pr_review, conductor_pr_review_clear, conductor_pr_recover, " +
|
|
614
|
+
"conductor_label), conductor_release and conductor_install remain available.",
|
|
571
615
|
);
|
|
572
616
|
}
|
|
573
617
|
}
|
|
@@ -591,6 +635,8 @@ async function decide(deps: VerbDeps, channel: VerbChannel, raw: unknown): Promi
|
|
|
591
635
|
return installVerb(deps, project, channel, args, refuse, allow);
|
|
592
636
|
case "conductor_pr_review":
|
|
593
637
|
return prReviewVerb(deps, project, channel, args, refuse, allow);
|
|
638
|
+
case "conductor_pr_review_clear":
|
|
639
|
+
return prReviewClearVerb(deps, project, channel, args, refuse, allow);
|
|
594
640
|
case "conductor_pr_recover":
|
|
595
641
|
return prRecoverVerb(deps, project, channel, args, refuse, allow);
|
|
596
642
|
case "conductor_pr_status":
|
|
@@ -970,6 +1016,96 @@ async function prMergeVerb(
|
|
|
970
1016
|
}
|
|
971
1017
|
}
|
|
972
1018
|
|
|
1019
|
+
// Exact-head review gate (#888): green checks say nothing about the durable
|
|
1020
|
+
// review lifecycle, and #886 merged through exactly that blind spot — CI
|
|
1021
|
+
// green at the live head while a review revision had settled `failed` on
|
|
1022
|
+
// that same head hours earlier. Consulted here, before the lock and every
|
|
1023
|
+
// `gh` call, from the same local rows the review verb writes: any round not
|
|
1024
|
+
// yet settled (queued, or dispatched and crashed mid-review) and any round
|
|
1025
|
+
// settled `failed` at exactly the requested head refuses the merge, unless a
|
|
1026
|
+
// `conductor_pr_review_clear` recorded at or after that round's own last
|
|
1027
|
+
// activity has dispositioned it (#913). A finding tied only to an older head
|
|
1028
|
+
// never appears here — clearance follows the exact head, so a corrected push
|
|
1029
|
+
// is never poisoned.
|
|
1030
|
+
// A head an adjudication decided against never merges (#876), whatever else
|
|
1031
|
+
// is green. The disposition normally closes the PR, so this rarely fires —
|
|
1032
|
+
// but "rarely" is not "never": a close that failed, or a race between the
|
|
1033
|
+
// disposition pass and this call, must not leave a rejected diff mergeable.
|
|
1034
|
+
// Read from the same durable row status renders, so the refusal and the board
|
|
1035
|
+
// cannot disagree.
|
|
1036
|
+
const decided = deps.store.reviewAdjudicationForHead(project.name, prUrl, headSha);
|
|
1037
|
+
if (decided !== undefined && decided.settledAt !== undefined && decided.state !== "cleared") {
|
|
1038
|
+
return refuse(
|
|
1039
|
+
"merge-blocked-by-review",
|
|
1040
|
+
`refused: the review-ceiling adjudication for ${prUrl} at ${headSha.slice(0, 12)} settled ` +
|
|
1041
|
+
`${decided.state}, not cleared${decided.evidence === undefined ? "" : ` — ${decided.evidence}`}. ` +
|
|
1042
|
+
"There is no further review round and no second adjudicator: this head does not merge. " +
|
|
1043
|
+
"Push a corrected head, which is adjudicated on its own merits.",
|
|
1044
|
+
issue,
|
|
1045
|
+
);
|
|
1046
|
+
}
|
|
1047
|
+
|
|
1048
|
+
// The exact-head review-evidence gate (#888), whose reasoning the block above
|
|
1049
|
+
// this one continues from: any round not yet settled (queued, or dispatched
|
|
1050
|
+
// and crashed mid-review) and any round settled `failed` at exactly the
|
|
1051
|
+
// requested head refuses the merge, unless a `conductor_pr_review_clear`
|
|
1052
|
+
// recorded at or after that round's own last activity has dispositioned it.
|
|
1053
|
+
const blocking = deps.store.mergeBlockingReviews(project.name, prUrl, headSha);
|
|
1054
|
+
if (blocking.length > 0) {
|
|
1055
|
+
const b = blocking[0]!;
|
|
1056
|
+
const phase =
|
|
1057
|
+
b.settledAt !== undefined
|
|
1058
|
+
? `settled ${b.outcome ?? "failed"} with unaddressed findings`
|
|
1059
|
+
: b.dispatchedAt !== undefined
|
|
1060
|
+
? "was dispatched and never finished (crashed mid-review)"
|
|
1061
|
+
: "is queued and has not been addressed";
|
|
1062
|
+
const more = blocking.length > 1 ? ` (+${blocking.length - 1} further blocking round(s) at this head)` : "";
|
|
1063
|
+
// The advice names only clearances that exist (#913). Re-reviewing the
|
|
1064
|
+
// same head was the old wording's suggestion and it provably does not
|
|
1065
|
+
// work: `enqueueReviewRevision` appends to the pending row or opens a new
|
|
1066
|
+
// unsettled one, so a fresh round at an unchanged head is another blocking
|
|
1067
|
+
// round — and at the review-round ceiling there is no further round to
|
|
1068
|
+
// spend at all.
|
|
1069
|
+
return refuse(
|
|
1070
|
+
"merge-blocked-by-review",
|
|
1071
|
+
`refused: ${prUrl} cannot merge at ${headSha.slice(0, 12)} — review round ${b.round} at this exact head ` +
|
|
1072
|
+
`${phase}${more}. Green checks do not clear durable review findings, and neither does re-reviewing ` +
|
|
1073
|
+
"this head: a fresh `conductor_pr_review` at an unchanged head opens (or appends to) another " +
|
|
1074
|
+
"blocking round rather than superseding one. The two clearances that exist are a corrected head " +
|
|
1075
|
+
"pushed by the run — clearance follows the exact head, so a different head is not poisoned by this " +
|
|
1076
|
+
"one — or an explicit `conductor_pr_review_clear` for this exact head, recorded with its reason.",
|
|
1077
|
+
issue,
|
|
1078
|
+
);
|
|
1079
|
+
}
|
|
1080
|
+
|
|
1081
|
+
// Active release-composition gate (#850): while the project has a release
|
|
1082
|
+
// declared as being assembled, only a PR named in that composition — or one
|
|
1083
|
+
// with an explicit recorded operator override — may merge into it. Placed
|
|
1084
|
+
// after identity resolution and before the merge lock and every tracker
|
|
1085
|
+
// call, so an unrelated green PR is refused without spending a gh round
|
|
1086
|
+
// trip: this is the mechanical form of "a patch contains no unrelated
|
|
1087
|
+
// merges", not a status warning. The override bypasses only this check —
|
|
1088
|
+
// authority, pause, base-red-freeze, exact-head, green-checks and
|
|
1089
|
+
// single-flight all still apply below.
|
|
1090
|
+
const composition = deps.store.activeReleaseComposition(project.name);
|
|
1091
|
+
if (composition !== undefined && !composition.allowedPrUrls.includes(prUrl)) {
|
|
1092
|
+
const overridden =
|
|
1093
|
+
deps.store.releaseCompositionOverride(project.name, composition.campaign, prUrl) !== undefined;
|
|
1094
|
+
if (!overridden) {
|
|
1095
|
+
const allowed =
|
|
1096
|
+
composition.allowedPrUrls.length === 0 ? "none yet" : composition.allowedPrUrls.join(", ");
|
|
1097
|
+
return refuse(
|
|
1098
|
+
"outside-active-release",
|
|
1099
|
+
`refused: ${project.name} is assembling release ${composition.campaign} and ${prUrl} is not in it ` +
|
|
1100
|
+
`(allowed so far: ${allowed}). Merging it would smuggle unrelated work into that release ` +
|
|
1101
|
+
"(outside-active-release). The guard clears only through `omp-conductor release-composition " +
|
|
1102
|
+
"complete|cancel`, or an explicit one-PR `omp-conductor release-composition override --pr <url> " +
|
|
1103
|
+
"--reason <why>` — this call's merge reason is not an override.",
|
|
1104
|
+
issue,
|
|
1105
|
+
);
|
|
1106
|
+
}
|
|
1107
|
+
}
|
|
1108
|
+
|
|
973
1109
|
// Taken before any network call, so two concurrent callers contend here
|
|
974
1110
|
// rather than both spending a `gh` round trip and racing at the merge.
|
|
975
1111
|
const holderId = randomUUID();
|
|
@@ -1184,8 +1320,21 @@ async function labelVerb(
|
|
|
1184
1320
|
return refuse("action-failed", `refused: the tracker rejected the label change:\n${why}`, ref.issue);
|
|
1185
1321
|
}
|
|
1186
1322
|
|
|
1323
|
+
// A promotion that committed has made an issue claimable *now*, so the
|
|
1324
|
+
// resident daemon is poked rather than leaving the candidate for the next
|
|
1325
|
+
// scheduled pass (#878): three claimable issues once sat against `workers
|
|
1326
|
+
// 0 / 3` for five minutes, which reads from outside exactly like a stalled
|
|
1327
|
+
// queue. Only the queue label earns it — no other label changes what is
|
|
1328
|
+
// dispatchable — and only after the tracker mutation above committed, so a
|
|
1329
|
+
// refused or failed change never pokes anything. The pass it prompts applies
|
|
1330
|
+
// every hold, drain, lane and budget gate as usual.
|
|
1331
|
+
const woken =
|
|
1332
|
+
action === "add" && label === project.queueLabel ? await wakeDispatch(project.name) : undefined;
|
|
1333
|
+
|
|
1187
1334
|
return allow(
|
|
1188
|
-
echo === undefined ? outcome : `${outcome} File lane: ${echo}.`,
|
|
1335
|
+
[echo === undefined ? outcome : `${outcome} File lane: ${echo}.`, woken]
|
|
1336
|
+
.filter((part) => part !== undefined)
|
|
1337
|
+
.join(" "),
|
|
1189
1338
|
undefined,
|
|
1190
1339
|
ref.issue,
|
|
1191
1340
|
);
|
|
@@ -1395,10 +1544,27 @@ async function releaseVerb(
|
|
|
1395
1544
|
}
|
|
1396
1545
|
|
|
1397
1546
|
const tag = args["tag"];
|
|
1547
|
+
const sha = args["sha"];
|
|
1548
|
+
// A pinned commit is only meaningful for the two tag shapes; anywhere else it
|
|
1549
|
+
// would be a decorative argument, which is exactly the silent fake this
|
|
1550
|
+
// guards against. Refused rather than ignored, so the caller learns.
|
|
1551
|
+
if (typeof sha === "string" && shape !== "git-tag" && shape !== "git-push-tags") {
|
|
1552
|
+
return refuse(
|
|
1553
|
+
"malformed-argument",
|
|
1554
|
+
`refused: sha pins a tag to one commit and only git-tag and git-push-tags cut tags; you asked for ${shape}.`,
|
|
1555
|
+
);
|
|
1556
|
+
}
|
|
1557
|
+
if (typeof sha === "string" && !FULL_OR_ABBREVIATED_SHA.test(sha)) {
|
|
1558
|
+
return refuse(
|
|
1559
|
+
"malformed-argument",
|
|
1560
|
+
`refused: ${JSON.stringify(sha)} is not a commit id. Pass the exact hex sha the tag must point at.`,
|
|
1561
|
+
);
|
|
1562
|
+
}
|
|
1398
1563
|
const outcome = await deps.actions.release({
|
|
1399
1564
|
shape,
|
|
1400
1565
|
repo,
|
|
1401
1566
|
...(typeof tag === "string" ? { tag } : {}),
|
|
1567
|
+
...(typeof sha === "string" ? { sha } : {}),
|
|
1402
1568
|
...(typeof artefact === "string" ? { artefact } : {}),
|
|
1403
1569
|
...(typeof environment === "string" ? { environment } : {}),
|
|
1404
1570
|
});
|
|
@@ -1857,11 +2023,46 @@ async function prReviewVerb(
|
|
|
1857
2023
|
const review = resolveReview(project);
|
|
1858
2024
|
const pendingForRun = deps.store.pendingReviewForRun(project.name, target.id);
|
|
1859
2025
|
if (pendingForRun === undefined && deps.store.latestReviewRound(project.name, target.id) >= review.maxRounds) {
|
|
1860
|
-
|
|
1861
|
-
|
|
1862
|
-
|
|
1863
|
-
|
|
1864
|
-
|
|
2026
|
+
// The ceiling no longer ends in an operator task (#873, #932). There are no
|
|
2027
|
+
// worker rounds left — that part is unchanged and is the whole point of a
|
|
2028
|
+
// ceiling — so what escalates is the AUTOMATION: this admits one bounded
|
|
2029
|
+
// adjudication under the project's configured stronger role, and the
|
|
2030
|
+
// daemon's next dispatch pass launches it.
|
|
2031
|
+
//
|
|
2032
|
+
// The head is the one already verified live and green above, so a moved head
|
|
2033
|
+
// has refused (`head-stale`) before reaching here: an adjudication is never
|
|
2034
|
+
// opened against a head nobody re-read. The findings being submitted travel
|
|
2035
|
+
// with it — they are the most relevant single input the adjudicator gets,
|
|
2036
|
+
// and dropping them on the floor would make it re-derive (or miss) exactly
|
|
2037
|
+
// the concern that provoked the escalation.
|
|
2038
|
+
const admitted = deps.store.openReviewAdjudication({
|
|
2039
|
+
project: project.name,
|
|
2040
|
+
issue,
|
|
2041
|
+
prUrl,
|
|
2042
|
+
headSha,
|
|
2043
|
+
role: review.adjudicator,
|
|
2044
|
+
findings,
|
|
2045
|
+
requestedAt: deps.now(),
|
|
2046
|
+
});
|
|
2047
|
+
if (admitted.kind === "refused") {
|
|
2048
|
+
return refuse(
|
|
2049
|
+
"review-in-flight",
|
|
2050
|
+
`refused: an adjudication is already live for ${prUrl} at ${admitted.record.headSha} ` +
|
|
2051
|
+
`(${admitted.record.state}), and this call names ${headSha}. That PR moved while it was being ` +
|
|
2052
|
+
"adjudicated; wait for the live adjudication to settle before returning a finding at the new head.",
|
|
2053
|
+
issue,
|
|
2054
|
+
);
|
|
2055
|
+
}
|
|
2056
|
+
// Idempotent by construction: a repeated call at the same head is told the
|
|
2057
|
+
// row it already has, never given a second adjudicator.
|
|
2058
|
+
return allow(
|
|
2059
|
+
admitted.kind === "created"
|
|
2060
|
+
? `review round ceiling reached (${review.maxRounds}) — recorded a final adjudication for ${prUrl} at ` +
|
|
2061
|
+
`${headSha} under the "${review.adjudicator}" model role. The daemon launches it on its next dispatch ` +
|
|
2062
|
+
"pass; do not review, merge, close or repair this PR by hand."
|
|
2063
|
+
: `an adjudication for ${prUrl} at ${headSha} already exists (${admitted.record.state}) under the ` +
|
|
2064
|
+
`"${admitted.record.role}" role — nothing further to record. The daemon owns it from here.`,
|
|
2065
|
+
undefined,
|
|
1865
2066
|
issue,
|
|
1866
2067
|
);
|
|
1867
2068
|
}
|
|
@@ -1922,6 +2123,155 @@ async function prReviewVerb(
|
|
|
1922
2123
|
);
|
|
1923
2124
|
}
|
|
1924
2125
|
|
|
2126
|
+
/**
|
|
2127
|
+
* Record that the durable review findings standing at one exact head are
|
|
2128
|
+
* settled, so the #888 merge gate stops refusing that head (#913).
|
|
2129
|
+
*
|
|
2130
|
+
* The gate it feeds had no reachable clearance at an unchanged head. Its own
|
|
2131
|
+
* refusal advised a fresh `conductor_pr_review`, and that provably does not
|
|
2132
|
+
* work: `enqueueReviewRevision` either appends the finding to the pending row
|
|
2133
|
+
* or inserts a new unsettled one, and both block. The only other clearances
|
|
2134
|
+
* are a corrected head — which needs a worker — or a round settling
|
|
2135
|
+
* `skipped`/`revised`, which nothing an orchestrator can call does. A PR at
|
|
2136
|
+
* the review-round ceiling was therefore permanently unmergeable at that head:
|
|
2137
|
+
* a gate that can deadlock a release with no way out.
|
|
2138
|
+
*
|
|
2139
|
+
* This is a separate verb rather than an argument on the merge, following
|
|
2140
|
+
* #850's composition override exactly: a merge call's own `reason` is not an
|
|
2141
|
+
* override, because an override that rides inside the act it authorizes leaves
|
|
2142
|
+
* no separable record of the judgement. Here the judgement is its own ledger
|
|
2143
|
+
* row, its own material event and its own append-only table row.
|
|
2144
|
+
*
|
|
2145
|
+
* Three gates, and each exists against a specific way of faking it:
|
|
2146
|
+
* - **ownership**, through the same `prReviewReadiness` the review verb uses,
|
|
2147
|
+
* but *without* its revisable-state requirement: the run legitimately sits
|
|
2148
|
+
* `pushed-green` here, and demanding revisability would make the clearance
|
|
2149
|
+
* unreachable for exactly the PRs it exists for;
|
|
2150
|
+
* - **exact head**, re-read live. Green is deliberately NOT required —
|
|
2151
|
+
* clearing findings and passing checks are different questions, and the
|
|
2152
|
+
* merge gate re-checks green itself immediately before merging;
|
|
2153
|
+
* - **non-vacuous**: a clearance at a head where nothing blocks is refused,
|
|
2154
|
+
* so one can never be banked "just in case" against a finding that has not
|
|
2155
|
+
* been written yet. Combined with the store's timestamp bound, a clearance
|
|
2156
|
+
* can only ever dispose of evidence that already existed when it was taken.
|
|
2157
|
+
*/
|
|
2158
|
+
async function prReviewClearVerb(
|
|
2159
|
+
deps: VerbDeps,
|
|
2160
|
+
project: ProjectConfig,
|
|
2161
|
+
channel: VerbChannel,
|
|
2162
|
+
args: Record<string, unknown>,
|
|
2163
|
+
refuse: Refuse,
|
|
2164
|
+
allow: Allow,
|
|
2165
|
+
): Promise<Verdict> {
|
|
2166
|
+
const prUrl = String(args["prUrl"]);
|
|
2167
|
+
const headSha = String(args["headSha"]);
|
|
2168
|
+
const reason = String(args["reason"]);
|
|
2169
|
+
|
|
2170
|
+
// Ownership only: the newest project-owned run that recorded this PR. A
|
|
2171
|
+
// `not-revisable` readiness is the *expected* answer — the run pushed green
|
|
2172
|
+
// and is waiting on the merge — so only `no-owner` refuses here.
|
|
2173
|
+
const readiness = prReviewReadiness(deps.store, project.name, prUrl, deps.now());
|
|
2174
|
+
if (readiness.kind === "no-owner") {
|
|
2175
|
+
return refuse(
|
|
2176
|
+
"pr-not-this-run",
|
|
2177
|
+
`refused: ${prUrl} is not a pull request any run in ${project.name} opened. ` +
|
|
2178
|
+
"A review clearance disposes of evidence recorded against a run-owned PR only.",
|
|
2179
|
+
);
|
|
2180
|
+
}
|
|
2181
|
+
const target = readiness.run;
|
|
2182
|
+
if (!prInProjectRouting(project, prUrl)) {
|
|
2183
|
+
return refuse(
|
|
2184
|
+
"pr-not-this-run",
|
|
2185
|
+
`refused: ${prUrl} is not in ${project.name}'s routed repositories ` +
|
|
2186
|
+
`(${Object.values(project.routing.repos).map(repoSlugFor).join(", ") || "none"}).`,
|
|
2187
|
+
target.issue,
|
|
2188
|
+
);
|
|
2189
|
+
}
|
|
2190
|
+
const issue = target.issue;
|
|
2191
|
+
|
|
2192
|
+
if (reason.trim() === "") {
|
|
2193
|
+
return refuse(
|
|
2194
|
+
"malformed-argument",
|
|
2195
|
+
"refused: conductor_pr_review_clear needs a non-empty reason — the reason IS the audit trail for " +
|
|
2196
|
+
"merging a head a review round refused, and an unexplained clearance is indistinguishable from " +
|
|
2197
|
+
"the blanket override this verb exists not to be.",
|
|
2198
|
+
issue,
|
|
2199
|
+
);
|
|
2200
|
+
}
|
|
2201
|
+
|
|
2202
|
+
const paused = orchestratorPauseRefusal(deps, channel, target, refuse);
|
|
2203
|
+
if (paused !== undefined) return paused;
|
|
2204
|
+
|
|
2205
|
+
// Exact head, read live. `verification.headSha` is the live head read in the
|
|
2206
|
+
// same request, so the comparison is against what is on the branch now — not
|
|
2207
|
+
// against the status verdict, which is why a red or pending PR is cleared
|
|
2208
|
+
// just as well as a green one. The merge gate is what insists on green.
|
|
2209
|
+
let verification: PrVerification | undefined;
|
|
2210
|
+
try {
|
|
2211
|
+
verification = await deps.tracker.verifyPr(prUrl, headSha);
|
|
2212
|
+
} catch (err) {
|
|
2213
|
+
const why = err instanceof Error ? err.message : String(err);
|
|
2214
|
+
return refuse("head-unresolvable", `refused: the live head of ${prUrl} could not be read (${why}).`, issue);
|
|
2215
|
+
}
|
|
2216
|
+
if (verification === undefined) {
|
|
2217
|
+
return refuse(
|
|
2218
|
+
"head-unresolvable",
|
|
2219
|
+
`refused: the live head of ${prUrl} could not be resolved. Refusing rather than clearing findings ` +
|
|
2220
|
+
"against a head nobody re-read.",
|
|
2221
|
+
issue,
|
|
2222
|
+
);
|
|
2223
|
+
}
|
|
2224
|
+
if (verification.headSha.toLowerCase() !== headSha.toLowerCase()) {
|
|
2225
|
+
return refuse(
|
|
2226
|
+
"head-stale",
|
|
2227
|
+
`refused: ${prUrl} is at ${verification.headSha}, not ${headSha}. A clearance names the exact head ` +
|
|
2228
|
+
"whose findings it disposes of; the branch moved, so those findings no longer stand between this PR " +
|
|
2229
|
+
"and a merge. Re-read the head and re-review it.",
|
|
2230
|
+
issue,
|
|
2231
|
+
);
|
|
2232
|
+
}
|
|
2233
|
+
|
|
2234
|
+
// Non-vacuous. Read after the head check on purpose: if the caller named a
|
|
2235
|
+
// head that moved, the honest refusal is the stale head, not "nothing to
|
|
2236
|
+
// clear".
|
|
2237
|
+
const blocking = deps.store.mergeBlockingReviews(project.name, prUrl, headSha);
|
|
2238
|
+
if (blocking.length === 0) {
|
|
2239
|
+
return refuse(
|
|
2240
|
+
"review-clearance-vacuous",
|
|
2241
|
+
`refused: nothing blocks merging ${prUrl} at ${headSha.slice(0, 12)} — there is no review evidence ` +
|
|
2242
|
+
"at this head to dispose of. A clearance is a disposition of specific findings, never a standing " +
|
|
2243
|
+
"pass recorded ahead of them: a later round at this head would not be cleared by it anyway.",
|
|
2244
|
+
issue,
|
|
2245
|
+
);
|
|
2246
|
+
}
|
|
2247
|
+
|
|
2248
|
+
const cleared = deps.store.recordReviewClearance({
|
|
2249
|
+
project: project.name,
|
|
2250
|
+
prUrl,
|
|
2251
|
+
headSha,
|
|
2252
|
+
by: "orchestrator",
|
|
2253
|
+
reason,
|
|
2254
|
+
at: deps.now(),
|
|
2255
|
+
});
|
|
2256
|
+
deps.store.recordMaterialEvent({
|
|
2257
|
+
project: project.name,
|
|
2258
|
+
category: "review-clearance",
|
|
2259
|
+
summary:
|
|
2260
|
+
`#${issue} review findings cleared at ${headSha.slice(0, 12)} ` +
|
|
2261
|
+
`(round(s) ${blocking.map((row) => row.round).join(", ")}): ${reason}`,
|
|
2262
|
+
evidence: prUrl,
|
|
2263
|
+
occurredAt: cleared.at,
|
|
2264
|
+
recordedAt: cleared.at,
|
|
2265
|
+
});
|
|
2266
|
+
return allow(
|
|
2267
|
+
`cleared ${blocking.length} blocking review round(s) (${blocking.map((row) => row.round).join(", ")}) ` +
|
|
2268
|
+
`for ${prUrl} at ${headSha}: ${reason}. conductor_pr_merge may now proceed at this exact head; every ` +
|
|
2269
|
+
"other gate still applies, and a finding recorded after this clearance blocks again.",
|
|
2270
|
+
headSha,
|
|
2271
|
+
issue,
|
|
2272
|
+
);
|
|
2273
|
+
}
|
|
2274
|
+
|
|
1925
2275
|
/**
|
|
1926
2276
|
* The run states a missing-PR recovery may act on (#806).
|
|
1927
2277
|
*
|
|
@@ -1947,6 +2297,15 @@ const RECOVERABLE_RUN_STATES: Record<string, true> = {
|
|
|
1947
2297
|
/** A complete commit SHA, as current workers record it. */
|
|
1948
2298
|
const FULL_HEAD_SHA = /^[0-9a-f]{40}$/i;
|
|
1949
2299
|
|
|
2300
|
+
/**
|
|
2301
|
+
* A commit id a release may pin a tag to (#695): full, or a git-legal
|
|
2302
|
+
* abbreviation of at least 7 hex — the same lower bound the recovery path uses,
|
|
2303
|
+
* because shorter prefixes are ambiguous. Syntax only; whether the commit
|
|
2304
|
+
* exists and is on the released branch is the privileged half's answer, which
|
|
2305
|
+
* needs the mirror.
|
|
2306
|
+
*/
|
|
2307
|
+
const FULL_OR_ABBREVIATED_SHA = /^[0-9a-f]{7,40}$/i;
|
|
2308
|
+
|
|
1950
2309
|
/**
|
|
1951
2310
|
* A legacy abbreviated commit SHA (#836): the `git rev-parse --short` output
|
|
1952
2311
|
* an older worker generation recorded instead of the full SHA. Git defaults
|
|
@@ -2416,6 +2775,86 @@ async function prRecoverVerb(
|
|
|
2416
2775
|
// through and replace it with a real PR at the verified head.
|
|
2417
2776
|
}
|
|
2418
2777
|
|
|
2778
|
+
// The lane gate (#925), and the last check before anything is published.
|
|
2779
|
+
//
|
|
2780
|
+
// #899 released a worker-free branch's files: its worker is finished, nothing
|
|
2781
|
+
// is writing through it, and holding its whole diff forever idled the fleet.
|
|
2782
|
+
// The consequence is that by the time recovery runs, another worker may
|
|
2783
|
+
// legitimately hold those same files — and publishing this frozen branch then
|
|
2784
|
+
// puts an older diff into review, and into the merge queue, beside a live
|
|
2785
|
+
// worker still rewriting it. That is the one reactivation nothing else
|
|
2786
|
+
// catches: the merge gate reads checks and heads, never who else is mid-edit.
|
|
2787
|
+
//
|
|
2788
|
+
// This half fails CLOSED, unlike admission's interlock, and the asymmetry is
|
|
2789
|
+
// the point. Admission runs unattended on every tick, so refusing there on an
|
|
2790
|
+
// unreadable probe would stall the queue on a transient. Recovery is one
|
|
2791
|
+
// deliberate call whose refusal is legible and retryable: it costs a retry and
|
|
2792
|
+
// names the file, the holder and when to try again. Between "publish
|
|
2793
|
+
// silently on an unproven lane" and "say what could not be read", only the
|
|
2794
|
+
// second is defensible for a call that ends in a public pull request.
|
|
2795
|
+
const leases = deps.store.leasedRuns(project.name).filter((run) => run.repo === target.repo && run.issue !== issue);
|
|
2796
|
+
if (leases.length > 0) {
|
|
2797
|
+
const base = routed.defaultBranch;
|
|
2798
|
+
const mirror = join(project.mirrorRoot, `${target.repo}.git`);
|
|
2799
|
+
const laneOf = async (run: RunRecord): Promise<LaneFile[]> =>
|
|
2800
|
+
deps.lane.probeRunLane({
|
|
2801
|
+
worktree: run.worktree,
|
|
2802
|
+
baseRef: `refs/remotes/origin/${base}`,
|
|
2803
|
+
...(run.branch === "" ? {} : { branchRef: `refs/heads/${run.branch}` }),
|
|
2804
|
+
mirror,
|
|
2805
|
+
});
|
|
2806
|
+
let carried: LaneFile[];
|
|
2807
|
+
try {
|
|
2808
|
+
carried = await laneOf(target);
|
|
2809
|
+
} catch (err) {
|
|
2810
|
+
return refuse(
|
|
2811
|
+
"recovery-lane-unprovable",
|
|
2812
|
+
`refused: the files preserved branch ${target.branch} carries could not be read ` +
|
|
2813
|
+
`(${err instanceof Error ? err.message : String(err)}), and ${leases.length} live mutation lease(s) are ` +
|
|
2814
|
+
`writing in ${identity}. Recovery will not publish a branch whose overlap with live work is unknown — ` +
|
|
2815
|
+
"retry, or wait for those runs to settle.",
|
|
2816
|
+
issue,
|
|
2817
|
+
);
|
|
2818
|
+
}
|
|
2819
|
+
// A branch that carries nothing cannot collide with anything: no read of
|
|
2820
|
+
// the leaseholders is needed, and none is made.
|
|
2821
|
+
if (carried.length > 0) {
|
|
2822
|
+
const wanted = new Set(carried.map(({ file }) => file));
|
|
2823
|
+
for (const run of leases) {
|
|
2824
|
+
let held: LaneFile[];
|
|
2825
|
+
try {
|
|
2826
|
+
held = await laneOf(run);
|
|
2827
|
+
} catch (err) {
|
|
2828
|
+
return refuse(
|
|
2829
|
+
"recovery-lane-unprovable",
|
|
2830
|
+
`refused: the lane of live run ${run.id} (#${run.issue}, ${run.state}) could not be read ` +
|
|
2831
|
+
`(${err instanceof Error ? err.message : String(err)}), so recovery cannot prove ${target.branch} does ` +
|
|
2832
|
+
"not overlap work in flight. Retry, or wait for that run to settle.",
|
|
2833
|
+
issue,
|
|
2834
|
+
);
|
|
2835
|
+
}
|
|
2836
|
+
// The run's own persisted declaration counts too: a leaseholder admitted
|
|
2837
|
+
// on a lane it has not written yet still owns those files, exactly as
|
|
2838
|
+
// admission's occupancy reads it (#744).
|
|
2839
|
+
const occupies = [
|
|
2840
|
+
...held.map(({ file, source }) => ({ file, source: source as LaneSource | "declared" })),
|
|
2841
|
+
...(run.lane?.files ?? []).map((file) => ({ file, source: "declared" as const })),
|
|
2842
|
+
];
|
|
2843
|
+
const collision = occupies.find(({ file }) => wanted.has(file));
|
|
2844
|
+
if (collision !== undefined) {
|
|
2845
|
+
return refuse(
|
|
2846
|
+
"recovery-lane-occupied",
|
|
2847
|
+
`refused: ${collision.file} is held by run ${run.id} for #${run.issue} (${run.state}, ${collision.source}), ` +
|
|
2848
|
+
`and preserved branch ${target.branch} carries it. Publishing now would put this frozen diff into ` +
|
|
2849
|
+
"review beside a worker still rewriting that file. The lease releases when that run settles — recover " +
|
|
2850
|
+
"then, or land its work first.",
|
|
2851
|
+
issue,
|
|
2852
|
+
);
|
|
2853
|
+
}
|
|
2854
|
+
}
|
|
2855
|
+
}
|
|
2856
|
+
}
|
|
2857
|
+
|
|
2419
2858
|
// The PR's own voice comes from durable state only: the issue title and the
|
|
2420
2859
|
// run's settlement report. The closing keyword must name the TRACKER
|
|
2421
2860
|
// repository — the issue lives there, not in the routed code repo, and
|
package/src/wake.ts
ADDED
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The one bounded client for the resident daemon's loopback wake (#878).
|
|
3
|
+
*
|
|
4
|
+
* Extracted from `resume` because three different successful transitions leave
|
|
5
|
+
* queued work waiting for the next scheduled pass — a promotion to the queue
|
|
6
|
+
* label, an unblock that restores eligibility, and `resume` itself — and each
|
|
7
|
+
* one waiting up to five minutes reads, from outside, exactly like a stalled
|
|
8
|
+
* queue. On 2026-08-21T17:57Z three claimable issues sat unadmitted against
|
|
9
|
+
* `workers 0 / 3` until the 18:02:33Z pass, and the operator reasonably asked
|
|
10
|
+
* whether a second dispatcher was needed. It was not: the resident daemon
|
|
11
|
+
* already owns claims and capacity, and `POST /wake` already exists — the
|
|
12
|
+
* transitions simply never used it.
|
|
13
|
+
*
|
|
14
|
+
* What a wake is NOT: it starts no pass of its own, claims nothing, and skips
|
|
15
|
+
* nothing. It shortens a sleep. The pass it prompts re-reads every pause file,
|
|
16
|
+
* drain fence, lane interlock, budget and routing rule exactly as a scheduled
|
|
17
|
+
* pass does, so a project that is held stays held. Anything else — a shorter
|
|
18
|
+
* interval, a second `daemon --once`, waking only from the tick — makes the
|
|
19
|
+
* delay look smaller while leaving idle slots or breaking single-owner safety.
|
|
20
|
+
*
|
|
21
|
+
* Every failure is honest and non-fatal: the caller's own mutation already
|
|
22
|
+
* committed, so a missing or unreachable daemon degrades to "the next scheduled
|
|
23
|
+
* pass will claim" rather than an error that suggests the mutation failed.
|
|
24
|
+
*/
|
|
25
|
+
|
|
26
|
+
import { livingDaemon } from "./lifecycle.ts";
|
|
27
|
+
|
|
28
|
+
/** What the wake attempt did, as the line a human reads after the mutation. */
|
|
29
|
+
export async function wakeDispatch(projectName: string): Promise<string> {
|
|
30
|
+
const daemon = livingDaemon();
|
|
31
|
+
if (daemon === undefined) {
|
|
32
|
+
return "daemon not running — claiming starts when the daemon next starts or ticks";
|
|
33
|
+
}
|
|
34
|
+
let response: Response;
|
|
35
|
+
try {
|
|
36
|
+
response = await fetch(`http://127.0.0.1:${daemon.port}/wake`, {
|
|
37
|
+
method: "POST",
|
|
38
|
+
headers: { "content-type": "application/json" },
|
|
39
|
+
body: JSON.stringify({ project: projectName }),
|
|
40
|
+
});
|
|
41
|
+
} catch {
|
|
42
|
+
return "daemon wake failed — daemon unreachable; the next scheduled pass will claim";
|
|
43
|
+
}
|
|
44
|
+
if (!response.ok) {
|
|
45
|
+
return `daemon wake failed (HTTP ${response.status}); the next scheduled pass will claim`;
|
|
46
|
+
}
|
|
47
|
+
return "daemon dispatch loop woken — claiming starts on an immediate pass";
|
|
48
|
+
}
|