omp-conductor 0.17.0 → 0.17.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/REFERENCE.md +1 -1
- package/package.json +1 -1
- package/schema/config.schema.json +16 -0
- package/src/admission.ts +159 -43
- package/src/availability.ts +27 -1
- package/src/briefs/worker.md +2 -0
- package/src/commands/arm.ts +6 -3
- package/src/commands/message.ts +32 -4
- package/src/config-schema.ts +20 -0
- package/src/config.ts +37 -0
- package/src/daemon.ts +1240 -18
- package/src/doctor.ts +310 -22
- package/src/escalate.ts +560 -57
- package/src/failure-class.ts +56 -13
- package/src/fleet.ts +169 -32
- package/src/gitops.ts +103 -24
- package/src/lifecycle.ts +7 -2
- package/src/orchestrator-tick.ts +327 -151
- package/src/release-policy.ts +177 -5
- package/src/setup-host.ts +193 -4
- package/src/setup-wizard.ts +1182 -78
- package/src/setup.ts +60 -3
- package/src/status-render.ts +11 -1
- package/src/store.ts +333 -12
- package/src/tracker/github.ts +562 -13
- package/src/types.ts +204 -2
- package/src/upgrade.ts +50 -19
- package/src/verbs/actions.ts +66 -18
- package/src/verbs/protocol.ts +45 -0
- package/src/verbs/server.ts +212 -11
- package/src/worker.ts +26 -0
- package/systemd/omp-conductor-recover.sh +73 -0
- package/systemd/recover-unit-test.sh +61 -0
package/src/verbs/server.ts
CHANGED
|
@@ -54,6 +54,7 @@ import type {
|
|
|
54
54
|
ReleaseRequirement,
|
|
55
55
|
ReleaseShape,
|
|
56
56
|
RepoTarget,
|
|
57
|
+
ReviewReason,
|
|
57
58
|
RunRecord,
|
|
58
59
|
Store,
|
|
59
60
|
Tracker,
|
|
@@ -167,7 +168,6 @@ export interface VerbActions {
|
|
|
167
168
|
fields: { title?: string; body?: string },
|
|
168
169
|
): Promise<{ ok: true } | { ok: false; stderr: string }>;
|
|
169
170
|
mergePr(prUrl: string, headSha: string): Promise<ActionOutcome>;
|
|
170
|
-
setLabel(issue: number, label: string, action: "add" | "remove"): Promise<ActionOutcome>;
|
|
171
171
|
release(execution: ReleaseExecution): Promise<ActionOutcome>;
|
|
172
172
|
/**
|
|
173
173
|
* Request the fleet upgrade itself to `version`: refuse when npm does not
|
|
@@ -298,6 +298,12 @@ export interface ReleaseFacts {
|
|
|
298
298
|
* must wait on a process; these clear by themselves on the next sweep.
|
|
299
299
|
*/
|
|
300
300
|
unconfirmedMerges?: number;
|
|
301
|
+
/**
|
|
302
|
+
* The unsettled runs themselves, routed-repo and issue each, so the refusal
|
|
303
|
+
* names what is blocking instead of a bare count (#603). Absent when the
|
|
304
|
+
* caller could not gather them (the plain wording is used then).
|
|
305
|
+
*/
|
|
306
|
+
blockingRuns?: { repo: string; issue: number }[];
|
|
301
307
|
/** Queue depth, or `undefined` when the tracker could not be read. */
|
|
302
308
|
queueDepth: number | undefined;
|
|
303
309
|
/** Current live-head workflow verdict for the released routed repository. */
|
|
@@ -311,7 +317,7 @@ export function releaseRequirementRefusal(
|
|
|
311
317
|
facts: ReleaseFacts,
|
|
312
318
|
): string | undefined {
|
|
313
319
|
for (const requirement of requires) {
|
|
314
|
-
if (requirement === "runs-settled" && facts.unsettledRuns > 0) {
|
|
320
|
+
if ((requirement === "runs-settled" || requirement === "fleet-runs-settled") && facts.unsettledRuns > 0) {
|
|
315
321
|
// The same blocker reads differently depending on WHY the row is still
|
|
316
322
|
// active: a live worker is the operator's wait, an unconfirmed merge or
|
|
317
323
|
// an unrecorded sweep clears by itself on the next settle pass. The
|
|
@@ -328,9 +334,16 @@ export function releaseRequirementRefusal(
|
|
|
328
334
|
if (open > 0) parts.push(`${open} pull request(s) still open`);
|
|
329
335
|
if (unconfirmed > 0) parts.push(`${unconfirmed} run(s) whose PR merge the settle sweep has not yet recorded`);
|
|
330
336
|
if (remaining > 0) parts.push(`${remaining} run(s) with no merged PR (closed or none yet)`);
|
|
337
|
+
// Name the runs behind the count so the operator can see at a glance
|
|
338
|
+
// whether this is the release's own work or someone else's (#603).
|
|
339
|
+
const blocking =
|
|
340
|
+
facts.blockingRuns === undefined || facts.blockingRuns.length === 0
|
|
341
|
+
? ""
|
|
342
|
+
: ` — blocking: ${facts.blockingRuns.map((r) => `${r.repo} #${r.issue}`).join(", ")}`;
|
|
331
343
|
return (
|
|
332
|
-
`policy.release.requires includes
|
|
333
|
-
(parts.length === 0 ? "" : ` — ${parts.join("; ")}`)
|
|
344
|
+
`policy.release.requires includes ${requirement} and ${facts.unsettledRuns} run(s) have not settled` +
|
|
345
|
+
(parts.length === 0 ? "" : ` — ${parts.join("; ")}`) +
|
|
346
|
+
blocking
|
|
334
347
|
);
|
|
335
348
|
}
|
|
336
349
|
if (requirement === "no-open-prs" && facts.openPrs > 0) {
|
|
@@ -505,6 +518,7 @@ async function decide(deps: VerbDeps, channel: VerbChannel, raw: unknown): Promi
|
|
|
505
518
|
(verb === "conductor_pr_merge" ||
|
|
506
519
|
verb === "conductor_pr_update_branch" ||
|
|
507
520
|
verb === "conductor_pr_update" ||
|
|
521
|
+
verb === "conductor_pr_review" ||
|
|
508
522
|
verb === "conductor_label");
|
|
509
523
|
const stopped =
|
|
510
524
|
spec.mutating && verb !== "conductor_release" && verb !== "conductor_install" && !orchestratorCompletion
|
|
@@ -540,6 +554,8 @@ async function decide(deps: VerbDeps, channel: VerbChannel, raw: unknown): Promi
|
|
|
540
554
|
return releaseVerb(deps, project, channel, args, refuse, allow);
|
|
541
555
|
case "conductor_install":
|
|
542
556
|
return installVerb(deps, project, channel, args, refuse, allow);
|
|
557
|
+
case "conductor_pr_review":
|
|
558
|
+
return prReviewVerb(deps, project, channel, args, refuse, allow);
|
|
543
559
|
case "conductor_pr_status":
|
|
544
560
|
return prStatusVerb(deps, project, channel, args, refuse, allow);
|
|
545
561
|
}
|
|
@@ -797,7 +813,15 @@ async function prUpdateBranchVerb(
|
|
|
797
813
|
if (!outcome.ok) {
|
|
798
814
|
return refuse("action-failed", `refused: gh could not update the branch:\n${outcome.stderr}`, target?.issue);
|
|
799
815
|
}
|
|
800
|
-
|
|
816
|
+
// The REST endpoint answers 202 Accepted: the refresh runs asynchronously, so
|
|
817
|
+
// this call produces no head SHA and the reply/ledger must not claim one —
|
|
818
|
+
// the head the action observed is the pre-update guard head, never a result
|
|
819
|
+
// (#655 review). The caller re-reads PR and check state before merging.
|
|
820
|
+
return allow(
|
|
821
|
+
`requested a base-branch update of ${prUrl} from its observed head; PR and check state must be re-read before merging.`,
|
|
822
|
+
undefined,
|
|
823
|
+
target?.issue,
|
|
824
|
+
);
|
|
801
825
|
}
|
|
802
826
|
|
|
803
827
|
async function prMergeVerb(
|
|
@@ -1064,9 +1088,21 @@ async function labelVerb(
|
|
|
1064
1088
|
}
|
|
1065
1089
|
|
|
1066
1090
|
const action = args["action"] === "remove" ? "remove" : "add";
|
|
1067
|
-
|
|
1068
|
-
|
|
1069
|
-
|
|
1091
|
+
// The queue label is the fleet's lock, so it must not ride the GraphQL
|
|
1092
|
+
// surface a provider outage just broke: `gh issue edit` is a GraphQL
|
|
1093
|
+
// mutation in gh 2.86, and #642 comment 4 measured a label withdrawal
|
|
1094
|
+
// refused by a GraphQL 503 leaving two overlapping issues claimable. The
|
|
1095
|
+
// tracker's label methods are core REST with noop classification for the
|
|
1096
|
+
// already-holding end state — the guarded path.
|
|
1097
|
+
try {
|
|
1098
|
+
if (action === "add") {
|
|
1099
|
+
await deps.tracker.addLabel(ref.issue, label);
|
|
1100
|
+
} else {
|
|
1101
|
+
await deps.tracker.removeLabel(ref.issue, label);
|
|
1102
|
+
}
|
|
1103
|
+
} catch (err) {
|
|
1104
|
+
const why = err instanceof Error ? err.message : String(err);
|
|
1105
|
+
return refuse("action-failed", `refused: the tracker rejected the label change:\n${why}`, ref.issue);
|
|
1070
1106
|
}
|
|
1071
1107
|
return allow(`${action === "add" ? "added" : "removed"} ${label} on #${ref.issue}.`, undefined, ref.issue);
|
|
1072
1108
|
}
|
|
@@ -1151,22 +1187,36 @@ async function releaseVerb(
|
|
|
1151
1187
|
// this release by holding. Fail-closed stays fail-closed: a row whose PR
|
|
1152
1188
|
// state cannot be read is still unsettled, and an unknown PR is still
|
|
1153
1189
|
// treated as open.
|
|
1190
|
+
//
|
|
1191
|
+
// The run facts are scoped to the released repo (#603): a run in another
|
|
1192
|
+
// routed repo cannot invalidate a tag on this one, so it must not gate this
|
|
1193
|
+
// release. A genuinely suite-wide shape — a pin or manifest that consumes
|
|
1194
|
+
// several repos — opts back into project-wide strictness through the named
|
|
1195
|
+
// `fleet-runs-settled` requirement, which widens the scope to every active
|
|
1196
|
+
// run. `no-open-prs` shares the scope: the branch being released is this
|
|
1197
|
+
// repo's, so an open PR elsewhere is not "against" it either.
|
|
1198
|
+
const fleetWide = policy.release.requires.includes("fleet-runs-settled");
|
|
1154
1199
|
const wantsRunFacts =
|
|
1155
1200
|
policy.release.requires.includes("runs-settled") ||
|
|
1201
|
+
fleetWide ||
|
|
1156
1202
|
policy.release.requires.includes("no-open-prs");
|
|
1157
|
-
|
|
1158
|
-
let
|
|
1203
|
+
const scoped = fleetWide ? active : active.filter((r) => r.repo === repoName);
|
|
1204
|
+
let unsettledRuns = scoped.length;
|
|
1205
|
+
let openPrs = scoped.filter((r) => r.prUrl !== undefined).length;
|
|
1159
1206
|
let liveWorkers: number | undefined;
|
|
1160
1207
|
let unconfirmedMerges: number | undefined;
|
|
1208
|
+
let blockingRuns: { repo: string; issue: number }[] | undefined;
|
|
1161
1209
|
if (wantsRunFacts) {
|
|
1162
1210
|
let unsettled = 0;
|
|
1163
1211
|
let open = 0;
|
|
1164
1212
|
let live = 0;
|
|
1165
1213
|
let unconfirmed = 0;
|
|
1166
|
-
|
|
1214
|
+
const blocking: { repo: string; issue: number }[] = [];
|
|
1215
|
+
for (const run of scoped) {
|
|
1167
1216
|
if (run.state === "claimed" || run.state === "running") {
|
|
1168
1217
|
unsettled += 1;
|
|
1169
1218
|
live += 1;
|
|
1219
|
+
blocking.push({ repo: run.repo, issue: run.issue });
|
|
1170
1220
|
// A live worker's PR is not yet confirmable; count it as open, the
|
|
1171
1221
|
// same fail-closed answer the old row-count gave it.
|
|
1172
1222
|
if (run.prUrl !== undefined) open += 1;
|
|
@@ -1175,6 +1225,7 @@ async function releaseVerb(
|
|
|
1175
1225
|
// pushed-pending / pushed-green: the worker is finished; the PR decides.
|
|
1176
1226
|
if (run.prUrl === undefined) {
|
|
1177
1227
|
unsettled += 1; // nothing to read; the sweep cannot settle it either
|
|
1228
|
+
blocking.push({ repo: run.repo, issue: run.issue });
|
|
1178
1229
|
continue;
|
|
1179
1230
|
}
|
|
1180
1231
|
let pr: PrState | undefined;
|
|
@@ -1185,6 +1236,7 @@ async function releaseVerb(
|
|
|
1185
1236
|
}
|
|
1186
1237
|
if (pr === "merged") continue; // settled in fact; not an open PR
|
|
1187
1238
|
unsettled += 1;
|
|
1239
|
+
blocking.push({ repo: run.repo, issue: run.issue });
|
|
1188
1240
|
if (pr === "open") open += 1;
|
|
1189
1241
|
else if (pr === undefined) {
|
|
1190
1242
|
// Cannot confirm the merge (or its absence): fail closed, and count
|
|
@@ -1200,12 +1252,14 @@ async function releaseVerb(
|
|
|
1200
1252
|
openPrs = open;
|
|
1201
1253
|
liveWorkers = live;
|
|
1202
1254
|
unconfirmedMerges = unconfirmed;
|
|
1255
|
+
blockingRuns = blocking;
|
|
1203
1256
|
}
|
|
1204
1257
|
const unmet = releaseRequirementRefusal(policy.release.requires, {
|
|
1205
1258
|
unsettledRuns,
|
|
1206
1259
|
openPrs,
|
|
1207
1260
|
...(liveWorkers === undefined ? {} : { liveWorkers }),
|
|
1208
1261
|
...(unconfirmedMerges === undefined ? {} : { unconfirmedMerges }),
|
|
1262
|
+
...(blockingRuns === undefined ? {} : { blockingRuns }),
|
|
1209
1263
|
queueDepth,
|
|
1210
1264
|
...(health === undefined ? {} : { baseCheck: health.verdict }),
|
|
1211
1265
|
...(health?.verdict === "red" && health.detail !== undefined
|
|
@@ -1509,6 +1563,153 @@ async function prUpdateVerb(
|
|
|
1509
1563
|
return allow(`updated ${prUrl}.`, undefined, issue);
|
|
1510
1564
|
}
|
|
1511
1565
|
|
|
1566
|
+
/**
|
|
1567
|
+
* Return one green, run-owned pull request to its worker with blocking
|
|
1568
|
+
* findings (#677): the transport that replaces the manual close-PR →
|
|
1569
|
+
* comment-on-issue → unblock → continuation dance.
|
|
1570
|
+
*
|
|
1571
|
+
* Everything is recorded durably BEFORE anything is woken: the findings, the
|
|
1572
|
+
* exact reviewed head, the round number, and the target run/session. The wake
|
|
1573
|
+
* itself is the daemon's next dispatch pass (the verb has no handle on the
|
|
1574
|
+
* worker machinery, and the CLI path must behave exactly like the embedded
|
|
1575
|
+
* one), and the run row is reused — same branch, same PR, same attempt, same
|
|
1576
|
+
* session directory — so a revision round never creates a new attempt and
|
|
1577
|
+
* never touches the failed-attempt or continuation budgets.
|
|
1578
|
+
*
|
|
1579
|
+
* The duplicate in-flight guard has two halves: while a revision is queued
|
|
1580
|
+
* but not yet dispatched, the pending `review_revisions` row refuses the
|
|
1581
|
+
* second request atomically inside `claimReviewRevision`; once dispatched,
|
|
1582
|
+
* the run row itself is `running` (claimed from `pushed-green` by the
|
|
1583
|
+
* dispatch pass), which this verb refuses by name.
|
|
1584
|
+
*/
|
|
1585
|
+
async function prReviewVerb(
|
|
1586
|
+
deps: VerbDeps,
|
|
1587
|
+
project: ProjectConfig,
|
|
1588
|
+
channel: VerbChannel,
|
|
1589
|
+
args: Record<string, unknown>,
|
|
1590
|
+
refuse: Refuse,
|
|
1591
|
+
allow: Allow,
|
|
1592
|
+
): Promise<Verdict> {
|
|
1593
|
+
const prUrl = String(args["prUrl"]);
|
|
1594
|
+
const headSha = String(args["headSha"]);
|
|
1595
|
+
const findings = String(args["findings"]);
|
|
1596
|
+
const reason = String(args["reason"]) as ReviewReason;
|
|
1597
|
+
|
|
1598
|
+
// The run whose pushed-green row owns the PR — same resolution as merge: the
|
|
1599
|
+
// newest attempt that recorded this PR within the recent-history window. A
|
|
1600
|
+
// PR no run of this project opened (or one outside the routed repos) cannot
|
|
1601
|
+
// be returned to a worker this project can resume.
|
|
1602
|
+
const target = runForPr(deps, project.name, prUrl);
|
|
1603
|
+
if (target === undefined) {
|
|
1604
|
+
return refuse(
|
|
1605
|
+
"pr-not-this-run",
|
|
1606
|
+
`refused: ${prUrl} is not a pull request any run in ${project.name} opened. ` +
|
|
1607
|
+
"A review revision acts on a run-owned PR only.",
|
|
1608
|
+
);
|
|
1609
|
+
}
|
|
1610
|
+
if (!prInProjectRouting(project, prUrl)) {
|
|
1611
|
+
return refuse(
|
|
1612
|
+
"pr-not-this-run",
|
|
1613
|
+
`refused: ${prUrl} is not in ${project.name}'s routed repositories ` +
|
|
1614
|
+
`(${Object.values(project.routing.repos).map(repoSlugFor).join(", ") || "none"}).`,
|
|
1615
|
+
target.issue,
|
|
1616
|
+
);
|
|
1617
|
+
}
|
|
1618
|
+
const issue = target.issue;
|
|
1619
|
+
|
|
1620
|
+
if (findings.trim() === "") {
|
|
1621
|
+
return refuse(
|
|
1622
|
+
"malformed-argument",
|
|
1623
|
+
"refused: conductor_pr_review needs a non-empty findings string — an empty revision returns the worker nothing to fix.",
|
|
1624
|
+
issue,
|
|
1625
|
+
);
|
|
1626
|
+
}
|
|
1627
|
+
|
|
1628
|
+
// The revisable state is a settled green run. Any other state is a worker in
|
|
1629
|
+
// flight (the original run or an earlier revision) or a run whose PR no
|
|
1630
|
+
// longer waits on revision — the detail names the state field that produced
|
|
1631
|
+
// the refusal.
|
|
1632
|
+
if (target.state !== "pushed-green") {
|
|
1633
|
+
const live = target.state === "running" || target.state === "claimed";
|
|
1634
|
+
return refuse(
|
|
1635
|
+
"review-in-flight",
|
|
1636
|
+
`refused: run ${target.id} is ${target.state}, not pushed-green — ${
|
|
1637
|
+
live
|
|
1638
|
+
? `a worker (the original run or an earlier revision) is still live on ${prUrl}; wait for it to settle before returning it.`
|
|
1639
|
+
: `a review revision starts from a settled green run, and this one is ${target.state}.`
|
|
1640
|
+
}`,
|
|
1641
|
+
issue,
|
|
1642
|
+
);
|
|
1643
|
+
}
|
|
1644
|
+
|
|
1645
|
+
const paused = orchestratorPauseRefusal(deps, channel, target, refuse);
|
|
1646
|
+
if (paused !== undefined) return paused;
|
|
1647
|
+
|
|
1648
|
+
// Exact-head, re-read at request time: the orchestrator reviewed a specific
|
|
1649
|
+
// head, and a PR that moved since is not the thing it reviewed. Same verdict
|
|
1650
|
+
// the merge gate uses, so "green at headSha" here is the same fact the merge
|
|
1651
|
+
// would have required.
|
|
1652
|
+
let verification: PrVerification | undefined;
|
|
1653
|
+
try {
|
|
1654
|
+
verification = await deps.tracker.verifyPr(prUrl, headSha);
|
|
1655
|
+
} catch (err) {
|
|
1656
|
+
const why = err instanceof Error ? err.message : String(err);
|
|
1657
|
+
return refuse(
|
|
1658
|
+
"head-unresolvable",
|
|
1659
|
+
`refused: the live head of ${prUrl} could not be read (${why}).`,
|
|
1660
|
+
issue,
|
|
1661
|
+
);
|
|
1662
|
+
}
|
|
1663
|
+
if (verification === undefined) {
|
|
1664
|
+
return refuse(
|
|
1665
|
+
"head-unresolvable",
|
|
1666
|
+
`refused: the live head of ${prUrl} could not be resolved. Refusing rather than returning a PR you did not re-read.`,
|
|
1667
|
+
issue,
|
|
1668
|
+
);
|
|
1669
|
+
}
|
|
1670
|
+
if (verification.status !== "green") {
|
|
1671
|
+
const stale = verification.status === "failed" && isHeadMismatch(verification.reason);
|
|
1672
|
+
return refuse(
|
|
1673
|
+
stale ? "head-stale" : "checks-not-green",
|
|
1674
|
+
stale
|
|
1675
|
+
? `refused: ${verification.reason}. You reviewed ${headSha}; that is not what is on the branch now. Re-read the head and re-review.`
|
|
1676
|
+
: `refused: ${prUrl} is not green at ${headSha} — ${verification.reason}. A revision is returned to a green PR only.`,
|
|
1677
|
+
issue,
|
|
1678
|
+
);
|
|
1679
|
+
}
|
|
1680
|
+
|
|
1681
|
+
// Durable, and refused atomically against a concurrent duplicate: the row is
|
|
1682
|
+
// persisted before any wake, and a second request for the same run bumps
|
|
1683
|
+
// into the pending row rather than racing it.
|
|
1684
|
+
const round = deps.store.latestReviewRound(project.name, target.id) + 1;
|
|
1685
|
+
const recorded = deps.store.createReviewRevision({
|
|
1686
|
+
project: project.name,
|
|
1687
|
+
runId: target.id,
|
|
1688
|
+
issue,
|
|
1689
|
+
prUrl,
|
|
1690
|
+
headSha,
|
|
1691
|
+
findings,
|
|
1692
|
+
round,
|
|
1693
|
+
reason,
|
|
1694
|
+
...(target.sessionFile === undefined ? {} : { sessionFile: target.sessionFile }),
|
|
1695
|
+
requestedAt: deps.now(),
|
|
1696
|
+
});
|
|
1697
|
+
if (recorded === undefined) {
|
|
1698
|
+
return refuse(
|
|
1699
|
+
"review-in-flight",
|
|
1700
|
+
`refused: a review revision is already pending for ${prUrl} (run ${target.id}); exactly one revision is in flight per PR.`,
|
|
1701
|
+
issue,
|
|
1702
|
+
);
|
|
1703
|
+
}
|
|
1704
|
+
|
|
1705
|
+
return allow(
|
|
1706
|
+
`recorded review round ${round} for ${prUrl} at ${headSha}; the daemon will resume run ${target.id}'s session` +
|
|
1707
|
+
`${target.sessionFile === undefined ? "" : ` (${target.sessionFile})`} on its next dispatch pass.`,
|
|
1708
|
+
undefined,
|
|
1709
|
+
issue,
|
|
1710
|
+
);
|
|
1711
|
+
}
|
|
1712
|
+
|
|
1512
1713
|
// ------------------------------------------------------------------ the listener
|
|
1513
1714
|
|
|
1514
1715
|
export interface VerbListener {
|
package/src/worker.ts
CHANGED
|
@@ -74,6 +74,32 @@ export const ORPHAN_RESUME_PROMPT =
|
|
|
74
74
|
"then continue exactly where you left off: re-check the outcome of your last action before repeating it, and keep " +
|
|
75
75
|
"working your original brief to the same report contract.";
|
|
76
76
|
|
|
77
|
+
/**
|
|
78
|
+
* What a review-revision worker is told instead of re-sending its brief (#677).
|
|
79
|
+
*
|
|
80
|
+
* Same rule as {@link ORPHAN_RESUME_PROMPT}: the original brief is already in
|
|
81
|
+
* the resumed transcript, and re-sending it is how a resumed worker re-does
|
|
82
|
+
* work it already did. What makes this a revision rather than a resume is the
|
|
83
|
+
* deliverable contract: the PR stays open, the branch stays the run's own, and
|
|
84
|
+
* the session ends only when the same final-report shape says the revised head
|
|
85
|
+
* is green. One literal so tests can pin it.
|
|
86
|
+
*/
|
|
87
|
+
export const REVIEW_REVISION_PROMPT =
|
|
88
|
+
"The orchestrator reviewed your green pull request and found blocking findings. Continue this same session " +
|
|
89
|
+
"on this same run: same branch, same pull request — do not close or reopen it, and do not open another. " +
|
|
90
|
+
"Address exactly the findings below, push with conductor_push, verify the checks with conductor_pr_status, " +
|
|
91
|
+
"and finish with the same final-report contract as before (state: pushed-green, pr:, head:). " +
|
|
92
|
+
"Your original brief is already in this transcript; redo only what the findings implicate.";
|
|
93
|
+
|
|
94
|
+
/**
|
|
95
|
+
* The full opening prompt for one review revision round: the pinned continuation
|
|
96
|
+
* text plus the orchestrator's findings verbatim. Exported so a unit test can
|
|
97
|
+
* pin the exact handoff without standing up a session.
|
|
98
|
+
*/
|
|
99
|
+
export function renderReviewRevisionPrompt(findings: string, round: number): string {
|
|
100
|
+
return `${REVIEW_REVISION_PROMPT}\n\nReview round ${round} — blocking findings:\n\n${findings}`;
|
|
101
|
+
}
|
|
102
|
+
|
|
77
103
|
export interface WorkerOpts {
|
|
78
104
|
brief: string;
|
|
79
105
|
cwd: string;
|
|
@@ -84,6 +84,10 @@ JOURNAL_LINES=${RECOVER_JOURNAL_LINES:-200}
|
|
|
84
84
|
VERIFY_POLL_S=${RECOVER_VERIFY_POLL_S:-2}
|
|
85
85
|
VERIFY_TRIES=${RECOVER_VERIFY_TRIES:-10}
|
|
86
86
|
PROJECT=${RECOVER_PROJECT:-}
|
|
87
|
+
# The bound on the post-restore re-arm. `arm` with a claim-only proof performs
|
|
88
|
+
# no Telegram send or wait, so it completes in milliseconds; the timeout exists
|
|
89
|
+
# so a misbehaving conductor can never hang this bounded playbook (#613).
|
|
90
|
+
ARM_TIMEOUT_S=${RECOVER_ARM_TIMEOUT_S:-30}
|
|
87
91
|
|
|
88
92
|
# --------------------------------------------------------------------------
|
|
89
93
|
# output
|
|
@@ -298,6 +302,55 @@ choose_action() { # <failed-unit>
|
|
|
298
302
|
printf 'NONE|\n'
|
|
299
303
|
}
|
|
300
304
|
|
|
305
|
+
# --------------------------------------------------------------------------
|
|
306
|
+
# re-arm — the one follow-on after a successful restoration (#613)
|
|
307
|
+
# --------------------------------------------------------------------------
|
|
308
|
+
|
|
309
|
+
# Whether the scoped project opted into claim-only arming. The unit renders
|
|
310
|
+
# RECOVER_PROJECT only for an unambiguous single-project host (#510/#530), and
|
|
311
|
+
# "proof" is a value unique to the project's "arm" block, so two exact-key
|
|
312
|
+
# greps decide it on the machine-written config. Any unreadable, hand-formatted
|
|
313
|
+
# or legacy shape (no "arm" key at all) reads as challenge — the explicit human
|
|
314
|
+
# gate — which is the fail-safe direction: recovery never arms a project that
|
|
315
|
+
# did not opt into claim-only.
|
|
316
|
+
project_arms_claim_only() { # <config-path>
|
|
317
|
+
local path=${1:-$CONFIG_PATH}
|
|
318
|
+
[[ -f $path ]] || return 1
|
|
319
|
+
grep -qE '"arm"[[:space:]]*:' "$path" || return 1
|
|
320
|
+
grep -qE '"proof"[[:space:]]*:[[:space:]]*"claim-only"' "$path"
|
|
321
|
+
}
|
|
322
|
+
|
|
323
|
+
# Re-arm the scoped project through the ordinary `arm` path, unattended only
|
|
324
|
+
# when the project opted into claim-only proof: that proof performs no Telegram
|
|
325
|
+
# send or wait, so the command completes headless. A challenge project is never
|
|
326
|
+
# poked — its gate is a human answering the challenge — and the caller's
|
|
327
|
+
# escalation says so plainly instead. Prints exactly one outcome line (the
|
|
328
|
+
# progress line goes to stderr, so the captured return value is the outcome).
|
|
329
|
+
rearm_if_claim_only() {
|
|
330
|
+
if [[ -z $PROJECT ]]; then
|
|
331
|
+
printf 'no scoped project (RECOVER_PROJECT unset) — nothing to re-arm\n'
|
|
332
|
+
return 0
|
|
333
|
+
fi
|
|
334
|
+
if ! project_arms_claim_only "$CONFIG_PATH"; then
|
|
335
|
+
printf 'project %s keeps the challenge proof — arming stays an explicit human gate\n' "$PROJECT"
|
|
336
|
+
return 0
|
|
337
|
+
fi
|
|
338
|
+
log "re-arming $PROJECT via the ordinary arm path (claim-only proof)"
|
|
339
|
+
if dry_run; then
|
|
340
|
+
printf 'dry run: would run %s arm --project %s (claim-only, bounded by %ss)\n' "$CONDUCTOR" "$PROJECT" "$ARM_TIMEOUT_S"
|
|
341
|
+
return 0
|
|
342
|
+
fi
|
|
343
|
+
local out code
|
|
344
|
+
out=$(timeout "$ARM_TIMEOUT_S" "$CONDUCTOR" arm --project "$PROJECT" 2>&1)
|
|
345
|
+
code=$?
|
|
346
|
+
if (( code == 0 )); then
|
|
347
|
+
printf 're-armed %s via the claim-only proof\n' "$PROJECT"
|
|
348
|
+
return 0
|
|
349
|
+
fi
|
|
350
|
+
printf 're-arm of %s failed (exit %s): %s\n' "$PROJECT" "$code" "$out"
|
|
351
|
+
return 0
|
|
352
|
+
}
|
|
353
|
+
|
|
301
354
|
# --------------------------------------------------------------------------
|
|
302
355
|
# main
|
|
303
356
|
# --------------------------------------------------------------------------
|
|
@@ -318,6 +371,13 @@ run_recovery() {
|
|
|
318
371
|
IFS='|' read -r action_name action_arg <<<"$(choose_action "$failed")"
|
|
319
372
|
say "plan: failed unit $failed; action: $action_name"
|
|
320
373
|
fi
|
|
374
|
+
if [[ -z $PROJECT ]]; then
|
|
375
|
+
say "plan: re-arm — none (RECOVER_PROJECT unset)"
|
|
376
|
+
elif project_arms_claim_only "$CONFIG_PATH"; then
|
|
377
|
+
say "plan: re-arm — $CONDUCTOR arm --project $PROJECT (claim-only proof, bounded by ${ARM_TIMEOUT_S}s)"
|
|
378
|
+
else
|
|
379
|
+
say "plan: re-arm — none ($PROJECT keeps the challenge proof; arming is an explicit human gate)"
|
|
380
|
+
fi
|
|
321
381
|
say "plan: escalation — $(escalation_text "$failed" "${action_name:-none}" "dry run" "DRY-RUN")"
|
|
322
382
|
return 0
|
|
323
383
|
fi
|
|
@@ -411,6 +471,19 @@ run_recovery() {
|
|
|
411
471
|
bump_failed_attempts
|
|
412
472
|
fi
|
|
413
473
|
|
|
474
|
+
# The one follow-on a verified restoration earns (#613): re-arm the scoped
|
|
475
|
+
# project through the ordinary `arm` command. Claim-only completes headless;
|
|
476
|
+
# a challenge project stays an explicit human gate and the escalation below
|
|
477
|
+
# says so plainly. Only after `verify` passed — writing an arm marker for a
|
|
478
|
+
# fleet that is still down is the improvisation this playbook never does.
|
|
479
|
+
local rearm_line="not attempted (recovery did not verify the fleet restored)"
|
|
480
|
+
if (( ok == 0 )); then
|
|
481
|
+
rearm_line=$(rearm_if_claim_only)
|
|
482
|
+
fi
|
|
483
|
+
say "re-arm: $rearm_line"
|
|
484
|
+
append_evidence "$ev" "re-arm: $rearm_line"
|
|
485
|
+
outcome="$outcome; re-arm: $rearm_line"
|
|
486
|
+
|
|
414
487
|
if escalate "$failed" "$action_name" "$outcome" "$ev"; then
|
|
415
488
|
if (( ok == 0 )); then
|
|
416
489
|
return 0
|
|
@@ -97,6 +97,9 @@ case "$1" in
|
|
|
97
97
|
--version)
|
|
98
98
|
echo "0.15.11"
|
|
99
99
|
;;
|
|
100
|
+
arm)
|
|
101
|
+
echo "armed demo via fixture"
|
|
102
|
+
;;
|
|
100
103
|
report)
|
|
101
104
|
echo "report 485-abc queued for demo (tier2)"
|
|
102
105
|
;;
|
|
@@ -147,6 +150,12 @@ marker() { # <case-dir> -> attempts count or 0
|
|
|
147
150
|
fi
|
|
148
151
|
}
|
|
149
152
|
|
|
153
|
+
# The minimal machine-written config shape the re-arm decision greps: the arm
|
|
154
|
+
# block with the chosen proof, as `saveConfig` would write it.
|
|
155
|
+
write_project_config() { # <case-dir> <proof>
|
|
156
|
+
printf '{\n "version": 2,\n "projects": [\n { "name": "demo", "arm": { "proof": "%s" } }\n ]\n}\n' "$2" >"$1/state/config.json"
|
|
157
|
+
}
|
|
158
|
+
|
|
150
159
|
run_recover() { # <case-dir> [env assignments…]
|
|
151
160
|
local d="$1"
|
|
152
161
|
shift
|
|
@@ -300,6 +309,58 @@ check 'ROLLBACK escalates as well' \
|
|
|
300
309
|
check 'the pre-upgrade snapshot survives the rollback for the operator' \
|
|
301
310
|
'{"preUpgrade":true}' "$(cat "$d/state/backups/config"/config.json.pre-upgrade-*)"
|
|
302
311
|
|
|
312
|
+
# ---------------------------------------------------------------------------
|
|
313
|
+
# case — the post-restore re-arm (#613): a verified restoration re-arms a
|
|
314
|
+
# claim-only project through the ordinary arm command; a challenge project is
|
|
315
|
+
# never poked and the escalation says the human gate is explicit.
|
|
316
|
+
# ---------------------------------------------------------------------------
|
|
317
|
+
|
|
318
|
+
d=$(newcase rearm-claim-only)
|
|
319
|
+
write_project_config "$d" claim-only
|
|
320
|
+
unit_state "$d" herdr-fleet.service failed
|
|
321
|
+
unit_state "$d" omp-conductor.service active
|
|
322
|
+
code=$(run_with_heal "$d" herdr-fleet.service)
|
|
323
|
+
check 'claim-only re-arm exits 0 when the unit heals' '0' "$code"
|
|
324
|
+
check 'claim-only recovery re-arms via the ordinary arm path' \
|
|
325
|
+
'omp-conductor arm --project demo' "$(grep '^omp-conductor arm' "$d/calls")"
|
|
326
|
+
check 'the escalation names the claim-only re-arm' \
|
|
327
|
+
're-armed demo via the claim-only proof' "$(grep '^omp-conductor report' "$d/calls" | head -n 1)"
|
|
328
|
+
check 'the evidence bundle records the re-arm' \
|
|
329
|
+
're-arm: re-armed demo via the claim-only proof' "$(cat "$d/state/recovery"/evidence-*)"
|
|
330
|
+
|
|
331
|
+
d=$(newcase rearm-challenge)
|
|
332
|
+
write_project_config "$d" challenge
|
|
333
|
+
unit_state "$d" herdr-fleet.service failed
|
|
334
|
+
unit_state "$d" omp-conductor.service active
|
|
335
|
+
code=$(run_with_heal "$d" herdr-fleet.service)
|
|
336
|
+
check 'a challenge project exits 0 when the unit heals' '0' "$code"
|
|
337
|
+
check 'a challenge project is never poked with arm' \
|
|
338
|
+
'' "$(grep '^omp-conductor arm' "$d/calls" || true)"
|
|
339
|
+
check 'the escalation says the challenge gate is explicit' \
|
|
340
|
+
'arming stays an explicit human gate' "$(grep '^omp-conductor report' "$d/calls" | head -n 1)"
|
|
341
|
+
|
|
342
|
+
# A project whose config predates the key reads as challenge: no "arm" block.
|
|
343
|
+
d=$(newcase rearm-legacy-config)
|
|
344
|
+
unit_state "$d" herdr-fleet.service failed
|
|
345
|
+
unit_state "$d" omp-conductor.service active
|
|
346
|
+
code=$(run_with_heal "$d" herdr-fleet.service)
|
|
347
|
+
check 'a legacy config without arm.proof is never poked with arm' \
|
|
348
|
+
'' "$(grep '^omp-conductor arm' "$d/calls" || true)"
|
|
349
|
+
check 'a legacy config keeps the human gate in the escalation' \
|
|
350
|
+
'arming stays an explicit human gate' "$(grep '^omp-conductor report' "$d/calls" | head -n 1)"
|
|
351
|
+
|
|
352
|
+
# RESTART_PEER never verifies a restoration, so even a claim-only project must
|
|
353
|
+
# not be re-armed while the daemon is still down.
|
|
354
|
+
d=$(newcase rearm-unverified)
|
|
355
|
+
write_project_config "$d" claim-only
|
|
356
|
+
unit_state "$d" omp-conductor.service failed
|
|
357
|
+
unit_state "$d" herdr-fleet.service active
|
|
358
|
+
code=$(run_recover "$d")
|
|
359
|
+
check 'an unverified recovery never re-arms' \
|
|
360
|
+
're-arm: not attempted' "$(grep 're-arm:' "$d/out" | head -n 1)"
|
|
361
|
+
check 'an unverified recovery calls no arm' \
|
|
362
|
+
'' "$(grep '^omp-conductor arm' "$d/calls" || true)"
|
|
363
|
+
|
|
303
364
|
# ---------------------------------------------------------------------------
|
|
304
365
|
# case RESTART_PEER and the bound: attempts cap at two, then escalate-only.
|
|
305
366
|
# A daemon crash with no upgrade evidence must never get a blind daemon start.
|