omp-conductor 0.16.2 → 0.17.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (52) hide show
  1. package/README.md +38 -4
  2. package/REFERENCE.md +18 -12
  3. package/package.json +2 -1
  4. package/schema/config.schema.json +16 -0
  5. package/src/admission.ts +159 -43
  6. package/src/availability.ts +27 -1
  7. package/src/briefs/worker.md +2 -0
  8. package/src/clack-ui.ts +83 -0
  9. package/src/command-manifest.ts +16 -7
  10. package/src/commands/arm.ts +11 -3
  11. package/src/commands/decision.ts +17 -7
  12. package/src/commands/doctor.ts +18 -1
  13. package/src/commands/hold.ts +9 -7
  14. package/src/commands/ledger.ts +25 -4
  15. package/src/commands/message.ts +32 -4
  16. package/src/commands/setup.ts +61 -10
  17. package/src/commands/stats.ts +9 -5
  18. package/src/commands/status.ts +32 -5
  19. package/src/commands/tail.ts +13 -1
  20. package/src/commands/watch.ts +16 -7
  21. package/src/config-schema.ts +20 -0
  22. package/src/config.ts +37 -0
  23. package/src/daemon.ts +1240 -18
  24. package/src/doctor.ts +310 -22
  25. package/src/escalate.ts +560 -57
  26. package/src/failure-class.ts +56 -13
  27. package/src/fleet.ts +224 -47
  28. package/src/gitops.ts +103 -24
  29. package/src/lifecycle.ts +7 -2
  30. package/src/orchestrator-tick.ts +372 -157
  31. package/src/privileged.ts +3 -0
  32. package/src/release-policy.ts +177 -5
  33. package/src/setup-answers.ts +135 -0
  34. package/src/setup-host.ts +193 -4
  35. package/src/setup-install.ts +2 -0
  36. package/src/setup-probe.ts +1 -0
  37. package/src/setup-wizard.ts +1296 -101
  38. package/src/setup.ts +60 -3
  39. package/src/status-render.ts +11 -1
  40. package/src/store.ts +333 -12
  41. package/src/tracker/github.ts +562 -13
  42. package/src/types.ts +204 -2
  43. package/src/ui/progress.ts +32 -0
  44. package/src/ui/style.ts +11 -0
  45. package/src/upgrade.ts +50 -19
  46. package/src/verbs/actions.ts +66 -18
  47. package/src/verbs/protocol.ts +45 -0
  48. package/src/verbs/server.ts +212 -11
  49. package/src/wizard-ui.ts +14 -5
  50. package/src/worker.ts +26 -0
  51. package/systemd/omp-conductor-recover.sh +73 -0
  52. package/systemd/recover-unit-test.sh +61 -0
@@ -54,6 +54,7 @@ import type {
54
54
  ReleaseRequirement,
55
55
  ReleaseShape,
56
56
  RepoTarget,
57
+ ReviewReason,
57
58
  RunRecord,
58
59
  Store,
59
60
  Tracker,
@@ -167,7 +168,6 @@ export interface VerbActions {
167
168
  fields: { title?: string; body?: string },
168
169
  ): Promise<{ ok: true } | { ok: false; stderr: string }>;
169
170
  mergePr(prUrl: string, headSha: string): Promise<ActionOutcome>;
170
- setLabel(issue: number, label: string, action: "add" | "remove"): Promise<ActionOutcome>;
171
171
  release(execution: ReleaseExecution): Promise<ActionOutcome>;
172
172
  /**
173
173
  * Request the fleet upgrade itself to `version`: refuse when npm does not
@@ -298,6 +298,12 @@ export interface ReleaseFacts {
298
298
  * must wait on a process; these clear by themselves on the next sweep.
299
299
  */
300
300
  unconfirmedMerges?: number;
301
+ /**
302
+ * The unsettled runs themselves, routed-repo and issue each, so the refusal
303
+ * names what is blocking instead of a bare count (#603). Absent when the
304
+ * caller could not gather them (the plain wording is used then).
305
+ */
306
+ blockingRuns?: { repo: string; issue: number }[];
301
307
  /** Queue depth, or `undefined` when the tracker could not be read. */
302
308
  queueDepth: number | undefined;
303
309
  /** Current live-head workflow verdict for the released routed repository. */
@@ -311,7 +317,7 @@ export function releaseRequirementRefusal(
311
317
  facts: ReleaseFacts,
312
318
  ): string | undefined {
313
319
  for (const requirement of requires) {
314
- if (requirement === "runs-settled" && facts.unsettledRuns > 0) {
320
+ if ((requirement === "runs-settled" || requirement === "fleet-runs-settled") && facts.unsettledRuns > 0) {
315
321
  // The same blocker reads differently depending on WHY the row is still
316
322
  // active: a live worker is the operator's wait, an unconfirmed merge or
317
323
  // an unrecorded sweep clears by itself on the next settle pass. The
@@ -328,9 +334,16 @@ export function releaseRequirementRefusal(
328
334
  if (open > 0) parts.push(`${open} pull request(s) still open`);
329
335
  if (unconfirmed > 0) parts.push(`${unconfirmed} run(s) whose PR merge the settle sweep has not yet recorded`);
330
336
  if (remaining > 0) parts.push(`${remaining} run(s) with no merged PR (closed or none yet)`);
337
+ // Name the runs behind the count so the operator can see at a glance
338
+ // whether this is the release's own work or someone else's (#603).
339
+ const blocking =
340
+ facts.blockingRuns === undefined || facts.blockingRuns.length === 0
341
+ ? ""
342
+ : ` — blocking: ${facts.blockingRuns.map((r) => `${r.repo} #${r.issue}`).join(", ")}`;
331
343
  return (
332
- `policy.release.requires includes runs-settled and ${facts.unsettledRuns} run(s) have not settled` +
333
- (parts.length === 0 ? "" : ` — ${parts.join("; ")}`)
344
+ `policy.release.requires includes ${requirement} and ${facts.unsettledRuns} run(s) have not settled` +
345
+ (parts.length === 0 ? "" : ` — ${parts.join("; ")}`) +
346
+ blocking
334
347
  );
335
348
  }
336
349
  if (requirement === "no-open-prs" && facts.openPrs > 0) {
@@ -505,6 +518,7 @@ async function decide(deps: VerbDeps, channel: VerbChannel, raw: unknown): Promi
505
518
  (verb === "conductor_pr_merge" ||
506
519
  verb === "conductor_pr_update_branch" ||
507
520
  verb === "conductor_pr_update" ||
521
+ verb === "conductor_pr_review" ||
508
522
  verb === "conductor_label");
509
523
  const stopped =
510
524
  spec.mutating && verb !== "conductor_release" && verb !== "conductor_install" && !orchestratorCompletion
@@ -540,6 +554,8 @@ async function decide(deps: VerbDeps, channel: VerbChannel, raw: unknown): Promi
540
554
  return releaseVerb(deps, project, channel, args, refuse, allow);
541
555
  case "conductor_install":
542
556
  return installVerb(deps, project, channel, args, refuse, allow);
557
+ case "conductor_pr_review":
558
+ return prReviewVerb(deps, project, channel, args, refuse, allow);
543
559
  case "conductor_pr_status":
544
560
  return prStatusVerb(deps, project, channel, args, refuse, allow);
545
561
  }
@@ -797,7 +813,15 @@ async function prUpdateBranchVerb(
797
813
  if (!outcome.ok) {
798
814
  return refuse("action-failed", `refused: gh could not update the branch:\n${outcome.stderr}`, target?.issue);
799
815
  }
800
- return allow(`updated ${prUrl} with its base branch. Re-check its checks before merging.`, outcome.sha, target?.issue);
816
+ // The REST endpoint answers 202 Accepted: the refresh runs asynchronously, so
817
+ // this call produces no head SHA and the reply/ledger must not claim one —
818
+ // the head the action observed is the pre-update guard head, never a result
819
+ // (#655 review). The caller re-reads PR and check state before merging.
820
+ return allow(
821
+ `requested a base-branch update of ${prUrl} from its observed head; PR and check state must be re-read before merging.`,
822
+ undefined,
823
+ target?.issue,
824
+ );
801
825
  }
802
826
 
803
827
  async function prMergeVerb(
@@ -1064,9 +1088,21 @@ async function labelVerb(
1064
1088
  }
1065
1089
 
1066
1090
  const action = args["action"] === "remove" ? "remove" : "add";
1067
- const outcome = await deps.actions.setLabel(ref.issue, label, action);
1068
- if (!outcome.ok) {
1069
- return refuse("action-failed", `refused: the tracker rejected the label change:\n${outcome.stderr}`, ref.issue);
1091
+ // The queue label is the fleet's lock, so it must not ride the GraphQL
1092
+ // surface a provider outage just broke: `gh issue edit` is a GraphQL
1093
+ // mutation in gh 2.86, and #642 comment 4 measured a label withdrawal
1094
+ // refused by a GraphQL 503 leaving two overlapping issues claimable. The
1095
+ // tracker's label methods are core REST with noop classification for the
1096
+ // already-holding end state — the guarded path.
1097
+ try {
1098
+ if (action === "add") {
1099
+ await deps.tracker.addLabel(ref.issue, label);
1100
+ } else {
1101
+ await deps.tracker.removeLabel(ref.issue, label);
1102
+ }
1103
+ } catch (err) {
1104
+ const why = err instanceof Error ? err.message : String(err);
1105
+ return refuse("action-failed", `refused: the tracker rejected the label change:\n${why}`, ref.issue);
1070
1106
  }
1071
1107
  return allow(`${action === "add" ? "added" : "removed"} ${label} on #${ref.issue}.`, undefined, ref.issue);
1072
1108
  }
@@ -1151,22 +1187,36 @@ async function releaseVerb(
1151
1187
  // this release by holding. Fail-closed stays fail-closed: a row whose PR
1152
1188
  // state cannot be read is still unsettled, and an unknown PR is still
1153
1189
  // treated as open.
1190
+ //
1191
+ // The run facts are scoped to the released repo (#603): a run in another
1192
+ // routed repo cannot invalidate a tag on this one, so it must not gate this
1193
+ // release. A genuinely suite-wide shape — a pin or manifest that consumes
1194
+ // several repos — opts back into project-wide strictness through the named
1195
+ // `fleet-runs-settled` requirement, which widens the scope to every active
1196
+ // run. `no-open-prs` shares the scope: the branch being released is this
1197
+ // repo's, so an open PR elsewhere is not "against" it either.
1198
+ const fleetWide = policy.release.requires.includes("fleet-runs-settled");
1154
1199
  const wantsRunFacts =
1155
1200
  policy.release.requires.includes("runs-settled") ||
1201
+ fleetWide ||
1156
1202
  policy.release.requires.includes("no-open-prs");
1157
- let unsettledRuns = active.length;
1158
- let openPrs = active.filter((r) => r.prUrl !== undefined).length;
1203
+ const scoped = fleetWide ? active : active.filter((r) => r.repo === repoName);
1204
+ let unsettledRuns = scoped.length;
1205
+ let openPrs = scoped.filter((r) => r.prUrl !== undefined).length;
1159
1206
  let liveWorkers: number | undefined;
1160
1207
  let unconfirmedMerges: number | undefined;
1208
+ let blockingRuns: { repo: string; issue: number }[] | undefined;
1161
1209
  if (wantsRunFacts) {
1162
1210
  let unsettled = 0;
1163
1211
  let open = 0;
1164
1212
  let live = 0;
1165
1213
  let unconfirmed = 0;
1166
- for (const run of active) {
1214
+ const blocking: { repo: string; issue: number }[] = [];
1215
+ for (const run of scoped) {
1167
1216
  if (run.state === "claimed" || run.state === "running") {
1168
1217
  unsettled += 1;
1169
1218
  live += 1;
1219
+ blocking.push({ repo: run.repo, issue: run.issue });
1170
1220
  // A live worker's PR is not yet confirmable; count it as open, the
1171
1221
  // same fail-closed answer the old row-count gave it.
1172
1222
  if (run.prUrl !== undefined) open += 1;
@@ -1175,6 +1225,7 @@ async function releaseVerb(
1175
1225
  // pushed-pending / pushed-green: the worker is finished; the PR decides.
1176
1226
  if (run.prUrl === undefined) {
1177
1227
  unsettled += 1; // nothing to read; the sweep cannot settle it either
1228
+ blocking.push({ repo: run.repo, issue: run.issue });
1178
1229
  continue;
1179
1230
  }
1180
1231
  let pr: PrState | undefined;
@@ -1185,6 +1236,7 @@ async function releaseVerb(
1185
1236
  }
1186
1237
  if (pr === "merged") continue; // settled in fact; not an open PR
1187
1238
  unsettled += 1;
1239
+ blocking.push({ repo: run.repo, issue: run.issue });
1188
1240
  if (pr === "open") open += 1;
1189
1241
  else if (pr === undefined) {
1190
1242
  // Cannot confirm the merge (or its absence): fail closed, and count
@@ -1200,12 +1252,14 @@ async function releaseVerb(
1200
1252
  openPrs = open;
1201
1253
  liveWorkers = live;
1202
1254
  unconfirmedMerges = unconfirmed;
1255
+ blockingRuns = blocking;
1203
1256
  }
1204
1257
  const unmet = releaseRequirementRefusal(policy.release.requires, {
1205
1258
  unsettledRuns,
1206
1259
  openPrs,
1207
1260
  ...(liveWorkers === undefined ? {} : { liveWorkers }),
1208
1261
  ...(unconfirmedMerges === undefined ? {} : { unconfirmedMerges }),
1262
+ ...(blockingRuns === undefined ? {} : { blockingRuns }),
1209
1263
  queueDepth,
1210
1264
  ...(health === undefined ? {} : { baseCheck: health.verdict }),
1211
1265
  ...(health?.verdict === "red" && health.detail !== undefined
@@ -1509,6 +1563,153 @@ async function prUpdateVerb(
1509
1563
  return allow(`updated ${prUrl}.`, undefined, issue);
1510
1564
  }
1511
1565
 
1566
+ /**
1567
+ * Return one green, run-owned pull request to its worker with blocking
1568
+ * findings (#677): the transport that replaces the manual close-PR →
1569
+ * comment-on-issue → unblock → continuation dance.
1570
+ *
1571
+ * Everything is recorded durably BEFORE anything is woken: the findings, the
1572
+ * exact reviewed head, the round number, and the target run/session. The wake
1573
+ * itself is the daemon's next dispatch pass (the verb has no handle on the
1574
+ * worker machinery, and the CLI path must behave exactly like the embedded
1575
+ * one), and the run row is reused — same branch, same PR, same attempt, same
1576
+ * session directory — so a revision round never creates a new attempt and
1577
+ * never touches the failed-attempt or continuation budgets.
1578
+ *
1579
+ * The duplicate in-flight guard has two halves: while a revision is queued
1580
+ * but not yet dispatched, the pending `review_revisions` row refuses the
1581
+ * second request atomically inside `claimReviewRevision`; once dispatched,
1582
+ * the run row itself is `running` (claimed from `pushed-green` by the
1583
+ * dispatch pass), which this verb refuses by name.
1584
+ */
1585
+ async function prReviewVerb(
1586
+ deps: VerbDeps,
1587
+ project: ProjectConfig,
1588
+ channel: VerbChannel,
1589
+ args: Record<string, unknown>,
1590
+ refuse: Refuse,
1591
+ allow: Allow,
1592
+ ): Promise<Verdict> {
1593
+ const prUrl = String(args["prUrl"]);
1594
+ const headSha = String(args["headSha"]);
1595
+ const findings = String(args["findings"]);
1596
+ const reason = String(args["reason"]) as ReviewReason;
1597
+
1598
+ // The run whose pushed-green row owns the PR — same resolution as merge: the
1599
+ // newest attempt that recorded this PR within the recent-history window. A
1600
+ // PR no run of this project opened (or one outside the routed repos) cannot
1601
+ // be returned to a worker this project can resume.
1602
+ const target = runForPr(deps, project.name, prUrl);
1603
+ if (target === undefined) {
1604
+ return refuse(
1605
+ "pr-not-this-run",
1606
+ `refused: ${prUrl} is not a pull request any run in ${project.name} opened. ` +
1607
+ "A review revision acts on a run-owned PR only.",
1608
+ );
1609
+ }
1610
+ if (!prInProjectRouting(project, prUrl)) {
1611
+ return refuse(
1612
+ "pr-not-this-run",
1613
+ `refused: ${prUrl} is not in ${project.name}'s routed repositories ` +
1614
+ `(${Object.values(project.routing.repos).map(repoSlugFor).join(", ") || "none"}).`,
1615
+ target.issue,
1616
+ );
1617
+ }
1618
+ const issue = target.issue;
1619
+
1620
+ if (findings.trim() === "") {
1621
+ return refuse(
1622
+ "malformed-argument",
1623
+ "refused: conductor_pr_review needs a non-empty findings string — an empty revision returns the worker nothing to fix.",
1624
+ issue,
1625
+ );
1626
+ }
1627
+
1628
+ // The revisable state is a settled green run. Any other state is a worker in
1629
+ // flight (the original run or an earlier revision) or a run whose PR no
1630
+ // longer waits on revision — the detail names the state field that produced
1631
+ // the refusal.
1632
+ if (target.state !== "pushed-green") {
1633
+ const live = target.state === "running" || target.state === "claimed";
1634
+ return refuse(
1635
+ "review-in-flight",
1636
+ `refused: run ${target.id} is ${target.state}, not pushed-green — ${
1637
+ live
1638
+ ? `a worker (the original run or an earlier revision) is still live on ${prUrl}; wait for it to settle before returning it.`
1639
+ : `a review revision starts from a settled green run, and this one is ${target.state}.`
1640
+ }`,
1641
+ issue,
1642
+ );
1643
+ }
1644
+
1645
+ const paused = orchestratorPauseRefusal(deps, channel, target, refuse);
1646
+ if (paused !== undefined) return paused;
1647
+
1648
+ // Exact-head, re-read at request time: the orchestrator reviewed a specific
1649
+ // head, and a PR that moved since is not the thing it reviewed. Same verdict
1650
+ // the merge gate uses, so "green at headSha" here is the same fact the merge
1651
+ // would have required.
1652
+ let verification: PrVerification | undefined;
1653
+ try {
1654
+ verification = await deps.tracker.verifyPr(prUrl, headSha);
1655
+ } catch (err) {
1656
+ const why = err instanceof Error ? err.message : String(err);
1657
+ return refuse(
1658
+ "head-unresolvable",
1659
+ `refused: the live head of ${prUrl} could not be read (${why}).`,
1660
+ issue,
1661
+ );
1662
+ }
1663
+ if (verification === undefined) {
1664
+ return refuse(
1665
+ "head-unresolvable",
1666
+ `refused: the live head of ${prUrl} could not be resolved. Refusing rather than returning a PR you did not re-read.`,
1667
+ issue,
1668
+ );
1669
+ }
1670
+ if (verification.status !== "green") {
1671
+ const stale = verification.status === "failed" && isHeadMismatch(verification.reason);
1672
+ return refuse(
1673
+ stale ? "head-stale" : "checks-not-green",
1674
+ stale
1675
+ ? `refused: ${verification.reason}. You reviewed ${headSha}; that is not what is on the branch now. Re-read the head and re-review.`
1676
+ : `refused: ${prUrl} is not green at ${headSha} — ${verification.reason}. A revision is returned to a green PR only.`,
1677
+ issue,
1678
+ );
1679
+ }
1680
+
1681
+ // Durable, and refused atomically against a concurrent duplicate: the row is
1682
+ // persisted before any wake, and a second request for the same run bumps
1683
+ // into the pending row rather than racing it.
1684
+ const round = deps.store.latestReviewRound(project.name, target.id) + 1;
1685
+ const recorded = deps.store.createReviewRevision({
1686
+ project: project.name,
1687
+ runId: target.id,
1688
+ issue,
1689
+ prUrl,
1690
+ headSha,
1691
+ findings,
1692
+ round,
1693
+ reason,
1694
+ ...(target.sessionFile === undefined ? {} : { sessionFile: target.sessionFile }),
1695
+ requestedAt: deps.now(),
1696
+ });
1697
+ if (recorded === undefined) {
1698
+ return refuse(
1699
+ "review-in-flight",
1700
+ `refused: a review revision is already pending for ${prUrl} (run ${target.id}); exactly one revision is in flight per PR.`,
1701
+ issue,
1702
+ );
1703
+ }
1704
+
1705
+ return allow(
1706
+ `recorded review round ${round} for ${prUrl} at ${headSha}; the daemon will resume run ${target.id}'s session` +
1707
+ `${target.sessionFile === undefined ? "" : ` (${target.sessionFile})`} on its next dispatch pass.`,
1708
+ undefined,
1709
+ issue,
1710
+ );
1711
+ }
1712
+
1512
1713
  // ------------------------------------------------------------------ the listener
1513
1714
 
1514
1715
  export interface VerbListener {
package/src/wizard-ui.ts CHANGED
@@ -24,6 +24,15 @@
24
24
  * (`Cancelled`). The terminal UI guarantees the readline interface is closed on
25
25
  * every exit path so the verb cannot hang the terminal.
26
26
  */
27
+ export interface PromptOptions {
28
+ /** Stable answer-file key. This is a permanent CLI API, not display copy. */
29
+ key: string;
30
+ }
31
+
32
+ export interface SelectOptions extends PromptOptions {
33
+ initialIndex?: number;
34
+ }
35
+
27
36
  export interface WizardUi {
28
37
  notify(message: string, type?: "info" | "warning" | "error"): void;
29
38
  /**
@@ -32,17 +41,17 @@ export interface WizardUi {
32
41
  * `Cancelled` and abandons the run. Collapsing the two is how Ctrl-C at a
33
42
  * confirm used to record a silent "no" and carry on to the next question.
34
43
  */
35
- confirm(title: string, message: string): Promise<boolean | undefined>;
44
+ confirm(title: string, message: string, promptOptions: PromptOptions): Promise<boolean | undefined>;
36
45
  /** Single-line text prompt. `undefined` dismisses it; an empty submit accepts
37
46
  * the placeholder. */
38
- input(title: string, placeholder?: string): Promise<string | undefined>;
47
+ input(title: string, placeholder: string | undefined, promptOptions: PromptOptions): Promise<string | undefined>;
39
48
  /** Single-choice list. On an interactive TTY the current option is rendered
40
49
  * inline and moved with ↑/↓ or j/k; on a pipe it stays the numbered wall.
41
50
  * Resolves the chosen option's label, or `undefined` when dismissed. */
42
51
  select(
43
52
  title: string,
44
53
  options: { label: string; description?: string }[],
45
- dialogOptions?: { initialIndex?: number },
54
+ dialogOptions: SelectOptions,
46
55
  ): Promise<string | undefined>;
47
56
  }
48
57
 
@@ -83,8 +92,8 @@ import type { Readable, Writable } from "node:stream";
83
92
  * rather than only through a spawned CLI; production callers pass nothing.
84
93
  */
85
94
  export interface TerminalUi extends WizardUi {
86
- /** Releases stdin. Idempotent; call it from a `finally`. */
87
- close(): void;
95
+ /** Releases stdin; interactive drivers may render the outcome message. Idempotent. */
96
+ close(message?: string): void;
88
97
  }
89
98
 
90
99
  export function terminalUi(io: { input?: Readable; output?: Writable } = {}): TerminalUi {
package/src/worker.ts CHANGED
@@ -74,6 +74,32 @@ export const ORPHAN_RESUME_PROMPT =
74
74
  "then continue exactly where you left off: re-check the outcome of your last action before repeating it, and keep " +
75
75
  "working your original brief to the same report contract.";
76
76
 
77
+ /**
78
+ * What a review-revision worker is told instead of re-sending its brief (#677).
79
+ *
80
+ * Same rule as {@link ORPHAN_RESUME_PROMPT}: the original brief is already in
81
+ * the resumed transcript, and re-sending it is how a resumed worker re-does
82
+ * work it already did. What makes this a revision rather than a resume is the
83
+ * deliverable contract: the PR stays open, the branch stays the run's own, and
84
+ * the session ends only when the same final-report shape says the revised head
85
+ * is green. One literal so tests can pin it.
86
+ */
87
+ export const REVIEW_REVISION_PROMPT =
88
+ "The orchestrator reviewed your green pull request and found blocking findings. Continue this same session " +
89
+ "on this same run: same branch, same pull request — do not close or reopen it, and do not open another. " +
90
+ "Address exactly the findings below, push with conductor_push, verify the checks with conductor_pr_status, " +
91
+ "and finish with the same final-report contract as before (state: pushed-green, pr:, head:). " +
92
+ "Your original brief is already in this transcript; redo only what the findings implicate.";
93
+
94
+ /**
95
+ * The full opening prompt for one review revision round: the pinned continuation
96
+ * text plus the orchestrator's findings verbatim. Exported so a unit test can
97
+ * pin the exact handoff without standing up a session.
98
+ */
99
+ export function renderReviewRevisionPrompt(findings: string, round: number): string {
100
+ return `${REVIEW_REVISION_PROMPT}\n\nReview round ${round} — blocking findings:\n\n${findings}`;
101
+ }
102
+
77
103
  export interface WorkerOpts {
78
104
  brief: string;
79
105
  cwd: string;
@@ -84,6 +84,10 @@ JOURNAL_LINES=${RECOVER_JOURNAL_LINES:-200}
84
84
  VERIFY_POLL_S=${RECOVER_VERIFY_POLL_S:-2}
85
85
  VERIFY_TRIES=${RECOVER_VERIFY_TRIES:-10}
86
86
  PROJECT=${RECOVER_PROJECT:-}
87
+ # The bound on the post-restore re-arm. `arm` with a claim-only proof performs
88
+ # no Telegram send or wait, so it completes in milliseconds; the timeout exists
89
+ # so a misbehaving conductor can never hang this bounded playbook (#613).
90
+ ARM_TIMEOUT_S=${RECOVER_ARM_TIMEOUT_S:-30}
87
91
 
88
92
  # --------------------------------------------------------------------------
89
93
  # output
@@ -298,6 +302,55 @@ choose_action() { # <failed-unit>
298
302
  printf 'NONE|\n'
299
303
  }
300
304
 
305
+ # --------------------------------------------------------------------------
306
+ # re-arm — the one follow-on after a successful restoration (#613)
307
+ # --------------------------------------------------------------------------
308
+
309
+ # Whether the scoped project opted into claim-only arming. The unit renders
310
+ # RECOVER_PROJECT only for an unambiguous single-project host (#510/#530), and
311
+ # "proof" is a value unique to the project's "arm" block, so two exact-key
312
+ # greps decide it on the machine-written config. Any unreadable, hand-formatted
313
+ # or legacy shape (no "arm" key at all) reads as challenge — the explicit human
314
+ # gate — which is the fail-safe direction: recovery never arms a project that
315
+ # did not opt into claim-only.
316
+ project_arms_claim_only() { # <config-path>
317
+ local path=${1:-$CONFIG_PATH}
318
+ [[ -f $path ]] || return 1
319
+ grep -qE '"arm"[[:space:]]*:' "$path" || return 1
320
+ grep -qE '"proof"[[:space:]]*:[[:space:]]*"claim-only"' "$path"
321
+ }
322
+
323
+ # Re-arm the scoped project through the ordinary `arm` path, unattended only
324
+ # when the project opted into claim-only proof: that proof performs no Telegram
325
+ # send or wait, so the command completes headless. A challenge project is never
326
+ # poked — its gate is a human answering the challenge — and the caller's
327
+ # escalation says so plainly instead. Prints exactly one outcome line (the
328
+ # progress line goes to stderr, so the captured return value is the outcome).
329
+ rearm_if_claim_only() {
330
+ if [[ -z $PROJECT ]]; then
331
+ printf 'no scoped project (RECOVER_PROJECT unset) — nothing to re-arm\n'
332
+ return 0
333
+ fi
334
+ if ! project_arms_claim_only "$CONFIG_PATH"; then
335
+ printf 'project %s keeps the challenge proof — arming stays an explicit human gate\n' "$PROJECT"
336
+ return 0
337
+ fi
338
+ log "re-arming $PROJECT via the ordinary arm path (claim-only proof)"
339
+ if dry_run; then
340
+ printf 'dry run: would run %s arm --project %s (claim-only, bounded by %ss)\n' "$CONDUCTOR" "$PROJECT" "$ARM_TIMEOUT_S"
341
+ return 0
342
+ fi
343
+ local out code
344
+ out=$(timeout "$ARM_TIMEOUT_S" "$CONDUCTOR" arm --project "$PROJECT" 2>&1)
345
+ code=$?
346
+ if (( code == 0 )); then
347
+ printf 're-armed %s via the claim-only proof\n' "$PROJECT"
348
+ return 0
349
+ fi
350
+ printf 're-arm of %s failed (exit %s): %s\n' "$PROJECT" "$code" "$out"
351
+ return 0
352
+ }
353
+
301
354
  # --------------------------------------------------------------------------
302
355
  # main
303
356
  # --------------------------------------------------------------------------
@@ -318,6 +371,13 @@ run_recovery() {
318
371
  IFS='|' read -r action_name action_arg <<<"$(choose_action "$failed")"
319
372
  say "plan: failed unit $failed; action: $action_name"
320
373
  fi
374
+ if [[ -z $PROJECT ]]; then
375
+ say "plan: re-arm — none (RECOVER_PROJECT unset)"
376
+ elif project_arms_claim_only "$CONFIG_PATH"; then
377
+ say "plan: re-arm — $CONDUCTOR arm --project $PROJECT (claim-only proof, bounded by ${ARM_TIMEOUT_S}s)"
378
+ else
379
+ say "plan: re-arm — none ($PROJECT keeps the challenge proof; arming is an explicit human gate)"
380
+ fi
321
381
  say "plan: escalation — $(escalation_text "$failed" "${action_name:-none}" "dry run" "DRY-RUN")"
322
382
  return 0
323
383
  fi
@@ -411,6 +471,19 @@ run_recovery() {
411
471
  bump_failed_attempts
412
472
  fi
413
473
 
474
+ # The one follow-on a verified restoration earns (#613): re-arm the scoped
475
+ # project through the ordinary `arm` command. Claim-only completes headless;
476
+ # a challenge project stays an explicit human gate and the escalation below
477
+ # says so plainly. Only after `verify` passed — writing an arm marker for a
478
+ # fleet that is still down is the improvisation this playbook never does.
479
+ local rearm_line="not attempted (recovery did not verify the fleet restored)"
480
+ if (( ok == 0 )); then
481
+ rearm_line=$(rearm_if_claim_only)
482
+ fi
483
+ say "re-arm: $rearm_line"
484
+ append_evidence "$ev" "re-arm: $rearm_line"
485
+ outcome="$outcome; re-arm: $rearm_line"
486
+
414
487
  if escalate "$failed" "$action_name" "$outcome" "$ev"; then
415
488
  if (( ok == 0 )); then
416
489
  return 0
@@ -97,6 +97,9 @@ case "$1" in
97
97
  --version)
98
98
  echo "0.15.11"
99
99
  ;;
100
+ arm)
101
+ echo "armed demo via fixture"
102
+ ;;
100
103
  report)
101
104
  echo "report 485-abc queued for demo (tier2)"
102
105
  ;;
@@ -147,6 +150,12 @@ marker() { # <case-dir> -> attempts count or 0
147
150
  fi
148
151
  }
149
152
 
153
+ # The minimal machine-written config shape the re-arm decision greps: the arm
154
+ # block with the chosen proof, as `saveConfig` would write it.
155
+ write_project_config() { # <case-dir> <proof>
156
+ printf '{\n "version": 2,\n "projects": [\n { "name": "demo", "arm": { "proof": "%s" } }\n ]\n}\n' "$2" >"$1/state/config.json"
157
+ }
158
+
150
159
  run_recover() { # <case-dir> [env assignments…]
151
160
  local d="$1"
152
161
  shift
@@ -300,6 +309,58 @@ check 'ROLLBACK escalates as well' \
300
309
  check 'the pre-upgrade snapshot survives the rollback for the operator' \
301
310
  '{"preUpgrade":true}' "$(cat "$d/state/backups/config"/config.json.pre-upgrade-*)"
302
311
 
312
+ # ---------------------------------------------------------------------------
313
+ # case — the post-restore re-arm (#613): a verified restoration re-arms a
314
+ # claim-only project through the ordinary arm command; a challenge project is
315
+ # never poked and the escalation says the human gate is explicit.
316
+ # ---------------------------------------------------------------------------
317
+
318
+ d=$(newcase rearm-claim-only)
319
+ write_project_config "$d" claim-only
320
+ unit_state "$d" herdr-fleet.service failed
321
+ unit_state "$d" omp-conductor.service active
322
+ code=$(run_with_heal "$d" herdr-fleet.service)
323
+ check 'claim-only re-arm exits 0 when the unit heals' '0' "$code"
324
+ check 'claim-only recovery re-arms via the ordinary arm path' \
325
+ 'omp-conductor arm --project demo' "$(grep '^omp-conductor arm' "$d/calls")"
326
+ check 'the escalation names the claim-only re-arm' \
327
+ 're-armed demo via the claim-only proof' "$(grep '^omp-conductor report' "$d/calls" | head -n 1)"
328
+ check 'the evidence bundle records the re-arm' \
329
+ 're-arm: re-armed demo via the claim-only proof' "$(cat "$d/state/recovery"/evidence-*)"
330
+
331
+ d=$(newcase rearm-challenge)
332
+ write_project_config "$d" challenge
333
+ unit_state "$d" herdr-fleet.service failed
334
+ unit_state "$d" omp-conductor.service active
335
+ code=$(run_with_heal "$d" herdr-fleet.service)
336
+ check 'a challenge project exits 0 when the unit heals' '0' "$code"
337
+ check 'a challenge project is never poked with arm' \
338
+ '' "$(grep '^omp-conductor arm' "$d/calls" || true)"
339
+ check 'the escalation says the challenge gate is explicit' \
340
+ 'arming stays an explicit human gate' "$(grep '^omp-conductor report' "$d/calls" | head -n 1)"
341
+
342
+ # A project whose config predates the key reads as challenge: no "arm" block.
343
+ d=$(newcase rearm-legacy-config)
344
+ unit_state "$d" herdr-fleet.service failed
345
+ unit_state "$d" omp-conductor.service active
346
+ code=$(run_with_heal "$d" herdr-fleet.service)
347
+ check 'a legacy config without arm.proof is never poked with arm' \
348
+ '' "$(grep '^omp-conductor arm' "$d/calls" || true)"
349
+ check 'a legacy config keeps the human gate in the escalation' \
350
+ 'arming stays an explicit human gate' "$(grep '^omp-conductor report' "$d/calls" | head -n 1)"
351
+
352
+ # RESTART_PEER never verifies a restoration, so even a claim-only project must
353
+ # not be re-armed while the daemon is still down.
354
+ d=$(newcase rearm-unverified)
355
+ write_project_config "$d" claim-only
356
+ unit_state "$d" omp-conductor.service failed
357
+ unit_state "$d" herdr-fleet.service active
358
+ code=$(run_recover "$d")
359
+ check 'an unverified recovery never re-arms' \
360
+ 're-arm: not attempted' "$(grep 're-arm:' "$d/out" | head -n 1)"
361
+ check 'an unverified recovery calls no arm' \
362
+ '' "$(grep '^omp-conductor arm' "$d/calls" || true)"
363
+
303
364
  # ---------------------------------------------------------------------------
304
365
  # case RESTART_PEER and the bound: attempts cap at two, then escalate-only.
305
366
  # A daemon crash with no upgrade evidence must never get a blind daemon start.