omp-conductor 0.17.0 → 0.18.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (51) hide show
  1. package/REFERENCE.md +12 -8
  2. package/package.json +1 -1
  3. package/schema/config.schema.json +40 -1
  4. package/src/admission.ts +263 -44
  5. package/src/ask.ts +39 -3
  6. package/src/availability.ts +27 -1
  7. package/src/backups.ts +2 -2
  8. package/src/briefs/orchestrator.md +1 -0
  9. package/src/briefs/worker.md +38 -19
  10. package/src/command-help.ts +8 -1
  11. package/src/command-manifest.ts +5 -2
  12. package/src/commands/arm.ts +6 -3
  13. package/src/commands/message.ts +32 -4
  14. package/src/commands/watch.ts +62 -3
  15. package/src/config-schema.ts +53 -0
  16. package/src/config.ts +97 -1
  17. package/src/daemon.ts +1479 -1483
  18. package/src/decisions.ts +51 -6
  19. package/src/depends-on.ts +261 -1
  20. package/src/diff-flags.ts +350 -0
  21. package/src/digest-schedule.ts +37 -0
  22. package/src/doctor.ts +310 -22
  23. package/src/escalate.ts +560 -57
  24. package/src/failure-class.ts +71 -15
  25. package/src/fleet.ts +189 -34
  26. package/src/gitops.ts +103 -24
  27. package/src/graph-health.ts +20 -7
  28. package/src/graph.ts +313 -68
  29. package/src/lifecycle.ts +43 -7
  30. package/src/omp.ts +42 -0
  31. package/src/orchestrator-tick.ts +430 -162
  32. package/src/release-policy.ts +177 -5
  33. package/src/routing.ts +11 -3
  34. package/src/session-host.ts +16 -0
  35. package/src/settlement.ts +1728 -0
  36. package/src/setup-host.ts +193 -4
  37. package/src/setup-install.ts +91 -30
  38. package/src/setup-wizard.ts +1257 -78
  39. package/src/setup.ts +153 -6
  40. package/src/status-render.ts +36 -4
  41. package/src/store.ts +411 -17
  42. package/src/tracker/github.ts +607 -12
  43. package/src/types.ts +331 -5
  44. package/src/upgrade.ts +50 -19
  45. package/src/verbs/actions.ts +66 -18
  46. package/src/verbs/protocol.ts +45 -0
  47. package/src/verbs/server.ts +270 -13
  48. package/src/worker.ts +239 -6
  49. package/src/worktree.ts +115 -8
  50. package/systemd/omp-conductor-recover.sh +73 -0
  51. package/systemd/recover-unit-test.sh +61 -0
package/src/worker.ts CHANGED
@@ -30,6 +30,77 @@ const GITHUB_PR_URL_PATTERN = /https:\/\/github\.com\/([^/\s]+\/[^/\s]+)\/pull\/
30
30
  /** `{{KEY}}` placeholders in a brief template. */
31
31
  const PLACEHOLDER_PATTERN = /\{\{([A-Za-z0-9_]+)\}\}/g;
32
32
 
33
+ /**
34
+ * The structured settlement a worker yields at the end of a run (#540).
35
+ *
36
+ * Fields decided here, not per project. The daemon stays the authority on
37
+ * green — it verifies the PR itself — so this is the worker's *claim*,
38
+ * cross-checked exactly as the prose report it replaces: a `green` claim
39
+ * must carry the PR URL and the head SHA the worker actually watched go
40
+ * green, and anything less fails closed.
41
+ */
42
+ export interface WorkerSettlement {
43
+ status: "green" | "blocked" | "failed";
44
+ /** The run's pull request URL. Required with `status: "green"`. */
45
+ prUrl?: string;
46
+ /** The branch the PR was pushed from. */
47
+ branch?: string;
48
+ /** The 40-hex head SHA the green CI verdict was observed at. Required with `status: "green"`. */
49
+ headSha?: string;
50
+ /** What changed and why — the narrative a reviewer reads. */
51
+ summary: string;
52
+ /** What a decision or credential this run needed (blocked runs). */
53
+ blockers?: string[];
54
+ /** Commands run as evidence, exactly as executed. */
55
+ proof?: string[];
56
+ }
57
+
58
+ /**
59
+ * The JSON Schema form of {@link WorkerSettlement}, handed to the harness as
60
+ * the session's `outputSchema` (#540).
61
+ *
62
+ * Used two ways, and both must stay honest to the same contract:
63
+ * `requireYieldTool` puts the `yield` tool in front of the worker with this
64
+ * schema as its description, and `outputSchemaMode: "permissive"` means a
65
+ * violation is a fallback, never a lost report — the worker's text path must
66
+ * still settle a run whose yield does not parse.
67
+ */
68
+ export const WORKER_SETTLEMENT_SCHEMA = {
69
+ $schema: "https://json-schema.org/draft/2020-12/schema",
70
+ type: "object",
71
+ description:
72
+ "The run's settlement: what this worker claims it achieved. The daemon verifies the PR itself; " +
73
+ "this is the claim, cross-checked like the settlement report it replaces.",
74
+ additionalProperties: false,
75
+ required: ["status", "summary"],
76
+ properties: {
77
+ status: {
78
+ type: "string",
79
+ enum: ["green", "blocked", "failed"],
80
+ description:
81
+ "green: pushed and the checks you watched are green. blocked: a decision, credential or " +
82
+ "repo fact is missing. failed: the run could not complete.",
83
+ },
84
+ prUrl: { type: "string", description: "The run's pull request URL — required with status green." },
85
+ branch: { type: "string", description: "The branch the PR was pushed from." },
86
+ headSha: {
87
+ type: "string",
88
+ description: "The 40-character head SHA you observed green — required with status green.",
89
+ },
90
+ summary: { type: "string", description: "One short paragraph: what changed and why." },
91
+ blockers: {
92
+ type: "array",
93
+ items: { type: "string" },
94
+ description: "What is missing or uncertain, one item per blocker (blocked runs).",
95
+ },
96
+ proof: {
97
+ type: "array",
98
+ items: { type: "string" },
99
+ description: "The commands you ran as evidence, exactly as executed.",
100
+ },
101
+ },
102
+ } as const;
103
+
33
104
  function scheduleWallClock(callback: () => void, delayMs: number): () => void {
34
105
  const timer = setTimeout(callback, delayMs);
35
106
  return () => clearTimeout(timer);
@@ -74,6 +145,32 @@ export const ORPHAN_RESUME_PROMPT =
74
145
  "then continue exactly where you left off: re-check the outcome of your last action before repeating it, and keep " +
75
146
  "working your original brief to the same report contract.";
76
147
 
148
+ /**
149
+ * What a review-revision worker is told instead of re-sending its brief (#677).
150
+ *
151
+ * Same rule as {@link ORPHAN_RESUME_PROMPT}: the original brief is already in
152
+ * the resumed transcript, and re-sending it is how a resumed worker re-does
153
+ * work it already did. What makes this a revision rather than a resume is the
154
+ * deliverable contract: the PR stays open, the branch stays the run's own, and
155
+ * the session ends only when the same final-report shape says the revised head
156
+ * is green. One literal so tests can pin it.
157
+ */
158
+ export const REVIEW_REVISION_PROMPT =
159
+ "The orchestrator reviewed your green pull request and found blocking findings. Continue this same session " +
160
+ "on this same run: same branch, same pull request — do not close or reopen it, and do not open another. " +
161
+ "Address exactly the findings below, push with conductor_push, verify the checks with conductor_pr_status, " +
162
+ "and finish with the same final-report contract as before (yield your settlement: status green, prUrl, headSha). " +
163
+ "Your original brief is already in this transcript; redo only what the findings implicate.";
164
+
165
+ /**
166
+ * The full opening prompt for one review revision round: the pinned continuation
167
+ * text plus the orchestrator's findings verbatim. Exported so a unit test can
168
+ * pin the exact handoff without standing up a session.
169
+ */
170
+ export function renderReviewRevisionPrompt(findings: string, round: number): string {
171
+ return `${REVIEW_REVISION_PROMPT}\n\nReview round ${round} — blocking findings:\n\n${findings}`;
172
+ }
173
+
77
174
  export interface WorkerOpts {
78
175
  brief: string;
79
176
  cwd: string;
@@ -179,6 +276,13 @@ export interface WorkerResult {
179
276
  headSha?: string;
180
277
  turns: number;
181
278
  spendUsd: number;
279
+ /**
280
+ * The run's settlement text. Today this is the worker's own last words; a
281
+ * run that yielded a structured {@link WorkerSettlement} instead carries the
282
+ * canonical rendering of that parsed object (#540) — the narrative lives in
283
+ * its `summary`, transcribed here field for field rather than regexed out of
284
+ * prose.
285
+ */
182
286
  report: string;
183
287
  /** In-session HTTP 429 responses the session recorded (stopReason "error",
184
288
  * errorStatus 429), counted as the messages streamed in. A healthy run
@@ -251,12 +355,34 @@ export function renderBrief(template: string, vars: Record<string, string>): str
251
355
  * must carry both a PR URL and the head SHA observed after CI; the daemon then
252
356
  * asks the tracker to verify those facts independently. Missing or malformed
253
357
  * evidence fails closed.
358
+ *
359
+ * A parsed {@link WorkerSettlement} wins over the text whenever it is present
360
+ * (#540): the schema is the contract now, and a misleadingly regular-looking
361
+ * prose block must not out-vote the object the worker actually yielded. The
362
+ * text path is unchanged underneath, so a run that never yields settles
363
+ * exactly as it did before.
254
364
  */
255
- export function deriveResult(report: string, repoSlug?: string): {
365
+ export function deriveResult(
366
+ report: string,
367
+ repoSlug?: string,
368
+ structured?: WorkerSettlement,
369
+ ): {
256
370
  state: RunState;
257
371
  prUrl?: string;
258
372
  headSha?: string;
259
373
  } {
374
+ if (structured !== undefined && (structured.status === "green" || structured.status === "blocked" || structured.status === "failed")) {
375
+ const prUrl = structured.prUrl;
376
+ const headSha = structured.headSha?.toLowerCase();
377
+ if (structured.status === "green" && prUrl !== undefined && headSha !== undefined) {
378
+ return { state: "pushed-green", prUrl, headSha };
379
+ }
380
+ return {
381
+ state: structured.status === "blocked" ? "blocked" : "failed",
382
+ ...(prUrl === undefined ? {} : { prUrl }),
383
+ ...(headSha === undefined ? {} : { headSha }),
384
+ };
385
+ }
260
386
  const structuredPrUrl = PR_URL_PATTERN.exec(report)?.[1];
261
387
  const headSha = HEAD_SHA_PATTERN.exec(report)?.[1]?.toLowerCase();
262
388
  if (PUSHED_GREEN_PATTERN.test(report) && structuredPrUrl !== undefined && headSha !== undefined) {
@@ -279,6 +405,81 @@ export function deriveResult(report: string, repoSlug?: string): {
279
405
  };
280
406
  }
281
407
 
408
+ /**
409
+ * Parse a worker's structured settlement out of one assistant message's
410
+ * content (#540).
411
+ *
412
+ * The worker calls the `yield` tool with `{ result: { data: <settlement> } }`
413
+ * and the harness records that call as a `toolCall` content block on the
414
+ * assistant message — the same content `reportText` flattens. Parsed
415
+ * permissively on purpose: `outputSchemaMode` is permissive, so a yield the
416
+ * schema rejected must still fall back to the text path rather than vanish.
417
+ * The acceptance bar mirrors the schema's own `required`: a recognizable
418
+ * `status` and a string `summary`; every other field is adopted when it has
419
+ * the right type and dropped otherwise. Exported so the run loop's precedence
420
+ * (yield over prose) is pinned by a unit test.
421
+ */
422
+ export function structuredSettlement(content: unknown): WorkerSettlement | undefined {
423
+ if (!Array.isArray(content)) return undefined;
424
+ let parsed: WorkerSettlement | undefined;
425
+ for (const block of content) {
426
+ if (field(block, "type") !== "toolCall") continue;
427
+ if (field(block, "name") !== "yield") continue;
428
+ const args = field(block, "arguments");
429
+ if (args === null || typeof args !== "object") continue;
430
+ const result = field(args, "result");
431
+ if (result === null || typeof result !== "object") continue;
432
+ const data = field(result, "data");
433
+ if (data === null || typeof data !== "object" || Array.isArray(data)) continue;
434
+ const record = data as Record<string, unknown>;
435
+ const status = record.status;
436
+ if (status !== "green" && status !== "blocked" && status !== "failed") continue;
437
+ const summary = record.summary;
438
+ if (typeof summary !== "string") continue;
439
+ const settlement: WorkerSettlement = { status, summary };
440
+ for (const key of ["prUrl", "branch", "headSha"] as const) {
441
+ const value = record[key];
442
+ if (typeof value === "string" && value !== "") settlement[key] = value;
443
+ }
444
+ for (const key of ["blockers", "proof"] as const) {
445
+ const value = record[key];
446
+ if (Array.isArray(value) && value.every((item) => typeof item === "string")) {
447
+ settlement[key] = value;
448
+ }
449
+ }
450
+ parsed = settlement;
451
+ }
452
+ return parsed;
453
+ }
454
+
455
+ /**
456
+ * Render a parsed {@link WorkerSettlement} as the run's stored report (#540).
457
+ *
458
+ * A worker that yields structured output may write little or no prose after
459
+ * the yield call, and the prose shape is no longer the contract — so the
460
+ * report the store keeps for such a run is this canonical rendering of the
461
+ * parsed object itself: the same fields a reviewer would have had to regex
462
+ * out of prose, derived from the object, field for field. The daemon's own
463
+ * postfixes — the PR-diff `changed:` line and the reliability sentence — are
464
+ * appended to it exactly as they are to a prose report.
465
+ */
466
+ export function renderSettlement(settlement: WorkerSettlement): string {
467
+ const lines: string[] = [`status: ${settlement.status}`];
468
+ if (settlement.prUrl !== undefined) lines.push(`pr: ${settlement.prUrl}`);
469
+ if (settlement.branch !== undefined) lines.push(`branch: ${settlement.branch}`);
470
+ if (settlement.headSha !== undefined) lines.push(`head: ${settlement.headSha}`);
471
+ if (settlement.summary !== "") lines.push("", settlement.summary);
472
+ if (settlement.blockers !== undefined && settlement.blockers.length > 0) {
473
+ lines.push("", "blockers:");
474
+ for (const blocker of settlement.blockers) lines.push(` - ${blocker}`);
475
+ }
476
+ if (settlement.proof !== undefined && settlement.proof.length > 0) {
477
+ lines.push("", "proof:");
478
+ for (const item of settlement.proof) lines.push(` - ${item}`);
479
+ }
480
+ return lines.join("\n");
481
+ }
482
+
282
483
  /**
283
484
  * Did this report state a verdict at all?
284
485
  *
@@ -354,6 +555,13 @@ export async function runWorker(
354
555
  ...(o.verbSocketPath === undefined ? {} : { verbSocketPath: o.verbSocketPath }),
355
556
  ...(o.onChildLog === undefined ? {} : { onChildLog: o.onChildLog }),
356
557
  ...(o.maySpawn === undefined ? {} : { maySpawn: o.maySpawn }),
558
+ // The structured settlement contract (#540): the worker's `yield` tool
559
+ // validates its `data` payload against this schema, permissively — an
560
+ // invalid or absent yield falls back to the text path, never a lost
561
+ // report. Always-on for workers: one schema, and no per-project shape.
562
+ outputSchema: WORKER_SETTLEMENT_SCHEMA,
563
+ outputSchemaMode: "permissive",
564
+ requireYieldTool: true,
357
565
  });
358
566
  } catch (err) {
359
567
  // The pre-spawn gate closed (#374): a daemon stop landed while the session
@@ -447,6 +655,10 @@ export async function runWorker(
447
655
  let spendUsd = 0;
448
656
  let provider429Count = 0;
449
657
  let report = "";
658
+ // The newest structured settlement the worker yielded (#540). A terminal
659
+ // `yield` ends the run, so anything parsed here is the run's own claim, and
660
+ // any later message is stale-polling chatter, never a newer verdict.
661
+ let structured: WorkerSettlement | undefined;
450
662
  // Which model/provider actually wrote the newest assistant message. Last
451
663
  // assistant message wins: that is the durable answer even for a run that
452
664
  // never failed over (#535 slice 1, read off the message field which is where
@@ -603,15 +815,22 @@ export async function runWorker(
603
815
  session.on("message_end", (event) => {
604
816
  const message = field(event, "message");
605
817
  if (field(message, "role") !== "assistant") return;
818
+ const content = field(message, "content");
606
819
  // Keep the newest non-empty assistant text: whatever the worker said last
607
820
  // is its report, whether it finished cleanly or was cut off.
608
- const text = reportText(field(message, "content"));
821
+ const text = reportText(content);
609
822
  if (text !== "") {
610
823
  report = text;
611
- const stated = deriveResult(text, o.repoSlug);
612
- if (stated.state === "pushed-green" && stated.prUrl !== undefined && stated.headSha !== undefined) {
613
- claim = { prUrl: stated.prUrl, headSha: stated.headSha };
614
- }
824
+ }
825
+ // The structured settlement (#540): the `yield` tool call rides the same
826
+ // message's content as a `toolCall` block. Newest yield wins; the
827
+ // derivation itself prefers it over prose, so a yield and a misleadingly
828
+ // regular-looking text block on the same message agree in its favour.
829
+ const yielded = structuredSettlement(content);
830
+ if (yielded !== undefined) structured = yielded;
831
+ const stated = deriveResult(text, o.repoSlug, yielded);
832
+ if (stated.state === "pushed-green" && stated.prUrl !== undefined && stated.headSha !== undefined) {
833
+ claim = { prUrl: stated.prUrl, headSha: stated.headSha };
615
834
  }
616
835
 
617
836
  // Real cost lives on assistant messages as `usage.cost.total` (live hermes
@@ -776,6 +995,20 @@ export async function runWorker(
776
995
  }));
777
996
  }
778
997
 
998
+ // A structured yield is the run's settlement: it wins over the last text —
999
+ // the worker may have written nothing after the yield call, and the stored
1000
+ // report is the canonical rendering of the parsed object, not prose the
1001
+ // regex has to recover (#540).
1002
+ if (structured !== undefined) {
1003
+ return withMetrics(withSessionFacts({
1004
+ ...deriveResult(report, o.repoSlug, structured),
1005
+ turns,
1006
+ spendUsd,
1007
+ provider429Count,
1008
+ report: renderSettlement(structured),
1009
+ }));
1010
+ }
1011
+
779
1012
  // An explicit later verdict always wins: a worker that pushed green and then
780
1013
  // stopped to ask a question means the question. The earlier claim is only
781
1014
  // restored when the last thing said was not a verdict at all.
package/src/worktree.ts CHANGED
@@ -759,12 +759,87 @@ export async function removeWorktree(
759
759
 
760
760
  export type RetainedWorktreeCleanup =
761
761
  | { kind: "removed" }
762
- | { kind: "retained"; reason: "dirty" | "unpushed" | "unknown"; detail: string };
762
+ | {
763
+ kind: "retained";
764
+ reason: "dirty" | "unpushed" | "unknown" | "quarantined";
765
+ detail: string;
766
+ };
767
+
768
+ /**
769
+ * Repairs a run repository whose `objects/info/alternates` names a path that no
770
+ * longer exists, re-pointing the dangling entry at this project's current
771
+ * mirror. Worktrees created before mirrors moved into per-project roots keep
772
+ * the old flat path (`<home>/mirrors/<repo>.git`); the store they borrow from
773
+ * is gone, so any fetch against them dies on unresolved deltas (#728). The
774
+ * repair belongs here — in the code that discovers the stale entry — rather
775
+ * than in a one-off host edit, so the next layout migration heals itself too.
776
+ *
777
+ * Only missing entries are rewritten: a valid alternate is left alone and a
778
+ * worktree with no alternates file is already sound. When the store cannot be
779
+ * made sound — the git dir is unresolvable, the file cannot be read or
780
+ * written, or a missing path survives the rewrite — the caller must not fetch
781
+ * into the broken state; a quarantine detail is returned instead.
782
+ */
783
+ async function repairAlternates(
784
+ worktreePath: string,
785
+ mirrorPath: string,
786
+ ): Promise<{ kind: "ok" } | { kind: "quarantine"; detail: string }> {
787
+ const common = await runGit(["rev-parse", "--git-common-dir"], worktreePath);
788
+ if (common.code !== 0) {
789
+ return {
790
+ kind: "quarantine",
791
+ detail: `cannot resolve the git dir: ${common.stderr.trim() || common.stdout.trim() || "no output"}`,
792
+ };
793
+ }
794
+ const raw = common.stdout.trim();
795
+ if (raw === "") {
796
+ return { kind: "quarantine", detail: "cannot resolve the git dir: git printed nothing" };
797
+ }
798
+ // `--git-common-dir` is relative for linked worktrees; resolve against the
799
+ // tree so bare-mirror layouts and plain clones both land on the alternates.
800
+ const commonDir = raw.startsWith("/") ? raw : join(worktreePath, raw);
801
+ const alternates = join(commonDir, "objects", "info", "alternates");
802
+ if (!existsSync(alternates)) return { kind: "ok" };
803
+
804
+ const current = join(mirrorPath, "objects");
805
+ let content: string;
806
+ try {
807
+ content = readFileSync(alternates, "utf8");
808
+ } catch (err) {
809
+ return { kind: "quarantine", detail: `${alternates} cannot be read: ${err instanceof Error ? err.message : String(err)}` };
810
+ }
811
+
812
+ let changed = false;
813
+ const rewritten = content.split("\n").map((line) => {
814
+ const candidate = line.trim();
815
+ if (candidate === "" || candidate === current || existsSync(candidate)) return line;
816
+ changed = true;
817
+ return current;
818
+ });
819
+ if (changed) {
820
+ try {
821
+ writeFileSync(alternates, rewritten.join("\n"));
822
+ } catch (err) {
823
+ return { kind: "quarantine", detail: `${alternates} cannot be rewritten: ${err instanceof Error ? err.message : String(err)}` };
824
+ }
825
+ }
826
+
827
+ const dangling = rewritten.map((line) => line.trim()).filter((path) => path !== "" && !existsSync(path));
828
+ if (dangling.length > 0) {
829
+ return {
830
+ kind: "quarantine",
831
+ detail: `${alternates} still names missing path(s): ${dangling.join(", ")}`,
832
+ };
833
+ }
834
+ return { kind: "ok" };
835
+ }
763
836
 
764
837
  /**
765
838
  * Reap a terminal run's tree and local mirror branch without deleting the only
766
839
  * copy of work. Tracker state is proved by the caller; this function proves the
767
- * local half after refreshing remote refs. Any ambiguity retains everything.
840
+ * local half after refreshing remote refs. Any ambiguity retains everything;
841
+ * a remote read that failed is ambiguity, never a pushed/unpushed claim, and
842
+ * the unpushed verdicts below are reachable only from reads that succeeded.
768
843
  */
769
844
  export async function cleanupRetainedWorktree(
770
845
  mirrorPath: string,
@@ -779,6 +854,20 @@ export async function cleanupRetainedWorktree(
779
854
 
780
855
  try {
781
856
  if (existsSync(worktreePath)) {
857
+ // A run repo created before mirrors moved into per-project roots may
858
+ // borrow objects from a path that no longer exists. Fetching into it is
859
+ // a guaranteed failure, so repair the alternates first — and when the
860
+ // store cannot be made sound, quarantine the tree instead of fetching
861
+ // into the broken state.
862
+ const repair = await repairAlternates(worktreePath, mirrorPath);
863
+ if (repair.kind === "quarantine") {
864
+ return {
865
+ kind: "retained",
866
+ reason: "quarantined",
867
+ detail: `worktree quarantined: ${repair.detail}`,
868
+ };
869
+ }
870
+
782
871
  const dirty = await git(["status", "--porcelain"], worktreePath);
783
872
  if (dirty !== "") {
784
873
  return { kind: "retained", reason: "dirty", detail: "worktree has uncommitted changes" };
@@ -794,8 +883,18 @@ export async function cleanupRetainedWorktree(
794
883
  }
795
884
 
796
885
  // A deleted remote branch can make a pushed commit look local-only until
797
- // the default branch is fetched. Failure is ambiguity, never permission.
798
- await git(["fetch", "--prune", "origin"], mirrorPath);
886
+ // the default branch is fetched. A failed fetch is ambiguity, never
887
+ // permission: without fresh remote refs the unpushed verdicts below are
888
+ // unreachable and the run is retained on an explicit unknown.
889
+ try {
890
+ await git(["fetch", "--prune", "origin"], mirrorPath);
891
+ } catch (err) {
892
+ return {
893
+ kind: "retained",
894
+ reason: "unknown",
895
+ detail: `remote refs could not be refreshed: ${err instanceof Error ? err.message : String(err)}`,
896
+ };
897
+ }
799
898
 
800
899
  const ref = `refs/heads/${branch}`;
801
900
 
@@ -806,10 +905,18 @@ export async function cleanupRetainedWorktree(
806
905
  // data loss #121 exists to prevent, reintroduced by the move to per-run
807
906
  // repositories, so it is checked where the objects actually are.
808
907
  if (existsSync(worktreePath)) {
809
- await git(
810
- ["fetch", "--no-tags", mirrorPath, "+refs/remotes/origin/*:refs/remotes/origin/*"],
811
- worktreePath,
812
- );
908
+ try {
909
+ await git(
910
+ ["fetch", "--no-tags", mirrorPath, "+refs/remotes/origin/*:refs/remotes/origin/*"],
911
+ worktreePath,
912
+ );
913
+ } catch (err) {
914
+ return {
915
+ kind: "retained",
916
+ reason: "unknown",
917
+ detail: `the run repository's remote refs could not be read: ${err instanceof Error ? err.message : String(err)}`,
918
+ };
919
+ }
813
920
  const runUnique = await git(["rev-list", ref, "--not", "--remotes"], worktreePath);
814
921
  if (runUnique !== "") {
815
922
  return {
@@ -84,6 +84,10 @@ JOURNAL_LINES=${RECOVER_JOURNAL_LINES:-200}
84
84
  VERIFY_POLL_S=${RECOVER_VERIFY_POLL_S:-2}
85
85
  VERIFY_TRIES=${RECOVER_VERIFY_TRIES:-10}
86
86
  PROJECT=${RECOVER_PROJECT:-}
87
+ # The bound on the post-restore re-arm. `arm` with a claim-only proof performs
88
+ # no Telegram send or wait, so it completes in milliseconds; the timeout exists
89
+ # so a misbehaving conductor can never hang this bounded playbook (#613).
90
+ ARM_TIMEOUT_S=${RECOVER_ARM_TIMEOUT_S:-30}
87
91
 
88
92
  # --------------------------------------------------------------------------
89
93
  # output
@@ -298,6 +302,55 @@ choose_action() { # <failed-unit>
298
302
  printf 'NONE|\n'
299
303
  }
300
304
 
305
+ # --------------------------------------------------------------------------
306
+ # re-arm — the one follow-on after a successful restoration (#613)
307
+ # --------------------------------------------------------------------------
308
+
309
+ # Whether the scoped project opted into claim-only arming. The unit renders
310
+ # RECOVER_PROJECT only for an unambiguous single-project host (#510/#530), and
311
+ # "proof" is a value unique to the project's "arm" block, so two exact-key
312
+ # greps decide it on the machine-written config. Any unreadable, hand-formatted
313
+ # or legacy shape (no "arm" key at all) reads as challenge — the explicit human
314
+ # gate — which is the fail-safe direction: recovery never arms a project that
315
+ # did not opt into claim-only.
316
+ project_arms_claim_only() { # <config-path>
317
+ local path=${1:-$CONFIG_PATH}
318
+ [[ -f $path ]] || return 1
319
+ grep -qE '"arm"[[:space:]]*:' "$path" || return 1
320
+ grep -qE '"proof"[[:space:]]*:[[:space:]]*"claim-only"' "$path"
321
+ }
322
+
323
+ # Re-arm the scoped project through the ordinary `arm` path, unattended only
324
+ # when the project opted into claim-only proof: that proof performs no Telegram
325
+ # send or wait, so the command completes headless. A challenge project is never
326
+ # poked — its gate is a human answering the challenge — and the caller's
327
+ # escalation says so plainly instead. Prints exactly one outcome line (the
328
+ # progress line goes to stderr, so the captured return value is the outcome).
329
+ rearm_if_claim_only() {
330
+ if [[ -z $PROJECT ]]; then
331
+ printf 'no scoped project (RECOVER_PROJECT unset) — nothing to re-arm\n'
332
+ return 0
333
+ fi
334
+ if ! project_arms_claim_only "$CONFIG_PATH"; then
335
+ printf 'project %s keeps the challenge proof — arming stays an explicit human gate\n' "$PROJECT"
336
+ return 0
337
+ fi
338
+ log "re-arming $PROJECT via the ordinary arm path (claim-only proof)"
339
+ if dry_run; then
340
+ printf 'dry run: would run %s arm --project %s (claim-only, bounded by %ss)\n' "$CONDUCTOR" "$PROJECT" "$ARM_TIMEOUT_S"
341
+ return 0
342
+ fi
343
+ local out code
344
+ out=$(timeout "$ARM_TIMEOUT_S" "$CONDUCTOR" arm --project "$PROJECT" 2>&1)
345
+ code=$?
346
+ if (( code == 0 )); then
347
+ printf 're-armed %s via the claim-only proof\n' "$PROJECT"
348
+ return 0
349
+ fi
350
+ printf 're-arm of %s failed (exit %s): %s\n' "$PROJECT" "$code" "$out"
351
+ return 0
352
+ }
353
+
301
354
  # --------------------------------------------------------------------------
302
355
  # main
303
356
  # --------------------------------------------------------------------------
@@ -318,6 +371,13 @@ run_recovery() {
318
371
  IFS='|' read -r action_name action_arg <<<"$(choose_action "$failed")"
319
372
  say "plan: failed unit $failed; action: $action_name"
320
373
  fi
374
+ if [[ -z $PROJECT ]]; then
375
+ say "plan: re-arm — none (RECOVER_PROJECT unset)"
376
+ elif project_arms_claim_only "$CONFIG_PATH"; then
377
+ say "plan: re-arm — $CONDUCTOR arm --project $PROJECT (claim-only proof, bounded by ${ARM_TIMEOUT_S}s)"
378
+ else
379
+ say "plan: re-arm — none ($PROJECT keeps the challenge proof; arming is an explicit human gate)"
380
+ fi
321
381
  say "plan: escalation — $(escalation_text "$failed" "${action_name:-none}" "dry run" "DRY-RUN")"
322
382
  return 0
323
383
  fi
@@ -411,6 +471,19 @@ run_recovery() {
411
471
  bump_failed_attempts
412
472
  fi
413
473
 
474
+ # The one follow-on a verified restoration earns (#613): re-arm the scoped
475
+ # project through the ordinary `arm` command. Claim-only completes headless;
476
+ # a challenge project stays an explicit human gate and the escalation below
477
+ # says so plainly. Only after `verify` passed — writing an arm marker for a
478
+ # fleet that is still down is the improvisation this playbook never does.
479
+ local rearm_line="not attempted (recovery did not verify the fleet restored)"
480
+ if (( ok == 0 )); then
481
+ rearm_line=$(rearm_if_claim_only)
482
+ fi
483
+ say "re-arm: $rearm_line"
484
+ append_evidence "$ev" "re-arm: $rearm_line"
485
+ outcome="$outcome; re-arm: $rearm_line"
486
+
414
487
  if escalate "$failed" "$action_name" "$outcome" "$ev"; then
415
488
  if (( ok == 0 )); then
416
489
  return 0
@@ -97,6 +97,9 @@ case "$1" in
97
97
  --version)
98
98
  echo "0.15.11"
99
99
  ;;
100
+ arm)
101
+ echo "armed demo via fixture"
102
+ ;;
100
103
  report)
101
104
  echo "report 485-abc queued for demo (tier2)"
102
105
  ;;
@@ -147,6 +150,12 @@ marker() { # <case-dir> -> attempts count or 0
147
150
  fi
148
151
  }
149
152
 
153
+ # The minimal machine-written config shape the re-arm decision greps: the arm
154
+ # block with the chosen proof, as `saveConfig` would write it.
155
+ write_project_config() { # <case-dir> <proof>
156
+ printf '{\n "version": 2,\n "projects": [\n { "name": "demo", "arm": { "proof": "%s" } }\n ]\n}\n' "$2" >"$1/state/config.json"
157
+ }
158
+
150
159
  run_recover() { # <case-dir> [env assignments…]
151
160
  local d="$1"
152
161
  shift
@@ -300,6 +309,58 @@ check 'ROLLBACK escalates as well' \
300
309
  check 'the pre-upgrade snapshot survives the rollback for the operator' \
301
310
  '{"preUpgrade":true}' "$(cat "$d/state/backups/config"/config.json.pre-upgrade-*)"
302
311
 
312
+ # ---------------------------------------------------------------------------
313
+ # case — the post-restore re-arm (#613): a verified restoration re-arms a
314
+ # claim-only project through the ordinary arm command; a challenge project is
315
+ # never poked and the escalation says the human gate is explicit.
316
+ # ---------------------------------------------------------------------------
317
+
318
+ d=$(newcase rearm-claim-only)
319
+ write_project_config "$d" claim-only
320
+ unit_state "$d" herdr-fleet.service failed
321
+ unit_state "$d" omp-conductor.service active
322
+ code=$(run_with_heal "$d" herdr-fleet.service)
323
+ check 'claim-only re-arm exits 0 when the unit heals' '0' "$code"
324
+ check 'claim-only recovery re-arms via the ordinary arm path' \
325
+ 'omp-conductor arm --project demo' "$(grep '^omp-conductor arm' "$d/calls")"
326
+ check 'the escalation names the claim-only re-arm' \
327
+ 're-armed demo via the claim-only proof' "$(grep '^omp-conductor report' "$d/calls" | head -n 1)"
328
+ check 'the evidence bundle records the re-arm' \
329
+ 're-arm: re-armed demo via the claim-only proof' "$(cat "$d/state/recovery"/evidence-*)"
330
+
331
+ d=$(newcase rearm-challenge)
332
+ write_project_config "$d" challenge
333
+ unit_state "$d" herdr-fleet.service failed
334
+ unit_state "$d" omp-conductor.service active
335
+ code=$(run_with_heal "$d" herdr-fleet.service)
336
+ check 'a challenge project exits 0 when the unit heals' '0' "$code"
337
+ check 'a challenge project is never poked with arm' \
338
+ '' "$(grep '^omp-conductor arm' "$d/calls" || true)"
339
+ check 'the escalation says the challenge gate is explicit' \
340
+ 'arming stays an explicit human gate' "$(grep '^omp-conductor report' "$d/calls" | head -n 1)"
341
+
342
+ # A project whose config predates the key reads as challenge: no "arm" block.
343
+ d=$(newcase rearm-legacy-config)
344
+ unit_state "$d" herdr-fleet.service failed
345
+ unit_state "$d" omp-conductor.service active
346
+ code=$(run_with_heal "$d" herdr-fleet.service)
347
+ check 'a legacy config without arm.proof is never poked with arm' \
348
+ '' "$(grep '^omp-conductor arm' "$d/calls" || true)"
349
+ check 'a legacy config keeps the human gate in the escalation' \
350
+ 'arming stays an explicit human gate' "$(grep '^omp-conductor report' "$d/calls" | head -n 1)"
351
+
352
+ # RESTART_PEER never verifies a restoration, so even a claim-only project must
353
+ # not be re-armed while the daemon is still down.
354
+ d=$(newcase rearm-unverified)
355
+ write_project_config "$d" claim-only
356
+ unit_state "$d" omp-conductor.service failed
357
+ unit_state "$d" herdr-fleet.service active
358
+ code=$(run_recover "$d")
359
+ check 'an unverified recovery never re-arms' \
360
+ 're-arm: not attempted' "$(grep 're-arm:' "$d/out" | head -n 1)"
361
+ check 'an unverified recovery calls no arm' \
362
+ '' "$(grep '^omp-conductor arm' "$d/calls" || true)"
363
+
303
364
  # ---------------------------------------------------------------------------
304
365
  # case RESTART_PEER and the bound: attempts cap at two, then escalate-only.
305
366
  # A daemon crash with no upgrade evidence must never get a blind daemon start.