omp-conductor 0.19.5 → 0.19.7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -42,7 +42,7 @@ import { randomUUID } from "node:crypto";
42
42
  import { createServer, type Server, type Socket } from "node:net";
43
43
  import { join } from "node:path";
44
44
 
45
- import { resolvePolicy, resolveReleaseGrants, resolveReview } from "../config.ts";
45
+ import { resolvePolicy, resolveReleaseGrants, resolveReview, resolveSharedInstallAuthority } from "../config.ts";
46
46
  import { wakeDispatch } from "../wake.ts";
47
47
  // The mediated release is the drain's closing gesture (#791): a successful
48
48
  // release ends this project's bounded release window, so the privileged half
@@ -60,6 +60,8 @@ import {
60
60
  import { releaseRefusal } from "../release-policy.ts";
61
61
  import { PR_LOOKUP_WINDOW_MS, REVISABLE_RUN_STATES, prReviewReadiness } from "../decisions.ts";
62
62
  import { LIVE_STATES } from "../store.ts";
63
+ import { parseToSpecEvidence } from "../to-spec.ts";
64
+ import { DENIED_RELEASE_GRANTS } from "../types.ts";
63
65
  import type {
64
66
  FileLane,
65
67
  IssueComment,
@@ -71,6 +73,7 @@ import type {
71
73
  ReleaseRequirement,
72
74
  ReleaseShape,
73
75
  RepoTarget,
76
+ ResolvedGrants,
74
77
  ReviewReason,
75
78
  RunRecord,
76
79
  RunState,
@@ -172,6 +175,9 @@ export interface ReleaseExecution {
172
175
  export interface InstallExecution {
173
176
  /** The published semver to pin all three surfaces to, exactly as npm has it. */
174
177
  version: string;
178
+ /** Shared host authority and scope, carried into the detached report. */
179
+ holder: ResolvedGrants["install"];
180
+ projects: readonly string[];
175
181
  }
176
182
 
177
183
  /**
@@ -222,6 +228,11 @@ export interface VerbDeps {
222
228
  * unreadable, and cannot pick up an operator's edit either.
223
229
  */
224
230
  project: () => ProjectConfig;
231
+ /**
232
+ * Every project served by the shared daemon, re-read from the same config.
233
+ * Host-global mutations use this set rather than the calling socket's project.
234
+ */
235
+ projects: () => readonly ProjectConfig[];
225
236
  store: Store;
226
237
  tracker: Tracker;
227
238
  actions: VerbActions;
@@ -632,7 +643,7 @@ async function decide(deps: VerbDeps, channel: VerbChannel, raw: unknown): Promi
632
643
  case "conductor_release":
633
644
  return releaseVerb(deps, project, channel, args, refuse, allow);
634
645
  case "conductor_install":
635
- return installVerb(deps, project, channel, args, refuse, allow);
646
+ return installVerb(deps, channel, args, refuse, allow);
636
647
  case "conductor_pr_review":
637
648
  return prReviewVerb(deps, project, channel, args, refuse, allow);
638
649
  case "conductor_pr_review_clear":
@@ -1288,10 +1299,23 @@ async function labelVerb(
1288
1299
  let malformed: string | undefined;
1289
1300
  if (action === "add" && label === project.queueLabel) {
1290
1301
  const lane = await laneForEcho(deps, ref.issue);
1291
- if (lane !== undefined) {
1302
+ if (lane.readable) {
1292
1303
  malformed = lane.malformed;
1293
1304
  echo = laneEcho(lane.lane);
1294
1305
  }
1306
+ // The durable grooming verdict is an admission contract only while the
1307
+ // dispatch brief still matches it (#1036): a stale issue body once
1308
+ // promoted beside a PROMOTABLE verdict whose fileLane had moved on, and
1309
+ // the run dispatched on the stale lane. So before any tracker mutation or
1310
+ // daemon wake, a promotable row must verify against the same parse the
1311
+ // echo reports — exact in both directions, order-insensitive. The syntax
1312
+ // refusal below keeps precedence: fix the section first, then the contract.
1313
+ if (malformed === undefined) {
1314
+ const mismatch = promotionBriefRefusal(deps, project, ref.issue, lane);
1315
+ if (mismatch !== undefined) {
1316
+ return refuse("promotion-brief-mismatch", mismatch, ref.issue);
1317
+ }
1318
+ }
1295
1319
  }
1296
1320
  if (malformed !== undefined) {
1297
1321
  return refuse(
@@ -1344,32 +1368,99 @@ async function labelVerb(
1344
1368
  * The effective file lane admission will enforce for one issue, as the
1345
1369
  * one-line echo (#724), plus the malformed write-lane section marker (#825):
1346
1370
  * the body plus the whole comment thread, the same inputs `effectiveLane`
1347
- * reads at admission. `undefined` when either read fails or the tracker cannot
1348
- * produce the issue — the promotion is never blocked by its own feedback.
1371
+ * reads at admission. `readable` says whether the issue and thread were read
1372
+ * at all — the echo stays best-effort (an unreadable issue never blocks its
1373
+ * own promotion, #724), but the promotion-brief gate (#1036) must be able to
1374
+ * tell "no lane declared" from "could not look", because behind a durable
1375
+ * `promotable` verdict the second is a fail-closed refusal rather than
1376
+ * feedback.
1349
1377
  */
1350
- async function laneForEcho(deps: VerbDeps, issue: number): Promise<
1351
- | { lane: FileLane | undefined; malformed: string | undefined }
1352
- | undefined
1353
- > {
1378
+ async function laneForEcho(deps: VerbDeps, issue: number): Promise<{
1379
+ readable: boolean;
1380
+ lane: FileLane | undefined;
1381
+ malformed: string | undefined;
1382
+ }> {
1354
1383
  let body: string;
1355
1384
  let comments: IssueComment[];
1356
1385
  try {
1357
1386
  const row = await deps.tracker.getIssue(issue);
1358
- if (row === undefined) return undefined;
1387
+ if (row === undefined) return { readable: false, lane: undefined, malformed: undefined };
1359
1388
  body = row.body;
1360
1389
  comments = await deps.tracker.listComments(issue);
1361
1390
  } catch {
1362
- return undefined;
1391
+ return { readable: false, lane: undefined, malformed: undefined };
1363
1392
  }
1364
1393
  const lane = effectiveLane(body, comments);
1365
- if (lane !== undefined) return { lane, malformed: undefined };
1394
+ if (lane !== undefined) return { readable: true, lane, malformed: undefined };
1366
1395
  // No declaration parsed anywhere. If a clearly delimited write-lane section
1367
1396
  // exists anyway, the section tried to declare and failed — the echo must
1368
1397
  // refuse, never claim fail-open.
1369
1398
  const heading =
1370
1399
  writeLaneSectionHeading(body) ??
1371
1400
  comments.map((c) => writeLaneSectionHeading(c.body)).find((h) => h !== undefined);
1372
- return { lane: undefined, malformed: heading };
1401
+ return { readable: true, lane: undefined, malformed: heading };
1402
+ }
1403
+
1404
+ /**
1405
+ * The promotion-brief gate (#1036): a durable `promotable` grooming verdict is
1406
+ * an admission contract only while the dispatch brief still matches it, so a
1407
+ * queue-label add must prove the match before any tracker mutation or daemon
1408
+ * wake. Compares the lane admission would enforce (the same parse the echo
1409
+ * reports) against the verdict's `fileLane` as deduplicated,
1410
+ * order-independent path sets, exact in both directions — missing and extra
1411
+ * paths refuse alike. Returns the refusal detail, or `undefined` when
1412
+ * promotion may proceed: no grooming row, a non-`promotable` row, or a
1413
+ * verified match.
1414
+ *
1415
+ * Fail-closed by design: evidence behind a `promotable` row that does not
1416
+ * recover as a strict PROMOTABLE result (`parseToSpecEvidence`, the same
1417
+ * restart round-trip the selection side trusts), or an issue that cannot be
1418
+ * read to compare at all, refuses the promotion rather than dispatching on an
1419
+ * unverifiable brief. The gate never edits the issue — it names both path
1420
+ * sets and points at the verdict's `proposedBrief`; correcting the dispatch
1421
+ * brief is an explicit orchestrator act.
1422
+ */
1423
+ function promotionBriefRefusal(
1424
+ deps: VerbDeps,
1425
+ project: ProjectConfig,
1426
+ issue: number,
1427
+ read: { readable: boolean; lane: FileLane | undefined },
1428
+ ): string | undefined {
1429
+ const row = deps.store.grooming(project.name, issue);
1430
+ if (row?.verdict !== "promotable") return undefined;
1431
+ const result = parseToSpecEvidence(row.evidence);
1432
+ if (result === undefined || result.verdict !== "PROMOTABLE") {
1433
+ return (
1434
+ `refused: ${project.queueLabel} was not added — #${issue}'s durable grooming row reads promotable, but its ` +
1435
+ "evidence does not recover as a strict PROMOTABLE to-spec result, so promotion cannot prove the dispatch " +
1436
+ "brief still matches what was groomed. Re-groom the issue so a fresh verdict replaces the row, then add the label again."
1437
+ );
1438
+ }
1439
+ if (!read.readable) {
1440
+ return (
1441
+ `refused: ${project.queueLabel} was not added — #${issue} carries a durable PROMOTABLE grooming verdict, but ` +
1442
+ "the issue could not be read, so the current brief cannot be compared with the verdict's file lane. " +
1443
+ "Retry when the tracker responds."
1444
+ );
1445
+ }
1446
+ // One closure so both sides of the comparison normalize in lockstep: a set
1447
+ // comparison whose halves dedupe differently lies silently (#1036).
1448
+ const normalizedPaths = (paths: readonly string[]): string[] =>
1449
+ [...new Set(paths.map((path) => path.trim()).filter((path) => path.length > 0))].sort();
1450
+ const brief = normalizedPaths(read.lane?.files ?? []);
1451
+ const verdict = normalizedPaths(result.fileLane);
1452
+ if (
1453
+ brief.length === verdict.length &&
1454
+ brief.every((path, index) => path === verdict[index])
1455
+ ) {
1456
+ return undefined;
1457
+ }
1458
+ return (
1459
+ `refused: ${project.queueLabel} was not added — #${issue}'s current write lane [${brief.join(", ")}] disagrees ` +
1460
+ `with its durable PROMOTABLE verdict's file lane [${verdict.join(", ")}]. The durable verdict is the admission ` +
1461
+ "contract: apply the verdict's proposedBrief to the issue (its ## Exact write lane included), then add the " +
1462
+ "label again. The gate never rewrites the issue itself."
1463
+ );
1373
1464
  }
1374
1465
 
1375
1466
  async function releaseVerb(
@@ -1660,31 +1751,50 @@ async function releaseVerb(
1660
1751
 
1661
1752
 
1662
1753
  /**
1663
- * The fleet-installs-itself request (#486).
1664
- *
1665
- * Gated like a release act: the caller must be an orchestrator (spec), and the
1666
- * `install` shape must be granted to the orchestrator (release policy) — the
1667
- * package floor's "nobody patches the running conductor" is the deny default
1668
- * an operator deliberately moves, never something this verb works around.
1669
- *
1670
- * Everything past the gate is the privileged half's job: `actions.install`
1671
- * refussses a version npm does not expose and starts the detached transient
1672
- * unit. This verb records the request durably (the journal the post-restart
1673
- * tick verifies against) and tells the caller where the evidence will land.
1754
+ * The install is host-global: one package tree and daemon serve every project.
1755
+ * Authority is therefore the consensus holder across the configured project
1756
+ * set, never whichever project happened to invoke the verb (#1018). Mixed
1757
+ * holders refuse before version lookup or transient-unit creation.
1674
1758
  */
1675
1759
  async function installVerb(
1676
1760
  deps: VerbDeps,
1677
- project: ProjectConfig,
1678
1761
  channel: VerbChannel,
1679
1762
  args: Record<string, unknown>,
1680
1763
  refuse: Refuse,
1681
1764
  allow: Allow,
1682
1765
  ): Promise<Verdict> {
1683
- const grants = resolveReleaseGrants(project);
1684
- const granted = releaseRefusal(grants, channel.role, "install");
1766
+ let projects: readonly ProjectConfig[];
1767
+ try {
1768
+ projects = deps.projects();
1769
+ } catch (err) {
1770
+ const why = err instanceof Error ? err.message : String(err);
1771
+ return refuse("config-unreadable", `refused: shared install authority could not be read (${why}).`);
1772
+ }
1773
+ if (projects.length === 0) {
1774
+ return refuse("config-unreadable", "refused: shared install authority has no configured projects.");
1775
+ }
1776
+ const authority = resolveSharedInstallAuthority(projects);
1777
+ const holderDetail = authority.entries.map((entry) => `${entry.project}=${entry.holder}`).join(", ");
1778
+ if (authority.holder === undefined) {
1779
+ return refuse(
1780
+ "release-not-granted",
1781
+ `refused: conductor_install is host-global, but configured install holders conflict (${holderDetail}). ` +
1782
+ "No dispatch pause or transient install unit was created.",
1783
+ );
1784
+ }
1785
+ const globalHolder = authority.holder;
1786
+ const granted = releaseRefusal(
1787
+ { ...DENIED_RELEASE_GRANTS, install: globalHolder },
1788
+ channel.role,
1789
+ "install",
1790
+ );
1685
1791
  if (granted !== undefined) {
1686
- return refuse("release-not-granted", `refused: ${granted.reason}`);
1792
+ return refuse(
1793
+ "release-not-granted",
1794
+ `refused: ${granted.reason} Shared-host holders: ${holderDetail}.`,
1795
+ );
1687
1796
  }
1797
+ const affectedProjects = authority.entries.map((entry) => entry.project);
1688
1798
 
1689
1799
  const version = args["version"];
1690
1800
  if (typeof version !== "string" || version.trim().length === 0) {
@@ -1695,16 +1805,21 @@ async function installVerb(
1695
1805
  );
1696
1806
  }
1697
1807
 
1698
- const outcome = await deps.actions.install({ version: version.trim() });
1808
+ const outcome = await deps.actions.install({
1809
+ version: version.trim(),
1810
+ holder: globalHolder,
1811
+ projects: affectedProjects,
1812
+ });
1699
1813
  if (!outcome.ok) {
1700
1814
  return refuse("action-failed", `refused: the install request failed:\n${outcome.stderr}`);
1701
1815
  }
1702
1816
  const unit = outcome.detail;
1703
1817
  return allow(
1704
- `install requested: omp-conductor@${version} will be installed by a detached unit` +
1818
+ `install requested by host-global holder ${globalHolder}: omp-conductor@${version} will be installed ` +
1819
+ `for projects ${affectedProjects.join(", ")} by a detached unit` +
1705
1820
  `${unit === undefined ? "" : ` (${unit})`} that survives the restart. ` +
1706
1821
  "The first tick after the fleet restarts verifies version, /healthz, ticks, pane and doctor, " +
1707
- "and the outcome is reported through the durable outbox.",
1822
+ "and the outcome is reported through the durable outbox.",
1708
1823
  );
1709
1824
  }
1710
1825
 
@@ -61,6 +61,7 @@ PROG=omp-conductor-recover
61
61
  SYSTEMCTL=${SYSTEMCTL_BIN:-systemctl}
62
62
  JOURNALCTL=${JOURNALCTL_BIN:-journalctl}
63
63
  CONDUCTOR=${OMP_CONDUCTOR_BIN:-omp-conductor}
64
+ BUN=${BUN_BIN:-bun}
64
65
 
65
66
  # The fleet's conductor home: where config.json, the sqlite store and the
66
67
  # upgrade's durable snapshots live. The unit renders OMP_CONDUCTOR_HOME and
@@ -84,6 +85,12 @@ JOURNAL_LINES=${RECOVER_JOURNAL_LINES:-200}
84
85
  VERIFY_POLL_S=${RECOVER_VERIFY_POLL_S:-2}
85
86
  VERIFY_TRIES=${RECOVER_VERIFY_TRIES:-10}
86
87
  PROJECT=${RECOVER_PROJECT:-}
88
+ # Recovery action scope and durable-report ownership are deliberately separate.
89
+ # `RECOVER_PROJECT` is only safe on a single-project host; the report still
90
+ # needs one configured project to own its outbox row on a shared host.
91
+ REPORT_PROJECT=${RECOVER_REPORT_PROJECT:-$PROJECT}
92
+ AFFECTED_PROJECTS=${RECOVER_PROJECTS:-$PROJECT}
93
+ REPORT_ROUTING_NOTE=
87
94
  # The bound on the post-restore re-arm. `arm` with a claim-only proof performs
88
95
  # no Telegram send or wait, so it completes in milliseconds; the timeout exists
89
96
  # so a misbehaving conductor can never hang this bounded playbook (#613).
@@ -190,6 +197,40 @@ append_evidence() { # <evidence path> <text>
190
197
  printf '%s\n' "$2" >>"$1" 2>/dev/null || true
191
198
  }
192
199
 
200
+
201
+ # Resolve the host-global report owner from the live machine-written config.
202
+ # Bun is not an extra prerequisite: omp-conductor itself runs under Bun, and
203
+ # the recovery unit's canonical PATH already contains that executable. When a
204
+ # manual/degraded invocation cannot read the config, the script still attempts
205
+ # the legacy unscoped report and records that degradation in the evidence.
206
+ resolve_report_scope() {
207
+ if [[ -n $REPORT_PROJECT && -n $AFFECTED_PROJECTS ]]; then
208
+ return 0
209
+ fi
210
+ local lines first all
211
+ if lines=$(
212
+ "$BUN" -e '
213
+ const path = Bun.argv.at(-1);
214
+ const config = await Bun.file(path).json();
215
+ const names = Array.isArray(config.projects)
216
+ ? config.projects.map((project) => project?.name).filter((name) => typeof name === "string" && name.length > 0)
217
+ : [];
218
+ if (names.length === 0) process.exit(2);
219
+ console.log(names[0]);
220
+ console.log(names.join(","));
221
+ ' "$CONFIG_PATH" 2>/dev/null
222
+ ); then
223
+ first=${lines%%$'\n'*}
224
+ all=${lines#*$'\n'}
225
+ REPORT_PROJECT=${REPORT_PROJECT:-$first}
226
+ AFFECTED_PROJECTS=${AFFECTED_PROJECTS:-$all}
227
+ fi
228
+ if [[ -z $REPORT_PROJECT || -z $AFFECTED_PROJECTS ]]; then
229
+ AFFECTED_PROJECTS=${AFFECTED_PROJECTS:-unknown}
230
+ REPORT_ROUTING_NOTE="configured project resolution failed for $CONFIG_PATH; attempting the report without a project selector"
231
+ fi
232
+ return 0
233
+ }
193
234
  # --------------------------------------------------------------------------
194
235
  # escalation — always, through the durable outbox (#123, #288)
195
236
  # --------------------------------------------------------------------------
@@ -199,23 +240,27 @@ append_evidence() { # <evidence path> <text>
199
240
  # (the durable outbox) before anything is sent, and the daemon owns delivery
200
241
  # with bounded retries once it is back. This script only ever enqueues.
201
242
  escalation_text() { # <failed-unit or empty> <action> <outcome> <evidence>
243
+ local projects=${AFFECTED_PROJECTS:-unknown}
202
244
  if [[ ${1:-none} == none ]]; then
203
245
  printf '%s\n' \
204
- "fleet recovery (#485): a recovery trigger fired but no fleet unit was failing when it ran — the fleet got itself up or was started externally; evidence=$4"
246
+ "fleet recovery (#485): a recovery trigger fired but no fleet unit was failing when it ran — the fleet got itself up or was started externally; affected-projects=$projects; evidence=$4"
205
247
  return
206
248
  fi
207
249
  printf '%s\n' \
208
- "fleet recovery (#485): $1 entered failed state; recovery=${2:-none}; outcome=${3}; evidence=${4}; next: read the evidence file (and its doctor output) before touching the fleet"
250
+ "fleet recovery (#485): $1 entered failed state; affected-projects=$projects; recovery=${2:-none}; outcome=${3}; evidence=${4}; next: read the evidence file (and its doctor output) before touching the fleet"
209
251
  }
210
252
 
211
253
  escalate() { # <failed-unit or empty> <action-name> <outcome> <evidence-path>
212
254
  local unit=${1:-none} text args
213
- text=$(escalation_text "$unit" "$2" "$3" "$4")
214
- if [[ -n $PROJECT ]]; then
215
- args=("$CONDUCTOR" report --kind tier2 --project "$PROJECT" --text "$text")
216
- else
217
- args=("$CONDUCTOR" report --kind tier2 --text "$text")
255
+ resolve_report_scope
256
+ if [[ -n $REPORT_ROUTING_NOTE ]]; then
257
+ log "$REPORT_ROUTING_NOTE"
258
+ append_evidence "$4" "report routing: $REPORT_ROUTING_NOTE"
218
259
  fi
260
+ text=$(escalation_text "$unit" "$2" "$3" "$4")
261
+ args=("$CONDUCTOR" report --kind tier2)
262
+ [[ -n $REPORT_PROJECT ]] && args+=(--project "$REPORT_PROJECT")
263
+ args+=(--text "$text")
219
264
  say "escalation: ${args[*]}"
220
265
  if dry_run; then
221
266
  return 0
@@ -365,6 +410,9 @@ run_recovery() {
365
410
  done
366
411
 
367
412
  if dry_run; then
413
+ resolve_report_scope
414
+ say "plan: report owner — ${REPORT_PROJECT:-unscoped}; affected projects — $AFFECTED_PROJECTS"
415
+ [[ -n $REPORT_ROUTING_NOTE ]] && say "plan: report routing degraded — $REPORT_ROUTING_NOTE"
368
416
  say "dry run: nothing will be written or executed"
369
417
  [[ -z $failed ]] && say "plan: no unit failed — escalate only"
370
418
  if [[ -n $failed ]]; then
@@ -50,6 +50,7 @@ fi
50
50
 
51
51
  tmp=$(mktemp -d "${TMPDIR:-/tmp}/omp-recover-test.XXXXXX")
52
52
  trap 'rm -rf "$tmp"' EXIT
53
+ bun_bin=$(command -v bun)
53
54
  mkdir -p "$tmp/bin"
54
55
 
55
56
  # --------------------------------------------------------------------------
@@ -156,6 +157,18 @@ write_project_config() { # <case-dir> <proof>
156
157
  printf '{\n "version": 2,\n "projects": [\n { "name": "demo", "arm": { "proof": "%s" } }\n ]\n}\n' "$2" >"$1/state/config.json"
157
158
  }
158
159
 
160
+ write_multi_project_config() { # <case-dir>
161
+ cat >"$1/state/config.json" <<'JSON'
162
+ {
163
+ "version": 2,
164
+ "projects": [
165
+ { "name": "alpha" },
166
+ { "name": "beta" }
167
+ ]
168
+ }
169
+ JSON
170
+ }
171
+
159
172
  run_recover() { # <case-dir> [env assignments…]
160
173
  local d="$1"
161
174
  shift
@@ -182,7 +195,9 @@ run_recover() { # <case-dir> [env assignments…]
182
195
  # marks the named unit active, which is what a real peer restart does to a
183
196
  # healthy peer), so tests wanting daemon-side healing pass HEAL_ON_START=<unit>
184
197
  # and the stub's `start` flips that unit back to active.
185
- run_with_heal() { # <case-dir> <heal-on-start>
198
+ run_with_heal() { # <case-dir> <heal-on-start> [env assignments…]
199
+ local d="$1" heal="$2"
200
+ shift 2
186
201
  (
187
202
  cd "$here" || exit 1
188
203
  env \
@@ -190,14 +205,15 @@ run_with_heal() { # <case-dir> <heal-on-start>
190
205
  SYSTEMCTL_BIN="$tmp/bin/systemctl" \
191
206
  JOURNALCTL_BIN="$tmp/bin/journalctl" \
192
207
  OMP_CONDUCTOR_BIN="$tmp/bin/omp-conductor" \
193
- RECOVER_STATE_DIR="$1/state" \
208
+ RECOVER_STATE_DIR="$d/state" \
194
209
  RECOVER_PROJECT=demo \
195
210
  RECOVER_VERIFY_POLL_S=0 \
196
211
  RECOVER_VERIFY_TRIES=3 \
197
- STUB_CALLS="$1/calls" \
198
- STUB_UNITS="$1/units" \
199
- STUB_HEAL_ON_START="$2" \
200
- bash "$recover" >"$1/out" 2>"$1/stderr"
212
+ STUB_CALLS="$d/calls" \
213
+ STUB_UNITS="$d/units" \
214
+ STUB_HEAL_ON_START="$heal" \
215
+ "$@" \
216
+ bash "$recover" >"$d/out" 2>"$d/stderr"
201
217
  )
202
218
  echo $?
203
219
  }
@@ -412,6 +428,83 @@ check 'a trigger with nothing failed escalates (fleet came back)' \
412
428
  check 'a trigger with nothing failed exits 0' '0' "$code"
413
429
  check 'a trigger with nothing failed clears the old counter' '0' "$(marker "$d")"
414
430
 
431
+ # ---------------------------------------------------------------------------
432
+ # case — a host-global recovery on a multi-project config still has one
433
+ # deterministic outbox owner, while its body names every affected project.
434
+ # ---------------------------------------------------------------------------
435
+
436
+ d=$(newcase multi-project-report)
437
+ write_multi_project_config "$d"
438
+ unit_state "$d" omp-conductor.service active
439
+ unit_state "$d" herdr-fleet.service active
440
+ code=$(run_recover "$d" \
441
+ RECOVER_PROJECT= \
442
+ RECOVER_REPORT_PROJECT= \
443
+ RECOVER_PROJECTS= \
444
+ BUN_BIN="$bun_bin")
445
+ report_call=$(grep '^omp-conductor report' "$d/calls" | head -n 1)
446
+ check 'multi-project recovery exits after queuing its report' '0' "$code"
447
+ check 'multi-project recovery uses the first configured outbox owner' \
448
+ '--project alpha' "$report_call"
449
+ check 'multi-project recovery names every affected project' \
450
+ 'affected-projects=alpha,beta' "$report_call"
451
+ check 'multi-project recovery queues exactly one report' \
452
+ '1' "$(grep -c '^omp-conductor report' "$d/calls")"
453
+
454
+ d=$(newcase multi-project-recovery-action)
455
+ write_multi_project_config "$d"
456
+ unit_state "$d" omp-conductor.service active
457
+ unit_state "$d" herdr-fleet.service failed
458
+ code=$(run_with_heal "$d" herdr-fleet.service \
459
+ RECOVER_PROJECT= \
460
+ RECOVER_REPORT_PROJECT= \
461
+ RECOVER_PROJECTS= \
462
+ BUN_BIN="$bun_bin")
463
+ report_call=$(grep '^omp-conductor report' "$d/calls" | head -n 1)
464
+ check 'verified multi-project recovery action exits cleanly' '0' "$code"
465
+ check 'verified multi-project recovery action queues one scoped report' \
466
+ '1' "$(grep -c '^omp-conductor report' "$d/calls")"
467
+ check 'verified multi-project recovery action uses configured owner' \
468
+ '--project alpha' "$report_call"
469
+ check 'verified multi-project recovery action names every project' \
470
+ 'affected-projects=alpha,beta' "$report_call"
471
+
472
+ d=$(newcase multi-project-dry-run)
473
+ write_multi_project_config "$d"
474
+ unit_state "$d" omp-conductor.service active
475
+ unit_state "$d" herdr-fleet.service failed
476
+ code=$(run_recover "$d" \
477
+ RECOVER_PROJECT= \
478
+ RECOVER_REPORT_PROJECT= \
479
+ RECOVER_PROJECTS= \
480
+ RECOVER_DRY_RUN=1 \
481
+ BUN_BIN="$bun_bin")
482
+ check 'multi-project dry run exits cleanly' '0' "$code"
483
+ check 'multi-project dry run names its report owner' \
484
+ 'plan: report owner — alpha' "$(cat "$d/out")"
485
+ check 'multi-project dry run names every affected project' \
486
+ 'affected projects — alpha,beta' "$(cat "$d/out")"
487
+
488
+ d=$(newcase report-routing-fallback)
489
+ unit_state "$d" omp-conductor.service active
490
+ unit_state "$d" herdr-fleet.service active
491
+ code=$(run_recover "$d" \
492
+ RECOVER_PROJECT= \
493
+ RECOVER_REPORT_PROJECT= \
494
+ RECOVER_PROJECTS= \
495
+ BUN_BIN="$bun_bin")
496
+ report_call=$(grep '^omp-conductor report' "$d/calls" | head -n 1)
497
+ check 'unreadable config routing still attempts a durable report' \
498
+ 'omp-conductor report --kind tier2 --text' "$report_call"
499
+ if [[ $report_call == *"--project "* ]]; then
500
+ fail "routing fallback does not guess an unconfigured project — $report_call"
501
+ else
502
+ pass 'routing fallback does not guess an unconfigured project'
503
+ fi
504
+ check 'routing fallback remains a successful recovered run' '0' "$code"
505
+ check 'routing fallback is preserved in durable evidence' \
506
+ 'attempting the report without a project selector' "$(cat "$d/state/recovery"/evidence-*)"
507
+
415
508
  # ---------------------------------------------------------------------------
416
509
  # case DRY-RUN — the operator's rehearsal mode prints decisions and mutates
417
510
  # nothing, not even the recovery dir or the fixture call log.