omp-conductor 0.19.5 → 0.19.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/REFERENCE.md +17 -1
- package/package.json +1 -1
- package/src/cli.ts +2 -0
- package/src/command-help.ts +18 -8
- package/src/command-manifest.ts +22 -5
- package/src/commands/context.ts +1 -0
- package/src/commands/report.ts +63 -0
- package/src/commands/restore-db.ts +1 -1
- package/src/commands/snapshot-db.ts +67 -0
- package/src/commands/upgrade-install.ts +1 -1
- package/src/commands/upgrade.ts +2 -2
- package/src/commands/verb.ts +1 -0
- package/src/commands/watch.ts +14 -2
- package/src/config.ts +17 -0
- package/src/daemon.ts +146 -35
- package/src/doctor.ts +53 -15
- package/src/escalate.ts +39 -21
- package/src/fleet.ts +979 -143
- package/src/orchestrator-tick.ts +14 -3
- package/src/setup-host.ts +10 -10
- package/src/setup-wizard.ts +1 -1
- package/src/status-render.ts +10 -0
- package/src/store.ts +338 -22
- package/src/types.ts +51 -3
- package/src/upgrade-journal.ts +35 -1
- package/src/upgrade-verify.ts +27 -6
- package/src/upgrade.ts +301 -22
- package/src/verbs/actions.ts +6 -1
- package/src/verbs/server.ts +146 -31
- package/systemd/omp-conductor-recover.sh +55 -7
- package/systemd/recover-unit-test.sh +99 -6
package/src/verbs/server.ts
CHANGED
|
@@ -42,7 +42,7 @@ import { randomUUID } from "node:crypto";
|
|
|
42
42
|
import { createServer, type Server, type Socket } from "node:net";
|
|
43
43
|
import { join } from "node:path";
|
|
44
44
|
|
|
45
|
-
import { resolvePolicy, resolveReleaseGrants, resolveReview } from "../config.ts";
|
|
45
|
+
import { resolvePolicy, resolveReleaseGrants, resolveReview, resolveSharedInstallAuthority } from "../config.ts";
|
|
46
46
|
import { wakeDispatch } from "../wake.ts";
|
|
47
47
|
// The mediated release is the drain's closing gesture (#791): a successful
|
|
48
48
|
// release ends this project's bounded release window, so the privileged half
|
|
@@ -60,6 +60,8 @@ import {
|
|
|
60
60
|
import { releaseRefusal } from "../release-policy.ts";
|
|
61
61
|
import { PR_LOOKUP_WINDOW_MS, REVISABLE_RUN_STATES, prReviewReadiness } from "../decisions.ts";
|
|
62
62
|
import { LIVE_STATES } from "../store.ts";
|
|
63
|
+
import { parseToSpecEvidence } from "../to-spec.ts";
|
|
64
|
+
import { DENIED_RELEASE_GRANTS } from "../types.ts";
|
|
63
65
|
import type {
|
|
64
66
|
FileLane,
|
|
65
67
|
IssueComment,
|
|
@@ -71,6 +73,7 @@ import type {
|
|
|
71
73
|
ReleaseRequirement,
|
|
72
74
|
ReleaseShape,
|
|
73
75
|
RepoTarget,
|
|
76
|
+
ResolvedGrants,
|
|
74
77
|
ReviewReason,
|
|
75
78
|
RunRecord,
|
|
76
79
|
RunState,
|
|
@@ -172,6 +175,9 @@ export interface ReleaseExecution {
|
|
|
172
175
|
export interface InstallExecution {
|
|
173
176
|
/** The published semver to pin all three surfaces to, exactly as npm has it. */
|
|
174
177
|
version: string;
|
|
178
|
+
/** Shared host authority and scope, carried into the detached report. */
|
|
179
|
+
holder: ResolvedGrants["install"];
|
|
180
|
+
projects: readonly string[];
|
|
175
181
|
}
|
|
176
182
|
|
|
177
183
|
/**
|
|
@@ -222,6 +228,11 @@ export interface VerbDeps {
|
|
|
222
228
|
* unreadable, and cannot pick up an operator's edit either.
|
|
223
229
|
*/
|
|
224
230
|
project: () => ProjectConfig;
|
|
231
|
+
/**
|
|
232
|
+
* Every project served by the shared daemon, re-read from the same config.
|
|
233
|
+
* Host-global mutations use this set rather than the calling socket's project.
|
|
234
|
+
*/
|
|
235
|
+
projects: () => readonly ProjectConfig[];
|
|
225
236
|
store: Store;
|
|
226
237
|
tracker: Tracker;
|
|
227
238
|
actions: VerbActions;
|
|
@@ -632,7 +643,7 @@ async function decide(deps: VerbDeps, channel: VerbChannel, raw: unknown): Promi
|
|
|
632
643
|
case "conductor_release":
|
|
633
644
|
return releaseVerb(deps, project, channel, args, refuse, allow);
|
|
634
645
|
case "conductor_install":
|
|
635
|
-
return installVerb(deps,
|
|
646
|
+
return installVerb(deps, channel, args, refuse, allow);
|
|
636
647
|
case "conductor_pr_review":
|
|
637
648
|
return prReviewVerb(deps, project, channel, args, refuse, allow);
|
|
638
649
|
case "conductor_pr_review_clear":
|
|
@@ -1288,10 +1299,23 @@ async function labelVerb(
|
|
|
1288
1299
|
let malformed: string | undefined;
|
|
1289
1300
|
if (action === "add" && label === project.queueLabel) {
|
|
1290
1301
|
const lane = await laneForEcho(deps, ref.issue);
|
|
1291
|
-
if (lane
|
|
1302
|
+
if (lane.readable) {
|
|
1292
1303
|
malformed = lane.malformed;
|
|
1293
1304
|
echo = laneEcho(lane.lane);
|
|
1294
1305
|
}
|
|
1306
|
+
// The durable grooming verdict is an admission contract only while the
|
|
1307
|
+
// dispatch brief still matches it (#1036): a stale issue body once
|
|
1308
|
+
// promoted beside a PROMOTABLE verdict whose fileLane had moved on, and
|
|
1309
|
+
// the run dispatched on the stale lane. So before any tracker mutation or
|
|
1310
|
+
// daemon wake, a promotable row must verify against the same parse the
|
|
1311
|
+
// echo reports — exact in both directions, order-insensitive. The syntax
|
|
1312
|
+
// refusal below keeps precedence: fix the section first, then the contract.
|
|
1313
|
+
if (malformed === undefined) {
|
|
1314
|
+
const mismatch = promotionBriefRefusal(deps, project, ref.issue, lane);
|
|
1315
|
+
if (mismatch !== undefined) {
|
|
1316
|
+
return refuse("promotion-brief-mismatch", mismatch, ref.issue);
|
|
1317
|
+
}
|
|
1318
|
+
}
|
|
1295
1319
|
}
|
|
1296
1320
|
if (malformed !== undefined) {
|
|
1297
1321
|
return refuse(
|
|
@@ -1344,32 +1368,99 @@ async function labelVerb(
|
|
|
1344
1368
|
* The effective file lane admission will enforce for one issue, as the
|
|
1345
1369
|
* one-line echo (#724), plus the malformed write-lane section marker (#825):
|
|
1346
1370
|
* the body plus the whole comment thread, the same inputs `effectiveLane`
|
|
1347
|
-
* reads at admission. `
|
|
1348
|
-
*
|
|
1371
|
+
* reads at admission. `readable` says whether the issue and thread were read
|
|
1372
|
+
* at all — the echo stays best-effort (an unreadable issue never blocks its
|
|
1373
|
+
* own promotion, #724), but the promotion-brief gate (#1036) must be able to
|
|
1374
|
+
* tell "no lane declared" from "could not look", because behind a durable
|
|
1375
|
+
* `promotable` verdict the second is a fail-closed refusal rather than
|
|
1376
|
+
* feedback.
|
|
1349
1377
|
*/
|
|
1350
|
-
async function laneForEcho(deps: VerbDeps, issue: number): Promise<
|
|
1351
|
-
|
|
1352
|
-
| undefined
|
|
1353
|
-
|
|
1378
|
+
async function laneForEcho(deps: VerbDeps, issue: number): Promise<{
|
|
1379
|
+
readable: boolean;
|
|
1380
|
+
lane: FileLane | undefined;
|
|
1381
|
+
malformed: string | undefined;
|
|
1382
|
+
}> {
|
|
1354
1383
|
let body: string;
|
|
1355
1384
|
let comments: IssueComment[];
|
|
1356
1385
|
try {
|
|
1357
1386
|
const row = await deps.tracker.getIssue(issue);
|
|
1358
|
-
if (row === undefined) return undefined;
|
|
1387
|
+
if (row === undefined) return { readable: false, lane: undefined, malformed: undefined };
|
|
1359
1388
|
body = row.body;
|
|
1360
1389
|
comments = await deps.tracker.listComments(issue);
|
|
1361
1390
|
} catch {
|
|
1362
|
-
return undefined;
|
|
1391
|
+
return { readable: false, lane: undefined, malformed: undefined };
|
|
1363
1392
|
}
|
|
1364
1393
|
const lane = effectiveLane(body, comments);
|
|
1365
|
-
if (lane !== undefined) return { lane, malformed: undefined };
|
|
1394
|
+
if (lane !== undefined) return { readable: true, lane, malformed: undefined };
|
|
1366
1395
|
// No declaration parsed anywhere. If a clearly delimited write-lane section
|
|
1367
1396
|
// exists anyway, the section tried to declare and failed — the echo must
|
|
1368
1397
|
// refuse, never claim fail-open.
|
|
1369
1398
|
const heading =
|
|
1370
1399
|
writeLaneSectionHeading(body) ??
|
|
1371
1400
|
comments.map((c) => writeLaneSectionHeading(c.body)).find((h) => h !== undefined);
|
|
1372
|
-
return { lane: undefined, malformed: heading };
|
|
1401
|
+
return { readable: true, lane: undefined, malformed: heading };
|
|
1402
|
+
}
|
|
1403
|
+
|
|
1404
|
+
/**
|
|
1405
|
+
* The promotion-brief gate (#1036): a durable `promotable` grooming verdict is
|
|
1406
|
+
* an admission contract only while the dispatch brief still matches it, so a
|
|
1407
|
+
* queue-label add must prove the match before any tracker mutation or daemon
|
|
1408
|
+
* wake. Compares the lane admission would enforce (the same parse the echo
|
|
1409
|
+
* reports) against the verdict's `fileLane` as deduplicated,
|
|
1410
|
+
* order-independent path sets, exact in both directions — missing and extra
|
|
1411
|
+
* paths refuse alike. Returns the refusal detail, or `undefined` when
|
|
1412
|
+
* promotion may proceed: no grooming row, a non-`promotable` row, or a
|
|
1413
|
+
* verified match.
|
|
1414
|
+
*
|
|
1415
|
+
* Fail-closed by design: evidence behind a `promotable` row that does not
|
|
1416
|
+
* recover as a strict PROMOTABLE result (`parseToSpecEvidence`, the same
|
|
1417
|
+
* restart round-trip the selection side trusts), or an issue that cannot be
|
|
1418
|
+
* read to compare at all, refuses the promotion rather than dispatching on an
|
|
1419
|
+
* unverifiable brief. The gate never edits the issue — it names both path
|
|
1420
|
+
* sets and points at the verdict's `proposedBrief`; correcting the dispatch
|
|
1421
|
+
* brief is an explicit orchestrator act.
|
|
1422
|
+
*/
|
|
1423
|
+
function promotionBriefRefusal(
|
|
1424
|
+
deps: VerbDeps,
|
|
1425
|
+
project: ProjectConfig,
|
|
1426
|
+
issue: number,
|
|
1427
|
+
read: { readable: boolean; lane: FileLane | undefined },
|
|
1428
|
+
): string | undefined {
|
|
1429
|
+
const row = deps.store.grooming(project.name, issue);
|
|
1430
|
+
if (row?.verdict !== "promotable") return undefined;
|
|
1431
|
+
const result = parseToSpecEvidence(row.evidence);
|
|
1432
|
+
if (result === undefined || result.verdict !== "PROMOTABLE") {
|
|
1433
|
+
return (
|
|
1434
|
+
`refused: ${project.queueLabel} was not added — #${issue}'s durable grooming row reads promotable, but its ` +
|
|
1435
|
+
"evidence does not recover as a strict PROMOTABLE to-spec result, so promotion cannot prove the dispatch " +
|
|
1436
|
+
"brief still matches what was groomed. Re-groom the issue so a fresh verdict replaces the row, then add the label again."
|
|
1437
|
+
);
|
|
1438
|
+
}
|
|
1439
|
+
if (!read.readable) {
|
|
1440
|
+
return (
|
|
1441
|
+
`refused: ${project.queueLabel} was not added — #${issue} carries a durable PROMOTABLE grooming verdict, but ` +
|
|
1442
|
+
"the issue could not be read, so the current brief cannot be compared with the verdict's file lane. " +
|
|
1443
|
+
"Retry when the tracker responds."
|
|
1444
|
+
);
|
|
1445
|
+
}
|
|
1446
|
+
// One closure so both sides of the comparison normalize in lockstep: a set
|
|
1447
|
+
// comparison whose halves dedupe differently lies silently (#1036).
|
|
1448
|
+
const normalizedPaths = (paths: readonly string[]): string[] =>
|
|
1449
|
+
[...new Set(paths.map((path) => path.trim()).filter((path) => path.length > 0))].sort();
|
|
1450
|
+
const brief = normalizedPaths(read.lane?.files ?? []);
|
|
1451
|
+
const verdict = normalizedPaths(result.fileLane);
|
|
1452
|
+
if (
|
|
1453
|
+
brief.length === verdict.length &&
|
|
1454
|
+
brief.every((path, index) => path === verdict[index])
|
|
1455
|
+
) {
|
|
1456
|
+
return undefined;
|
|
1457
|
+
}
|
|
1458
|
+
return (
|
|
1459
|
+
`refused: ${project.queueLabel} was not added — #${issue}'s current write lane [${brief.join(", ")}] disagrees ` +
|
|
1460
|
+
`with its durable PROMOTABLE verdict's file lane [${verdict.join(", ")}]. The durable verdict is the admission ` +
|
|
1461
|
+
"contract: apply the verdict's proposedBrief to the issue (its ## Exact write lane included), then add the " +
|
|
1462
|
+
"label again. The gate never rewrites the issue itself."
|
|
1463
|
+
);
|
|
1373
1464
|
}
|
|
1374
1465
|
|
|
1375
1466
|
async function releaseVerb(
|
|
@@ -1660,31 +1751,50 @@ async function releaseVerb(
|
|
|
1660
1751
|
|
|
1661
1752
|
|
|
1662
1753
|
/**
|
|
1663
|
-
* The
|
|
1664
|
-
*
|
|
1665
|
-
*
|
|
1666
|
-
*
|
|
1667
|
-
* package floor's "nobody patches the running conductor" is the deny default
|
|
1668
|
-
* an operator deliberately moves, never something this verb works around.
|
|
1669
|
-
*
|
|
1670
|
-
* Everything past the gate is the privileged half's job: `actions.install`
|
|
1671
|
-
* refussses a version npm does not expose and starts the detached transient
|
|
1672
|
-
* unit. This verb records the request durably (the journal the post-restart
|
|
1673
|
-
* tick verifies against) and tells the caller where the evidence will land.
|
|
1754
|
+
* The install is host-global: one package tree and daemon serve every project.
|
|
1755
|
+
* Authority is therefore the consensus holder across the configured project
|
|
1756
|
+
* set, never whichever project happened to invoke the verb (#1018). Mixed
|
|
1757
|
+
* holders refuse before version lookup or transient-unit creation.
|
|
1674
1758
|
*/
|
|
1675
1759
|
async function installVerb(
|
|
1676
1760
|
deps: VerbDeps,
|
|
1677
|
-
project: ProjectConfig,
|
|
1678
1761
|
channel: VerbChannel,
|
|
1679
1762
|
args: Record<string, unknown>,
|
|
1680
1763
|
refuse: Refuse,
|
|
1681
1764
|
allow: Allow,
|
|
1682
1765
|
): Promise<Verdict> {
|
|
1683
|
-
|
|
1684
|
-
|
|
1766
|
+
let projects: readonly ProjectConfig[];
|
|
1767
|
+
try {
|
|
1768
|
+
projects = deps.projects();
|
|
1769
|
+
} catch (err) {
|
|
1770
|
+
const why = err instanceof Error ? err.message : String(err);
|
|
1771
|
+
return refuse("config-unreadable", `refused: shared install authority could not be read (${why}).`);
|
|
1772
|
+
}
|
|
1773
|
+
if (projects.length === 0) {
|
|
1774
|
+
return refuse("config-unreadable", "refused: shared install authority has no configured projects.");
|
|
1775
|
+
}
|
|
1776
|
+
const authority = resolveSharedInstallAuthority(projects);
|
|
1777
|
+
const holderDetail = authority.entries.map((entry) => `${entry.project}=${entry.holder}`).join(", ");
|
|
1778
|
+
if (authority.holder === undefined) {
|
|
1779
|
+
return refuse(
|
|
1780
|
+
"release-not-granted",
|
|
1781
|
+
`refused: conductor_install is host-global, but configured install holders conflict (${holderDetail}). ` +
|
|
1782
|
+
"No dispatch pause or transient install unit was created.",
|
|
1783
|
+
);
|
|
1784
|
+
}
|
|
1785
|
+
const globalHolder = authority.holder;
|
|
1786
|
+
const granted = releaseRefusal(
|
|
1787
|
+
{ ...DENIED_RELEASE_GRANTS, install: globalHolder },
|
|
1788
|
+
channel.role,
|
|
1789
|
+
"install",
|
|
1790
|
+
);
|
|
1685
1791
|
if (granted !== undefined) {
|
|
1686
|
-
return refuse(
|
|
1792
|
+
return refuse(
|
|
1793
|
+
"release-not-granted",
|
|
1794
|
+
`refused: ${granted.reason} Shared-host holders: ${holderDetail}.`,
|
|
1795
|
+
);
|
|
1687
1796
|
}
|
|
1797
|
+
const affectedProjects = authority.entries.map((entry) => entry.project);
|
|
1688
1798
|
|
|
1689
1799
|
const version = args["version"];
|
|
1690
1800
|
if (typeof version !== "string" || version.trim().length === 0) {
|
|
@@ -1695,16 +1805,21 @@ async function installVerb(
|
|
|
1695
1805
|
);
|
|
1696
1806
|
}
|
|
1697
1807
|
|
|
1698
|
-
const outcome = await deps.actions.install({
|
|
1808
|
+
const outcome = await deps.actions.install({
|
|
1809
|
+
version: version.trim(),
|
|
1810
|
+
holder: globalHolder,
|
|
1811
|
+
projects: affectedProjects,
|
|
1812
|
+
});
|
|
1699
1813
|
if (!outcome.ok) {
|
|
1700
1814
|
return refuse("action-failed", `refused: the install request failed:\n${outcome.stderr}`);
|
|
1701
1815
|
}
|
|
1702
1816
|
const unit = outcome.detail;
|
|
1703
1817
|
return allow(
|
|
1704
|
-
`install requested: omp-conductor@${version} will be installed
|
|
1818
|
+
`install requested by host-global holder ${globalHolder}: omp-conductor@${version} will be installed ` +
|
|
1819
|
+
`for projects ${affectedProjects.join(", ")} by a detached unit` +
|
|
1705
1820
|
`${unit === undefined ? "" : ` (${unit})`} that survives the restart. ` +
|
|
1706
1821
|
"The first tick after the fleet restarts verifies version, /healthz, ticks, pane and doctor, " +
|
|
1707
|
-
"and the outcome is reported through the durable outbox.",
|
|
1822
|
+
"and the outcome is reported through the durable outbox.",
|
|
1708
1823
|
);
|
|
1709
1824
|
}
|
|
1710
1825
|
|
|
@@ -61,6 +61,7 @@ PROG=omp-conductor-recover
|
|
|
61
61
|
SYSTEMCTL=${SYSTEMCTL_BIN:-systemctl}
|
|
62
62
|
JOURNALCTL=${JOURNALCTL_BIN:-journalctl}
|
|
63
63
|
CONDUCTOR=${OMP_CONDUCTOR_BIN:-omp-conductor}
|
|
64
|
+
BUN=${BUN_BIN:-bun}
|
|
64
65
|
|
|
65
66
|
# The fleet's conductor home: where config.json, the sqlite store and the
|
|
66
67
|
# upgrade's durable snapshots live. The unit renders OMP_CONDUCTOR_HOME and
|
|
@@ -84,6 +85,12 @@ JOURNAL_LINES=${RECOVER_JOURNAL_LINES:-200}
|
|
|
84
85
|
VERIFY_POLL_S=${RECOVER_VERIFY_POLL_S:-2}
|
|
85
86
|
VERIFY_TRIES=${RECOVER_VERIFY_TRIES:-10}
|
|
86
87
|
PROJECT=${RECOVER_PROJECT:-}
|
|
88
|
+
# Recovery action scope and durable-report ownership are deliberately separate.
|
|
89
|
+
# `RECOVER_PROJECT` is only safe on a single-project host; the report still
|
|
90
|
+
# needs one configured project to own its outbox row on a shared host.
|
|
91
|
+
REPORT_PROJECT=${RECOVER_REPORT_PROJECT:-$PROJECT}
|
|
92
|
+
AFFECTED_PROJECTS=${RECOVER_PROJECTS:-$PROJECT}
|
|
93
|
+
REPORT_ROUTING_NOTE=
|
|
87
94
|
# The bound on the post-restore re-arm. `arm` with a claim-only proof performs
|
|
88
95
|
# no Telegram send or wait, so it completes in milliseconds; the timeout exists
|
|
89
96
|
# so a misbehaving conductor can never hang this bounded playbook (#613).
|
|
@@ -190,6 +197,40 @@ append_evidence() { # <evidence path> <text>
|
|
|
190
197
|
printf '%s\n' "$2" >>"$1" 2>/dev/null || true
|
|
191
198
|
}
|
|
192
199
|
|
|
200
|
+
|
|
201
|
+
# Resolve the host-global report owner from the live machine-written config.
|
|
202
|
+
# Bun is not an extra prerequisite: omp-conductor itself runs under Bun, and
|
|
203
|
+
# the recovery unit's canonical PATH already contains that executable. When a
|
|
204
|
+
# manual/degraded invocation cannot read the config, the script still attempts
|
|
205
|
+
# the legacy unscoped report and records that degradation in the evidence.
|
|
206
|
+
resolve_report_scope() {
|
|
207
|
+
if [[ -n $REPORT_PROJECT && -n $AFFECTED_PROJECTS ]]; then
|
|
208
|
+
return 0
|
|
209
|
+
fi
|
|
210
|
+
local lines first all
|
|
211
|
+
if lines=$(
|
|
212
|
+
"$BUN" -e '
|
|
213
|
+
const path = Bun.argv.at(-1);
|
|
214
|
+
const config = await Bun.file(path).json();
|
|
215
|
+
const names = Array.isArray(config.projects)
|
|
216
|
+
? config.projects.map((project) => project?.name).filter((name) => typeof name === "string" && name.length > 0)
|
|
217
|
+
: [];
|
|
218
|
+
if (names.length === 0) process.exit(2);
|
|
219
|
+
console.log(names[0]);
|
|
220
|
+
console.log(names.join(","));
|
|
221
|
+
' "$CONFIG_PATH" 2>/dev/null
|
|
222
|
+
); then
|
|
223
|
+
first=${lines%%$'\n'*}
|
|
224
|
+
all=${lines#*$'\n'}
|
|
225
|
+
REPORT_PROJECT=${REPORT_PROJECT:-$first}
|
|
226
|
+
AFFECTED_PROJECTS=${AFFECTED_PROJECTS:-$all}
|
|
227
|
+
fi
|
|
228
|
+
if [[ -z $REPORT_PROJECT || -z $AFFECTED_PROJECTS ]]; then
|
|
229
|
+
AFFECTED_PROJECTS=${AFFECTED_PROJECTS:-unknown}
|
|
230
|
+
REPORT_ROUTING_NOTE="configured project resolution failed for $CONFIG_PATH; attempting the report without a project selector"
|
|
231
|
+
fi
|
|
232
|
+
return 0
|
|
233
|
+
}
|
|
193
234
|
# --------------------------------------------------------------------------
|
|
194
235
|
# escalation — always, through the durable outbox (#123, #288)
|
|
195
236
|
# --------------------------------------------------------------------------
|
|
@@ -199,23 +240,27 @@ append_evidence() { # <evidence path> <text>
|
|
|
199
240
|
# (the durable outbox) before anything is sent, and the daemon owns delivery
|
|
200
241
|
# with bounded retries once it is back. This script only ever enqueues.
|
|
201
242
|
escalation_text() { # <failed-unit or empty> <action> <outcome> <evidence>
|
|
243
|
+
local projects=${AFFECTED_PROJECTS:-unknown}
|
|
202
244
|
if [[ ${1:-none} == none ]]; then
|
|
203
245
|
printf '%s\n' \
|
|
204
|
-
"fleet recovery (#485): a recovery trigger fired but no fleet unit was failing when it ran — the fleet got itself up or was started externally; evidence=$4"
|
|
246
|
+
"fleet recovery (#485): a recovery trigger fired but no fleet unit was failing when it ran — the fleet got itself up or was started externally; affected-projects=$projects; evidence=$4"
|
|
205
247
|
return
|
|
206
248
|
fi
|
|
207
249
|
printf '%s\n' \
|
|
208
|
-
"fleet recovery (#485): $1 entered failed state; recovery=${2:-none}; outcome=${3}; evidence=${4}; next: read the evidence file (and its doctor output) before touching the fleet"
|
|
250
|
+
"fleet recovery (#485): $1 entered failed state; affected-projects=$projects; recovery=${2:-none}; outcome=${3}; evidence=${4}; next: read the evidence file (and its doctor output) before touching the fleet"
|
|
209
251
|
}
|
|
210
252
|
|
|
211
253
|
escalate() { # <failed-unit or empty> <action-name> <outcome> <evidence-path>
|
|
212
254
|
local unit=${1:-none} text args
|
|
213
|
-
|
|
214
|
-
if [[ -n $
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
args=("$CONDUCTOR" report --kind tier2 --text "$text")
|
|
255
|
+
resolve_report_scope
|
|
256
|
+
if [[ -n $REPORT_ROUTING_NOTE ]]; then
|
|
257
|
+
log "$REPORT_ROUTING_NOTE"
|
|
258
|
+
append_evidence "$4" "report routing: $REPORT_ROUTING_NOTE"
|
|
218
259
|
fi
|
|
260
|
+
text=$(escalation_text "$unit" "$2" "$3" "$4")
|
|
261
|
+
args=("$CONDUCTOR" report --kind tier2)
|
|
262
|
+
[[ -n $REPORT_PROJECT ]] && args+=(--project "$REPORT_PROJECT")
|
|
263
|
+
args+=(--text "$text")
|
|
219
264
|
say "escalation: ${args[*]}"
|
|
220
265
|
if dry_run; then
|
|
221
266
|
return 0
|
|
@@ -365,6 +410,9 @@ run_recovery() {
|
|
|
365
410
|
done
|
|
366
411
|
|
|
367
412
|
if dry_run; then
|
|
413
|
+
resolve_report_scope
|
|
414
|
+
say "plan: report owner — ${REPORT_PROJECT:-unscoped}; affected projects — $AFFECTED_PROJECTS"
|
|
415
|
+
[[ -n $REPORT_ROUTING_NOTE ]] && say "plan: report routing degraded — $REPORT_ROUTING_NOTE"
|
|
368
416
|
say "dry run: nothing will be written or executed"
|
|
369
417
|
[[ -z $failed ]] && say "plan: no unit failed — escalate only"
|
|
370
418
|
if [[ -n $failed ]]; then
|
|
@@ -50,6 +50,7 @@ fi
|
|
|
50
50
|
|
|
51
51
|
tmp=$(mktemp -d "${TMPDIR:-/tmp}/omp-recover-test.XXXXXX")
|
|
52
52
|
trap 'rm -rf "$tmp"' EXIT
|
|
53
|
+
bun_bin=$(command -v bun)
|
|
53
54
|
mkdir -p "$tmp/bin"
|
|
54
55
|
|
|
55
56
|
# --------------------------------------------------------------------------
|
|
@@ -156,6 +157,18 @@ write_project_config() { # <case-dir> <proof>
|
|
|
156
157
|
printf '{\n "version": 2,\n "projects": [\n { "name": "demo", "arm": { "proof": "%s" } }\n ]\n}\n' "$2" >"$1/state/config.json"
|
|
157
158
|
}
|
|
158
159
|
|
|
160
|
+
write_multi_project_config() { # <case-dir>
|
|
161
|
+
cat >"$1/state/config.json" <<'JSON'
|
|
162
|
+
{
|
|
163
|
+
"version": 2,
|
|
164
|
+
"projects": [
|
|
165
|
+
{ "name": "alpha" },
|
|
166
|
+
{ "name": "beta" }
|
|
167
|
+
]
|
|
168
|
+
}
|
|
169
|
+
JSON
|
|
170
|
+
}
|
|
171
|
+
|
|
159
172
|
run_recover() { # <case-dir> [env assignments…]
|
|
160
173
|
local d="$1"
|
|
161
174
|
shift
|
|
@@ -182,7 +195,9 @@ run_recover() { # <case-dir> [env assignments…]
|
|
|
182
195
|
# marks the named unit active, which is what a real peer restart does to a
|
|
183
196
|
# healthy peer), so tests wanting daemon-side healing pass HEAL_ON_START=<unit>
|
|
184
197
|
# and the stub's `start` flips that unit back to active.
|
|
185
|
-
run_with_heal() { # <case-dir> <heal-on-start>
|
|
198
|
+
run_with_heal() { # <case-dir> <heal-on-start> [env assignments…]
|
|
199
|
+
local d="$1" heal="$2"
|
|
200
|
+
shift 2
|
|
186
201
|
(
|
|
187
202
|
cd "$here" || exit 1
|
|
188
203
|
env \
|
|
@@ -190,14 +205,15 @@ run_with_heal() { # <case-dir> <heal-on-start>
|
|
|
190
205
|
SYSTEMCTL_BIN="$tmp/bin/systemctl" \
|
|
191
206
|
JOURNALCTL_BIN="$tmp/bin/journalctl" \
|
|
192
207
|
OMP_CONDUCTOR_BIN="$tmp/bin/omp-conductor" \
|
|
193
|
-
RECOVER_STATE_DIR="$
|
|
208
|
+
RECOVER_STATE_DIR="$d/state" \
|
|
194
209
|
RECOVER_PROJECT=demo \
|
|
195
210
|
RECOVER_VERIFY_POLL_S=0 \
|
|
196
211
|
RECOVER_VERIFY_TRIES=3 \
|
|
197
|
-
STUB_CALLS="$
|
|
198
|
-
STUB_UNITS="$
|
|
199
|
-
STUB_HEAL_ON_START="$
|
|
200
|
-
|
|
212
|
+
STUB_CALLS="$d/calls" \
|
|
213
|
+
STUB_UNITS="$d/units" \
|
|
214
|
+
STUB_HEAL_ON_START="$heal" \
|
|
215
|
+
"$@" \
|
|
216
|
+
bash "$recover" >"$d/out" 2>"$d/stderr"
|
|
201
217
|
)
|
|
202
218
|
echo $?
|
|
203
219
|
}
|
|
@@ -412,6 +428,83 @@ check 'a trigger with nothing failed escalates (fleet came back)' \
|
|
|
412
428
|
check 'a trigger with nothing failed exits 0' '0' "$code"
|
|
413
429
|
check 'a trigger with nothing failed clears the old counter' '0' "$(marker "$d")"
|
|
414
430
|
|
|
431
|
+
# ---------------------------------------------------------------------------
|
|
432
|
+
# case — a host-global recovery on a multi-project config still has one
|
|
433
|
+
# deterministic outbox owner, while its body names every affected project.
|
|
434
|
+
# ---------------------------------------------------------------------------
|
|
435
|
+
|
|
436
|
+
d=$(newcase multi-project-report)
|
|
437
|
+
write_multi_project_config "$d"
|
|
438
|
+
unit_state "$d" omp-conductor.service active
|
|
439
|
+
unit_state "$d" herdr-fleet.service active
|
|
440
|
+
code=$(run_recover "$d" \
|
|
441
|
+
RECOVER_PROJECT= \
|
|
442
|
+
RECOVER_REPORT_PROJECT= \
|
|
443
|
+
RECOVER_PROJECTS= \
|
|
444
|
+
BUN_BIN="$bun_bin")
|
|
445
|
+
report_call=$(grep '^omp-conductor report' "$d/calls" | head -n 1)
|
|
446
|
+
check 'multi-project recovery exits after queuing its report' '0' "$code"
|
|
447
|
+
check 'multi-project recovery uses the first configured outbox owner' \
|
|
448
|
+
'--project alpha' "$report_call"
|
|
449
|
+
check 'multi-project recovery names every affected project' \
|
|
450
|
+
'affected-projects=alpha,beta' "$report_call"
|
|
451
|
+
check 'multi-project recovery queues exactly one report' \
|
|
452
|
+
'1' "$(grep -c '^omp-conductor report' "$d/calls")"
|
|
453
|
+
|
|
454
|
+
d=$(newcase multi-project-recovery-action)
|
|
455
|
+
write_multi_project_config "$d"
|
|
456
|
+
unit_state "$d" omp-conductor.service active
|
|
457
|
+
unit_state "$d" herdr-fleet.service failed
|
|
458
|
+
code=$(run_with_heal "$d" herdr-fleet.service \
|
|
459
|
+
RECOVER_PROJECT= \
|
|
460
|
+
RECOVER_REPORT_PROJECT= \
|
|
461
|
+
RECOVER_PROJECTS= \
|
|
462
|
+
BUN_BIN="$bun_bin")
|
|
463
|
+
report_call=$(grep '^omp-conductor report' "$d/calls" | head -n 1)
|
|
464
|
+
check 'verified multi-project recovery action exits cleanly' '0' "$code"
|
|
465
|
+
check 'verified multi-project recovery action queues one scoped report' \
|
|
466
|
+
'1' "$(grep -c '^omp-conductor report' "$d/calls")"
|
|
467
|
+
check 'verified multi-project recovery action uses configured owner' \
|
|
468
|
+
'--project alpha' "$report_call"
|
|
469
|
+
check 'verified multi-project recovery action names every project' \
|
|
470
|
+
'affected-projects=alpha,beta' "$report_call"
|
|
471
|
+
|
|
472
|
+
d=$(newcase multi-project-dry-run)
|
|
473
|
+
write_multi_project_config "$d"
|
|
474
|
+
unit_state "$d" omp-conductor.service active
|
|
475
|
+
unit_state "$d" herdr-fleet.service failed
|
|
476
|
+
code=$(run_recover "$d" \
|
|
477
|
+
RECOVER_PROJECT= \
|
|
478
|
+
RECOVER_REPORT_PROJECT= \
|
|
479
|
+
RECOVER_PROJECTS= \
|
|
480
|
+
RECOVER_DRY_RUN=1 \
|
|
481
|
+
BUN_BIN="$bun_bin")
|
|
482
|
+
check 'multi-project dry run exits cleanly' '0' "$code"
|
|
483
|
+
check 'multi-project dry run names its report owner' \
|
|
484
|
+
'plan: report owner — alpha' "$(cat "$d/out")"
|
|
485
|
+
check 'multi-project dry run names every affected project' \
|
|
486
|
+
'affected projects — alpha,beta' "$(cat "$d/out")"
|
|
487
|
+
|
|
488
|
+
d=$(newcase report-routing-fallback)
|
|
489
|
+
unit_state "$d" omp-conductor.service active
|
|
490
|
+
unit_state "$d" herdr-fleet.service active
|
|
491
|
+
code=$(run_recover "$d" \
|
|
492
|
+
RECOVER_PROJECT= \
|
|
493
|
+
RECOVER_REPORT_PROJECT= \
|
|
494
|
+
RECOVER_PROJECTS= \
|
|
495
|
+
BUN_BIN="$bun_bin")
|
|
496
|
+
report_call=$(grep '^omp-conductor report' "$d/calls" | head -n 1)
|
|
497
|
+
check 'unreadable config routing still attempts a durable report' \
|
|
498
|
+
'omp-conductor report --kind tier2 --text' "$report_call"
|
|
499
|
+
if [[ $report_call == *"--project "* ]]; then
|
|
500
|
+
fail "routing fallback does not guess an unconfigured project — $report_call"
|
|
501
|
+
else
|
|
502
|
+
pass 'routing fallback does not guess an unconfigured project'
|
|
503
|
+
fi
|
|
504
|
+
check 'routing fallback remains a successful recovered run' '0' "$code"
|
|
505
|
+
check 'routing fallback is preserved in durable evidence' \
|
|
506
|
+
'attempting the report without a project selector' "$(cat "$d/state/recovery"/evidence-*)"
|
|
507
|
+
|
|
415
508
|
# ---------------------------------------------------------------------------
|
|
416
509
|
# case DRY-RUN — the operator's rehearsal mode prints decisions and mutates
|
|
417
510
|
# nothing, not even the recovery dir or the fixture call log.
|