omp-conductor 0.19.5 → 0.19.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/cli.ts +2 -0
- package/src/command-help.ts +18 -8
- package/src/command-manifest.ts +22 -5
- package/src/commands/context.ts +1 -0
- package/src/commands/report.ts +63 -0
- package/src/commands/restore-db.ts +1 -1
- package/src/commands/snapshot-db.ts +67 -0
- package/src/commands/upgrade-install.ts +1 -1
- package/src/commands/upgrade.ts +2 -2
- package/src/commands/verb.ts +1 -0
- package/src/commands/watch.ts +14 -2
- package/src/config.ts +17 -0
- package/src/daemon.ts +23 -3
- package/src/doctor.ts +36 -3
- package/src/fleet.ts +51 -3
- package/src/orchestrator-tick.ts +14 -3
- package/src/setup-host.ts +10 -10
- package/src/setup-wizard.ts +1 -1
- package/src/status-render.ts +10 -0
- package/src/store.ts +296 -22
- package/src/types.ts +32 -3
- package/src/upgrade-journal.ts +35 -1
- package/src/upgrade-verify.ts +27 -6
- package/src/upgrade.ts +301 -22
- package/src/verbs/actions.ts +6 -1
- package/src/verbs/server.ts +54 -20
- package/systemd/omp-conductor-recover.sh +55 -7
- package/systemd/recover-unit-test.sh +99 -6
package/src/types.ts
CHANGED
|
@@ -275,8 +275,8 @@ export const RELEASE_SHAPES = [
|
|
|
275
275
|
"github-release",
|
|
276
276
|
"deploy",
|
|
277
277
|
/**
|
|
278
|
-
* Replace the running conductor itself: the
|
|
279
|
-
* and the Herdr plugin, pinned to one published release and executed
|
|
278
|
+
* Replace the running conductor itself: the discovered CLI package, the omp
|
|
279
|
+
* plugin and the Herdr plugin, pinned to one published release and executed
|
|
280
280
|
* detached. This is the "nobody patches the running conductor" boundary
|
|
281
281
|
* moved deliberately, not worked around — a session that may cut a release
|
|
282
282
|
* still cannot install one until an operator grants this shape too.
|
|
@@ -2354,7 +2354,7 @@ export type ReportKind = (typeof REPORT_KINDS)[number];
|
|
|
2354
2354
|
* `failed` is the bounded retry budget running out, and is itself news: a
|
|
2355
2355
|
* report nobody can deliver escalates as tier 2 in its own right.
|
|
2356
2356
|
*/
|
|
2357
|
-
export const REPORT_DELIVERY_STATES = ["pending", "sending", "delivered", "failed"] as const;
|
|
2357
|
+
export const REPORT_DELIVERY_STATES = ["pending", "sending", "delivered", "failed", "withdrawn"] as const;
|
|
2358
2358
|
|
|
2359
2359
|
export type ReportDeliveryState = (typeof REPORT_DELIVERY_STATES)[number];
|
|
2360
2360
|
|
|
@@ -2411,6 +2411,9 @@ export interface ReportRecord {
|
|
|
2411
2411
|
deliveredAt?: number;
|
|
2412
2412
|
/** Bounded text of the last known failure, surfaced verbatim by `status`. */
|
|
2413
2413
|
lastError?: string;
|
|
2414
|
+
withdrawnAt?: number;
|
|
2415
|
+
withdrawnBy?: string;
|
|
2416
|
+
withdrawReason?: string;
|
|
2414
2417
|
}
|
|
2415
2418
|
|
|
2416
2419
|
/** What a caller hands over. The store owns identity, state and every timestamp. */
|
|
@@ -2431,6 +2434,22 @@ export interface ReportEnqueue {
|
|
|
2431
2434
|
deduped: boolean;
|
|
2432
2435
|
}
|
|
2433
2436
|
|
|
2437
|
+
export interface HandoffWithdrawal {
|
|
2438
|
+
id: string;
|
|
2439
|
+
project: string;
|
|
2440
|
+
target: "report" | "notice";
|
|
2441
|
+
actor: string;
|
|
2442
|
+
reason: string;
|
|
2443
|
+
at: number;
|
|
2444
|
+
summary: string;
|
|
2445
|
+
detail: string;
|
|
2446
|
+
}
|
|
2447
|
+
|
|
2448
|
+
export type HandoffWithdrawalResult =
|
|
2449
|
+
| { kind: "withdrawn"; withdrawal: HandoffWithdrawal }
|
|
2450
|
+
| { kind: "not-found" }
|
|
2451
|
+
| { kind: "refused"; target: "report" | "notice"; state: string };
|
|
2452
|
+
|
|
2434
2453
|
/** Maximum ledger rows of each kind placed in one digest prompt. Older rows
|
|
2435
2454
|
* stay first, so a busy day drains deterministically over later digests. */
|
|
2436
2455
|
export const DIGEST_BACKLOG_LIMIT = 20;
|
|
@@ -3076,6 +3095,16 @@ export interface Store {
|
|
|
3076
3095
|
categories?: readonly InterruptCategory[],
|
|
3077
3096
|
urgent?: boolean,
|
|
3078
3097
|
): HeldNotice[];
|
|
3098
|
+
/** Withdraw one unsent report or unassigned held notice, preserving an audit row. */
|
|
3099
|
+
withdrawHandoff(
|
|
3100
|
+
id: string,
|
|
3101
|
+
project: string,
|
|
3102
|
+
actor: string,
|
|
3103
|
+
reason: string,
|
|
3104
|
+
at: number,
|
|
3105
|
+
): HandoffWithdrawalResult;
|
|
3106
|
+
/** Recent successful withdrawals, newest first, for status/audit. */
|
|
3107
|
+
handoffWithdrawals(project: string, limit?: number): HandoffWithdrawal[];
|
|
3079
3108
|
/** Record the install identities this host carried on this pass (#919).
|
|
3080
3109
|
* One row, replaced: the question is "what is installed now", and a history
|
|
3081
3110
|
* of installs is what the upgrade journal already keeps. */
|
package/src/upgrade-journal.ts
CHANGED
|
@@ -22,6 +22,7 @@
|
|
|
22
22
|
import { appendFileSync, existsSync, mkdirSync, readFileSync } from "node:fs";
|
|
23
23
|
import { join } from "node:path";
|
|
24
24
|
import { stateDir } from "./config.ts";
|
|
25
|
+
import type { ResolvedGrants } from "./types.ts";
|
|
25
26
|
|
|
26
27
|
/**
|
|
27
28
|
* Where the fleet's upgrade journal lives: one JSONL file per host, because
|
|
@@ -80,15 +81,37 @@ export interface UpgradeJournalEntry {
|
|
|
80
81
|
initial?: JournalFleetState;
|
|
81
82
|
/** The projects the install covers, in the form the upgrade engine uses. */
|
|
82
83
|
selectors?: readonly (string | null)[];
|
|
84
|
+
/** Host-global install authority resolved before the detached unit started. */
|
|
85
|
+
installHolder?: ResolvedGrants["install"];
|
|
86
|
+
/** Every configured project affected by the host-global install, report-only. */
|
|
87
|
+
installProjects?: readonly string[];
|
|
83
88
|
/** Verification evidence when the post-restart tick closes the transaction. */
|
|
84
89
|
checks?: readonly UpgradeCheck[];
|
|
85
90
|
}
|
|
86
91
|
|
|
92
|
+
/** Paths and roots discovered before the first install mutation (#1019). */
|
|
93
|
+
export interface InstallSnapshotManagement {
|
|
94
|
+
cli: {
|
|
95
|
+
executable: string;
|
|
96
|
+
target: string;
|
|
97
|
+
root: string;
|
|
98
|
+
};
|
|
99
|
+
omp: {
|
|
100
|
+
root: string;
|
|
101
|
+
lock: string;
|
|
102
|
+
};
|
|
103
|
+
herdr: {
|
|
104
|
+
root?: string;
|
|
105
|
+
installRoot: string;
|
|
106
|
+
};
|
|
107
|
+
}
|
|
108
|
+
|
|
87
109
|
/** The installed identities a rollback must restore, verbatim. */
|
|
88
110
|
export interface InstallSnapshotSurfaces {
|
|
89
111
|
cliVersion: string;
|
|
90
112
|
ompVersion?: string;
|
|
91
113
|
herdrSource?: string;
|
|
114
|
+
management?: InstallSnapshotManagement;
|
|
92
115
|
}
|
|
93
116
|
|
|
94
117
|
export function upgradeJournalPath(root = stateDir()): string {
|
|
@@ -167,7 +190,12 @@ export type UnitSpawnResult = { code: number; stdout: string; stderr: string };
|
|
|
167
190
|
export type UnitSpawnFn = (argv: readonly string[]) => Promise<UnitSpawnResult>;
|
|
168
191
|
|
|
169
192
|
export type UnitCommand =
|
|
170
|
-
| {
|
|
193
|
+
| {
|
|
194
|
+
kind: "install";
|
|
195
|
+
version: string;
|
|
196
|
+
holder?: ResolvedGrants["install"];
|
|
197
|
+
projects?: readonly string[];
|
|
198
|
+
}
|
|
171
199
|
| { kind: "rollback"; version: string };
|
|
172
200
|
|
|
173
201
|
export interface LaunchedUnit {
|
|
@@ -199,6 +227,12 @@ export async function launchTransientUnit(
|
|
|
199
227
|
const setenv = [`PATH=${executorPath(env)}`, `OMP_CONDUCTOR_UNIT=${unit}`];
|
|
200
228
|
const session = env["HERDR_SESSION"];
|
|
201
229
|
if (session !== undefined && session.length > 0) setenv.push(`HERDR_SESSION=${session}`);
|
|
230
|
+
if (command.kind === "install" && command.holder !== undefined) {
|
|
231
|
+
setenv.push(`OMP_CONDUCTOR_INSTALL_HOLDER=${command.holder}`);
|
|
232
|
+
}
|
|
233
|
+
if (command.kind === "install" && command.projects !== undefined) {
|
|
234
|
+
setenv.push(`OMP_CONDUCTOR_INSTALL_PROJECTS=${command.projects.join(",")}`);
|
|
235
|
+
}
|
|
202
236
|
const argv = [
|
|
203
237
|
"systemd-run",
|
|
204
238
|
`--unit=${unit}`,
|
package/src/upgrade-verify.ts
CHANGED
|
@@ -203,6 +203,8 @@ export interface PendingUpgradeRequest {
|
|
|
203
203
|
initialPaused?: boolean;
|
|
204
204
|
pauseKey?: string;
|
|
205
205
|
selectors: readonly (string | undefined)[];
|
|
206
|
+
installHolder?: UpgradeJournalEntry["installHolder"];
|
|
207
|
+
installProjects?: readonly string[];
|
|
206
208
|
initial?: UpgradeJournalEntry["initial"];
|
|
207
209
|
configBackup?: string;
|
|
208
210
|
previous?: JournalSurfaces;
|
|
@@ -241,15 +243,29 @@ export function pendingUpgradeRequest(
|
|
|
241
243
|
gitHead: request.gitHead,
|
|
242
244
|
initialPaused: snapshot?.initialPaused,
|
|
243
245
|
pauseKey: snapshot?.pauseKey,
|
|
244
|
-
selectors: (snapshot?.selectors ?? []).map((selector) => selector ?? undefined),
|
|
246
|
+
selectors: (snapshot?.selectors ?? request.selectors ?? []).map((selector) => selector ?? undefined),
|
|
245
247
|
pauseSince: pausePhase?.pauseSince,
|
|
246
248
|
initial: snapshot?.initial,
|
|
247
249
|
configBackup: snapshot?.configBackup,
|
|
248
250
|
previous: snapshot?.previous,
|
|
251
|
+
installHolder: request.installHolder,
|
|
252
|
+
installProjects: request.installProjects,
|
|
249
253
|
...(snapshotAt !== undefined && Number.isFinite(snapshotAt) ? { reloadAfterMs: snapshotAt } : {}),
|
|
250
254
|
};
|
|
251
255
|
}
|
|
252
256
|
|
|
257
|
+
function reportProjects(request: PendingUpgradeRequest): readonly (string | undefined)[] {
|
|
258
|
+
return request.installProjects ?? request.selectors;
|
|
259
|
+
}
|
|
260
|
+
|
|
261
|
+
function installAttribution(request: PendingUpgradeRequest): string {
|
|
262
|
+
const projects =
|
|
263
|
+
reportProjects(request).length === 0
|
|
264
|
+
? "all configured projects"
|
|
265
|
+
: reportProjects(request).map((selector) => selector ?? "(default)").join(", ");
|
|
266
|
+
return `Host-global install holder: ${request.installHolder ?? "unknown"}; affected projects: ${projects}.`;
|
|
267
|
+
}
|
|
268
|
+
|
|
253
269
|
export interface UpgradeRecoveryStatus {
|
|
254
270
|
failedVersion: string;
|
|
255
271
|
phase: string;
|
|
@@ -593,10 +609,11 @@ export async function verifyPendingUpgrade(deps: UpgradeVerifyDeps): Promise<Upg
|
|
|
593
609
|
// without guessing that an upgrade-owned sentinel still belongs to it.
|
|
594
610
|
const abortedAt = now();
|
|
595
611
|
deps.enqueue({
|
|
596
|
-
project:
|
|
612
|
+
project: reportProject(reportProjects(request)),
|
|
597
613
|
kind: "tier2",
|
|
598
614
|
body: [
|
|
599
615
|
`Fleet upgrade to omp-conductor@${request.version} never started (journal: ${upgradeJournalPath(root)}).`,
|
|
616
|
+
installAttribution(request),
|
|
600
617
|
"The detached install unit died before touching any surface.",
|
|
601
618
|
"Dispatch may still hold an unidentifiable install pause; check `status` and clear it explicitly with `omp-conductor resume`.",
|
|
602
619
|
`Re-run the request, or install by hand with \`omp-conductor upgrade --to ${request.version}\`.`,
|
|
@@ -644,10 +661,11 @@ export async function verifyPendingUpgrade(deps: UpgradeVerifyDeps): Promise<Upg
|
|
|
644
661
|
// after the outcome, the next tick can finish the idempotent resume.
|
|
645
662
|
const verifiedAt = now();
|
|
646
663
|
deps.enqueue({
|
|
647
|
-
project:
|
|
664
|
+
project: reportProject(reportProjects(request)),
|
|
648
665
|
kind: "material",
|
|
649
666
|
body: [
|
|
650
667
|
`omp-conductor@${request.version} is installed and verified on the first tick after the restart.`,
|
|
668
|
+
installAttribution(request),
|
|
651
669
|
"",
|
|
652
670
|
...checks.map((check) => `${check.ok ? "ok" : "FAIL"} ${check.name}${check.detail === undefined ? "" : ` — ${check.detail}`}`),
|
|
653
671
|
"",
|
|
@@ -716,10 +734,11 @@ async function verifyRecoveredUpgrade(
|
|
|
716
734
|
|
|
717
735
|
const recoveredAt = now();
|
|
718
736
|
deps.enqueue({
|
|
719
|
-
project:
|
|
737
|
+
project: reportProject(reportProjects(request)),
|
|
720
738
|
kind: "material",
|
|
721
739
|
body: [
|
|
722
740
|
`The failed upgrade to omp-conductor@${request.version} is recovered.`,
|
|
741
|
+
installAttribution(request),
|
|
723
742
|
"",
|
|
724
743
|
...checks.map((check) => `ok ${check.name} — ${check.detail ?? "matched"}`),
|
|
725
744
|
"",
|
|
@@ -925,10 +944,11 @@ async function triggerUpgradeRollback(
|
|
|
925
944
|
// and must not be quietly mixed.
|
|
926
945
|
journal({ kind: "outcome", phase: "rollback-unavailable", ok: false, detail: spawned.stderr });
|
|
927
946
|
deps.enqueue({
|
|
928
|
-
project:
|
|
947
|
+
project: reportProject(reportProjects(request)),
|
|
929
948
|
kind: "tier2",
|
|
930
949
|
body: [
|
|
931
950
|
`Fleet upgrade to omp-conductor@${request.version} is INCOMPLETE (${why}) and its detached rollback could not start.`,
|
|
951
|
+
installAttribution(request),
|
|
932
952
|
`Reason: ${spawned.stderr}`,
|
|
933
953
|
"",
|
|
934
954
|
"Surfaces may be at mixed versions; do not resume dispatch.",
|
|
@@ -941,10 +961,11 @@ async function triggerUpgradeRollback(
|
|
|
941
961
|
}
|
|
942
962
|
journal({ kind: "outcome", phase: "rollback-requested", ok: false, detail: why });
|
|
943
963
|
deps.enqueue({
|
|
944
|
-
project:
|
|
964
|
+
project: reportProject(reportProjects(request)),
|
|
945
965
|
kind: "tier2",
|
|
946
966
|
body: [
|
|
947
967
|
`Fleet upgrade to omp-conductor@${request.version} did not verify (${why}) and is being rolled back by ${spawned.unit}.`,
|
|
968
|
+
installAttribution(request),
|
|
948
969
|
"Dispatch stays paused until the rollback lands and a later tick verifies the old version.",
|
|
949
970
|
"",
|
|
950
971
|
`Journal: ${upgradeJournalPath(root)}`,
|