omp-conductor 0.18.2 → 0.19.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (60) hide show
  1. package/README.md +105 -40
  2. package/REFERENCE.md +865 -30
  3. package/package.json +1 -1
  4. package/schema/config.schema.json +26 -0
  5. package/src/admission.ts +212 -26
  6. package/src/ask.ts +288 -1
  7. package/src/briefs/orchestrator.md +6 -5
  8. package/src/cli.ts +5 -1
  9. package/src/command-help.ts +9 -1
  10. package/src/command-manifest.ts +36 -3
  11. package/src/commands/arm.ts +5 -1
  12. package/src/commands/context.ts +2 -0
  13. package/src/commands/message.ts +26 -2
  14. package/src/commands/reconcile-units.ts +104 -0
  15. package/src/commands/release-composition.ts +232 -0
  16. package/src/commands/resume.ts +2 -27
  17. package/src/commands/setup.ts +101 -16
  18. package/src/commands/stats.ts +11 -30
  19. package/src/commands/tail.ts +31 -1
  20. package/src/commands/upgrade.ts +20 -3
  21. package/src/commands/verb.ts +2 -1
  22. package/src/config-schema.ts +19 -0
  23. package/src/config.ts +80 -0
  24. package/src/credential-class.ts +366 -0
  25. package/src/daemon.ts +1218 -288
  26. package/src/dashboard/app.js +504 -2
  27. package/src/dashboard/controls.ts +336 -0
  28. package/src/dashboard/index.html +30 -0
  29. package/src/dashboard/server.ts +271 -30
  30. package/src/dashboard/style.css +116 -0
  31. package/src/dashboard/transcript.ts +173 -0
  32. package/src/doctor.ts +377 -20
  33. package/src/failure-class.ts +59 -0
  34. package/src/fleet.ts +497 -15
  35. package/src/host.ts +6 -130
  36. package/src/omp.ts +29 -0
  37. package/src/orchestrator-tick.ts +343 -88
  38. package/src/pause.ts +233 -0
  39. package/src/settlement.ts +159 -2
  40. package/src/setup-answers.ts +97 -0
  41. package/src/setup-host.ts +321 -1155
  42. package/src/setup-install.ts +204 -27
  43. package/src/setup-wizard.ts +111 -50
  44. package/src/setup.ts +33 -0
  45. package/src/spend-telemetry.ts +117 -0
  46. package/src/stats.ts +35 -0
  47. package/src/status-render.ts +348 -19
  48. package/src/store.ts +1229 -55
  49. package/src/telegram-freshness.ts +269 -0
  50. package/src/to-spec.ts +27 -0
  51. package/src/types.ts +697 -4
  52. package/src/unblock.ts +22 -0
  53. package/src/unit-reconcile.ts +303 -0
  54. package/src/upgrade-verify.ts +8 -1
  55. package/src/upgrade.ts +299 -12
  56. package/src/verbs/actions.ts +124 -10
  57. package/src/verbs/protocol.ts +70 -2
  58. package/src/verbs/server.ts +447 -8
  59. package/src/wake.ts +48 -0
  60. package/src/worker.ts +403 -3
package/src/upgrade.ts CHANGED
@@ -1,4 +1,5 @@
1
1
  import { existsSync, readFileSync } from "node:fs";
2
+ import { join } from "node:path";
2
3
  import { backupTimestamp, copyToUniqueBackup } from "./backups.ts";
3
4
  import { inspectBriefLayout, type BriefLayout } from "./brief-upgrade.ts";
4
5
  import { pauseInstance, setPaused, statusSnapshot, type AdmissionAckRecord } from "./daemon.ts";
@@ -23,6 +24,7 @@ import {
23
24
  appendJournal,
24
25
  readUpgradeJournal,
25
26
  upgradeJournalPath,
27
+ type UpgradeCheck,
26
28
  type UpgradeJournalEntry,
27
29
  } from "./upgrade-journal.ts";
28
30
  import {
@@ -43,6 +45,13 @@ import {
43
45
  const PACKAGE = "omp-conductor";
44
46
  const HERDR_PLUGIN = "herdr-conductor";
45
47
  const HERDR_SOURCE = "TerrifiedBug/conductor/herdr";
48
+
49
+ /** The package root inside the repo, for a bootstrap install by git ref. */
50
+ const PACKAGE_SOURCE = "TerrifiedBug/conductor/omp";
51
+
52
+ /** A 40-hex commit: the only identity shape a bootstrap accepts. A branch or a
53
+ * tag is a moving target, and "the exact build I verified" is the whole point. */
54
+ const COMMIT_SHA = /^[0-9a-f]{40}$/i;
46
55
  const HERDR_UNIT = "herdr-fleet.service";
47
56
  const DRAIN_POLL_MS = 5_000;
48
57
  const RECOVERY_POLL_MS = 2_000;
@@ -50,6 +59,22 @@ const RECOVERY_ATTEMPTS = 30;
50
59
 
51
60
  export interface UpgradeOptions {
52
61
  version?: string;
62
+ /**
63
+ * Bootstrap from an exact source/build identity instead of a published semver
64
+ * (#908) — the path that exists because a reliability release cannot be cut
65
+ * when the *installed* conductor is the thing that is broken: publishing
66
+ * needs working workers, and the workers are what is broken.
67
+ *
68
+ * `sha` is the whole identity (a 40-hex commit, never a range), and `source`
69
+ * is a checkout of exactly that commit — its only job is to be the tree the
70
+ * package's own checks run against before anything is installed. The install
71
+ * itself is by git ref, so the identity is globally attestable rather than
72
+ * dependent on one machine's directory.
73
+ *
74
+ * Mutually exclusive with {@link version}: two identity kinds in one call is
75
+ * an ambiguity, not a merge.
76
+ */
77
+ bootstrap?: { sha: string; source: string };
53
78
  project?: string;
54
79
  /**
55
80
  * Run as the detached fleet installer (#486): journal every surface the
@@ -71,6 +96,15 @@ export interface UpgradeResult {
71
96
 
72
97
  export interface UpgradeDeps {
73
98
  run(command: string, args: readonly string[]): Promise<UpgradeCommandResult>;
99
+ /**
100
+ * A command run in a named working directory (#908): the bootstrap's checks
101
+ * run inside the source tree, and `run` is deliberately cwd-free everywhere
102
+ * else so no other step can depend on where it was invoked from.
103
+ */
104
+ runIn(cwd: string, command: string, args: readonly string[]): Promise<UpgradeCommandResult>;
105
+ /** Read one file as text. Injected so a bootstrap's manifest read is
106
+ * testable without a checkout on disk. */
107
+ readFile(path: string): string;
74
108
  snapshot(project?: string): { liveWorkers: number };
75
109
  layers(project?: string): FleetLayers;
76
110
  brief(project?: string): { kind: BriefLayout["kind"]; current: boolean };
@@ -145,6 +179,8 @@ export interface UpgradeDeps {
145
179
  * post-restart verifier and the rollback unit run processes the same way. */
146
180
  export const DEFAULT_DEPS: UpgradeDeps = {
147
181
  run: runCommand,
182
+ runIn: (cwd, command, args) => runCommand(command, args, cwd),
183
+ readFile: (path) => readFileSync(path, "utf8"),
148
184
  snapshot: statusSnapshot,
149
185
  layers: fleetLayers,
150
186
  brief: (projectName) => {
@@ -191,7 +227,7 @@ function commandLine(command: string, args: readonly string[]): string {
191
227
  }
192
228
 
193
229
  async function mustRun(
194
- deps: UpgradeDeps,
230
+ deps: SurfaceProbeDeps,
195
231
  command: string,
196
232
  args: readonly string[],
197
233
  ): Promise<UpgradeCommandResult> {
@@ -222,6 +258,108 @@ function parseRegistry(raw: string): { version: string; gitHead: string } {
222
258
  return { version, gitHead };
223
259
  }
224
260
 
261
+ /**
262
+ * One release identity plus how to install it (#908).
263
+ *
264
+ * Two kinds, one shape downstream: everything after this — the currentness
265
+ * comparison, the journal, the install, the attestation and the rollback —
266
+ * reads a `{ version, gitHead }` pair and an install spec, so a bootstrap is a
267
+ * second accepted identity rather than a parallel installer.
268
+ */
269
+ interface ReleaseIdentityPlan {
270
+ version: string;
271
+ gitHead: string;
272
+ /** What `bun add -g` / `omp plugin install` are given for this identity. */
273
+ packageSpec: string;
274
+ /** Named checks already run to earn this identity, journalled as evidence. */
275
+ checks: UpgradeCheck[];
276
+ /** True for a source/build identity: npm does not have this version. */
277
+ bootstrap: boolean;
278
+ }
279
+
280
+ /** The version a source tree declares — the identity's other half, read from
281
+ * the tree itself because npm has never seen it. */
282
+ function sourceVersion(deps: UpgradeDeps, source: string): string {
283
+ const manifest = join(source, "omp", "package.json");
284
+ let raw: string;
285
+ try {
286
+ raw = deps.readFile(manifest);
287
+ } catch (err) {
288
+ throw new Error(
289
+ `bootstrap refused: cannot read ${manifest} (${err instanceof Error ? err.message : String(err)}) — ` +
290
+ "--source must be a checkout of this repository",
291
+ );
292
+ }
293
+ let parsed: unknown;
294
+ try {
295
+ parsed = JSON.parse(raw);
296
+ } catch {
297
+ throw new Error(`bootstrap refused: ${manifest} is not valid JSON`);
298
+ }
299
+ const version = parsed !== null && typeof parsed === "object" ? Reflect.get(parsed, "version") : undefined;
300
+ if (typeof version !== "string" || version.length === 0) {
301
+ throw new Error(`bootstrap refused: ${manifest} declares no version`);
302
+ }
303
+ return version;
304
+ }
305
+
306
+ /**
307
+ * Resolve which release this run installs, and prove a bootstrap identity
308
+ * before it can touch anything (#908).
309
+ *
310
+ * The three refusals are the deliverable, not the install: a sha that is not a
311
+ * sha, a source tree that is not at that sha, and a source tree whose own
312
+ * checks fail. Each names the identity and what failed, and each happens before
313
+ * the first surface is replaced — an unverified tree must never become a live
314
+ * install, which is the difference between this and the `bun add github:...`
315
+ * POLICY forbids.
316
+ */
317
+ async function resolveIdentity(deps: UpgradeDeps, options: UpgradeOptions): Promise<ReleaseIdentityPlan> {
318
+ const bootstrap = options.bootstrap;
319
+ if (bootstrap === undefined) {
320
+ const requested = options.version === undefined ? `${PACKAGE}@latest` : `${PACKAGE}@${options.version}`;
321
+ const release = parseRegistry(
322
+ (await mustRun(deps, "npm", ["view", requested, "version", "gitHead", "--json"])).stdout,
323
+ );
324
+ return { ...release, packageSpec: `${PACKAGE}@${release.version}`, checks: [], bootstrap: false };
325
+ }
326
+ if (options.version !== undefined) {
327
+ throw new Error("bootstrap refused: pass either --to VERSION or --bootstrap SHA, never both");
328
+ }
329
+ const sha = bootstrap.sha.toLowerCase();
330
+ if (!COMMIT_SHA.test(sha)) {
331
+ throw new Error(`bootstrap refused: "${bootstrap.sha}" is not a 40-character commit sha`);
332
+ }
333
+ const checks: UpgradeCheck[] = [];
334
+ // The tree must BE that commit. Without this the identity is a label the
335
+ // operator typed, and the checks below would attest a different build than
336
+ // the one installed.
337
+ const head = await deps.run("git", ["-C", bootstrap.source, "rev-parse", "HEAD"]);
338
+ const at = head.stdout.trim().toLowerCase();
339
+ if (head.code !== 0 || at !== sha) {
340
+ checks.push({ name: "source-at-sha", ok: false, detail: at.length === 0 ? head.stderr.trim() : at });
341
+ throw new Error(
342
+ `bootstrap refused: ${bootstrap.source} is at ${at.length === 0 ? "an unreadable HEAD" : at}, not ${sha} — ` +
343
+ "the checks would attest a different build than the one installed",
344
+ );
345
+ }
346
+ checks.push({ name: "source-at-sha", ok: true, detail: sha });
347
+ const version = sourceVersion(deps, bootstrap.source);
348
+ // The package's own declared gate, against the source, before anything is
349
+ // installed. A failure refuses: a partially verified tree must not go live.
350
+ deps.log(`bootstrap: running the package checks against ${bootstrap.source} (${sha.slice(0, 12)})`);
351
+ const gate = await deps.runIn(join(bootstrap.source, "omp"), "bun", ["run", "check"]);
352
+ if (gate.code !== 0) {
353
+ const detail = (gate.stderr.trim() || gate.stdout.trim()).split("\n").slice(-3).join("; ");
354
+ checks.push({ name: "package-checks", ok: false, detail });
355
+ throw new Error(
356
+ `bootstrap refused: \`bun run check\` failed against ${bootstrap.source} at ${sha} — ${detail}`,
357
+ );
358
+ }
359
+ checks.push({ name: "package-checks", ok: true });
360
+ return { version, gitHead: sha, packageSpec: `github:${PACKAGE_SOURCE}#${sha}`, checks, bootstrap: true };
361
+ }
362
+
225
363
  function ompPluginVersion(raw: string): string | undefined {
226
364
  let parsed: unknown;
227
365
  try {
@@ -255,13 +393,43 @@ function herdrPluginSource(raw: string): string | undefined {
255
393
  return source;
256
394
  }
257
395
 
258
- interface InstalledSurfaces {
396
+ /** The three installed identities one host carries: the Bun-global CLI/daemon
397
+ * tree, the omp plugin, and the herdr recovery plugin's pin. */
398
+ export interface InstalledSurfaces {
259
399
  cliVersion: string;
260
400
  ompVersion?: string;
261
401
  herdrSource?: string;
262
402
  }
263
403
 
264
- async function inspectSurfaces(deps: UpgradeDeps): Promise<InstalledSurfaces> {
404
+ /** What reading the three surfaces needs — deliberately narrower than
405
+ * {@link UpgradeDeps} so a read-only consumer (`doctor`'s surface-parity
406
+ * finding, #904) can supply three fields instead of a whole transaction. */
407
+ export type SurfaceProbeDeps = Pick<UpgradeDeps, "run" | "log" | "env">;
408
+
409
+ /**
410
+ * Read the CLI, omp-plugin and herdr-plugin identities actually installed on
411
+ * this host. Exported because `upgrade` is not the only thing that needs to
412
+ * know whether the three agree: a host whose installs are manual diverges
413
+ * silently between transactions, and `doctor` reads them through this one
414
+ * seam rather than growing a second implementation (#904).
415
+ */
416
+ export async function inspectSurfaces(
417
+ deps: SurfaceProbeDeps,
418
+ /** `readHerdr: false` skips the herdr session probe and plugin read
419
+ * entirely, so a host with no herdr on PATH can still be asked what its
420
+ * CLI and omp plugin are (#904). The upgrade transaction always reads all
421
+ * three — a release pins every surface. */
422
+ opts: { readHerdr?: boolean } = {},
423
+ ): Promise<InstalledSurfaces> {
424
+ if (opts.readHerdr === false) {
425
+ const [cliOnly, ompOnly] = await Promise.all([
426
+ mustRun(deps, "omp-conductor", ["--version"]),
427
+ mustRun(deps, "omp", ["plugin", "list", "--json"]),
428
+ ]);
429
+ const version = cliOnly.stdout.trim();
430
+ if (version.length === 0) throw new Error("installed omp-conductor CLI has no version");
431
+ return { cliVersion: version, ompVersion: ompPluginVersion(ompOnly.stdout) };
432
+ }
265
433
  // HERDR_SESSION is authoritative. Otherwise prefer DEFAULT_HERDR_SESSION, with
266
434
  // #320's one-release bridge to a populated legacy "fleet" session.
267
435
  let session = resolveHerdrSession(deps.env);
@@ -302,12 +470,25 @@ async function inspectSurfaces(deps: UpgradeDeps): Promise<InstalledSurfaces> {
302
470
  };
303
471
  }
304
472
 
305
- function expectedHerdrSource(gitHead: string): string {
473
+ export function expectedHerdrSource(gitHead: string): string {
306
474
  return `github:${HERDR_SOURCE}@${gitHead}`;
307
475
  }
308
476
 
477
+ /**
478
+ * The published identity of one release: its version and the exact commit it
479
+ * was cut from, straight from the registry metadata `upgrade` already trusts.
480
+ * Exported so a read-only consumer can answer "which commit should this
481
+ * host's herdr plugin be pinned to?" without re-deriving the mapping (#904).
482
+ */
483
+ export async function releaseIdentity(
484
+ deps: SurfaceProbeDeps,
485
+ version: string,
486
+ ): Promise<{ version: string; gitHead: string }> {
487
+ return parseRegistry((await mustRun(deps, "npm", ["view", `${PACKAGE}@${version}`, "version", "gitHead", "--json"])).stdout);
488
+ }
489
+
309
490
  function surfacesCurrent(
310
- surfaces: Awaited<ReturnType<typeof inspectSurfaces>>,
491
+ surfaces: InstalledSurfaces,
311
492
  version: string,
312
493
  gitHead: string,
313
494
  ): boolean {
@@ -992,6 +1173,26 @@ export async function rollbackUpgrade(
992
1173
  }
993
1174
  }
994
1175
 
1176
+ // The units the forward transaction reconciled (#905). Restoring them is the
1177
+ // same operation in the other direction: the *restored* CLI renders the
1178
+ // previous version's templates, so running its own reconcile puts the
1179
+ // pre-upgrade bytes back at every destination — and leaves anything the
1180
+ // operator edited alone, because that drift is protected on both legs.
1181
+ //
1182
+ // Ordered before the restarts below, exactly as the forward leg is, and
1183
+ // best-effort: a rollback that restored three surfaces must not fail because
1184
+ // a unit file could not be rewritten. The failure is collected and reported
1185
+ // with the rest.
1186
+ if (installTouched) {
1187
+ const restoreUnits = await deps.run("omp-conductor", ["reconcile-units", "--yes"]);
1188
+ const summary = (restoreUnits.stdout.trim() || restoreUnits.stderr.trim()).split("\n").pop() ?? "";
1189
+ if (restoreUnits.code === 0) {
1190
+ deps.log(`rollback: ${summary.length === 0 ? "host units restored" : summary}`);
1191
+ } else {
1192
+ failures.push(`host units not restored: ${summary}`);
1193
+ }
1194
+ }
1195
+
995
1196
  if (herdrReloadStarted) {
996
1197
  await restore("restart herdr-fleet.service", "systemctl", ["restart", HERDR_UNIT]);
997
1198
  }
@@ -1053,7 +1254,16 @@ function logHostRuntimeDrift(deps: UpgradeDeps): void {
1053
1254
  if (plan.drift.length === 0) return;
1054
1255
  deps.log("host runtime:");
1055
1256
  for (const path of plan.drift) deps.log(` ${path} differs from this version's render`);
1056
- deps.log("fix: run `omp-conductor setup host` from the fleet account to re-install the host units");
1257
+ // Units are this transaction's own to reconcile (#905), and the reconcile
1258
+ // verb is bounded to them. Anything else the render owns — the herdr
1259
+ // pane-shell config, the herdr-conductor `config.env`, the worker identity —
1260
+ // still needs `setup host`, so the advisory names the right tool for each
1261
+ // rather than routing every drift into the flow that restarts (and kills)
1262
+ // the caller's own pane (#834).
1263
+ deps.log(
1264
+ "fix: `omp-conductor reconcile-units` re-installs the host units; anything else above needs " +
1265
+ "`omp-conductor setup host` from the fleet account",
1266
+ );
1057
1267
  }
1058
1268
 
1059
1269
  export async function upgradeConductor(
@@ -1069,14 +1279,29 @@ export async function upgradeConductor(
1069
1279
  throw new Error("run omp-conductor upgrade from a shell outside the target Herdr session");
1070
1280
  }
1071
1281
 
1072
- const requested = options.version === undefined ? `${PACKAGE}@latest` : `${PACKAGE}@${options.version}`;
1073
1282
  // Host-wide by default (#389): one daemon serves every configured project,
1074
1283
  // so an upgrade that restarts it drains them all and refreshes every brief.
1075
1284
  const scope = resolveScope(deps, "upgrade", options.project);
1076
1285
 
1077
- const release = parseRegistry(
1078
- (await mustRun(deps, "npm", ["view", requested, "version", "gitHead", "--json"])).stdout,
1079
- );
1286
+ // Published semver, or an exact source/build identity whose checks have
1287
+ // already passed (#908). Everything below reads the same pair either way.
1288
+ const identity = await resolveIdentity(deps, options);
1289
+ const release = { version: identity.version, gitHead: identity.gitHead };
1290
+ if (identity.bootstrap) {
1291
+ // The bootstrap's evidence lands in the journal before the first surface
1292
+ // moves, so an interrupted bootstrap leaves a record of what was verified
1293
+ // rather than an unexplained half-install.
1294
+ journal({
1295
+ kind: "request",
1296
+ ok: true,
1297
+ version: release.version,
1298
+ gitHead: release.gitHead,
1299
+ phase: "bootstrap",
1300
+ detail: `source identity from ${options.bootstrap?.source ?? "(source)"}`,
1301
+ checks: identity.checks,
1302
+ unit: deps.env["OMP_CONDUCTOR_UNIT"],
1303
+ });
1304
+ }
1080
1305
  if (detached) {
1081
1306
  // The durable request: written before anything can pause, so a unit that
1082
1307
  // dies after this line leaves a journal the returning process can act on
@@ -1096,6 +1321,18 @@ export async function upgradeConductor(
1096
1321
  ...deps.brief(selector),
1097
1322
  }));
1098
1323
  const installNeeded = !surfacesCurrent(surfaces, release.version, release.gitHead);
1324
+ // The host-unit baseline, read while the OLD package is still the one
1325
+ // rendering (#905). A destination already differing from the old render was
1326
+ // edited outside conductor, so the reconcile after the install must not
1327
+ // overwrite it; `undefined` means the baseline could not be established at
1328
+ // all, and the reconcile is skipped rather than guessing.
1329
+ const preInstallDrift = ((): string[] | undefined => {
1330
+ try {
1331
+ return [...deps.hostRuntime().drift];
1332
+ } catch {
1333
+ return undefined;
1334
+ }
1335
+ })();
1099
1336
  if (!installNeeded && briefs.every((b) => b.current)) {
1100
1337
  if (detached) {
1101
1338
  journal({
@@ -1209,10 +1446,10 @@ export async function upgradeConductor(
1209
1446
  if (installNeeded) {
1210
1447
  installTouched = true;
1211
1448
  deps.log(`install 1/3: Bun-global omp-conductor CLI → ${release.version}`);
1212
- await mustRun(deps, "bun", ["add", "-g", `${PACKAGE}@${release.version}`]);
1449
+ await mustRun(deps, "bun", ["add", "-g", identity.packageSpec]);
1213
1450
  journal({ kind: "phase", phase: "install", surface: "cli", ok: true, version: release.version, gitHead: release.gitHead });
1214
1451
  deps.log(`install 2/3: omp plugin omp-conductor → ${release.version}`);
1215
- await mustRun(deps, "omp", ["plugin", "install", `${PACKAGE}@${release.version}`]);
1452
+ await mustRun(deps, "omp", ["plugin", "install", identity.packageSpec]);
1216
1453
  journal({ kind: "phase", phase: "install", surface: "omp", ok: true, version: release.version, gitHead: release.gitHead });
1217
1454
  if (surfaces.herdrSource?.startsWith("local:")) {
1218
1455
  await mustRun(deps, "herdr", ["plugin", "unlink", HERDR_PLUGIN]);
@@ -1236,6 +1473,56 @@ export async function upgradeConductor(
1236
1473
  await upgradeBriefs(deps, briefs);
1237
1474
  journal({ kind: "phase", phase: "brief", surface: "brief", ok: true, version: release.version });
1238
1475
 
1476
+ // #905: the templates this release re-rendered. The render is CODE, so
1477
+ // this process — which loaded the previous version before replacing it —
1478
+ // cannot see the new one: the reconcile runs as a fresh child of the
1479
+ // just-installed CLI, which is also why it is spawned rather than called.
1480
+ // (Not an escalation of conductor itself: the child escalates only its own
1481
+ // bounded `install`/`daemon-reload` steps, which is exactly the boundary
1482
+ // `privileged.ts` draws.)
1483
+ //
1484
+ // It lands here, before the restarts below, so the daemon and the herdr
1485
+ // session come up reading the units this release actually ships — and
1486
+ // inside the same pause/drain window, so nothing is admitted against a
1487
+ // half-reconciled host.
1488
+ {
1489
+ // No baseline means no *refresh*: drift the operator made cannot be told
1490
+ // from drift the upgrade made, so nothing is overwritten. It does not
1491
+ // mean no *retirement* (#895) — a unit this release stopped shipping is
1492
+ // not a comparison against any render, and skipping the whole child
1493
+ // there left an affected host mounting an obsolete bind after every
1494
+ // upgrade, forever. So the child always runs; only its refresh half is
1495
+ // gated.
1496
+ const refreshable = preInstallDrift !== undefined;
1497
+ if (!refreshable) {
1498
+ deps.log(
1499
+ "host units: refresh skipped — this version's host state could not be read before the install, " +
1500
+ "so drift the operator made cannot be told from drift the upgrade made. Retirement still runs; " +
1501
+ "run `omp-conductor reconcile-units --dry-run` to see what differs.",
1502
+ );
1503
+ }
1504
+ const reconcile = await deps.run("omp-conductor", [
1505
+ "reconcile-units",
1506
+ "--yes",
1507
+ ...(refreshable ? [] : ["--no-refresh"]),
1508
+ // Destinations that already differed from the OLD render are the
1509
+ // operator's own edits, not this upgrade's doing: named here so the
1510
+ // child leaves them alone and reports them.
1511
+ ...(preInstallDrift ?? []).flatMap((path) => ["--protect", path]),
1512
+ ]);
1513
+ const summary = (reconcile.stdout.trim() || reconcile.stderr.trim()).split("\n").pop() ?? "";
1514
+ if (reconcile.code === 0) {
1515
+ deps.log(summary.length === 0 ? "host units: reconciled" : summary);
1516
+ journal({ kind: "phase", phase: "reconcile", surface: "units", ok: true, version: release.version, detail: summary });
1517
+ } else {
1518
+ // A failed reconcile is a drifted host, not a failed upgrade: the
1519
+ // packages are installed and verifiable, and the units are exactly as
1520
+ // they were. Say so precisely instead of rolling three surfaces back.
1521
+ deps.log(`host units: reconcile did not complete — ${summary}`);
1522
+ journal({ kind: "phase", phase: "reconcile", surface: "units", ok: false, version: release.version, detail: summary });
1523
+ }
1524
+ }
1525
+
1239
1526
  if (initial.herdr === "active") {
1240
1527
  herdrReloadStarted = true;
1241
1528
  deps.log("reload: restarting herdr-fleet.service and recovering the orchestrator pane");
@@ -321,6 +321,46 @@ export function githubVerbActions(
321
321
  : { ok: true, sha };
322
322
  };
323
323
 
324
+ /**
325
+ * Resolve a caller-pinned release commit and prove it belongs on the released
326
+ * branch (#695).
327
+ *
328
+ * Two questions, both fail-closed. `rev-parse <sha>^{commit}` answers "does
329
+ * this commit exist in the released repository at all" — an abbreviation is
330
+ * expanded here, so what the tag receives is always a full id. `merge-base
331
+ * --is-ancestor` answers the one that matters: a tag pointing at a commit
332
+ * that is not on the released branch is a release of code nobody merged, and
333
+ * that is the failure a decorative `sha` argument would create. The ancestor
334
+ * check accepts the branch tip itself, which is the ordinary case.
335
+ */
336
+ const pinnedReleaseCommit = async (
337
+ mirror: string,
338
+ repo: RepoTarget,
339
+ sha: string,
340
+ ): Promise<CommitOutcome> => {
341
+ const resolved = await readCommit(mirror, ["rev-parse", `${sha}^{commit}`], `pinned commit ${sha}`);
342
+ if (!resolved.ok) {
343
+ return {
344
+ ok: false,
345
+ stderr:
346
+ `refusing release: ${sha} is not a commit in ${repo.name}. ` +
347
+ `Pass a commit that exists on ${repo.defaultBranch}.\n${resolved.stderr}`,
348
+ };
349
+ }
350
+ const branchRef = `refs/remotes/origin/${repo.defaultBranch}`;
351
+ const reachable = await git(mirror, ["merge-base", "--is-ancestor", resolved.sha, branchRef]);
352
+ if (!reachable.result.ok) {
353
+ return {
354
+ ok: false,
355
+ stderr:
356
+ `refusing release: ${resolved.sha} is not reachable from ${repo.name}'s ${repo.defaultBranch}. ` +
357
+ "A tag must never point at a commit that is not on the released branch — check the sha, or " +
358
+ "wait for the commit to land.",
359
+ };
360
+ }
361
+ return resolved;
362
+ };
363
+
324
364
  const releaseTargetMoved = (repo: RepoTarget, target: string, live: string): { ok: false; stderr: string } => ({
325
365
  ok: false,
326
366
  stderr:
@@ -636,6 +676,38 @@ export function githubVerbActions(
636
676
  };
637
677
  },
638
678
 
679
+ // Closing a rejected PR is the one destructive half of an adjudication's
680
+ // disposition (#876), and it is REST rather than `gh pr close` for the same
681
+ // reason `updatePrBranch` is: a selective GraphQL outage must not leave a
682
+ // rejected PR open, permanently blocking the issue's reimplementation.
683
+ closePr: async (prUrl, comment) => {
684
+ const parts = prUrlParts(prUrl);
685
+ if (parts === undefined) {
686
+ return { ok: false, stderr: `could not parse ${prUrl} as a pull request URL` };
687
+ }
688
+ // The comment first, and only then the close: a closed PR still shows its
689
+ // comments, but a close that succeeds while the explanation fails leaves a
690
+ // PR nobody can account for. Ordering the durable explanation before the
691
+ // irreversible act is the same rule the merge path follows.
692
+ const said = await gh([
693
+ "api",
694
+ "--method",
695
+ "POST",
696
+ `repos/${parts.owner}/${parts.repo}/issues/${parts.number}/comments`,
697
+ "-f",
698
+ `body=${comment}`,
699
+ ]);
700
+ if (!said.ok) return { ok: false, stderr: said.stderr };
701
+ return gh([
702
+ "api",
703
+ "--method",
704
+ "PATCH",
705
+ `repos/${parts.owner}/${parts.repo}/pulls/${parts.number}`,
706
+ "-f",
707
+ "state=closed",
708
+ ]);
709
+ },
710
+
639
711
  updatePr: async (prUrl, fields) => {
640
712
  const argv = ["pr", "edit", prUrl];
641
713
  if (fields.title !== undefined) argv.push("--title", fields.title);
@@ -682,15 +754,31 @@ export function githubVerbActions(
682
754
  if (!prepared.ok) return prepared;
683
755
  const mirror = prepared.path;
684
756
  if (execution.shape === "git-tag") {
685
- const target = await readCommit(
686
- mirror,
687
- ["rev-parse", `refs/remotes/origin/${execution.repo.defaultBranch}^{commit}`],
688
- `refreshed ${execution.repo.defaultBranch}`,
689
- );
757
+ // A pinned commit (#695) is the whole point of the argument: the tag
758
+ // points where the caller says, so the branch moving afterwards — which
759
+ // is what reaching a quiet release window requires — cannot strand it.
760
+ // The moved-branch guards below exist to catch main shifting under an
761
+ // implicit "current main" target; for an explicit commit they would
762
+ // refuse exactly the case the pin was asked for, so they are skipped
763
+ // and the reachability check is the invariant instead.
764
+ const pinned =
765
+ execution.sha === undefined
766
+ ? undefined
767
+ : await pinnedReleaseCommit(mirror, execution.repo, execution.sha);
768
+ if (pinned !== undefined && !pinned.ok) return pinned;
769
+ const target =
770
+ pinned ??
771
+ (await readCommit(
772
+ mirror,
773
+ ["rev-parse", `refs/remotes/origin/${execution.repo.defaultBranch}^{commit}`],
774
+ `refreshed ${execution.repo.defaultBranch}`,
775
+ ));
690
776
  if (!target.ok) return target;
691
- const liveBefore = await liveDefaultHead(mirror, execution.repo);
692
- if (!liveBefore.ok) return liveBefore;
693
- if (target.sha !== liveBefore.sha) return releaseTargetMoved(execution.repo, target.sha, liveBefore.sha);
777
+ if (pinned === undefined) {
778
+ const liveBefore = await liveDefaultHead(mirror, execution.repo);
779
+ if (!liveBefore.ok) return liveBefore;
780
+ if (target.sha !== liveBefore.sha) return releaseTargetMoved(execution.repo, target.sha, liveBefore.sha);
781
+ }
694
782
 
695
783
  const versionReady = await releaseVersionRefusal(mirror, execution.repo, target.sha, tag);
696
784
  if (!versionReady.ok) return versionReady;
@@ -736,7 +824,12 @@ export function githubVerbActions(
736
824
  ]);
737
825
  if (!tagged.result.ok) return failed(tagged.result, tagged.argv);
738
826
 
739
- const liveAfter = await liveDefaultHead(mirror, execution.repo);
827
+ // Only meaningful for an implicit target: a pinned tag is *expected* to
828
+ // sit behind a branch that has moved on.
829
+ const liveAfter =
830
+ pinned === undefined
831
+ ? await liveDefaultHead(mirror, execution.repo)
832
+ : ({ ok: true, sha: target.sha } as const);
740
833
  if (!liveAfter.ok || liveAfter.sha !== target.sha) {
741
834
  const rollback = previousSha !== undefined
742
835
  ? await git(mirror, [
@@ -774,6 +867,27 @@ export function githubVerbActions(
774
867
  const target = await readCommit(mirror, ["rev-parse", `refs/tags/${tag}^{commit}`], `tag ${tag}`);
775
868
  if (!target.ok) return target;
776
869
  let tagSha = target.sha;
870
+ // A pinned push publishes the tag EXACTLY as it was cut (#695). Without
871
+ // this the retarget-to-live-main path below would quietly move a pinned
872
+ // tag onto whatever merged since — which is the same stranding, inverted:
873
+ // the caller would get a published tag pointing somewhere it never asked
874
+ // for. The pin is re-proven against the branch and against the local tag,
875
+ // so a mismatch is a refusal rather than a silent move.
876
+ const pinnedPush =
877
+ execution.sha === undefined
878
+ ? undefined
879
+ : await pinnedReleaseCommit(mirror, execution.repo, execution.sha);
880
+ if (pinnedPush !== undefined) {
881
+ if (!pinnedPush.ok) return pinnedPush;
882
+ if (pinnedPush.sha !== tagSha) {
883
+ return {
884
+ ok: false,
885
+ stderr:
886
+ `refusing release: ${tag} is at ${tagSha} locally, and you asked to publish it at ` +
887
+ `${pinnedPush.sha}. A published tag is never force-moved; cut the tag you mean to push.`,
888
+ };
889
+ }
890
+ }
777
891
  const live = await liveDefaultHead(mirror, execution.repo);
778
892
  if (!live.ok) return live;
779
893
 
@@ -795,7 +909,7 @@ export function githubVerbActions(
795
909
  if (!versionReady.ok) return versionReady;
796
910
 
797
911
  let oldSha: string | undefined;
798
- if (tagSha !== live.sha) {
912
+ if (pinnedPush === undefined && tagSha !== live.sha) {
799
913
  const fresh = await readCommit(
800
914
  mirror,
801
915
  ["rev-parse", `refs/remotes/origin/${execution.repo.defaultBranch}^{commit}`],