elanous 0.2.12 → 0.2.14

package/src/hq/lease.ts

@@ -0,0 +1,210 @@
+// HQ lease (본부 임대) — single writer across mbp · node-b · cloud-vm with a 2-of-3 quorum (OP 10-04 08:33 · 08:40).
+// The arbiter (cloud-vm) keeps one record; only the arbiter promotes the standby, and only after two consecutive
+// checks in which neither the arbiter nor the standby can reach the holder. Fenced work carries the generation.
+import { createHash } from 'node:crypto';
+import { existsSync, mkdirSync, readFileSync, renameSync, rmdirSync, statSync, writeFileSync } from 'node:fs';
+import { dirname } from 'node:path';
+import { spawnSync } from 'node:child_process';
+
+export const DEFAULT_TTL_SECONDS = 1500;
+/** A standby view older than this is not evidence (two heartbeats at the 10-minute cadence). */
+export const VIEW_FRESH_SECONDS = 1500;
+/** Consecutive dual-unreachable checks before the arbiter promotes the standby. */
+export const PROMOTE_AFTER_CHECKS = 2;
+
+export interface HostView { target: string; reachable: boolean; at: number }
+export interface LeaseRecord {
+  holder: string;
+  generation: number;
+  acquiredAt: number;
+  renewedAt: number;
+  ttlSeconds: number;
+  /** Each observer's latest view of the holder (arbiter and standby). */
+  views?: Record<string, HostView>;
+  /** Consecutive arbiter checks in which both arbiter and standby could not reach the holder. */
+  dualUnreachableStreak?: number;
+  lastCheckAt?: number;
+  promotedFrom?: string;
+}
+
+export function parseLease(raw: string | null): LeaseRecord | null {
+  if (!raw || !raw.trim()) return null;
+  const data = JSON.parse(raw) as LeaseRecord;
+  if (typeof data.holder !== 'string' || !Number.isSafeInteger(data.generation) || typeof data.renewedAt !== 'number') throw new Error('invalid lease record');
+  return data;
+}
+export const serializeLease = (record: LeaseRecord): string => `${JSON.stringify(record)}\n`;
+export const leaseExpired = (record: LeaseRecord, now: number): boolean => record.renewedAt + record.ttlSeconds < now;
+
+/** acquire: only when there is no lease or it has expired; a new holder gets the next generation. */
+export function decideAcquire(record: LeaseRecord | null, me: string, now: number, ttlSeconds = DEFAULT_TTL_SECONDS):
+  { ok: true; next: LeaseRecord } | { ok: false; reason: string } {
+  if (record && record.holder === me && !leaseExpired(record, now)) return { ok: true, next: { ...record, renewedAt: now, ttlSeconds } };
+  if (record && !leaseExpired(record, now)) return { ok: false, reason: `held by ${record.holder} (generation ${record.generation})` };
+  return { ok: true, next: { holder: me, generation: (record?.generation ?? 0) + 1, acquiredAt: now, renewedAt: now, ttlSeconds, views: {}, dualUnreachableStreak: 0 } };
+}
+
+export function decideRenew(record: LeaseRecord | null, me: string, generation: number | undefined, now: number):
+  { ok: true; next: LeaseRecord } | { ok: false; reason: string } {
+  if (!record) return { ok: false, reason: 'no lease' };
+  if (record.holder !== me) return { ok: false, reason: `held by ${record.holder} (generation ${record.generation})` };
+  if (generation !== undefined && generation < record.generation) return { ok: false, reason: `stale generation ${generation} < ${record.generation}` };
+  return { ok: true, next: { ...record, renewedAt: now, dualUnreachableStreak: 0 } };
+}
+
+export function decideRelease(record: LeaseRecord | null, me: string): { ok: true; next: LeaseRecord } | { ok: false; reason: string } {
+  if (!record || record.holder !== me) return { ok: false, reason: record ? `held by ${record.holder}` : 'no lease' };
+  return { ok: true, next: { ...record, renewedAt: 0 } };
+}
+
+/** The standby reports whether it can reach the holder (rule ②: both views go into the record). */
+export function recordView(record: LeaseRecord, observer: string, reachable: boolean, now: number): LeaseRecord {
+  return { ...record, views: { ...(record.views ?? {}), [observer]: { target: record.holder, reachable, at: now } } };
+}
+
+/**
+ * Arbiter check (rule ②). Records the arbiter's own view; counts a dual-unreachable check only when the standby's
+ * fresh view of the same holder also says unreachable; promotes the standby after PROMOTE_AFTER_CHECKS in a row.
+ */
+export function decideArbiterCheck(record: LeaseRecord, input: { arbiter: string; standby: string; arbiterReachesHolder: boolean; now: number }):
+  { next: LeaseRecord; promoted: boolean; streak: number } {
+  let next = recordView(record, input.arbiter, input.arbiterReachesHolder, input.now);
+  const view = next.views?.[input.standby];
+  const standbyDown = !!view && view.target === record.holder && !view.reachable && input.now - view.at <= VIEW_FRESH_SECONDS;
+  const dual = !input.arbiterReachesHolder && standbyDown && record.holder !== input.standby;
+  const streak = dual ? (record.dualUnreachableStreak ?? 0) + 1 : 0;
+  next = { ...next, dualUnreachableStreak: streak, lastCheckAt: input.now };
+  if (streak < PROMOTE_AFTER_CHECKS) return { next, promoted: false, streak };
+  return {
+    next: { holder: input.standby, generation: record.generation + 1, acquiredAt: input.now, renewedAt: input.now, ttlSeconds: record.ttlSeconds, views: {}, dualUnreachableStreak: 0, lastCheckAt: input.now, promotedFrom: record.holder },
+    promoted: true, streak,
+  };
+}
+
+/** What this host remembers between runs: its last confirmed generation and when quorum last confirmed it. */
+export interface LocalHqState { holder?: string; generation?: number; confirmedAt?: number; ttlSeconds?: number }
+
+export type FenceReason = 'holder' | 'quorum-standby' | 'within-ttl' | 'not-holder' | 'stale-generation' | 'no-quorum-expired' | 'no-lease' | 'no-lease-after-seen' | 'fail-open';
+/**
+ * Fence (rules ① · ③). Arbiter reachable → run only as the current holder at the current generation.
+ * Arbiter unreachable → the holder keeps running while it reaches the standby or its own confirmation is within TTL;
+ * a standby that reports a higher generation always blocks (a returning old HQ stays standby).
+ */
+export function decideFence(input: {
+  me: string; now: number; local: LocalHqState;
+  record: LeaseRecord | null | 'unreachable';
+  standbyReachable?: boolean; standbyGeneration?: number; failOpen?: boolean;
+  /** Highest generation this host ever saw — a vanished record after that is «unknown», not «no lease yet». */
+  seenGeneration?: number;
+}): { run: boolean; reason: FenceReason; generation?: number; holder?: string } {
+  const { me, now, local, record } = input;
+  if (record !== 'unreachable') {
+    if (!record) return { run: false, reason: input.seenGeneration !== undefined ? 'no-lease-after-seen' : 'no-lease', ...(input.seenGeneration !== undefined ? { generation: input.seenGeneration } : {}) };
+    if (record.holder !== me) return { run: false, reason: 'not-holder', holder: record.holder, generation: record.generation };
+    if (local.generation !== undefined && local.generation > record.generation) return { run: false, reason: 'stale-generation', holder: record.holder, generation: record.generation };
+    return { run: true, reason: 'holder', generation: record.generation, holder: me };
+  }
+  if (local.holder !== me || local.generation === undefined) return { run: false, reason: 'not-holder', holder: local.holder, generation: local.generation };
+  if (input.standbyGeneration !== undefined && input.standbyGeneration > local.generation) return { run: false, reason: 'stale-generation', generation: input.standbyGeneration };
+  if (input.standbyReachable) return { run: true, reason: 'quorum-standby', generation: local.generation, holder: me };
+  const ttl = local.ttlSeconds ?? DEFAULT_TTL_SECONDS;
+  if (local.confirmedAt !== undefined && now - local.confirmedAt <= ttl) return { run: true, reason: 'within-ttl', generation: local.generation, holder: me };
+  if (input.failOpen) return { run: true, reason: 'fail-open', generation: local.generation, holder: me };
+  return { run: false, reason: 'no-quorum-expired', generation: local.generation, holder: me };
+}
+
+// ── stores ────────────────────────────────────────────────────────────────────
+export interface LeaseStore {
+  /** null raw = no record · throws when the store cannot be reached. */
+  read(): { now: number; raw: string | null };
+  /** Compare-and-swap on the raw bytes; false when someone else wrote first. */
+  cas(expected: string | null, next: string): boolean;
+}
+export const sha = (raw: string | null): string => raw === null ? 'none' : createHash('sha256').update(raw).digest('hex');
+
+/** Arbiter-local store (the arbiter's own check runs next to the file). mkdir is the lock. */
+export function fileLeaseStore(path: string, clock: () => number = () => Math.floor(Date.now() / 1000)): LeaseStore {
+  const lock = `${path}.lock`;
+  return {
+    read: () => ({ now: clock(), raw: existsSync(path) ? readFileSync(path, 'utf8') : null }),
+    cas(expected, next) {
+      mkdirSync(dirname(path), { recursive: true });
+      for (let i = 0; ; i++) {
+        try { mkdirSync(lock); break; } catch {
+          try { if (Date.now() - statSync(lock).mtimeMs > 30_000) { rmdirSync(lock); continue; } } catch { /* raced */ }
+          if (i > 100) throw new Error('lease lock busy');
+          Bun.sleepSync(50);
+        }
+      }
+      try {
+        const current = existsSync(path) ? readFileSync(path, 'utf8') : null;
+        if (sha(current) !== sha(expected)) return false;
+        const tmp = `${path}.${process.pid}.tmp`;
+        writeFileSync(tmp, next, { mode: 0o600 });
+        renameSync(tmp, path);
+        return true;
+      } finally { try { rmdirSync(lock); } catch { /* already gone */ } }
+    },
+  };
+}
+
+export type SshRunner = (host: string, script: string, input?: string) => { status: number | null; stdout: string; stderr: string };
+export const defaultSshRunner: SshRunner = (host, script, input) => {
+  // ssh joins its arguments into one remote shell line — hand bash a single-quoted script.
+  const quoted = `'${script.replaceAll("'", "'\\''")}'`;
+  const r = spawnSync('ssh', ['-o', 'BatchMode=yes', '-o', 'ConnectTimeout=8', host, `bash -c ${quoted}`],
+    { input: input ?? '', encoding: 'utf8', timeout: 30_000 });
+  return { status: r.status, stdout: r.stdout ?? '', stderr: r.stderr ?? '' };
+};
+
+/** The arbiter itself (cloud-vm) runs the same lease script locally — `hq.arbiter: "local"` (drill 10-04: gcp has no ssh to itself). */
+export const localShellRunner: SshRunner = (_host, script, input) => {
+  const r = spawnSync('bash', ['-c', script], { input: input ?? '', encoding: 'utf8', timeout: 30_000 });
+  return { status: r.status, stdout: r.stdout ?? '', stderr: r.stderr ?? '' };
+};
+
+const REMOTE_DIR = '$HOME/.elanous-hq';
+/** Remote store over ssh: the remote side only compares a sha and renames under a mkdir lock — all decisions stay here. */
+export function sshLeaseStore(host: string, run: SshRunner = defaultSshRunner): LeaseStore {
+  return {
+    read() {
+      const r = run(host, `date +%s && echo ---- && { cat ${REMOTE_DIR}/lease.json 2>/dev/null || true; }`);
+      if (r.status !== 0) throw new Error(`arbiter ${host} unreachable: ${r.stderr.trim().split('\n')[0] ?? r.status}`);
+      const [head, ...rest] = r.stdout.split('----\n');
+      const now = Number.parseInt((head ?? '').trim(), 10);
+      if (!Number.isFinite(now)) throw new Error(`arbiter ${host} gave no clock`);
+      const raw = rest.join('----\n');
+      return { now, raw: raw.length ? raw : null };
+    },
+    cas(expected, next) {
+      const script = [
+        `set -e; d=${REMOTE_DIR}; f=$d/lease.json; mkdir -p "$d"`,
+        'i=0; until mkdir "$d/lock" 2>/dev/null; do i=$((i+1)); [ $i -gt 100 ] && { echo BUSY; exit 3; }; sleep 0.05; done',
+        'trap \'rmdir "$d/lock" 2>/dev/null || true\' EXIT',
+        'if [ -f "$f" ]; then cur=$( (sha256sum "$f" 2>/dev/null || shasum -a 256 "$f") | cut -d" " -f1); else cur=none; fi',
+        `[ "$cur" = "${sha(expected)}" ] || { echo CONFLICT; exit 0; }`,
+        'cat > "$f.tmp.$$"; chmod 600 "$f.tmp.$$"; mv "$f.tmp.$$" "$f"; echo OK',
+      ].join('\n');
+      const r = run(host, script, next);
+      if (r.status !== 0) throw new Error(`arbiter ${host} write failed: ${r.stdout.trim() || r.stderr.trim().split('\n')[0]}`);
+      return r.stdout.trim().endsWith('OK');
+    },
+  };
+}
+
+/** Home-to-home reachability (ssh true) — used only by the holder to reach the standby, never by the arbiter. */
+export const sshReachable = (host: string, run: SshRunner = defaultSshRunner): boolean => {
+  try { return run(host, 'true').status === 0; } catch { return false; }
+};
+
+/**
+ * Tailnet probe (OP 10-04 09:04): liveness = `tailscale ping`, application = nexus health over the tailnet.
+ * One «unreachable» observation needs BOTH to fail. No ssh — the arbiter holds no shell on home machines.
+ */
+export interface HostProbe { ping(host: string): boolean; health(host: string): boolean }
+export function probeReachable(host: string, probe: HostProbe): { reachable: boolean; ping: boolean; health: boolean } {
+  let ping = false; let health = false;
+  try { ping = probe.ping(host); } catch { ping = false; }
+  try { health = probe.health(host); } catch { health = false; }
+  return { reachable: ping || health, ping, health };
+}

package/src/hq/ledger-write-target.ts

@@ -0,0 +1,41 @@
+import { realpathSync } from 'node:fs';
+import { dirname, isAbsolute, join, relative, sep } from 'node:path';
+import { effectiveInstanceRoot, prodInstanceRoot } from '../instance/resolve.js';
+
+function physicalPath(path: string): string {
+  // Resolve one component at a time: lexical normalization of `link/..` before
+  // realpathSync would discard the link even though the filesystem follows it.
+  const raw = isAbsolute(path) ? path : `${process.cwd()}${sep}${path}`;
+  let current: string = sep;
+  for (const component of raw.split(sep)) {
+    if (!component || component === '.') continue;
+    if (component === '..') {
+      current = dirname(current);
+      continue;
+    }
+    const candidate = join(current, component);
+    try { current = realpathSync(candidate); }
+    catch (error) {
+      if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error;
+      current = candidate;
+    }
+  }
+  return current;
+}
+
+function within(root: string, target: string): boolean {
+  const suffix = relative(root, target);
+  return suffix === '' || (suffix !== '..' && !suffix.startsWith(`..${sep}`));
+}
+
+/** Only writes inside this process's non-operational instance universe bypass the HQ ledger lease. */
+export function isIsolatedLedgerWriteRoot(
+  ledgerRoot: string,
+  instanceRoot = effectiveInstanceRoot(),
+  operationalRoot = prodInstanceRoot(),
+): boolean {
+  const target = physicalPath(ledgerRoot);
+  const instance = physicalPath(instanceRoot);
+  const prod = physicalPath(operationalRoot);
+  return instance !== prod && within(instance, target) && !within(prod, target);
+}

package/src/hq/promote.ts

@@ -0,0 +1,301 @@
+// HQ promotion runbook: read-only preview by default; all remote mutations are behind --apply.
+import { spawnSync } from 'node:child_process';
+import { getUserConfig } from '../user-config.js';
+import { parseLease, sshLeaseStore, defaultSshRunner, localShellRunner } from './lease.js';
+
+export interface PromoteRunner {
+  lease(): { holder: string | null; generation: number | null; expired: boolean };
+  run(host: string, script: string): { status: number | null; stdout: string; stderr: string };
+}
+
+export interface PromoteLine { step: string; status: 'ready' | 'done' | 'failed'; measurement: string; command: string }
+export interface PromoteResult { ok: boolean; apply: boolean; host: string; lines: PromoteLine[] }
+
+const install = '"$HOME/.local/share/elanous/current/node_modules/elanous"';
+const state = '"$HOME/.elanous-hqstate"';
+const cli = 'elanous';
+// A bind to the same address as nexus distinguishes an available socket from an
+// occupied or uninspectable one; lsof's exit 1 alone cannot make that distinction.
+const portProbe = `command -v lsof >/dev/null || { echo 'lsof unavailable' >&2; exit 1; }
+python3 -c 'import os,socket,subprocess,sys; s=socket.socket(); s.bind(("127.0.0.1",0)); s.listen(); r=subprocess.run(["lsof","-nP","-t","-iTCP:"+str(s.getsockname()[1]),"-sTCP:LISTEN"],capture_output=True,text=True); s.close(); sys.exit(0 if r.returncode == 0 and r.stdout.strip() == str(os.getpid()) else "lsof control listener unreadable: "+r.stderr.strip())' || exit 1
+port_detail=$(lsof -nP -t -iTCP:31416 -sTCP:LISTEN 2>&1); port_rc=$?
+if [ "$port_rc" -ne 1 ] || [ -n "$port_detail" ]; then echo "port=31416 occupied or unreadable: $port_detail (lsof=$port_rc)" >&2; exit 1; fi
+python3 -c 'import socket; s=socket.socket(); s.bind(("127.0.0.1", 31416)); s.close(); print("port=31416 available")'`;
+// The drill uses short SSH aliases; host-local hq.hostName may use either the short or full name.
+const hostNamesFor = (host: string): string[] => host === 'node-b' ? ['node-b', 'MacStudioB1']
+  : host === 'mbp' ? ['mbp', 'MacBookProM5'] : [host];
+const checks: Array<[string, string]> = [
+  ['PWA build', `test -f "$HOME/.local/share/elanous/current/node_modules/elanous/apps/pwa/out/index.html" && echo 'index.html present'`],
+  ['MCP list', `set -o pipefail; codex mcp list --json | python3 -c 'import json,sys; data=json.load(sys.stdin); print("MCP entries="+str(len(data)))'`],
+  ['installed hq', `${cli} hq promote --help >/dev/null && python3 -c 'import json,pathlib; p=pathlib.Path.home()/".elanous-hqcfg/config.json"; print("host="+json.loads(p.read_text())["hq"]["hostName"])'`],
+];
+const stages: Array<[string, string]> = [
+  ['standby checksums', `cd ${install} && bun -e 'import("node:fs/promises").then(async ({ readlink }) => { const root = process.env.HOME + "/.elanous-standby"; const generations = { core: await readlink(root + "/core/latest"), big: await readlink(root + "/big/latest") }; const { verifyStandby } = await import("./scripts/hq/standby-verify.ts"); const result = await verifyStandby({ root, tiers: ["core", "big"], generations }); console.log(JSON.stringify(result)); if (!result.ok) process.exitCode = 1 }).catch(error => { console.error(error); process.exitCode = 1 })'`],
+  ['copy to promotion universe', `python3 - <<'PY'
+import hashlib, json, os, pathlib, shutil, tempfile
+home = pathlib.Path.home()
+standby = home / '.elanous-standby'
+target = home / '.elanous-hqstate'
+parent = target.parent
+stage = pathlib.Path(tempfile.mkdtemp(prefix='.elanous-hqstate-promote-', dir=parent))
+backup = None
+try:
+    copied = 0
+    for tier in ('core', 'big'):
+        generation = os.environ['HQ_PROMOTE_' + tier.upper()]
+        source = standby / tier / generation
+        if not source.is_dir() or source.is_symlink():
+            raise ValueError('missing or symlinked pinned generation: ' + tier)
+        manifest_bytes = (source / 'MANIFEST.json').read_bytes()
+        expected_manifest = os.environ['HQ_PROMOTE_' + tier.upper() + '_MANIFEST_SHA256']
+        if hashlib.sha256(manifest_bytes).hexdigest() != expected_manifest:
+            raise ValueError('verified manifest changed: ' + tier)
+        manifest = json.loads(manifest_bytes)
+        if manifest['tier'] != tier or manifest['generation'] != generation:
+            raise ValueError('generation changed: ' + tier)
+        for entry in manifest['entries']:
+            name = entry['path']
+            rel = pathlib.PurePosixPath(name)
+            if not name or rel.is_absolute() or '..' in rel.parts or name.startswith('./') or chr(92) in name or name in ('MANIFEST.json', 'SHA256SUMS'):
+                raise ValueError('invalid manifest path: ' + name)
+            src = source
+            for part in rel.parts:
+                src = src / part
+                if src.is_symlink():
+                    raise ValueError('symlinked source: ' + name)
+            if not src.is_file():
+                raise ValueError('missing source: ' + name)
+            dest = stage / name
+            if dest.parent.is_symlink():
+                raise ValueError('symlinked destination parent: ' + name)
+            dest.parent.mkdir(parents=True, exist_ok=True)
+            if dest.is_symlink():
+                raise ValueError('symlinked destination: ' + name)
+            shutil.copy2(src, dest)
+            digest = hashlib.sha256(dest.read_bytes()).hexdigest()
+            if digest != entry['sha256']:
+                raise ValueError('checksum changed during copy: ' + name)
+            copied += 1
+        if hashlib.sha256((source / 'MANIFEST.json').read_bytes()).hexdigest() != expected_manifest:
+            raise ValueError('verified manifest changed during copy: ' + tier)
+    for tier in ('core', 'big'):
+        source = standby / tier / os.environ['HQ_PROMOTE_' + tier.upper()]
+        expected_manifest = os.environ['HQ_PROMOTE_' + tier.upper() + '_MANIFEST_SHA256']
+        if hashlib.sha256((source / 'MANIFEST.json').read_bytes()).hexdigest() != expected_manifest:
+            raise ValueError('verified manifest changed before promotion: ' + tier)
+    if not (stage / 'release/features.sqlite').is_file() or not (stage / 'config.json').is_file():
+        raise ValueError('core promotion files missing')
+    if target.is_symlink() or (target.exists() and not target.is_dir()):
+        raise ValueError('promotion target is not a directory')
+    if target.exists():
+        backup = pathlib.Path(tempfile.mkdtemp(prefix='.elanous-hqstate-old-', dir=parent))
+        backup.rmdir()
+        target.rename(backup)
+    try:
+        stage.rename(target)
+    except BaseException:
+        if backup is not None:
+            backup.rename(target)
+            backup = None
+        raise
+    bytes_count = sum(p.stat().st_size for p in target.rglob('*') if p.is_file())
+    if backup is not None:
+        shutil.rmtree(backup)
+        backup = None
+    print('promotion files=' + str(copied) + ' bytes=' + str(bytes_count) + 'B')
+finally:
+    if stage.exists():
+        shutil.rmtree(stage)
+PY`],
+  ['target hq config', `python3 - <<'PY'
+import json, os, pathlib, tempfile
+source = pathlib.Path.home() / '.elanous-hqcfg/config.json'
+target = pathlib.Path.home() / '.elanous-hqstate/config.json'
+with source.open() as f: hq = json.load(f)['hq']
+with target.open() as f: config = json.load(f)
+assert isinstance(hq, dict) and hq.get('hostName') == os.environ['HQ_PROMOTE_HOST'], 'host-specific hq config mismatch'
+config['hq'] = hq
+with tempfile.NamedTemporaryFile(mode='w', dir=target.parent, prefix='config.promote-', delete=False) as f:
+    tmp = pathlib.Path(f.name)
+    json.dump(config, f)
+    f.write(chr(10))
+try:
+    os.chmod(tmp, 0o600)
+    os.replace(tmp, target)
+finally:
+    tmp.unlink(missing_ok=True)
+print('hq.hostName=' + hq['hostName'])
+PY`],
+  ['nexus health', `command -v lsof >/dev/null || { echo 'lsof unavailable: cannot identify nexus listener' >&2; exit 1; }
+${portProbe} || { echo 'port=31416 already occupied or unreadable' >&2; exit 1; }
+ELANOUS_STATE_DIR=${state} ${cli} --config-dir ${state} nexus run --port 31416 --http-host 127.0.0.1 --no-mcp --no-auto-build --no-auto-install --no-auto-restart --no-watch > "$HOME/.elanous-hqstate/nexus-promote.log" 2>&1 & pid=$!
+descendants() {
+  for child in $(pgrep -P "$1" 2>/dev/null); do descendants "$child"; echo "$child"; done
+}
+cleanup() {
+  tree=$(descendants "$pid")
+  for proc in $tree; do kill "$proc" 2>/dev/null || true; done
+  kill "$pid" 2>/dev/null || true
+  wait "$pid" 2>/dev/null || true
+  n=0
+  until ${portProbe} >/dev/null 2>&1; do
+    if [ "$n" -ge 10 ]; then echo 'port=31416 still occupied after cleanup' >&2; return 1; fi
+    n=$((n+1)); sleep 1
+  done
+  echo 'nexus stopped port=31416 released' >&2
+}
+trap 'cleanup' EXIT
+n=0
+code=000
+owner=''
+owned=0
+while [ "$n" -lt 15 ]; do
+  code=$(curl -s --max-time 2 -o /dev/null -w '%{http_code}' http://127.0.0.1:31416/v1/health) || code=000
+  listeners=$(lsof -nP -t -iTCP:31416 -sTCP:LISTEN 2> /dev/null) || listeners=''
+  owner=$(printf '%s\\n' "$listeners" | head -n 1)
+  ancestor=$owner
+  owned=0
+  while [ -n "$ancestor" ] && [ "$ancestor" != 1 ]; do
+    if [ "$ancestor" = "$pid" ]; then owned=1; break; fi
+    ancestor=$(ps -o ppid= -p "$ancestor" | tr -d '[:space:]')
+  done
+  if [ "$code" = 200 ] && [ "$owned" = 1 ] && [ "$listeners" = "$owner" ] && kill -0 "$pid" 2>/dev/null; then
+    trap - EXIT
+    echo "health=$code port=31416 pid=$pid listener_pid=$owner"; exit 0
+  fi
+  if ! kill -0 "$pid" 2>/dev/null; then break; fi
+  n=$((n+1)); sleep 2
+done
+echo "health=$code port=31416 nexus_pid=$pid listener_pid=$owner owned=$owned" >&2; exit 1`],
+  ['steward rescue', `ELANOUS_STATE_DIR=${state} ${cli} --config-dir ${state} steward mode rescue && ELANOUS_STATE_DIR=${state} ${cli} --config-dir ${state} steward mode --json`],
+];
+
+// One --apply per target host: stages run as separate ssh calls, so the lock is a directory that outlives each call.
+const lockDir = '"$HOME/.elanous-hq/promote.lock"';
+export const promoteLockAcquire = `mkdir -p "$HOME/.elanous-hq" && if mkdir ${lockDir} 2>/dev/null; then printf '%s %s\\n' "$PPID" "$(date -u +%Y-%m-%dT%H:%M:%SZ)" > ${lockDir}/owner; else echo "promote lock held: $(cat ${lockDir}/owner 2>/dev/null) · remove ${lockDir} only after confirming no promote is running" >&2; exit 1; fi`;
+export const promoteLockRelease = `rm -rf ${lockDir}`;
+
+export function defaultPromoteRunner(): PromoteRunner {
+  const config = getUserConfig().hq ?? {};
+  const arbiter = config.arbiter ?? 'cloud-vm';
+  const store = sshLeaseStore(arbiter, arbiter === 'local' ? localShellRunner : defaultSshRunner);
+  return {
+    lease() {
+      const { now, raw } = store.read();
+      const record = parseLease(raw);
+      return { holder: record?.holder ?? null, generation: record?.generation ?? null,
+        expired: !record || now - record.renewedAt > record.ttlSeconds };
+    },
+    run(host, script) {
+      const r = spawnSync('ssh', ['-o', 'BatchMode=yes', host, `bash -c '${script.replaceAll("'", "'\\''")}'`], { encoding: 'utf8', timeout: 120_000 });
+      return { status: r.status, stdout: r.stdout ?? '', stderr: r.stderr ?? String(r.error ?? '') };
+    },
+  };
+}
+
+export function promoteHq(host: string, apply = false, runner: PromoteRunner = defaultPromoteRunner()): PromoteResult {
+  if (!/^[a-zA-Z0-9][a-zA-Z0-9.-]*$/.test(host) || host.includes('..')) throw new Error('hq promote: invalid host');
+  const lines: PromoteLine[] = [];
+  const acceptedHostNames = hostNamesFor(host);
+  const result = (step: string, command: string, status: PromoteLine['status'], measurement: string) => {
+    lines.push({ step, status, measurement, command });
+  };
+  let preflightOk = true;
+  for (const [name, command] of checks) {
+    let r: ReturnType<PromoteRunner['run']>;
+    try { r = runner.run(host, command); } catch (error) { r = { status: null, stdout: '', stderr: String(error) }; }
+    const ok = r.status === 0 && (name !== 'installed hq' || acceptedHostNames.some(name => r.stdout.trim() === `host=${name}`));
+    if (!ok) preflightOk = false;
+    result(`preflight ${name}`, command, ok ? 'done' : 'failed', (r.status === 0 ? r.stdout : r.stderr).trim() || `exit=${r.status}`);
+  }
+  let lease: ReturnType<PromoteRunner['lease']>;
+  try { lease = runner.lease(); } catch (error) {
+    result('① lease', 'read arbiter lease', 'failed', String(error));
+    return { ok: false, apply, host, lines };
+  }
+  if (!acceptedHostNames.includes(lease.holder ?? '') || lease.expired) {
+    result('① lease', 'read arbiter lease', 'failed', `holder=${lease.holder ?? 'none'} generation=${lease.generation ?? 'none'} expired=${lease.expired}`);
+    return { ok: false, apply, host, lines };
+  }
+  result('① lease', 'read arbiter lease', 'done', `holder=${lease.holder} generation=${lease.generation} expired=false`);
+  const targetHostName = lines.find(line => line.step === 'preflight installed hq')?.measurement.trim().replace(/^host=/, '') ?? '';
+  if (!acceptedHostNames.includes(targetHostName) || targetHostName !== lease.holder) {
+    result('① lease', 'compare arbiter holder to target hq.hostName', 'failed', `holder=${lease.holder} hq.hostName=${targetHostName}`);
+    return { ok: false, apply, host, lines };
+  }
+  if (apply && !preflightOk) return { ok: false, apply, host, lines };
+  let pinned: { core: string; big: string; coreManifestSha256: string; bigManifestSha256: string } | null = null;
+  let locked = false;
+  try {
+  for (let i = 0; i < stages.length; i++) {
+    const [name, script] = stages[i];
+    const command = i === 1 && pinned
+      ? `HQ_PROMOTE_CORE=${pinned.core} HQ_PROMOTE_BIG=${pinned.big} HQ_PROMOTE_CORE_MANIFEST_SHA256=${pinned.coreManifestSha256} HQ_PROMOTE_BIG_MANIFEST_SHA256=${pinned.bigManifestSha256} ${script}`
+      : i === 2 ? `HQ_PROMOTE_HOST=${targetHostName} ${script}` : script;
+    if (apply && i > 0) {
+      try {
+        const current = runner.lease();
+        if (current.holder !== targetHostName || current.generation !== lease.generation || current.expired) {
+          result(`${'②③④⑤⑥'[i]} ${name}`, command, 'failed', `lease changed: holder=${current.holder ?? 'none'} generation=${current.generation ?? 'none'} expired=${current.expired}`);
+          return { ok: false, apply, host, lines };
+        }
+      } catch (error) {
+        result(`${'②③④⑤⑥'[i]} ${name}`, command, 'failed', `lease read failed: ${String(error)}`);
+        return { ok: false, apply, host, lines };
+      }
+    }
+    if (!apply && i > 0) {
+      result(`${'②③④⑤⑥'[i]} ${name}`, command, 'ready', `dry-run: would run on ${host}; no writes`);
+      continue;
+    }
+    if (apply && i === 1) {
+      // The host lock and the port probe share one call: the lock is held from before the copy until the finally below.
+      const lockedProbe = `${promoteLockAcquire} && ${portProbe}`;
+      let probe: ReturnType<PromoteRunner['run']>;
+      try { probe = runner.run(host, lockedProbe); } catch (error) { probe = { status: null, stdout: '', stderr: String(error) }; }
+      locked = !probe.stderr.includes('promote lock held');
+      if (!locked) {
+        result('③ copy to promotion universe', lockedProbe, 'failed', probe.stderr.trim());
+        return { ok: false, apply, host, lines };
+      }
+      if (probe.status !== 0 || probe.stdout.trim() !== 'port=31416 available') {
+        result('③ copy to promotion universe', portProbe, 'failed', `port=31416 occupied or unreadable before copy: ${probe.stderr.trim() || probe.stdout.trim() || `exit=${probe.status}`}`);
+        return { ok: false, apply, host, lines };
+      }
+    }
+    let r: ReturnType<PromoteRunner['run']>;
+    try { r = runner.run(host, command); } catch (error) { r = { status: null, stdout: '', stderr: String(error) }; }
+    let measurement = (r.status === 0 ? r.stdout : r.stderr).trim() || `exit=${r.status}`;
+    let ok = r.status === 0;
+    if (i === 1 && ok) ok = /promotion files=\d+ bytes=\d+B/.test(r.stdout);
+    if (i === 2 && ok) ok = r.stdout.trim() === `hq.hostName=${targetHostName}`;
+    if (i === 3 && ok) ok = /health=200 port=31416 pid=\d+ listener_pid=\d+\b/.test(r.stdout);
+    if (i === 4 && ok) ok = /"mode"\s*:\s*"rescue"/.test(r.stdout);
+    if (i === 0 && ok) {
+      try {
+        const verification = JSON.parse(r.stdout) as { ok?: boolean; tiers?: Array<{ tier: string; generation: string | null; manifestSha256: string | null; checked: number; entries: number | null }> };
+        ok = verification.ok === true && verification.tiers?.length === 2
+          && verification.tiers.map(t => t.tier).join(',') === 'core,big'
+          && verification.tiers.every(t => t.generation && /^[a-zA-Z0-9][a-zA-Z0-9_-]*$/.test(t.generation)
+            && typeof t.manifestSha256 === 'string' && /^[a-f0-9]{64}$/.test(t.manifestSha256)
+            && t.entries !== null && t.entries > 0 && t.checked === t.entries) === true;
+        if (ok) pinned = { core: verification.tiers![0].generation!, big: verification.tiers![1].generation!,
+          coreManifestSha256: verification.tiers![0].manifestSha256!, bigManifestSha256: verification.tiers![1].manifestSha256! };
+      } catch { ok = false; }
+      if (!ok) measurement = `checksum verification failed · ${measurement}`;
+    }
+    result(`${'②③④⑤⑥'[i]} ${name}`, command, ok ? 'done' : 'failed', measurement);
+    if (!ok) return { ok: false, apply, host, lines };
+  }
+  return { ok: preflightOk, apply, host, lines };
+  } finally {
+    if (locked) {
+      try { runner.run(host, promoteLockRelease); } catch { /* the lock names its owner for manual release */ }
+    }
+  }
+}
+
+export function formatPromote(result: PromoteResult): string {
+  return [`hq promote --to ${result.host} ${result.apply ? '--apply' : '(dry-run)'}`,
+    ...result.lines.map(line => `${line.step}: ${line.status} · ${line.measurement}${line.status === 'ready' ? ` · ${line.command}` : ''}`)].join('\n');
+}