elanous 0.2.15 → 0.2.17

package/src/hq/fence-wrapper.ts

@@ -0,0 +1,123 @@
+import { spawnSync } from 'node:child_process';
+import { chmodSync, constants, copyFileSync, lstatSync, mkdirSync, mkdtempSync, readFileSync, realpathSync, renameSync, rmSync, writeFileSync } from 'node:fs';
+import { dirname, isAbsolute, join } from 'node:path';
+import { homedir } from 'node:os';
+import { createPatch } from 'diff';
+
+const quote = (value: string): string => `'${value.replaceAll("'", "'\"'\"'")}'`;
+
+/** Only the installed entrypoint runs the fence; never use a checkout as a fallback.
+ * Arbiter (cloud-vm) outage retains the current lease and forbids promotion;
+ * this wrapper does not change the fence's lease or promotion decisions.
+ * cron runs with PATH=/usr/bin:/bin, so bun is baked in as an absolute path (OP 10-06 NO-GO). */
+export function renderHqFenceWrapper({ entry, configDir, bun, logFile }: { entry: string; configDir: string; bun: string; logFile?: string }): string {
+  if (!isAbsolute(bun)) throw new Error(`hq-fence: bun must be an absolute path: ${bun}`);
+  const log = logFile ?? join(configDir, 'logs', 'hq-fence.log');
+  return `#!/bin/sh
+# Arbiter (cloud-vm) outage: retain the lease, do not promote; hq fence owns that decision.
+ENTRY=${quote(entry)}
+CONFIG_DIR=${quote(configDir)}
+# cron has no PATH for bun — the absolute path was resolved when this wrapper was installed.
+BUN=${quote(bun)}
+LOG=${quote(log)}
+# Last resort when no alert path works: one line in a log file, no bun needed.
+log_line() {
+  mkdir -p "$(dirname "$LOG")" 2>/dev/null
+  printf '%s %s\\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$1" >> "$LOG" 2>/dev/null
+}
+# The alert goes through the same installed entry; PATH \`elanous\` is only a fallback when the entry itself is missing.
+# Alert bun: the installed one first, then usual absolute install locations (still no PATH lookup).
+alert_bun() {
+  for b in "$BUN" "$HOME/.bun/bin/bun" /opt/homebrew/bin/bun /usr/local/bin/bun; do
+    if [ -x "$b" ]; then echo "$b"; return 0; fi
+  done
+  return 1
+}
+alert() {
+  if [ -f "$ENTRY" ] && AB=$(alert_bun); then
+    if "$AB" "$ENTRY" --config-dir "$CONFIG_DIR" hq fence-wrapper alert "$1"; then return 0; fi
+    echo "hq-fence: owner alert failed: $1" >&2
+    log_line "owner alert failed: $1"
+    return 0
+  fi
+  if command -v elanous >/dev/null 2>&1 && elanous --config-dir "$CONFIG_DIR" hq fence-wrapper alert "$1"; then return 0; fi
+  echo "hq-fence: owner alert failed: $1" >&2
+  log_line "owner alert failed: $1"
+}
+if [ ! -x "$BUN" ]; then
+  echo "hq-fence: bun missing or not executable: $BUN" >&2
+  # Exactly one line on disk; the alert is tried only through an absolute fallback bun (never PATH).
+  log_line "bun missing or not executable: $BUN (rc=127)"
+  if [ -f "$ENTRY" ] && AB=$(alert_bun); then
+    "$AB" "$ENTRY" --config-dir "$CONFIG_DIR" hq fence-wrapper alert "hq-fence: bun missing or not executable: $BUN (rc=127)" || echo "hq-fence: owner alert failed (fallback bun $AB)" >&2
+  fi
+  exit 127
+fi
+if [ ! -f "$ENTRY" ] || [ ! -r "$ENTRY" ]; then
+  echo "hq-fence: installed entrypoint missing: $ENTRY" >&2
+  alert "hq-fence: installed entrypoint missing: $ENTRY"
+  exit 1
+fi
+cd "$(dirname "$ENTRY")/.." || { alert "hq-fence: installed directory missing"; exit 1; }
+if [ $# -lt 1 ] || [ -z "$1" ]; then
+  echo "hq-fence: role required" >&2
+  alert "hq-fence: role required"
+  exit 2
+fi
+ROLE=$1
+shift
+"$BUN" "$ENTRY" --config-dir "$CONFIG_DIR" hq fence --role "$ROLE" -- /bin/sh -c "$*"
+rc=$?
+if [ "$rc" -ne 0 ]; then alert "hq-fence: fence failed (rc=$rc, role=$ROLE)"; fi
+exit "$rc"
+`;
+}
+
+/** Absolute bun for the wrapper: this process when it is bun, else `command -v bun` resolved now (install time). */
+export function resolveHqFenceBun(deps: { isBun?: boolean; execPath?: string; lookup?: () => string } = {}): string {
+  const isBun = deps.isBun ?? typeof process.versions.bun === 'string';
+  const candidate = isBun
+    ? (deps.execPath ?? process.execPath)
+    : (deps.lookup ?? (() => spawnSync('/bin/sh', ['-c', 'command -v bun'], { encoding: 'utf8' }).stdout ?? ''))().trim();
+  if (!candidate || !isAbsolute(candidate)) throw new Error(`hq-fence: cannot resolve an absolute bun path (got ${JSON.stringify(candidate)})`);
+  return realpathSync(candidate);
+}
+
+export function installedHqFenceEntry(home = homedir()): string {
+  return join(home, '.local', 'share', 'elanous', 'current', 'node_modules', 'elanous', 'bin', 'elanous.mjs');
+}
+
+/** Literal \`cd <path>\` lines only — a worktree path built from a variable is not detected. */
+export function repositoryWorktreeCdWarning(body: string): string | undefined {
+  return body.split('\n').some(line => /^\s*cd\s+(?:['"]?[^\s'";]+\/)?(?:wt-[^\s'";]+|[^\s'";]+\/repo\.worktrees\/[^\s'";]+)(?:['"]?\s*(?:&&|\|\||;|#|$))/.test(line))
+    ? 'hq fence-audit: warning — hq-fence wrapper cd targets a repository worktree' : undefined;
+}
+
+export function installHqFenceWrapper(configDir: string, body: string, yes: boolean, deps: {
+  now?: () => Date;
+} = {}): { path: string; preview: string; backup?: string } {
+  const path = join(configDir, 'bin', 'hq-fence');
+  let present = false;
+  try { present = !!lstatSync(path); } catch (error) { if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error; }
+  if (present && !lstatSync(path).isFile()) throw new Error(`hq-fence: refusing non-regular wrapper: ${path}`);
+  const old = present ? readFileSync(path, 'utf8') : undefined;
+  const preview = `hq-fence proposed: ${path}\n${body}\n${createPatch(path, old ?? '', body, 'existing', 'proposed')}`;
+  if (!yes) return { path, preview };
+  let backup: string | undefined;
+  if (present) {
+    const backups = join(configDir, 'backups');
+    mkdirSync(backups, { recursive: true });
+    const stamp = (deps.now ?? (() => new Date()))().toISOString().replaceAll(':', '-');
+    backup = join(backups, `hq-fence.${stamp}`);
+    copyFileSync(path, backup, constants.COPYFILE_EXCL);
+  }
+  mkdirSync(dirname(path), { recursive: true });
+  const tempDir = mkdtempSync(join(dirname(path), '.hq-fence-'));
+  try {
+    const temp = join(tempDir, 'hq-fence');
+    writeFileSync(temp, body, { flag: 'wx', mode: 0o755 });
+    chmodSync(temp, 0o755);
+    renameSync(temp, path);
+  } finally { rmSync(tempDir, { recursive: true, force: true }); }
+  return { path, preview, ...(backup ? { backup } : {}) };
+}

package/src/hq/hq.ts

@@ -205,8 +205,16 @@ function mutate(r: Resolved, decide: (record: LeaseRecord | null, now: number) =
   return { ok: false, reason: 'lost the race five times', record: null };
 }
 
+/** Snapshot for read-only consumers: never advances the seen-generation marker or CASes the arbiter. */
+export function readHqLease(deps: HqDeps = {}) {
+  const { store } = resolve(deps);
+  const { now, raw } = store.read();
+  const record = parseLease(raw);
+  return { record, ageSeconds: record ? now - record.renewedAt : null, expired: record ? leaseExpired(record, now) : null };
+}
+
 export type LeaseAction = 'acquire' | 'renew' | 'status' | 'release';
-export function hqLease(action: LeaseAction, deps: HqDeps = {}, opts: { host?: string } = {}) {
+export function hqLease(action: LeaseAction, deps: HqDeps = {}, opts: { host?: string; expectedHolder?: string; expectedGeneration?: number } = {}) {
   const r0 = resolve(deps);
   const r = opts.host ? { ...r0, me: opts.host } : r0;
   if (action === 'status') {
@@ -215,9 +223,15 @@ export function hqLease(action: LeaseAction, deps: HqDeps = {}, opts: { host?: s
     noteSeen(r, record);
     return { ok: true as const, action, record, ageSeconds: record ? now - record.renewedAt : null, expired: record ? leaseExpired(record, now) : null };
   }
+  if (action === 'release' && ((opts.expectedHolder === undefined) !== (opts.expectedGeneration === undefined)
+    || (opts.expectedGeneration !== undefined && (!Number.isSafeInteger(opts.expectedGeneration) || opts.expectedGeneration < 1))
+    || (opts.expectedHolder !== undefined && !opts.expectedHolder)))
+    throw new Error('hq lease release: expected holder and positive generation must be supplied together');
   const local = readLocal(r.localPath);
+  const expected = opts.expectedHolder !== undefined && opts.expectedGeneration !== undefined
+    ? { holder: opts.expectedHolder, generation: opts.expectedGeneration } : undefined;
   const result = mutate(r, (record, now) => action === 'acquire' ? decideAcquire(record, r.me, now, r.ttl)
-    : action === 'renew' ? decideRenew(record, r.me, local.generation, now) : decideRelease(record, r.me));
+    : action === 'renew' ? decideRenew(record, r.me, local.generation, now) : decideRelease(record, r.me, expected));
   noteSeen(r, result.record);
   if (result.ok && action !== 'release') writeLocal(r.localPath, { holder: result.record.holder, generation: result.record.generation, confirmedAt: r.now(), ttlSeconds: result.record.ttlSeconds });
   if (result.ok && action === 'release') writeLocal(r.localPath, { ...local, holder: undefined, confirmedAt: undefined });

package/src/hq/lease.ts

@@ -39,8 +39,8 @@ export const leaseExpired = (record: LeaseRecord, now: number): boolean => recor
 /** acquire: only when there is no lease or it has expired; a new holder gets the next generation. */
 export function decideAcquire(record: LeaseRecord | null, me: string, now: number, ttlSeconds = DEFAULT_TTL_SECONDS):
   { ok: true; next: LeaseRecord } | { ok: false; reason: string } {
-  if (record && record.holder === me && !leaseExpired(record, now)) return { ok: true, next: { ...record, renewedAt: now, ttlSeconds } };
-  if (record && !leaseExpired(record, now)) return { ok: false, reason: `held by ${record.holder} (generation ${record.generation})` };
+  if (record && record.renewedAt !== 0 && record.holder === me && !leaseExpired(record, now)) return { ok: true, next: { ...record, renewedAt: now, ttlSeconds } };
+  if (record && record.renewedAt !== 0 && !leaseExpired(record, now)) return { ok: false, reason: `held by ${record.holder} (generation ${record.generation})` };
   return { ok: true, next: { holder: me, generation: (record?.generation ?? 0) + 1, acquiredAt: now, renewedAt: now, ttlSeconds, views: {}, dualUnreachableStreak: 0 } };
 }
 
@@ -52,8 +52,10 @@ export function decideRenew(record: LeaseRecord | null, me: string, generation:
   return { ok: true, next: { ...record, renewedAt: now, dualUnreachableStreak: 0 } };
 }
 
-export function decideRelease(record: LeaseRecord | null, me: string): { ok: true; next: LeaseRecord } | { ok: false; reason: string } {
+export function decideRelease(record: LeaseRecord | null, me: string, expected?: { holder: string; generation: number }): { ok: true; next: LeaseRecord } | { ok: false; reason: string } {
   if (!record || record.holder !== me) return { ok: false, reason: record ? `held by ${record.holder}` : 'no lease' };
+  if (expected && (record.holder !== expected.holder || record.generation !== expected.generation))
+    return { ok: false, reason: `lease changed: expected ${expected.holder} generation ${expected.generation}, observed ${record.holder} generation ${record.generation}` };
   return { ok: true, next: { ...record, renewedAt: 0 } };
 }
 

package/src/hq/transfer.ts

@@ -0,0 +1,185 @@
+import { appendFileSync, mkdirSync, readSync } from 'node:fs';
+import { join } from 'node:path';
+import { randomUUID } from 'node:crypto';
+import { getElanousConfigDir } from '../elanous-config-dir.js';
+import { getUserConfig } from '../user-config.js';
+import { defaultSshRunner, localShellRunner, parseLease, sshLeaseStore, type LeaseStore, type SshRunner } from './lease.js';
+import { promoteHq, type PromoteResult } from './promote.js';
+
+type Direction = 'promote' | 'handback';
+export interface TransferStep { step: string; status: 'done' | 'failed' | 'ready'; measurement: string }
+export interface TransferResult { ok: boolean; direction: Direction; from: string; to: string; runId: string; journal: string | null; steps: TransferStep[] }
+export interface TransferDeps {
+  store: LeaseStore;
+  run: SshRunner;
+  confirm: (question: string) => boolean;
+  promote: (host: string) => PromoteResult;
+  journalDir: string;
+}
+
+const aliases: Record<string, string[]> = { mbp: ['mbp', 'MacBookProM5'], node-b: ['node-b', 'MacStudioB1'] };
+const valid = (alias: string, holder: string) => aliases[alias]?.includes(holder) ?? false;
+const leaseCommand = (action: 'release' | 'acquire', expected?: { holder: string; generation: number }) =>
+  `elanous hq lease ${action}${expected ? ` --expected-holder '${expected.holder}' --expected-generation ${expected.generation}` : ''} --json`;
+const identityCommand = `python3 - <<'PY'
+import json, pathlib
+home=pathlib.Path.home()
+local=home/'.elanous-hq/host'
+config=home/'.elanous-hqcfg/config.json'
+name=local.read_text().strip() if local.exists() else json.loads(config.read_text())['hq']['hostName']
+print(name)
+PY`;
+const cronCommand = `crontab -l && printf '\\n---HQ-AUDIT---\\n' && elanous hq fence-audit --json`;
+const fenceCommand = `elanous hq fence --role cron -- sh -c 'printf "HQ_FENCE_ALLOWED:%s\\n" "$ELANOUS_HQ_GENERATION"'`;
+
+export function defaultTransferDeps(): TransferDeps {
+  const arbiter = getUserConfig().hq?.arbiter ?? 'cloud-vm';
+  return {
+    store: sshLeaseStore(arbiter, arbiter === 'local' ? localShellRunner : defaultSshRunner),
+    run: defaultSshRunner,
+    confirm: (question) => {
+      if (!process.stdin.isTTY) return false;
+      process.stderr.write(`${question}\nType TRANSFER to continue: `);
+      const buffer = Buffer.alloc(128);
+      const count = requireRead(buffer);
+      return buffer.toString('utf8', 0, count).trim() === 'TRANSFER';
+    },
+    promote: (host) => promoteHq(host, true),
+    journalDir: join(getElanousConfigDir(), 'hq', 'transfers'),
+  };
+}
+
+function requireRead(buffer: Buffer): number {
+  // Read one terminal line without treating EOF or a non-interactive caller as consent.
+  let size = 0;
+  while (size < buffer.length) {
+    const n = requireReadByte(buffer, size);
+    if (n === 0) break;
+    size += n;
+    if (buffer[size - 1] === 10) break;
+  }
+  return size;
+}
+
+function requireReadByte(buffer: Buffer, offset: number): number {
+  return readSync(0, buffer, offset, 1, null);
+}
+
+export function transferHq(direction: Direction, apply = false, deps: TransferDeps = defaultTransferDeps()): TransferResult {
+  const from = direction === 'promote' ? 'mbp' : 'node-b';
+  const to = direction === 'promote' ? 'node-b' : 'mbp';
+  const runId = randomUUID();
+  const journal = apply ? join(deps.journalDir, `${runId}.jsonl`) : null;
+  const steps: TransferStep[] = [];
+  const result = (ok: boolean): TransferResult => ({ ok, direction, from, to, runId, journal, steps });
+  const record = (step: string, status: TransferStep['status'], measurement: string) => {
+    const row = { step, status, measurement };
+    steps.push(row);
+    if (journal) {
+      mkdirSync(deps.journalDir, { recursive: true, mode: 0o700 });
+      appendFileSync(journal, `${JSON.stringify({ at: new Date().toISOString(), runId, direction, from, to, ...row })}\n`, { mode: 0o600 });
+    }
+    return status !== 'failed';
+  };
+  const lease = () => {
+    const { now, raw } = deps.store.read();
+    const current = parseLease(raw);
+    return { current, now };
+  };
+  const remote = (host: string, command: string) => {
+    const r = deps.run(host, command);
+    if (r.status !== 0) throw new Error(`${host}: ${r.stderr.trim() || r.stdout.trim() || `exit=${r.status}`}`);
+    return r.stdout.trim();
+  };
+  try {
+    const before = lease();
+    if (!before.current || !valid(from, before.current.holder) || before.current.renewedAt + before.current.ttlSeconds < before.now) {
+      record('① current lease', 'failed', `holder=${before.current?.holder ?? 'none'} generation=${before.current?.generation ?? 'none'} expired=${before.current ? before.current.renewedAt + before.current.ttlSeconds < before.now : true}`);
+      return result(false);
+    }
+    const source = remote(from, identityCommand);
+    const target = remote(to, identityCommand);
+    if (source !== before.current.holder || !valid(to, target) || source === target) {
+      record('② host identities', 'failed', `source=${source} holder=${before.current.holder} target=${target}`);
+      return result(false);
+    }
+    record('① current lease', 'done', `holder=${source} generation=${before.current.generation}`);
+    record('② host identities', 'done', `source=${source} target=${target}`);
+    for (const host of [from, to]) {
+      const inventory = remote(host, cronCommand);
+      const marker = '\n---HQ-AUDIT---\n';
+      const index = inventory.lastIndexOf(marker);
+      if (index < 0) throw new Error(`${host}: missing cron audit`);
+      const cron = inventory.slice(0, index);
+      const audit = JSON.parse(inventory.slice(index + marker.length)) as unknown;
+      const live = cron.split('\n').filter(line => line.trim() && !line.trimStart().startsWith('#'));
+      if (!Array.isArray(audit) || audit.length > 0 || !live.some(line => /hq heartbeat\b/.test(line)) || !live.some(line => /hq fence --role cron\b/.test(line))) {
+        record(`③ cron fence ${host}`, 'failed', `unfenced=${Array.isArray(audit) ? audit.length : 'invalid'} heartbeat=${live.some(line => /hq heartbeat\b/.test(line))} fenced=${live.some(line => /hq fence --role cron\b/.test(line))}`);
+        return result(false);
+      }
+      record(`③ cron fence ${host}`, 'done', `unfenced=0 heartbeat=true fenced=true`);
+    }
+    const oldFence = remote(from, fenceCommand);
+    const newFence = remote(to, fenceCommand);
+    if (oldFence !== `HQ_FENCE_ALLOWED:${before.current.generation}` || newFence.includes('HQ_FENCE_ALLOWED:')) {
+      record('④ pre-transfer fence', 'failed', `source=${oldFence} target=${newFence}`);
+      return result(false);
+    }
+    record('④ pre-transfer fence', 'done', `source=allowed generation=${before.current.generation} target=skipped`);
+    if (!apply) {
+      record('⑤ human confirmation', 'ready', 'dry-run: no writes; --apply requires interactive TRANSFER confirmation');
+      return result(true);
+    }
+    if (!deps.confirm(`Release ${source} generation ${before.current.generation}, acquire ${target}, and switch cron fence? This cannot undo work already sent.`)) {
+      record('⑤ human confirmation', 'failed', 'confirmation declined');
+      return result(false);
+    }
+    record('⑤ human confirmation', 'done', 'operator typed TRANSFER');
+    const still = lease().current;
+    if (!still || still.holder !== source || still.generation !== before.current.generation || still.renewedAt + still.ttlSeconds < deps.store.read().now) {
+      record('⑥ release old lease', 'failed', 'lease changed before release');
+      return result(false);
+    }
+    let released: { ok?: boolean; record?: { holder?: string; generation?: number } };
+    try { released = JSON.parse(remote(from, leaseCommand('release', { holder: source, generation: before.current.generation }))); }
+    catch (error) {
+      record('⑥ release old lease', 'failed', error instanceof Error ? error.message : String(error));
+      return result(false);
+    }
+    if (released.ok !== true || released.record?.holder !== source || released.record.generation !== before.current.generation) {
+      record('⑥ release old lease', 'failed', 'release not confirmed');
+      return result(false);
+    }
+    record('⑥ release old lease', 'done', `holder=${source} generation=${released.record.generation} released=true`);
+    const acquired = JSON.parse(remote(to, leaseCommand('acquire'))) as { ok?: boolean; record?: { holder?: string; generation?: number } };
+    const current = lease().current;
+    if (acquired.ok !== true || acquired.record?.holder !== target || acquired.record.generation !== before.current.generation + 1 || current?.holder !== target || current.generation !== acquired.record.generation) {
+      record('⑦ acquire new lease', 'failed', `expected=${target} generation=${before.current.generation + 1} observed=${current?.holder ?? 'none'} generation=${current?.generation ?? 'none'}; keep old host fenced, recover manually`);
+      return result(false);
+    }
+    record('⑦ acquire new lease', 'done', `holder=${target} generation=${current.generation}`);
+    if (direction === 'promote') {
+      const promoted = deps.promote(to);
+      for (const line of promoted.lines) record(`⑧ promotion ${line.step}`, line.status, line.measurement);
+      if (!promoted.ok) return result(false);
+    }
+    const sourceAfter = remote(from, fenceCommand);
+    const targetAfter = remote(to, fenceCommand);
+    const observed = lease().current;
+    if (sourceAfter.includes('HQ_FENCE_ALLOWED:') || targetAfter !== `HQ_FENCE_ALLOWED:${current.generation}` || observed?.holder !== target || observed.generation !== current.generation) {
+      record('⑨ final fence', 'failed', `source=${sourceAfter} target=${targetAfter} lease=${observed?.holder ?? 'none'} generation=${observed?.generation ?? 'none'}`);
+      return result(false);
+    }
+    record('⑨ final fence', 'done', `source=skipped target=allowed generation=${current.generation}`);
+    return result(true);
+  } catch (error) {
+    record('halt', 'failed', error instanceof Error ? error.message : String(error));
+    return result(false);
+  }
+}
+
+export function formatTransfer(result: TransferResult): string {
+  return [`hq ${result.direction}: ${result.ok ? 'ok' : 'failed'} · ${result.from} → ${result.to} · run=${result.runId}`,
+    ...result.steps.map(s => `${s.step}: ${s.status} · ${s.measurement}`),
+    ...(result.journal ? [`journal=${result.journal}`] : [])].join('\n');
+}