elanous 0.2.12 → 0.2.13

package/src/hq/hq.ts

@@ -0,0 +1,318 @@
+// HQ lease operations — CLI-facing glue over the pure decisions in ./lease.ts (HQ-HB · HQ-FENCE · 10-04).
+// Every outcome leaves one observation line (hq.lease / hq.fence) so the 12:00 drill is measurable from logs.
+import { chmodSync, existsSync, lstatSync, mkdirSync, mkdtempSync, readFileSync, renameSync, rmSync, writeFileSync } from 'node:fs';
+import { homedir, hostname } from 'node:os';
+import { dirname, join } from 'node:path';
+import { spawnSync } from 'node:child_process';
+import { debug } from '../debug/log.js';
+import { DEFAULT_NEXUS_HTTP_PORT } from '../nexus/default-port.js';
+import { getElanousConfigDir } from '../elanous-config-dir.js';
+import { getUserConfig, type HqConfig, type HqFenceRole } from '../user-config.js';
+import {
+  DEFAULT_TTL_SECONDS, decideAcquire, decideArbiterCheck, decideFence, decideRelease, decideRenew, defaultSshRunner,
+  fileLeaseStore, leaseExpired, parseLease, probeReachable, recordView, serializeLease, sshLeaseStore, sshReachable, localShellRunner,
+  type HostProbe, type LeaseRecord, type LeaseStore, type LocalHqState, type SshRunner,
+} from './lease.js';
+
+export const FENCE_ROLES: readonly HqFenceRole[] = ['telegram-poller', 'cron', 'seat-loop', 'release-run', 'conatus', 'git-push', 'ledger-cli'];
+
+export interface HqDeps {
+  config?: HqConfig;
+  store?: LeaseStore;
+  ssh?: SshRunner;
+  /** Holder/standby views and the arbiter's check use this tailnet probe — never ssh (OP 10-04 09:04). */
+  probe?: HostProbe;
+  localPath?: string;
+  /** Host-local identity, deliberately outside the replicated config/state directory. */
+  hostPath?: string;
+  /** Seen-generation marker (default hq.seenGenerationFile · ~/.elanous-hq/seen-generation). */
+  seenPath?: string;
+  now?: () => number;
+  log?: typeof debug.log;
+}
+
+interface Resolved { me: string; arbiter: string; standby: string; ttl: number; failOpen: Set<HqFenceRole>; store: LeaseStore; ssh: SshRunner; probe: HostProbe; localPath: string; seenPath: string; now: () => number; log: typeof debug.log }
+const defaultHostPath = () => join(homedir(), '.elanous-hq', 'host');
+
+function readHost(path: string): string | undefined {
+  try {
+    lstatSync(path);
+  } catch (error) {
+    if ((error as NodeJS.ErrnoException).code === 'ENOENT') return undefined;
+    throw error;
+  }
+  const text = readFileSync(path, 'utf8');
+  const name = text.endsWith('\n') ? text.slice(0, -1) : text;
+  if (!name || name !== name.trim() || /[\r\n]/.test(name)) {
+    throw new Error(`hq host: invalid host file: ${path}`);
+  }
+  return name;
+}
+
+/** Set this machine's identity outside the replicable config.json and HQ state universe. */
+export function hqHostSet(name: string, path = defaultHostPath()): string {
+  if (!name || name !== name.trim() || /[\r\n]/.test(name)) throw new Error('hq host set: name must be one nonempty line without surrounding whitespace');
+  mkdirSync(dirname(path), { recursive: true, mode: 0o700 });
+  const tempDir = mkdtempSync(`${path}.`);
+  try {
+    chmodSync(tempDir, 0o700);
+    const tmp = join(tempDir, 'host');
+    writeFileSync(tmp, `${name}\n`, { flag: 'wx', mode: 0o600 });
+    chmodSync(tmp, 0o600);
+    renameSync(tmp, path);
+  } finally {
+    rmSync(tempDir, { recursive: true, force: true });
+  }
+  return path;
+}
+
+function resolve(deps: HqDeps = {}): Resolved {
+  const config = deps.config ?? getUserConfig().hq ?? {};
+  const ssh = deps.ssh ?? defaultSshRunner;
+  const arbiter = config.arbiter ?? 'cloud-vm';
+  const local = readHost(deps.hostPath ?? defaultHostPath());
+  const me = local ?? config.hostName ?? hostname().replace(/\.local$/, '');
+  const log = deps.log ?? debug.log.bind(debug);
+  if (local && config.hostName && local !== config.hostName) {
+    try { log('hq.lease', 'hostname-mismatch', { local, config: config.hostName }); } catch { /* observation is fail-soft */ }
+  }
+  return {
+    me,
+    arbiter,
+    standby: config.standby ?? 'node-b',
+    ttl: config.ttlSeconds ?? DEFAULT_TTL_SECONDS,
+    failOpen: new Set(config.failOpenRoles ?? []),
+    store: deps.store ?? sshLeaseStore(arbiter, arbiter === 'local' ? localShellRunner : ssh),
+    ssh,
+    probe: deps.probe ?? tailnetProbe(config),
+    localPath: deps.localPath ?? join(getElanousConfigDir(), 'hq', 'local.json'),
+    seenPath: deps.seenPath ?? config.seenGenerationFile ?? (deps.localPath ? `${deps.localPath}.seen-generation` : join(homedir(), '.elanous-hq', 'seen-generation')),
+    now: deps.now ?? (() => Math.floor(Date.now() / 1000)),
+    log,
+  };
+}
+
+/** MagicDNS suffix from `tailscale status --json` (cached per process). */
+let dnsSuffix: string | null | undefined;
+function magicDnsSuffix(bin: string): string | undefined {
+  if (dnsSuffix !== undefined) return dnsSuffix ?? undefined;
+  const r = spawnSync(bin, ['status', '--json'], { encoding: 'utf8', timeout: 10_000 });
+  try { dnsSuffix = (JSON.parse(r.stdout || '{}') as { MagicDNSSuffix?: string }).MagicDNSSuffix?.replace(/\.$/, '') || null; } catch { dnsSuffix = null; }
+  return dnsSuffix ?? undefined;
+}
+
+/** Health URL of a remote host's nexus over the tailnet — its `tailscale serve` TLS mapping of the default nexus port
+ *  (resolveDaemonEndpoint only knows this machine's daemon; the arbiter probes another host). */
+export function healthUrlFor(host: string, config: HqConfig, suffix: () => string | undefined): string | undefined {
+  const pinned = config.healthUrls?.[host];
+  if (pinned) return pinned;
+  const domain = config.tailnetDomain ?? suffix();
+  return domain ? `https://${host}.${domain}:${DEFAULT_NEXUS_HTTP_PORT}/v1/health` : undefined;
+}
+
+/** Default probe: `tailscale ping -c 1` ⊕ curl of the nexus health (200 only). Both are tailnet-only, no ssh. */
+export function tailnetProbe(config: HqConfig): HostProbe {
+  const bin = config.tailscaleBin ?? 'tailscale';
+  const timeout = config.probeTimeoutSeconds ?? 5;
+  return {
+    ping(host) {
+      const r = spawnSync(bin, ['ping', '-c', '1', '--timeout', `${timeout}s`, host], { encoding: 'utf8', timeout: (timeout + 5) * 1000 });
+      return r.status === 0 && /\bpong\b/.test(r.stdout ?? '');
+    },
+    health(host) {
+      const url = healthUrlFor(host, config, () => magicDnsSuffix(bin));
+      if (!url) return false;
+      const r = spawnSync('curl', ['-s', '-o', '/dev/null', '-m', String(timeout), '-w', '%{http_code}', url], { encoding: 'utf8', timeout: (timeout + 5) * 1000 });
+      return (r.stdout ?? '').trim() === '200';
+    },
+  };
+}
+
+export function readLocal(path: string): LocalHqState {
+  try { return existsSync(path) ? JSON.parse(readFileSync(path, 'utf8')) as LocalHqState : {}; } catch { return {}; }
+}
+function writeLocal(path: string, state: LocalHqState): void {
+  mkdirSync(dirname(path), { recursive: true });
+  const tmp = `${path}.${process.pid}.tmp`;
+  writeFileSync(tmp, `${JSON.stringify(state)}\n`, { mode: 0o600 });
+  renameSync(tmp, path);
+}
+
+/** Highest lease generation this host ever observed (null = never). */
+export function readSeenGeneration(path: string): number | null {
+  try {
+    if (!existsSync(path)) return null;
+    const n = Number(readFileSync(path, 'utf8').trim());
+    return Number.isSafeInteger(n) && n > 0 ? n : null;
+  } catch { return null; }
+}
+/** Keep the max generation seen (atomic · 600). Fail-soft: a lost write never blocks the lease operation. */
+function noteSeen(r: Resolved, record: LeaseRecord | null | undefined): void {
+  if (!record || !Number.isSafeInteger(record.generation) || record.generation < 1) return;
+  try {
+    const prev = readSeenGeneration(r.seenPath);
+    if (prev !== null && prev >= record.generation) return;
+    mkdirSync(dirname(r.seenPath), { recursive: true, mode: 0o700 });
+    const tmp = `${r.seenPath}.${process.pid}.tmp`;
+    writeFileSync(tmp, `${record.generation}\n`, { mode: 0o600 });
+    renameSync(tmp, r.seenPath);
+  } catch (error) { observe(r, 'hq.lease', 'seen-write-failed', { error: String(error).slice(0, 200) }); }
+}
+
+/** `elanous hq seen` — wrappers use «never seen» (exit 3) to allow the pre-drill default-source path only. */
+export function hqSeen(deps: HqDeps = {}): { seen: boolean; generation: number | null; path: string } {
+  const r = resolve(deps);
+  const generation = readSeenGeneration(r.seenPath);
+  return { seen: generation !== null, generation, path: r.seenPath };
+}
+
+function observe(r: Resolved, category: 'hq.lease' | 'hq.fence', event: string, data: Record<string, unknown>): void {
+  try { r.log(category, event, { host: r.me, ...data }); } catch { /* observation is fail-soft */ }
+}
+
+/** Read → decide → CAS, retried on a lost race (two hosts racing cannot both win: only one CAS sees the old bytes). */
+function mutate(r: Resolved, decide: (record: LeaseRecord | null, now: number) => { ok: true; next: LeaseRecord } | { ok: false; reason: string }):
+  { ok: true; record: LeaseRecord } | { ok: false; reason: string; record: LeaseRecord | null } {
+  for (let attempt = 0; attempt < 5; attempt++) {
+    const { now, raw } = r.store.read();
+    const record = parseLease(raw);
+    const decided = decide(record, now);
+    if (!decided.ok) return { ok: false, reason: decided.reason, record };
+    if (r.store.cas(raw, serializeLease(decided.next))) return { ok: true, record: decided.next };
+  }
+  return { ok: false, reason: 'lost the race five times', record: null };
+}
+
+export type LeaseAction = 'acquire' | 'renew' | 'status' | 'release';
+export function hqLease(action: LeaseAction, deps: HqDeps = {}, opts: { host?: string } = {}) {
+  const r0 = resolve(deps);
+  const r = opts.host ? { ...r0, me: opts.host } : r0;
+  if (action === 'status') {
+    const { now, raw } = r.store.read();
+    const record = parseLease(raw);
+    noteSeen(r, record);
+    return { ok: true as const, action, record, ageSeconds: record ? now - record.renewedAt : null, expired: record ? leaseExpired(record, now) : null };
+  }
+  const local = readLocal(r.localPath);
+  const result = mutate(r, (record, now) => action === 'acquire' ? decideAcquire(record, r.me, now, r.ttl)
+    : action === 'renew' ? decideRenew(record, r.me, local.generation, now) : decideRelease(record, r.me));
+  noteSeen(r, result.record);
+  if (result.ok && action !== 'release') writeLocal(r.localPath, { holder: result.record.holder, generation: result.record.generation, confirmedAt: r.now(), ttlSeconds: result.record.ttlSeconds });
+  if (result.ok && action === 'release') writeLocal(r.localPath, { ...local, holder: undefined, confirmedAt: undefined });
+  observe(r, 'hq.lease', result.ok ? (action === 'acquire' ? 'acquired' : action === 'renew' ? 'renewed' : 'released') : `${action}-refused`,
+    result.ok ? { generation: result.record.generation } : { reason: result.reason });
+  return { action, ...result };
+}
+
+/**
+ * Heartbeat (every 10 min on mbp and node-b). Holder → renew at the arbiter, or (arbiter down) confirm quorum through the standby.
+ * Non-holder → report its view of the holder to the arbiter (rule ②) and adopt a generation the arbiter granted it.
+ */
+export function hqHeartbeat(deps: HqDeps = {}) {
+  const r = resolve(deps);
+  const local = readLocal(r.localPath);
+  let read: { now: number; raw: string | null };
+  try { read = r.store.read(); } catch (error) {
+    const standbyUp = local.holder === r.me ? sshReachable(r.standby, r.ssh) : false;
+    if (local.holder === r.me && standbyUp) writeLocal(r.localPath, { ...local, confirmedAt: r.now() });
+    const outcome = local.holder === r.me ? (standbyUp ? 'quorum-standby' : 'arbiter-unreachable') : 'arbiter-unreachable';
+    observe(r, 'hq.lease', outcome, { error: String(error).slice(0, 200), standbyReachable: standbyUp });
+    return { outcome };
+  }
+  const record = parseLease(read.raw);
+  noteSeen(r, record);
+  if (!record) { observe(r, 'hq.lease', 'no-lease', { seenGeneration: readSeenGeneration(r.seenPath) }); return { outcome: 'no-lease' as const }; }
+  if (record.holder === r.me) {
+    // The record names me as holder: a generation the arbiter granted me (promotion) is adopted, never refused.
+    const renewed = mutate(r, (current, now) => decideRenew(current, r.me, Math.max(local.generation ?? 0, current?.generation ?? 0), now));
+    if (renewed.ok && (local.generation ?? 0) < renewed.record.generation) observe(r, 'hq.lease', 'generation-adopted', { from: local.generation ?? null, to: renewed.record.generation });
+    if (renewed.ok) {
+      noteSeen(r, renewed.record);
+      writeLocal(r.localPath, { holder: r.me, generation: renewed.record.generation, confirmedAt: r.now(), ttlSeconds: renewed.record.ttlSeconds });
+      observe(r, 'hq.lease', 'renewed', { generation: renewed.record.generation });
+      return { outcome: 'renewed' as const, generation: renewed.record.generation };
+    }
+    observe(r, 'hq.lease', 'renew-refused', { reason: renewed.reason });
+    return { outcome: 'renew-refused' as const, reason: renewed.reason };
+  }
+  // Not the holder: report the view, keep the granted generation, stay standby.
+  const seen = probeReachable(record.holder, r.probe);
+  const reachable = seen.reachable;
+  mutate(r, (current) => current ? { ok: true, next: recordView(current, r.me, reachable, read.now) } : { ok: false, reason: 'no lease' });
+  if (leaseExpired(record, read.now)) observe(r, 'hq.lease', 'expired-seen', { holder: record.holder, generation: record.generation });
+  writeLocal(r.localPath, { holder: record.holder, generation: record.generation, ttlSeconds: record.ttlSeconds });
+  observe(r, 'hq.lease', 'standby', { holder: record.holder, generation: record.generation, holderReachable: reachable, ping: seen.ping, health: seen.health });
+  return { outcome: 'standby' as const, holder: record.holder, holderReachable: reachable };
+}
+
+/**
+ * Arbiter check (cron on cloud-vm every 10 min, next to the file): probe the holder over the tailnet
+ * (`tailscale ping` ⊕ nexus health · no ssh into home machines) and promote per rule ②.
+ */
+export function hqArbiterCheck(deps: HqDeps & { leasePath?: string } = {}) {
+  const r0 = resolve(deps);
+  const r = deps.store ? r0 : { ...r0, store: fileLeaseStore(deps.leasePath ?? join(process.env.HOME ?? '', '.elanous-hq', 'lease.json')) };
+  let outcome: { promoted: boolean; streak: number; holder?: string; generation?: number; ping?: boolean; health?: boolean } = { promoted: false, streak: 0 };
+  const result = mutate(r, (record, now) => {
+    if (!record) return { ok: false, reason: 'no lease' };
+    const seen = probeReachable(record.holder, r.probe);
+    const checked = decideArbiterCheck(record, { arbiter: r.me, standby: r.standby, arbiterReachesHolder: seen.reachable, now });
+    outcome = { promoted: checked.promoted, streak: checked.streak, holder: checked.next.holder, generation: checked.next.generation, ping: seen.ping, health: seen.health };
+    return { ok: true, next: checked.next };
+  });
+  observe(r, 'hq.lease', outcome.promoted ? 'promoted' : 'arbiter-check', { ...outcome, ok: result.ok });
+  return { ok: result.ok, ...outcome };
+}
+
+/** Standby's own generation, asked over ssh (rule ③ when the arbiter is down). */
+function standbyGeneration(r: Resolved): number | undefined {
+  const res = r.ssh(r.standby, 'cat "$HOME/.elanous/hq/local.json" 2>/dev/null || true');
+  try { const g = (JSON.parse(res.stdout || '{}') as LocalHqState).generation; return typeof g === 'number' ? g : undefined; } catch { return undefined; }
+}
+
+export function hqFenceDecision(role: HqFenceRole, deps: HqDeps = {}) {
+  const r = resolve(deps);
+  const local = readLocal(r.localPath);
+  let record: LeaseRecord | null | 'unreachable';
+  try { record = parseLease(r.store.read().raw); } catch { record = 'unreachable'; }
+  if (record !== 'unreachable') noteSeen(r, record);
+  const seenGeneration = readSeenGeneration(r.seenPath);
+  const standbyUp = record === 'unreachable' && local.holder === r.me ? sshReachable(r.standby, r.ssh) : undefined;
+  const decided = decideFence({
+    me: r.me, now: r.now(), local, record,
+    ...(standbyUp !== undefined ? { standbyReachable: standbyUp } : {}),
+    ...(standbyUp ? { standbyGeneration: standbyGeneration(r) } : {}),
+    failOpen: role !== 'ledger-cli' && r.failOpen.has(role),
+    ...(seenGeneration !== null ? { seenGeneration } : {}),
+  });
+  if (decided.run && record !== 'unreachable' && record) writeLocal(r.localPath, { holder: r.me, generation: record.generation, confirmedAt: r.now(), ttlSeconds: record.ttlSeconds });
+  observe(r, 'hq.fence', decided.run ? 'allowed' : 'skipped', { role, reason: decided.reason, holder: decided.holder, generation: decided.generation });
+  return decided;
+}
+
+/** CLI ledger writes fail closed after the first lease; bootstrap is allowed only before any generation was seen. */
+export function hqCliWriteAllowed(command: string, override = false, deps: HqDeps = {}): boolean {
+  const decision = hqFenceDecision('ledger-cli', deps);
+  const allowed = decision.run || decision.reason === 'no-lease';
+  const r = resolve(deps);
+  if (override) {
+    observe(r, 'hq.fence', 'cli-override', { command, reason: decision.reason, holder: decision.holder, generation: decision.generation, allowed });
+    console.error(`hq override: ${command} — ${decision.reason} (holder ${decision.holder ?? 'unknown'} gen ${decision.generation ?? 'unknown'})`);
+    return true;
+  }
+  if (allowed) return true;
+  observe(r, 'hq.fence', 'cli-refused', { command, reason: decision.reason, holder: decision.holder, generation: decision.generation });
+  console.error(`본부는 ${decision.holder ?? 'unknown'} gen ${decision.generation ?? 'unknown'} — 거기서 쓰거나 --hq-override(관측)`);
+  process.exitCode = 4;
+  return false;
+}
+
+/** Run `command` only when this host may write; the child sees ELANOUS_HQ_GENERATION. Skips exit 0. */
+export function hqFenceRun(role: HqFenceRole, command: string[], deps: HqDeps = {}): number {
+  const decided = hqFenceDecision(role, deps);
+  if (!decided.run) { console.error(`hq fence: skip ${role} — ${decided.reason}${decided.holder ? ` (holder ${decided.holder} gen ${decided.generation})` : ''}`); return 0; }
+  const [cmd, ...args] = command;
+  if (!cmd) throw new Error('hq fence: command required after --');
+  const r = spawnSync(cmd, args, { stdio: 'inherit', env: { ...process.env, ELANOUS_HQ_GENERATION: String(decided.generation ?? '') } });
+  return r.status ?? 1;
+}

package/src/hq/lease.ts

@@ -0,0 +1,210 @@
+// HQ lease (본부 임대) — single writer across mbp · node-b · cloud-vm with a 2-of-3 quorum (OP 10-04 08:33 · 08:40).
+// The arbiter (cloud-vm) keeps one record; only the arbiter promotes the standby, and only after two consecutive
+// checks in which neither the arbiter nor the standby can reach the holder. Fenced work carries the generation.
+import { createHash } from 'node:crypto';
+import { existsSync, mkdirSync, readFileSync, renameSync, rmdirSync, statSync, writeFileSync } from 'node:fs';
+import { dirname } from 'node:path';
+import { spawnSync } from 'node:child_process';
+
+export const DEFAULT_TTL_SECONDS = 1500;
+/** A standby view older than this is not evidence (two heartbeats at the 10-minute cadence). */
+export const VIEW_FRESH_SECONDS = 1500;
+/** Consecutive dual-unreachable checks before the arbiter promotes the standby. */
+export const PROMOTE_AFTER_CHECKS = 2;
+
+export interface HostView { target: string; reachable: boolean; at: number }
+export interface LeaseRecord {
+  holder: string;
+  generation: number;
+  acquiredAt: number;
+  renewedAt: number;
+  ttlSeconds: number;
+  /** Each observer's latest view of the holder (arbiter and standby). */
+  views?: Record<string, HostView>;
+  /** Consecutive arbiter checks in which both arbiter and standby could not reach the holder. */
+  dualUnreachableStreak?: number;
+  lastCheckAt?: number;
+  promotedFrom?: string;
+}
+
+export function parseLease(raw: string | null): LeaseRecord | null {
+  if (!raw || !raw.trim()) return null;
+  const data = JSON.parse(raw) as LeaseRecord;
+  if (typeof data.holder !== 'string' || !Number.isSafeInteger(data.generation) || typeof data.renewedAt !== 'number') throw new Error('invalid lease record');
+  return data;
+}
+export const serializeLease = (record: LeaseRecord): string => `${JSON.stringify(record)}\n`;
+export const leaseExpired = (record: LeaseRecord, now: number): boolean => record.renewedAt + record.ttlSeconds < now;
+
+/** acquire: only when there is no lease or it has expired; a new holder gets the next generation. */
+export function decideAcquire(record: LeaseRecord | null, me: string, now: number, ttlSeconds = DEFAULT_TTL_SECONDS):
+  { ok: true; next: LeaseRecord } | { ok: false; reason: string } {
+  if (record && record.holder === me && !leaseExpired(record, now)) return { ok: true, next: { ...record, renewedAt: now, ttlSeconds } };
+  if (record && !leaseExpired(record, now)) return { ok: false, reason: `held by ${record.holder} (generation ${record.generation})` };
+  return { ok: true, next: { holder: me, generation: (record?.generation ?? 0) + 1, acquiredAt: now, renewedAt: now, ttlSeconds, views: {}, dualUnreachableStreak: 0 } };
+}
+
+export function decideRenew(record: LeaseRecord | null, me: string, generation: number | undefined, now: number):
+  { ok: true; next: LeaseRecord } | { ok: false; reason: string } {
+  if (!record) return { ok: false, reason: 'no lease' };
+  if (record.holder !== me) return { ok: false, reason: `held by ${record.holder} (generation ${record.generation})` };
+  if (generation !== undefined && generation < record.generation) return { ok: false, reason: `stale generation ${generation} < ${record.generation}` };
+  return { ok: true, next: { ...record, renewedAt: now, dualUnreachableStreak: 0 } };
+}
+
+export function decideRelease(record: LeaseRecord | null, me: string): { ok: true; next: LeaseRecord } | { ok: false; reason: string } {
+  if (!record || record.holder !== me) return { ok: false, reason: record ? `held by ${record.holder}` : 'no lease' };
+  return { ok: true, next: { ...record, renewedAt: 0 } };
+}
+
+/** The standby reports whether it can reach the holder (rule ②: both views go into the record). */
+export function recordView(record: LeaseRecord, observer: string, reachable: boolean, now: number): LeaseRecord {
+  return { ...record, views: { ...(record.views ?? {}), [observer]: { target: record.holder, reachable, at: now } } };
+}
+
+/**
+ * Arbiter check (rule ②). Records the arbiter's own view; counts a dual-unreachable check only when the standby's
+ * fresh view of the same holder also says unreachable; promotes the standby after PROMOTE_AFTER_CHECKS in a row.
+ */
+export function decideArbiterCheck(record: LeaseRecord, input: { arbiter: string; standby: string; arbiterReachesHolder: boolean; now: number }):
+  { next: LeaseRecord; promoted: boolean; streak: number } {
+  let next = recordView(record, input.arbiter, input.arbiterReachesHolder, input.now);
+  const view = next.views?.[input.standby];
+  const standbyDown = !!view && view.target === record.holder && !view.reachable && input.now - view.at <= VIEW_FRESH_SECONDS;
+  const dual = !input.arbiterReachesHolder && standbyDown && record.holder !== input.standby;
+  const streak = dual ? (record.dualUnreachableStreak ?? 0) + 1 : 0;
+  next = { ...next, dualUnreachableStreak: streak, lastCheckAt: input.now };
+  if (streak < PROMOTE_AFTER_CHECKS) return { next, promoted: false, streak };
+  return {
+    next: { holder: input.standby, generation: record.generation + 1, acquiredAt: input.now, renewedAt: input.now, ttlSeconds: record.ttlSeconds, views: {}, dualUnreachableStreak: 0, lastCheckAt: input.now, promotedFrom: record.holder },
+    promoted: true, streak,
+  };
+}
+
+/** What this host remembers between runs: its last confirmed generation and when quorum last confirmed it. */
+export interface LocalHqState { holder?: string; generation?: number; confirmedAt?: number; ttlSeconds?: number }
+
+export type FenceReason = 'holder' | 'quorum-standby' | 'within-ttl' | 'not-holder' | 'stale-generation' | 'no-quorum-expired' | 'no-lease' | 'no-lease-after-seen' | 'fail-open';
+/**
+ * Fence (rules ① · ③). Arbiter reachable → run only as the current holder at the current generation.
+ * Arbiter unreachable → the holder keeps running while it reaches the standby or its own confirmation is within TTL;
+ * a standby that reports a higher generation always blocks (a returning old HQ stays standby).
+ */
+export function decideFence(input: {
+  me: string; now: number; local: LocalHqState;
+  record: LeaseRecord | null | 'unreachable';
+  standbyReachable?: boolean; standbyGeneration?: number; failOpen?: boolean;
+  /** Highest generation this host ever saw — a vanished record after that is «unknown», not «no lease yet». */
+  seenGeneration?: number;
+}): { run: boolean; reason: FenceReason; generation?: number; holder?: string } {
+  const { me, now, local, record } = input;
+  if (record !== 'unreachable') {
+    if (!record) return { run: false, reason: input.seenGeneration !== undefined ? 'no-lease-after-seen' : 'no-lease', ...(input.seenGeneration !== undefined ? { generation: input.seenGeneration } : {}) };
+    if (record.holder !== me) return { run: false, reason: 'not-holder', holder: record.holder, generation: record.generation };
+    if (local.generation !== undefined && local.generation > record.generation) return { run: false, reason: 'stale-generation', holder: record.holder, generation: record.generation };
+    return { run: true, reason: 'holder', generation: record.generation, holder: me };
+  }
+  if (local.holder !== me || local.generation === undefined) return { run: false, reason: 'not-holder', holder: local.holder, generation: local.generation };
+  if (input.standbyGeneration !== undefined && input.standbyGeneration > local.generation) return { run: false, reason: 'stale-generation', generation: input.standbyGeneration };
+  if (input.standbyReachable) return { run: true, reason: 'quorum-standby', generation: local.generation, holder: me };
+  const ttl = local.ttlSeconds ?? DEFAULT_TTL_SECONDS;
+  if (local.confirmedAt !== undefined && now - local.confirmedAt <= ttl) return { run: true, reason: 'within-ttl', generation: local.generation, holder: me };
+  if (input.failOpen) return { run: true, reason: 'fail-open', generation: local.generation, holder: me };
+  return { run: false, reason: 'no-quorum-expired', generation: local.generation, holder: me };
+}
+
+// ── stores ────────────────────────────────────────────────────────────────────
+export interface LeaseStore {
+  /** null raw = no record · throws when the store cannot be reached. */
+  read(): { now: number; raw: string | null };
+  /** Compare-and-swap on the raw bytes; false when someone else wrote first. */
+  cas(expected: string | null, next: string): boolean;
+}
+export const sha = (raw: string | null): string => raw === null ? 'none' : createHash('sha256').update(raw).digest('hex');
+
+/** Arbiter-local store (the arbiter's own check runs next to the file). mkdir is the lock. */
+export function fileLeaseStore(path: string, clock: () => number = () => Math.floor(Date.now() / 1000)): LeaseStore {
+  const lock = `${path}.lock`;
+  return {
+    read: () => ({ now: clock(), raw: existsSync(path) ? readFileSync(path, 'utf8') : null }),
+    cas(expected, next) {
+      mkdirSync(dirname(path), { recursive: true });
+      for (let i = 0; ; i++) {
+        try { mkdirSync(lock); break; } catch {
+          try { if (Date.now() - statSync(lock).mtimeMs > 30_000) { rmdirSync(lock); continue; } } catch { /* raced */ }
+          if (i > 100) throw new Error('lease lock busy');
+          Bun.sleepSync(50);
+        }
+      }
+      try {
+        const current = existsSync(path) ? readFileSync(path, 'utf8') : null;
+        if (sha(current) !== sha(expected)) return false;
+        const tmp = `${path}.${process.pid}.tmp`;
+        writeFileSync(tmp, next, { mode: 0o600 });
+        renameSync(tmp, path);
+        return true;
+      } finally { try { rmdirSync(lock); } catch { /* already gone */ } }
+    },
+  };
+}
+
+export type SshRunner = (host: string, script: string, input?: string) => { status: number | null; stdout: string; stderr: string };
+export const defaultSshRunner: SshRunner = (host, script, input) => {
+  // ssh joins its arguments into one remote shell line — hand bash a single-quoted script.
+  const quoted = `'${script.replaceAll("'", "'\\''")}'`;
+  const r = spawnSync('ssh', ['-o', 'BatchMode=yes', '-o', 'ConnectTimeout=8', host, `bash -c ${quoted}`],
+    { input: input ?? '', encoding: 'utf8', timeout: 30_000 });
+  return { status: r.status, stdout: r.stdout ?? '', stderr: r.stderr ?? '' };
+};
+
+/** The arbiter itself (cloud-vm) runs the same lease script locally — `hq.arbiter: "local"` (drill 10-04: gcp has no ssh to itself). */
+export const localShellRunner: SshRunner = (_host, script, input) => {
+  const r = spawnSync('bash', ['-c', script], { input: input ?? '', encoding: 'utf8', timeout: 30_000 });
+  return { status: r.status, stdout: r.stdout ?? '', stderr: r.stderr ?? '' };
+};
+
+const REMOTE_DIR = '$HOME/.elanous-hq';
+/** Remote store over ssh: the remote side only compares a sha and renames under a mkdir lock — all decisions stay here. */
+export function sshLeaseStore(host: string, run: SshRunner = defaultSshRunner): LeaseStore {
+  return {
+    read() {
+      const r = run(host, `date +%s && echo ---- && { cat ${REMOTE_DIR}/lease.json 2>/dev/null || true; }`);
+      if (r.status !== 0) throw new Error(`arbiter ${host} unreachable: ${r.stderr.trim().split('\n')[0] ?? r.status}`);
+      const [head, ...rest] = r.stdout.split('----\n');
+      const now = Number.parseInt((head ?? '').trim(), 10);
+      if (!Number.isFinite(now)) throw new Error(`arbiter ${host} gave no clock`);
+      const raw = rest.join('----\n');
+      return { now, raw: raw.length ? raw : null };
+    },
+    cas(expected, next) {
+      const script = [
+        `set -e; d=${REMOTE_DIR}; f=$d/lease.json; mkdir -p "$d"`,
+        'i=0; until mkdir "$d/lock" 2>/dev/null; do i=$((i+1)); [ $i -gt 100 ] && { echo BUSY; exit 3; }; sleep 0.05; done',
+        'trap \'rmdir "$d/lock" 2>/dev/null || true\' EXIT',
+        'if [ -f "$f" ]; then cur=$( (sha256sum "$f" 2>/dev/null || shasum -a 256 "$f") | cut -d" " -f1); else cur=none; fi',
+        `[ "$cur" = "${sha(expected)}" ] || { echo CONFLICT; exit 0; }`,
+        'cat > "$f.tmp.$$"; chmod 600 "$f.tmp.$$"; mv "$f.tmp.$$" "$f"; echo OK',
+      ].join('\n');
+      const r = run(host, script, next);
+      if (r.status !== 0) throw new Error(`arbiter ${host} write failed: ${r.stdout.trim() || r.stderr.trim().split('\n')[0]}`);
+      return r.stdout.trim().endsWith('OK');
+    },
+  };
+}
+
+/** Home-to-home reachability (ssh true) — used only by the holder to reach the standby, never by the arbiter. */
+export const sshReachable = (host: string, run: SshRunner = defaultSshRunner): boolean => {
+  try { return run(host, 'true').status === 0; } catch { return false; }
+};
+
+/**
+ * Tailnet probe (OP 10-04 09:04): liveness = `tailscale ping`, application = nexus health over the tailnet.
+ * One «unreachable» observation needs BOTH to fail. No ssh — the arbiter holds no shell on home machines.
+ */
+export interface HostProbe { ping(host: string): boolean; health(host: string): boolean }
+export function probeReachable(host: string, probe: HostProbe): { reachable: boolean; ping: boolean; health: boolean } {
+  let ping = false; let health = false;
+  try { ping = probe.ping(host); } catch { ping = false; }
+  try { health = probe.health(host); } catch { health = false; }
+  return { reachable: ping || health, ping, health };
+}

package/src/hq/ledger-write-target.ts

@@ -0,0 +1,41 @@
+import { realpathSync } from 'node:fs';
+import { dirname, isAbsolute, join, relative, sep } from 'node:path';
+import { effectiveInstanceRoot, prodInstanceRoot } from '../instance/resolve.js';
+
+function physicalPath(path: string): string {
+  // Resolve one component at a time: lexical normalization of `link/..` before
+  // realpathSync would discard the link even though the filesystem follows it.
+  const raw = isAbsolute(path) ? path : `${process.cwd()}${sep}${path}`;
+  let current: string = sep;
+  for (const component of raw.split(sep)) {
+    if (!component || component === '.') continue;
+    if (component === '..') {
+      current = dirname(current);
+      continue;
+    }
+    const candidate = join(current, component);
+    try { current = realpathSync(candidate); }
+    catch (error) {
+      if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error;
+      current = candidate;
+    }
+  }
+  return current;
+}
+
+function within(root: string, target: string): boolean {
+  const suffix = relative(root, target);
+  return suffix === '' || (suffix !== '..' && !suffix.startsWith(`..${sep}`));
+}
+
+/** Only writes inside this process's non-operational instance universe bypass the HQ ledger lease. */
+export function isIsolatedLedgerWriteRoot(
+  ledgerRoot: string,
+  instanceRoot = effectiveInstanceRoot(),
+  operationalRoot = prodInstanceRoot(),
+): boolean {
+  const target = physicalPath(ledgerRoot);
+  const instance = physicalPath(instanceRoot);
+  const prod = physicalPath(operationalRoot);
+  return instance !== prod && within(instance, target) && !within(prod, target);
+}