create-walle 0.10.2 → 0.10.3

package/template/coding-task-manager/lib/fleet-hub.js

@@ -0,0 +1,762 @@
+'use strict';
+
+// WP-2 (docs/fleet-auto-update-pooled-machines-2026-10-26.html §5.1 D-1/D-7/D-10, §6, §8 WP-2,
+// §9 S-8): the HUB reconciler — decide which box to update and send it the bundle.
+//
+// The box side (WP-0/WP-1, lib/fleet-update.js + bin/fleet-apply.js) already answers two questions:
+// `GET /api/fleet/status` ("what am I") and `POST /api/fleet/bundle?to=<sha>` ("apply this"). This
+// module is the only place that decides WHERE those bundles go, and it is deliberately split in two:
+//
+//   * `planFleetAction` — PURE. No I/O, no clock. Given the machine rows, the box statuses, the
+//     ancestry answers, the target and the hub's own persisted state, it returns one reason per box
+//     (first match wins) and at most one box to send to. A whole decision table is therefore testable
+//     with plain objects, and the ordering below is the specification, not an accident of evaluation.
+//   * `reconcileFleetOnce` — the one pass that does the I/O: read the target, ask every box, settle a
+//     previous send, compute ancestry, plan, and (when the plan says so) build a `git bundle` and POST
+//     it. Re-entrant by construction: a second call while one is running joins the first.
+//
+// Lead decisions that override the doc (2026-10-26):
+//   * There is NO `autoUpdate` setting (doc S-5 / AC-2d are dropped). A box opts out by not being a
+//     clean `main` checkout; the reasons `not_main`/`dirty` are the whole opt-out story.
+//   * OQ-3 cooldown: at most one bundle per box every 3 h, measured on the HUB clock from the hub's
+//     own `lastSentAt` for that box (never the box clock) — reason `cooldown`.
+//   * One attempt per (box, target): once the hub has sent target T to box B it never sends T to B
+//     again, whatever the verdict. A new hub `main` commit is a new target.
+//   * Serial: at most one box in flight across the whole pool (D-7).
+//
+// Lead review round 2 (2026-10-27) tightened four things, each with its own test:
+//   * the serial rule is not broken by a timing-out box — a box that still ANSWERS stays in flight
+//     however long it takes (`bin/fleet-apply.js` waits up to 24 h in `waiting_idle`); see step 4.
+//   * only the hub leads — `isFleetLeader`, checked before any fetch; see step 1b.
+//   * `current` means RUNNING the target, not just having it on disk — `restart_pending` otherwise.
+//   * the memoized target is re-read before a send is recorded, so a moved `main` cannot burn the
+//     one attempt for a target with a bundle the box would refuse; see step 7a.
+//
+// Every git call goes through `execFile`/`spawn` with an argv ARRAY (never a shell string), and every
+// network call carries an `AbortSignal.timeout` budget — the codebase's sanctioned non-timer deadline,
+// so this module owns no timer of its own.
+
+const { execFile, spawn } = require('node:child_process');
+const fsp = require('node:fs/promises');
+const path = require('node:path');
+
+const { HEALTH_STATES } = require('./machine-registry');
+const { FLEET_BUNDLE_MAX_BYTES, isFullSha, readFleetState, readHubTarget } = require('./fleet-update');
+const { isDevInstanceTag } = require('./dev-fault-injection');
+
+// The real launchers under a second name: both `buildFleetBundle` and `_execFileGit` keep the literal
+// call names (`spawn(...)`/`execFile(...)`) so bin/main-thread-lint.js keeps SEEING their launch sites
+// — a call written through an `*Impl` alias would be invisible to it, and an invisible process launch
+// is exactly what that ledger exists to prevent (same trick as lib/fleet-update.js).
+const defaultExecFile = execFile;
+const defaultSpawn = spawn;
+
+/** OQ-3/D-10 (user decision): at most one bundle per box every 3 h, on the hub clock. */
+const FLEET_COOLDOWN_MS = 3 * 60 * 60 * 1000;
+/**
+ * A send whose box went SILENT (no status answer at all) or whose helper died WITHOUT a verdict is
+ * abandoned after this and recorded as `timeout` — otherwise one stuck box would hold the serial lock
+ * forever and freeze the whole pool (D-7 has no other escape hatch).
+ *
+ * A box that still ANSWERS is never timed out, however long it takes: `bin/fleet-apply.js` waits up to
+ * its own `idleMaxMs` (24 h) in `waiting_idle` for real work to drain, so a hub-side 30 min timeout
+ * would abandon a perfectly healthy apply and, worse, advance the pool while the first box is still
+ * mid-apply. Only silence, or an `idle` box with no matching `last.to` (the helper never wrote its
+ * verdict), counts as a stall.
+ */
+const FLEET_IN_FLIGHT_TIMEOUT_MS = 30 * 60 * 1000;
+/** The status read's budget: a box that cannot answer `/api/fleet/status` in 5 s reads `unreachable`. */
+const FLEET_STATUS_TIMEOUT_MS = 5000;
+/** The bundle POST's budget (the worst live gap measured 2.3 MB / 0.49 s, §1.3). */
+const FLEET_SEND_TIMEOUT_MS = 120000;
+
+/** `git merge-base --is-ancestor` is a graph walk, not a transfer; 5 s is the §6 status budget. */
+const GIT_TIMEOUT_MS = 5000;
+
+/**
+ * The one health value a box must carry to be dialled (lib/machine-registry.js `HEALTH_STATES`).
+ * `unhealthy` is a box that answered but is degraded, and `unknown` is one this hub has never read:
+ * neither is a box we may write over (D-2). Read from the registry's own list so the two modules
+ * cannot drift apart silently.
+ */
+const REACHABLE_HEALTH = HEALTH_STATES.find((state) => state === 'reachable') || 'reachable';
+
+/** The repo root holding `.git` — `lib/` is two levels below it (`coding-task-manager/lib`). */
+const DEFAULT_REPO_ROOT = path.resolve(__dirname, '..', '..');
+
+/** `<dataDir>/fleet-hub.json` — the hub's own memory of what it has sent (doc §8 WP-2). */
+const HUB_STATE_FILE = 'fleet-hub.json';
+
+/**
+ * The planner's closed reason set, in decision order (§8 WP-2 AC-2a) plus `sending` for the one box a
+ * pass picks. Kept LOCAL to this module on purpose: `lib/fleet-update.js`'s `FLEET_REASONS` is the BOX
+ * side's vocabulary (its status route never says `updating`/`sending`), and a completeness test over
+ * this list must not force the box module to grow two words it does not use.
+ */
+const FLEET_HUB_REASONS = Object.freeze([
+  'unreachable',
+  'not_git',
+  'not_main',
+  // WP-3 (docs/fleet-origin-main-sync-skill-2026-10-31.html §5 R-1, rev-2 OQ-3): the box's target
+  // tree carries `landing.publish`, so the box syncs itself through origin and this hub never hands
+  // it a bundle — keeping both channels would re-open incident I-1 (two boxes publishing the same
+  // content). Decided before `current`/`behind`, because it is a fact about the BOX, not about how
+  // far its sha is from the hub's.
+  'origin_sync',
+  'current',
+  'restart_pending',
+  'dirty',
+  'diverged',
+  'updating',
+  'already_attempted',
+  'cooldown',
+  'another_box_updating',
+  'busy',
+  'behind',
+  'sending',
+]);
+
+// ── state helpers ────────────────────────────────────────────────────────────────────────────────
+
+/** The honest "nothing sent, nothing known" state — also what a missing/corrupt file reads as. */
+function _emptyHubState() { return { inFlight: null, boxes: {} }; }
+
+function hubStatePath(dataDir) { return path.join(String(dataDir || ''), HUB_STATE_FILE); }
+
+/**
+ * Coerce whatever came off disk (or a test) into the shape the planner relies on, so a torn file can
+ * never make `box.attempts[target]` throw. Unknown keys are dropped: this file is a cache of one
+ * decision, not a store of anything else.
+ */
+function _normalizeHubState(parsed) {
+  if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) return _emptyHubState();
+  const rawBoxes = parsed.boxes && typeof parsed.boxes === 'object' && !Array.isArray(parsed.boxes) ? parsed.boxes : {};
+  const boxes = {};
+  for (const [id, box] of Object.entries(rawBoxes)) {
+    if (!id || !box || typeof box !== 'object' || Array.isArray(box)) continue;
+    const attempts = {};
+    const rawAttempts = box.attempts && typeof box.attempts === 'object' && !Array.isArray(box.attempts) ? box.attempts : {};
+    for (const [attemptTarget, attempt] of Object.entries(rawAttempts)) {
+      if (!attemptTarget || !attempt || typeof attempt !== 'object') continue;
+      attempts[attemptTarget] = {
+        sentAt: Number(attempt.sentAt) || 0,
+        verdict: String(attempt.verdict == null ? '' : attempt.verdict),
+        reason: String(attempt.reason == null ? '' : attempt.reason),
+      };
+    }
+    boxes[id] = { lastSentAt: Number(box.lastSentAt) || 0, attempts };
+  }
+  const inFlight = parsed.inFlight && typeof parsed.inFlight === 'object'
+    && typeof parsed.inFlight.instanceId === 'string' && parsed.inFlight.instanceId
+    ? {
+      instanceId: parsed.inFlight.instanceId,
+      target: String(parsed.inFlight.target == null ? '' : parsed.inFlight.target),
+      sentAt: Number(parsed.inFlight.sentAt) || 0,
+    }
+    : null;
+  return { inFlight, boxes };
+}
+
+/** Warn ONCE per data dir per process: a corrupt file is replaced on the next write anyway. */
+const _corruptWarned = new Set();
+function _warnCorruptHubState(dataDir) {
+  const key = String(dataDir || '');
+  if (_corruptWarned.has(key)) return;
+  _corruptWarned.add(key);
+  // REPORT: the hub's send history is unreadable, so this pass starts from empty. The next write
+  // replaces the file; until then the only cost is that a pre-existing attempt may be re-attempted.
+  console.warn(`[fleet-hub] hub state at ${hubStatePath(dataDir)} is unreadable; starting from an empty state`);
+}
+
+/**
+ * `<dataDir>/fleet-hub.json`. Missing or corrupt ⇒ the empty state (never a throw into a sweep); a
+ * corrupt file is REPORTED once and replaced by the next `writeHubState`.
+ */
+async function readHubState(dataDir) {
+  if (!dataDir) return _emptyHubState();
+  let raw = '';
+  try {
+    raw = await fsp.readFile(hubStatePath(dataDir), 'utf8');
+  } catch {
+    // HANDLE: no hub state yet (first sweep on this hub), or an unreadable path — "nothing sent" is
+    // the truthful answer and is what makes the very next pass a normal send.
+    return _emptyHubState();
+  }
+  let parsed = null;
+  try {
+    parsed = JSON.parse(raw);
+  } catch {
+    // HANDLE: torn/garbage file — report once, start empty (see _warnCorruptHubState).
+    _warnCorruptHubState(dataDir);
+    return _emptyHubState();
+  }
+  if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) {
+    _warnCorruptHubState(dataDir);
+    return _emptyHubState();
+  }
+  return _normalizeHubState(parsed);
+}
+
+/** Atomic 0600 write: temp file in the same directory, then `rename` (the readFleetState pattern). */
+async function writeHubState(dataDir, state) {
+  const dir = String(dataDir || '');
+  await fsp.mkdir(dir, { recursive: true, mode: 0o700 });
+  const target = hubStatePath(dir);
+  const tmp = path.join(dir, `.fleet-hub.${process.pid}.${Date.now()}.tmp`);
+  await fsp.writeFile(tmp, JSON.stringify(_normalizeHubState(state)), { mode: 0o600 });
+  await fsp.rename(tmp, target);
+}
+
+// ── the pure planner (WP-2 §8 AC-2a) ─────────────────────────────────────────────────────────────
+
+/**
+ * One reason for one box, first match wins, in the order §8 WP-2 fixes (see FLEET_HUB_REASONS).
+ * `status` is the box's own `/api/fleet/status` body, or null when the GET failed.
+ */
+function _reasonFor({ machine, status, ancestry, target, state, id, clock }) {
+  // 1. unreachable — a health this hub cannot dial, or no status answer at all.
+  if (machine.health !== REACHABLE_HEALTH || !status) return 'unreachable';
+  // 2. not_git — an npm/release checkout has no bundle to fast-forward.
+  if (status.channel !== 'git') return 'not_git';
+  // 3. not_main — the box is on a branch; the fleet only ever advances `main`.
+  if (status.branch !== 'main') return 'not_main';
+  // 4. origin_sync — the box's target tree carries `landing.publish` (WP-3, R-1): its main is fed
+  //    from origin by its own CTM follower (`land sync --pull-only`), so it neither needs nor may
+  //    receive this hub's bundle (rev-2 OQ-3 retires the hub→box channel for exactly these boxes).
+  //    Checked before `current`: a self-syncing box is never reported as current/behind AT ALL, and
+  //    the hub's own view of its sha is no longer the deciding fact.
+  if (status.originSync && status.originSync.configured === true) return 'origin_sync';
+  // 5. current — at the target sha AND actually running it. A box whose `head` already moved but whose
+  //    `runningSha` did not has the new code on disk only (fleet-apply's busy_timeout path leaves
+  //    exactly that): it is NOT current, and it needs no bundle either — just its own restart.
+  if (status.head === target) return status.runningSha === target ? 'current' : 'restart_pending';
+  // 6. dirty — uncommitted tracked changes (untracked files never make a box ineligible).
+  if (status.clean !== true) return 'dirty';
+  // 7. diverged — the hub repo does not contain the box's head: `from..main` would not apply.
+  if (ancestry.get(id) !== true) return 'diverged';
+  // 8. updating — the box is mid-apply, or this hub already has a send out to it.
+  if (status.state !== 'idle') return 'updating';
+  if (state.inFlight && state.inFlight.instanceId === id) return 'updating';
+  const box = state.boxes ? state.boxes[id] : null;
+  // 9. already_attempted — one attempt per (box, target), whatever the verdict (lead decision).
+  if (box && box.attempts && box.attempts[target]) return 'already_attempted';
+  // 10. cooldown — at most one bundle per box every 3 h, on the hub clock (D-10/OQ-3).
+  const lastSentAt = box ? Number(box.lastSentAt) || 0 : 0;
+  if (lastSentAt > 0 && clock - lastSentAt < FLEET_COOLDOWN_MS) return 'cooldown';
+  // 11. another_box_updating — serial across the pool (D-7): the first box is the canary.
+  if (state.inFlight) return 'another_box_updating';
+  // 12. busy — the box's own restart guard is blocking (D-5). The box also waits on its own gate;
+  //     this check only avoids handing a bundle to a box that would immediately park it.
+  if (Number(status.guard && status.guard.blockingCount) > 0) return 'busy';
+  // 13. behind — eligible.
+  return 'behind';
+}
+
+/**
+ * The pure decision for ONE pass (no I/O, no `Date.now`). `machines` are the registry rows (only
+ * `enabled` rows are considered; the hub's own row never appears there), `statuses` maps instanceId
+ * to the box's `/api/fleet/status` body (null = the GET failed), `ancestry` maps instanceId to
+ * "the box's head is an ancestor of target in the hub repo", and `hubState` is the persisted state.
+ *
+ * Returns `{ send, decisions }`: `send` is the one instanceId to bundle (or null), and `decisions`
+ * maps EVERY enabled box to `{ reason, behind }` — `behind` is the pure "this box is not RUNNING the
+ * target" fact, independent of eligibility. A box that landed the target on disk without it running
+ * (`restart_pending`) is therefore `behind`, even though no bundle may be sent to it.
+ */
+function planFleetAction({ machines = [], statuses = new Map(), ancestry = new Map(), target = '', hubState = null, now = 0 } = {}) {
+  const decisions = new Map();
+  const state = hubState && typeof hubState === 'object' ? hubState : _emptyHubState();
+  const clock = Number(now) || 0;
+  const list = (Array.isArray(machines) ? machines : []).filter((machine) => machine && machine.enabled);
+  for (const machine of list) {
+    const id = String(machine.instanceId || '');
+    const status = statuses.get(id) || null;
+    const behind = !!(status && !(status.head === target && status.runningSha === target));
+    decisions.set(id, { reason: _reasonFor({ machine, status, ancestry, target, state, id, clock }), behind });
+  }
+  // send = the FIRST eligible box by ascending instanceId. Every other eligible box stays `behind`
+  // (not `another_box_updating`): nothing is in flight yet, so the next pass — with this one recorded
+  // — is what reports the serial wait.
+  const send = list
+    .map((machine) => String(machine.instanceId || ''))
+    .filter((id) => { const decision = decisions.get(id); return !!decision && decision.reason === 'behind'; })
+    .sort()[0] || null;
+  if (send) decisions.get(send).reason = 'sending';
+  return { send, decisions };
+}
+
+// ── git: the bundle build and the ancestry probe ─────────────────────────────────────────────────
+
+/** The production git runner: `execFile` with an argv ARRAY, an explicit timeout, no shell. */
+function _execFileGit(args, { cwd, timeout = GIT_TIMEOUT_MS } = {}) {
+  return new Promise((resolve, reject) => {
+    execFile('git', args, { cwd, timeout }, (error, stdout) => {
+      if (error) { reject(error); return; }
+      resolve(String(stdout == null ? '' : stdout));
+    });
+  });
+}
+
+/**
+ * One git call reduced to `{ code, stdout, stderr }` — never a throw. `runGit` has two legitimate
+ * shapes: the production runner above (rejects on a non-zero exit, with `error.code` = the exit code)
+ * and a test double that returns `{ code, stdout, stderr }`. Both read here, so a caller that must
+ * look at an EXIT CODE (`git merge-base --is-ancestor`) does not care which it got.
+ */
+async function _runGitCode(runGit, args, cwd, timeout = GIT_TIMEOUT_MS) {
+  try {
+    const out = await runGit(args, { cwd, timeout });
+    if (out && typeof out === 'object' && Number.isInteger(out.code)) {
+      return { code: out.code, stdout: String(out.stdout == null ? '' : out.stdout), stderr: String(out.stderr == null ? '' : out.stderr) };
+    }
+    return { code: 0, stdout: String(out == null ? '' : out), stderr: '' };
+  } catch (error) {
+    const code = error && Number.isInteger(error.code) ? error.code : 1;
+    const stderr = error && (error.stderr != null ? String(error.stderr) : String((error && error.message) || ''));
+    return { code, stdout: String((error && error.stdout) || ''), stderr };
+  }
+}
+
+/**
+ * D-3: `git bundle create - <from>..main`, the whole gap in one stream. The right-hand side MUST be
+ * the branch NAME `main` — git refuses to record a raw sha as the bundle's own ref, and a bundle with
+ * no ref is not one `git bundle verify`/`git fetch` can consume. stdout is collected into a Buffer and
+ * capped at `maxBytes`: past the cap the child is killed and the promise rejects with code `too_large`,
+ * so a pathological gap becomes a refusal on the hub, never an unbounded allocation.
+ */
+function buildFleetBundle({ repoRoot, from, maxBytes = FLEET_BUNDLE_MAX_BYTES, spawnImpl: spawn = defaultSpawn } = {}) {
+  return new Promise((resolve, reject) => {
+    let child;
+    try {
+      child = spawn('git', ['-C', repoRoot, 'bundle', 'create', '-', `${from}..main`], { stdio: ['ignore', 'pipe', 'pipe'] });
+    } catch (error) {
+      reject(error);
+      return;
+    }
+    const chunks = [];
+    const stderrChunks = [];
+    let total = 0;
+    let settled = false;
+    const fail = (error) => {
+      if (settled) return;
+      settled = true;
+      try { child.kill('SIGKILL'); } catch { /* expected: the child already exited (or never started) */ }
+      reject(error);
+    };
+    child.stdout.on('data', (chunk) => {
+      if (settled) return;
+      total += chunk.length;
+      if (total > maxBytes) {
+        const error = new Error(`fleet bundle for ${from} exceeds ${maxBytes} bytes`);
+        error.code = 'too_large';
+        fail(error);
+        return;
+      }
+      chunks.push(chunk);
+    });
+    child.stderr.on('data', (chunk) => { if (stderrChunks.length < 64) stderrChunks.push(chunk); });
+    child.on('error', fail);
+    child.on('close', (code) => {
+      if (settled) return;
+      settled = true;
+      if (code !== 0) {
+        const tail = Buffer.concat(stderrChunks).toString('utf8').trim().slice(-500);
+        const error = new Error(`git bundle create exited ${code}: ${tail}`);
+        error.code = 'bundle_failed';
+        reject(error);
+        return;
+      }
+      resolve(Buffer.concat(chunks));
+    });
+  });
+}
+
+// ── the reconciler ───────────────────────────────────────────────────────────────────────────────
+
+/** Re-entrancy: a call while one pass is running joins that pass (the sweep may fire twice). */
+let _reconciling = null;
+/** The last plan, for `fleetRowFor` — the persisted state is cached beside it so that call does no I/O. */
+let _lastPlan = null;
+let _hubStateCache = { dataDir: '', state: _emptyHubState() };
+
+function _joinUrl(baseUrl, pathname) {
+  const base = String(baseUrl == null ? '' : baseUrl).trim().replace(/\/+$/, '');
+  if (!base) return '';
+  try {
+    return new URL(pathname, base).toString();
+  } catch {
+    // expected: an unparseable machine url is a dial failure, never a throw out of a sweep
+    return '';
+  }
+}
+
+/** The codebase's sanctioned non-timer deadline (machines.js:157 and friends). */
+function _timeoutSignal(ms) {
+  return (typeof AbortSignal !== 'undefined' && typeof AbortSignal.timeout === 'function')
+    ? AbortSignal.timeout(ms)
+    : undefined;
+}
+
+function _ensureBox(state, id) {
+  if (!state.boxes || typeof state.boxes !== 'object') state.boxes = {};
+  const existing = state.boxes[id];
+  if (existing && typeof existing === 'object') {
+    if (!existing.attempts || typeof existing.attempts !== 'object') existing.attempts = {};
+    if (!Number.isFinite(Number(existing.lastSentAt))) existing.lastSentAt = 0;
+    return existing;
+  }
+  const created = { lastSentAt: 0, attempts: {} };
+  state.boxes[id] = created;
+  return created;
+}
+
+function _attemptFor(state, id, target) {
+  const box = state && state.boxes ? state.boxes[id] : null;
+  return (box && box.attempts && box.attempts[target]) || null;
+}
+
+function _recordAttempt(state, id, target, attempt) {
+  const box = _ensureBox(state, id);
+  box.attempts[target] = {
+    sentAt: Number(attempt.sentAt) || 0,
+    verdict: String(attempt.verdict == null ? '' : attempt.verdict),
+    reason: String(attempt.reason == null ? '' : attempt.reason),
+  };
+}
+
+async function _persist(dataDir, hubState) {
+  _hubStateCache = { dataDir: String(dataDir || ''), state: hubState };
+  if (!dataDir) return;
+  await writeHubState(dataDir, hubState);
+}
+
+/**
+ * Only the HUB leads (lead correction 2026-10-27). Pool members pull each other's rosters (the
+ * registry's P-5), so a BOX — the laptop, the Spark — also carries enabled machine rows for its peers,
+ * and would otherwise "reconcile" them with ITS OWN `main`: a box that is itself behind, or a /ctm-dev
+ * instance whose DB is a `--refresh` copy of the primary's, sending bundles to real boxes.
+ *
+ * Two refusals, both read-only:
+ *   * a dev instance tag (`/ctm-dev`) never leads — `dev-fault-injection.isDevInstanceTag` is the
+ *     codebase's one "is this the primary?" predicate (an unset/blank tag counts as primary, so the
+ *     real hub is never accidentally locked out);
+ *   * a box that has itself taken a fleet update (its own `<dataDir>/fleet/state.json` carries a
+ *     `last` verdict) follows a hub, so it is a follower wherever it runs.
+ */
+async function isFleetLeader({ dataDir = '', env = process.env } = {}) {
+  if (isDevInstanceTag(env && env.CTM_INSTANCE_TAG)) return false;
+  const boxState = await readFleetState(dataDir);
+  if (boxState && boxState.last) return false;
+  return true;
+}
+
+/**
+ * ONE status read. Any failure — no url, a transport error, a non-2xx, a body that is not JSON — is
+ * `null`, which the planner reports as `unreachable` (HANDLE at this boundary: a box that cannot
+ * answer is a box we must not write over, and never an exception out of the sweep).
+ */
+async function _readBoxStatus(machine, { fetchImpl, peerHeadersFor }) {
+  const url = _joinUrl(machine && machine.url, '/api/fleet/status');
+  if (!url || typeof fetchImpl !== 'function') return null;
+  const headers = typeof peerHeadersFor === 'function' ? (peerHeadersFor(machine.instanceId) || {}) : {};
+  let response = null;
+  try {
+    response = await fetchImpl(url, {
+      method: 'GET',
+      headers: { Accept: 'application/json', ...headers },
+      signal: _timeoutSignal(FLEET_STATUS_TIMEOUT_MS),
+    });
+  } catch {
+    // HANDLE: the box did not answer — the planner's `unreachable` reason is the whole verdict.
+    return null;
+  }
+  if (!response || !response.ok) return null;
+  try {
+    const body = await response.json();
+    return body && typeof body === 'object' ? body : null;
+  } catch {
+    // expected: a non-JSON 200 body is "no status", not a transport fault worth a different reason
+    return null;
+  }
+}
+
+/** The refusal's reason string: the box's own `reason`/`error` when it named one, else the status. */
+async function _refusalReason(response) {
+  if (!response) return 'no_response';
+  let body = null;
+  try {
+    body = typeof response.json === 'function' ? await response.json() : null;
+  } catch {
+    // expected: a non-JSON refusal is reported by its HTTP status, never as a parse failure of ours
+    body = null;
+  }
+  const named = body && typeof body === 'object' ? (body.reason || body.error) : null;
+  if (named) return String(named).slice(0, 120);
+  return `http_${Number(response.status) || 0}`;
+}
+
+/**
+ * Step 7: build the bundle and POST it. Called with the send ALREADY recorded (in-flight + attempt +
+ * `lastSentAt` persisted), so a crash mid-POST still counts as the one attempt for that target and
+ * still starts the cooldown. `202` leaves the send in flight for a later pass to settle from the box's
+ * own status; anything else is a refusal, recorded with the box's reason (or the HTTP status, or the
+ * error's code) — still the attempt, still the cooldown.
+ */
+async function _sendBundle({ machine, status, target, fetchImpl, peerHeadersFor, repoRoot, buildBundle, hubState, dataDir }) {
+  const id = String(machine.instanceId || '');
+  const sentAt = Number((_attemptFor(hubState, id, target) || {}).sentAt) || 0;
+  const refuse = async (reason) => {
+    _recordAttempt(hubState, id, target, { sentAt, verdict: 'refused', reason });
+    hubState.inFlight = null;
+    await _persist(dataDir, hubState);
+  };
+  const from = String((status && status.head) || '');
+  if (!isFullSha(from)) {
+    // REPORT: a send without a full box head is a planner/status contradiction — refuse it and say so.
+    console.warn(`[fleet-hub] no usable head for ${id}; refusing the send`);
+    await refuse('no_head');
+    return;
+  }
+  let bundle = null;
+  try {
+    bundle = await buildBundle({ repoRoot, from, maxBytes: FLEET_BUNDLE_MAX_BYTES });
+  } catch (error) {
+    // REPORT: a build failure is a hub-local fault; the code (or `bundle_failed`) is the recorded reason.
+    console.warn(`[fleet-hub] bundle build for ${id} failed: ${(error && error.message) || error}`);
+    await refuse((error && error.code) || 'bundle_failed');
+    return;
+  }
+  let response = null;
+  try {
+    response = await fetchImpl(_joinUrl(machine.url, `/api/fleet/bundle?to=${encodeURIComponent(target)}`), {
+      method: 'POST',
+      headers: {
+        ...(typeof peerHeadersFor === 'function' ? (peerHeadersFor(id) || {}) : {}),
+        'content-type': 'application/octet-stream',
+      },
+      body: bundle,
+      signal: _timeoutSignal(FLEET_SEND_TIMEOUT_MS),
+    });
+  } catch (error) {
+    // REPORT: the transport failed — record the code and let the next sweep try the next box.
+    console.warn(`[fleet-hub] bundle POST to ${id} failed: ${(error && error.message) || error}`);
+    await refuse((error && error.code) || 'send_failed');
+    return;
+  }
+  if (response && Number(response.status) === 202) return; // accepted: stays in flight until settled
+  await refuse(await _refusalReason(response));
+}
+
+/**
+ * One reconciler pass (WP-2 §8). Resolves `{ target, send, decisions }`; `send` is the instanceId whose
+ * bundle was POSTed this pass (the caller does NOT await that POST — see the sweep hook), or null.
+ */
+async function _reconcileOnce({
+  db,
+  fetchImpl = (typeof fetch === 'function' ? fetch : null),
+  peerHeadersFor = null,
+  repoRoot = DEFAULT_REPO_ROOT,
+  dataDir = '',
+  env = process.env,
+  now = Date.now,
+  runGit = _execFileGit,
+  buildBundle = buildFleetBundle,
+  listMachines = require('./machine-registry').listMachines,
+} = {}) {
+  const clock = () => Number(now()) || 0;
+
+  // 1. the target — the hub's local `refs/heads/main` (D-1). No target, no work at all.
+  const target = String((await readHubTarget({ repoRoot })) || '');
+  if (!target) return { target: '', send: null, decisions: new Map() };
+
+  // 1b. only a leader leads. A follower (a box, or a /ctm-dev instance) returns before ANY fetch, so
+  //     it can never hand a real box a bundle built from its own — possibly stale, possibly refreshed —
+  //     `main`. Nothing is recorded and no plan is published, so its rows stay `fleet: null`.
+  if (!(await isFleetLeader({ dataDir, env }))) {
+    return { target, send: null, decisions: new Map(), follower: true };
+  }
+
+  // 2. the boxes — zero enabled machines ⇒ zero network work (AC-2g).
+  const rows = (listMachines({ db }) || {}).machines || [];
+  const machines = rows.filter((machine) => machine && machine.enabled);
+  if (!machines.length) {
+    _lastPlan = { target, decisions: new Map(), statuses: new Map() };
+    return { target, send: null, decisions: _lastPlan.decisions };
+  }
+
+  const hubState = await readHubState(dataDir);
+  _hubStateCache = { dataDir: String(dataDir || ''), state: hubState };
+
+  // 3. ask every box what it is. Sequential on purpose: the pool is small, and a deterministic order
+  //    makes a sweep's request count and sequence trivial to reason about.
+  const statuses = new Map();
+  for (const machine of machines) {
+    statuses.set(String(machine.instanceId || ''), await _readBoxStatus(machine, { fetchImpl, peerHeadersFor }));
+  }
+
+  // 4. settle a previous send from the box's own status before planning anything new.
+  //
+  //    A box that still ANSWERS is never timed out (lead correction 2026-10-27): bin/fleet-apply.js
+  //    waits up to its own idleMaxMs = 24 h in `waiting_idle` for real work to drain, so while the box
+  //    reports anything other than `idle` the hub keeps the serial lock — however long that takes —
+  //    instead of abandoning a healthy apply and letting the pool advance behind it. `timeout` is
+  //    reserved for the two ways a send really dies: the box stops answering (status null), or its
+  //    helper exited without writing a verdict and the box is back to `idle` with no matching `last.to`.
+  let changed = false;
+  if (hubState.inFlight) {
+    const inFlight = hubState.inFlight;
+    const status = statuses.get(inFlight.instanceId) || null;
+    const previous = _attemptFor(hubState, inFlight.instanceId, inFlight.target) || { sentAt: 0 };
+    let verdict = '';
+    let reason = '';
+    if (status && status.head === inFlight.target && status.runningSha === inFlight.target && status.state === 'idle') {
+      verdict = 'updated';
+    } else if (status && status.state === 'idle' && status.last && status.last.to === inFlight.target) {
+      verdict = String(status.last.verdict || 'updated');
+      reason = String(status.last.reason || '');
+    } else if (
+      clock() - (Number(inFlight.sentAt) || 0) > FLEET_IN_FLIGHT_TIMEOUT_MS
+      && (!status || (status.state === 'idle' && !(status.last && status.last.to === inFlight.target)))
+    ) {
+      verdict = 'timeout';
+    }
+    if (verdict) {
+      _recordAttempt(hubState, inFlight.instanceId, inFlight.target, { sentAt: previous.sentAt, verdict, reason });
+      hubState.inFlight = null;
+      changed = true;
+    }
+  }
+
+  // 5. ancestry — `git merge-base --is-ancestor <boxHead> <target>` in the HUB repo. Exit 0 is the
+  //    only true; 1 (not an ancestor) and 128 (not a sha we hold) are both `diverged` (D-9).
+  const ancestry = new Map();
+  for (const [id, status] of statuses) {
+    if (!status || !isFullSha(status.head) || status.head === target) continue;
+    const result = await _runGitCode(runGit, ['merge-base', '--is-ancestor', String(status.head), target], repoRoot);
+    ancestry.set(id, result.code === 0);
+  }
+
+  // 6. plan.
+  const plan = planFleetAction({ machines, statuses, ancestry, target, hubState, now: clock() });
+  // `statuses` is kept beside the decisions so `fleetRowFor` can serve the box's own `head`,
+  // `runningSha` and `originSync` to the Machines card (WP-3 AC-3d) without a second network read.
+  _lastPlan = { target, decisions: plan.decisions, statuses };
+
+  // 7. send. The send is recorded (in-flight, attempt, cooldown stamp) and persisted BEFORE any
+  //    network call, so a crash mid-POST cannot produce a second attempt for the same target.
+  if (!plan.send) {
+    if (changed) await _persist(dataDir, hubState);
+    return { target, send: null, decisions: plan.decisions };
+  }
+
+  // 7a. The bundle-ref race (lead correction 2026-10-27). `readHubTarget` memoizes for 15 s, so `main`
+  //     can move between step 1 and the build below; `git bundle create - <from>..main` would then
+  //     produce a bundle whose tip is no longer `to`, the box refuses it (tip !== to) and the ONE
+  //     attempt allowed for that target is burned for nothing. Re-read the ref through `runGit` and
+  //     stand down for this pass — nothing is recorded, so the next pass plans against the new target.
+  if (plan.send) {
+    const ref = await _runGitCode(runGit, ['rev-parse', '--verify', 'refs/heads/main'], repoRoot);
+    const currentMain = String(ref.stdout || '').trim();
+    if (!isFullSha(currentMain) || currentMain !== target) {
+      // REPORT: the hub moved under this pass. The box is still `behind` — it is just not this pass's
+      // job to update it, and no attempt should be spent on a bundle the box would refuse.
+      console.warn(`[fleet-hub] hub main is ${currentMain || '(unknown)'}, not ${target}; standing down for this pass`);
+      plan.decisions.get(plan.send).reason = 'behind';
+      if (changed) await _persist(dataDir, hubState);
+      return { target, send: null, decisions: plan.decisions };
+    }
+  }
+
+  const machine = machines.find((row) => String(row.instanceId || '') === plan.send) || null;
+  const status = statuses.get(plan.send) || null;
+  const sentAt = clock();
+  const box = _ensureBox(hubState, plan.send);
+  box.lastSentAt = sentAt;
+  box.attempts[target] = { sentAt, verdict: 'sent', reason: '' };
+  hubState.inFlight = { instanceId: plan.send, target, sentAt };
+  await _persist(dataDir, hubState);
+  await _sendBundle({ machine, status, target, fetchImpl, peerHeadersFor, repoRoot, buildBundle, hubState, dataDir });
+  return { target, send: plan.send, decisions: plan.decisions };
+}
+
+/**
+ * Re-entrant entry point: a call while a pass is running returns that pass's promise (§8 WP-2, S-8 —
+ * the machines-only loop and the sync tick can both fire). Rejections propagate to the caller (the
+ * sweep hook REPORTs them); the in-flight slot is always cleared.
+ */
+function reconcileFleetOnce(options = {}) {
+  if (_reconciling) return _reconciling;
+  _reconciling = _reconcileOnce(options).finally(() => { _reconciling = null; });
+  return _reconciling;
+}
+
+/** The newest recorded attempt, whatever the target — the row's "last verdict" (AC-3a's card line). */
+function _lastVerdictFor(box) {
+  if (!box || !box.attempts || typeof box.attempts !== 'object') return null;
+  let best = null;
+  for (const [attemptTarget, attempt] of Object.entries(box.attempts)) {
+    if (!attempt || typeof attempt !== 'object') continue;
+    const entry = {
+      target: attemptTarget,
+      verdict: String(attempt.verdict == null ? '' : attempt.verdict),
+      reason: String(attempt.reason == null ? '' : attempt.reason),
+      sentAt: Number(attempt.sentAt) || 0,
+    };
+    if (!best || entry.sentAt >= best.sentAt) best = entry;
+  }
+  return best;
+}
+
+/**
+ * The per-row fleet field for `GET /api/machines` (S-3). Reads the last plan + the cached persisted
+ * state — NO I/O and NO git, because this is called once per row per list. `null` before the first
+ * pass (never a fabricated "current"), and `null` for a box the last plan did not consider.
+ *
+ * WP-3 (AC-3d): the row also carries the box's own `head`/`runningSha` and its `originSync` block
+ * (the box's `/api/fleet/status` answer, reused verbatim) — the Machines card's `origin_sync` line
+ * needs all three, and this is the one place the box's status reaches the row. A box that did not
+ * answer (or a pass that ran before this field existed) reads `null`, never a fabricated sha.
+ */
+function fleetRowFor(instanceId) {
+  const id = String(instanceId || '');
+  if (!id || !_lastPlan) return null;
+  const decision = _lastPlan.decisions.get(id);
+  if (!decision) return null;
+  const state = (_hubStateCache && _hubStateCache.state) || _emptyHubState();
+  const box = state.boxes ? state.boxes[id] : null;
+  const status = (_lastPlan.statuses && _lastPlan.statuses.get(id)) || null;
+  return {
+    target: _lastPlan.target,
+    reason: decision.reason,
+    behind: !!decision.behind,
+    inFlight: !!(state.inFlight && state.inFlight.instanceId === id),
+    lastVerdict: _lastVerdictFor(box),
+    head: status ? String(status.head == null ? '' : status.head) : null,
+    runningSha: status ? String(status.runningSha == null ? '' : status.runningSha) : null,
+    originSync: (status && status.originSync) || null,
+  };
+}
+
+function _resetFleetHubStateForTests() {
+  _reconciling = null;
+  _lastPlan = null;
+  _hubStateCache = { dataDir: '', state: _emptyHubState() };
+  _corruptWarned.clear();
+}
+
+module.exports = {
+  _resetFleetHubStateForTests,
+  FLEET_COOLDOWN_MS,
+  FLEET_HUB_REASONS,
+  FLEET_IN_FLIGHT_TIMEOUT_MS,
+  FLEET_SEND_TIMEOUT_MS,
+  FLEET_STATUS_TIMEOUT_MS,
+  buildFleetBundle,
+  fleetRowFor,
+  hubStatePath,
+  isFleetLeader,
+  planFleetAction,
+  readHubState,
+  reconcileFleetOnce,
+  writeHubState,
+};