heyspex 0.1.0 → 0.2.0

package/dev-factory/lib/fleet-queue-file.mjs

@@ -0,0 +1,846 @@
+#!/usr/bin/env node
+/**
+ * fleet-queue-file.mjs — the file-backed (JSONL) queue implementation for a single machine /
+ * shared filesystem. THIS FILE WAS fleet-queue.mjs, moved here verbatim as part of the Neon
+ * fleet-queue port (HS-XOCK phase 4) — logic is unchanged from the original single-file
+ * module; only this header and the export surface note below are new.
+ *
+ * fleet-queue.mjs is now a thin backend-selecting façade (file vs. neon) that re-exports the
+ * same six functions as async wrappers — see fleet-queue.mjs and fleet-queue-neon.mjs. THIS
+ * module remains the offline / no-dispatcher fallback and the target of its own `--self-test`.
+ *
+ * Owner ask (2026-07-12): "If I have one agent already sending research request in progress
+ * to Night Action, and another agent has research that needs to be done, that agent can
+ * submit it successfully to the research queue - instead of getting a fail or rejection.
+ * This queue can also be used when Night Action has an existing ollama process. Research can
+ * be sent to the queue to be presented to Night Action as soon as it is done." And: "this
+ * queue concept can be used across the whole fleet for different use cases."
+ *
+ * FIRST LANE: "research-firecrawl" — research requests that lose the Ollama-XOR-Firecrawl
+ * mode-lease race in night-action-mode.mjs's requestOrEnqueueFirecrawlMode() land here
+ * instead of failing (see night-action-mode.mjs). Lanes are free-form strings; any future
+ * machine-gated resource (a build slot, a breakdown job, a GPU node) reuses this same module
+ * under its own lane name — see the design note at the bottom of this file.
+ *
+ * STORAGE — event-sourced, append-only JSONL: `.heyspex/fleet-queue.jsonl` (repo root,
+ * resolved by walking up to the heyspex.yaml vault marker — same convention as
+ * dev-factory/lib/fleet-state.mjs). Every mutation (enqueue / claim / complete / fail) is
+ * appended as ONE JSON line via a fresh open("a") + write + close (mirrors
+ * dev-factory/lib/run-events.mjs) — never a whole-file rewrite. Current item state is derived
+ * by FOLDING an item's events in file order, the same event-sourcing shape
+ * .heyspex/claim-history.jsonl and .heyspex/focus-history.jsonl already use elsewhere in this
+ * repo for durable ledgers (append pattern confirmed against both before writing this).
+ *
+ * WHY EVENT-SOURCED APPEND OVER REWRITE-WITH-STATUS (the crash-safety justification the task
+ * asked for): a whole-file rewrite (read all items, mutate one, write a temp file, rename over
+ * the original) is atomic PER WRITE but requires a read-modify-write on the FULL file for every
+ * single mutation, and a crash between "read" and "rename" either loses the mutation or (worse,
+ * on a partial rename) can corrupt every other item's state too. An append-only event log makes
+ * each mutation a single new line: a crash mid-write can only ever cost the LAST line (which
+ * readAllEvents() below already tolerates — a corrupt/partial trailing line is skipped, not
+ * thrown, mirroring run-events.mjs), and it can NEVER corrupt or lose any prior item's history.
+ * The trade is that readers must fold history instead of reading a snapshot — cheap here since
+ * the queue is small and local.
+ *
+ * CONCURRENCY — double-claim rejection: enqueue/complete/fail don't need cross-process
+ * exclusion (each only ever touches its own new line or an item only its own claimant should be
+ * mutating), but claim() has a genuine race — two processes could both read "pending" for the
+ * same id before either appends its claim event. claim() (and complete/fail, for symmetry) take
+ * a short-lived, per-id, TTL-bounded advisory file lock (`<queue>.lock.<id>`, exclusive-create
+ * `wx`) around their read-decide-append critical section — true mutual exclusion for same-id
+ * ops across processes, while different ids proceed fully in parallel. A lock file left behind
+ * by a crashed holder is auto-broken after `staleLockMs` (default 5s) so the queue can never
+ * wedge on a dead process.
+ *
+ * CRASH-RECOVERY (claim TTL): a claim() that never gets a matching complete()/fail() within
+ * `CLAIM_TTL_MS` (default 15 min, override via options or HEYSPEX_FLEET_QUEUE_CLAIM_TTL_MS) is
+ * treated as STALE — effectiveStatus() reports it back as "pending" so any caller (including
+ * the original claimant retrying after a crash) can reclaim it. The stale claim's history is
+ * never deleted, only superseded by a fresh claim event — so `list()` still shows the full
+ * claim history for audit.
+ *
+ * API (all synchronous — the façade in fleet-queue.mjs wraps these as async for backend parity
+ * with fleet-queue-neon.mjs):
+ *   enqueue({ lane, payload, requestedBy }) -> { id, position }   // position = 1-indexed rank among pending items in that lane
+ *   nextPending(lane)                        -> item | null       // oldest pending (or stale-reclaimable) item in the lane
+ *   claim(id, claimant)                      -> { ok, reason? }   // rejects double-claim / terminal states
+ *   complete(id, result?)                    -> { ok, reason? }   // must currently be claimed
+ *   fail(id, reason?)                        -> { ok, reason? }   // from pending or claimed
+ *   list(lane?, status?)                     -> item[]            // oldest-first
+ *
+ * CLI: node dev-factory/lib/fleet-queue-file.mjs --self-test
+ */
+
+import { closeSync, existsSync, mkdirSync, openSync, readFileSync, rmSync, statSync, writeSync } from "node:fs";
+import { dirname, join } from "node:path";
+import { codeRepoRoot } from "./code-repo-root.mjs";
+
+// ── repo-root + store resolution: this queue holds CODE-repo state (fleet-queue.jsonl), not
+//    vault state, so it anchors on the code repo itself rather than walking for a vault marker.
+export function findRepoRoot() {
+  return codeRepoRoot();
+}
+
+function defaultQueuePath() {
+  const dir = join(findRepoRoot(), ".heyspex");
+  mkdirSync(dir, { recursive: true });
+  return process.env.HEYSPEX_FLEET_QUEUE_PATH || join(dir, "fleet-queue.jsonl");
+}
+
+export const CLAIM_TTL_MS = Number(process.env.HEYSPEX_FLEET_QUEUE_CLAIM_TTL_MS) || 15 * 60 * 1000; // 15 min
+
+/** First lane (owner ask 2026-07-12): research requests that lose the Ollama-XOR-Firecrawl
+ *  mode-lease race in night-action-mode.mjs land here instead of failing. */
+export const RESEARCH_FIRECRAWL_LANE = "research-firecrawl";
+
+// ── event log I/O (mirrors run-events.mjs: line-atomic append, corrupt-tail-tolerant read) ──────
+function appendEvent(queuePath, record) {
+  mkdirSync(dirname(queuePath), { recursive: true });
+  const line = `${JSON.stringify(record)}\n`;
+  const fd = openSync(queuePath, "a");
+  try {
+    writeSync(fd, line);
+  } finally {
+    closeSync(fd);
+  }
+  return record;
+}
+
+function readAllEvents(queuePath) {
+  if (!existsSync(queuePath)) return [];
+  const text = readFileSync(queuePath, "utf8");
+  const events = [];
+  for (const line of text.split("\n")) {
+    const trimmed = line.trim();
+    if (!trimmed) continue;
+    try {
+      events.push(JSON.parse(trimmed));
+    } catch {
+      // corrupt/partial trailing line (e.g. a crash mid-write) — skip, never throw
+    }
+  }
+  return events;
+}
+
+// ── per-id advisory lock (exclusive-create; auto-breaks a stale/crashed holder) ─────────────────
+function lockPathFor(queuePath, id) {
+  return `${queuePath}.lock.${id}`;
+}
+
+function acquireLock(queuePath, id, { timeoutMs = 3000, staleLockMs = 5000, pollMs = 15 } = {}) {
+  const lp = lockPathFor(queuePath, id);
+  mkdirSync(dirname(queuePath), { recursive: true });
+  const start = Date.now();
+  for (;;) {
+    try {
+      const fd = openSync(lp, "wx");
+      writeSync(fd, String(process.pid));
+      closeSync(fd);
+      return true;
+    } catch (e) {
+      if (e?.code !== "EEXIST") throw e;
+      try {
+        const st = statSync(lp);
+        if (Date.now() - st.mtimeMs > staleLockMs) {
+          rmSync(lp, { force: true }); // break a lock left by a crashed holder
+          continue;
+        }
+      } catch {
+        continue; // lock vanished between the failed create and the stat — retry immediately
+      }
+      if (Date.now() - start > timeoutMs) return false;
+      // best-effort short busy-wait; this is a local dev tool, not a high-throughput queue
+      const until = Date.now() + pollMs;
+      while (Date.now() < until) { /* spin */ }
+    }
+  }
+}
+
+function releaseLock(queuePath, id) {
+  try {
+    rmSync(lockPathFor(queuePath, id), { force: true });
+  } catch {
+    // already gone — fine
+  }
+}
+
+function withLock(queuePath, id, fn) {
+  if (!acquireLock(queuePath, id)) return { ok: false, reason: `fleet-queue: could not acquire lock for "${id}" (held too long)` };
+  try {
+    return fn();
+  } finally {
+    releaseLock(queuePath, id);
+  }
+}
+
+// ── state derivation (event fold) ─────────────────────────────────────────────────────────────
+function foldOne(events) {
+  let state = null;
+  for (const ev of events) {
+    if (ev.type === "enqueue") {
+      state = {
+        id: ev.id,
+        lane: ev.lane,
+        payload: ev.payload ?? null,
+        requestedBy: ev.requestedBy ?? null,
+        status: "pending",
+        enqueuedAt: ev.at,
+        claimant: null,
+        claimedAt: null,
+        result: null,
+        failReason: null,
+        completedAt: null,
+      };
+    } else if (!state) {
+      continue; // orphan event with no enqueue in this log slice — ignore
+    } else if (ev.type === "claim") {
+      state.status = "claimed";
+      state.claimant = ev.claimant ?? null;
+      state.claimedAt = ev.at;
+    } else if (ev.type === "complete") {
+      state.status = "completed";
+      state.result = ev.result ?? null;
+      state.completedAt = ev.at;
+    } else if (ev.type === "fail") {
+      state.status = "failed";
+      state.failReason = ev.reason ?? null;
+      state.completedAt = ev.at;
+    }
+  }
+  return state;
+}
+
+/** Raw folded state (claimed stays "claimed" even if stale) keyed by id, oldest enqueue first. */
+function deriveRaw(queuePath) {
+  const events = readAllEvents(queuePath);
+  const byId = new Map();
+  for (const ev of events) {
+    if (!ev?.id) continue;
+    if (!byId.has(ev.id)) byId.set(ev.id, []);
+    byId.get(ev.id).push(ev);
+  }
+  const out = new Map();
+  for (const [id, evs] of byId) {
+    const s = foldOne(evs);
+    if (s) out.set(id, s);
+  }
+  return out;
+}
+
+/** Folded state with the TTL rule applied: a claim older than ttlMs reports back as "pending"
+ *  (reclaimable) without erasing claimant/claimedAt history — callers can see it was stale. */
+function deriveEffective(queuePath, { now = Date.now(), ttlMs = CLAIM_TTL_MS } = {}) {
+  const raw = deriveRaw(queuePath);
+  const out = new Map();
+  for (const [id, s] of raw) {
+    const stale = s.status === "claimed" && now - Date.parse(s.claimedAt) > ttlMs;
+    out.set(id, stale ? { ...s, status: "pending", stale: true } : { ...s, stale: false });
+  }
+  return out;
+}
+
+function genId() {
+  return `fq_${Date.now().toString(36)}_${Math.random().toString(36).slice(2, 8)}`;
+}
+
+// ── public API ────────────────────────────────────────────────────────────────────────────────
+export function enqueue({ lane, payload = null, requestedBy = null, queuePath = defaultQueuePath(), now = Date.now() } = {}) {
+  if (!lane) throw new Error("fleet-queue.enqueue: lane is required");
+  const id = genId();
+  const at = new Date(now).toISOString();
+  appendEvent(queuePath, { type: "enqueue", id, lane, payload, requestedBy, at });
+  const items = [...deriveEffective(queuePath, { now }).values()]
+    .filter((s) => s.lane === lane && s.status === "pending")
+    .sort((a, b) => a.enqueuedAt.localeCompare(b.enqueuedAt));
+  const position = items.findIndex((s) => s.id === id) + 1;
+  return { id, position };
+}
+
+export function nextPending(lane, { queuePath = defaultQueuePath(), now = Date.now(), ttlMs = CLAIM_TTL_MS } = {}) {
+  const items = [...deriveEffective(queuePath, { now, ttlMs }).values()]
+    .filter((s) => s.lane === lane && s.status === "pending")
+    .sort((a, b) => a.enqueuedAt.localeCompare(b.enqueuedAt));
+  return items[0] ?? null;
+}
+
+export function claim(id, claimant, { queuePath = defaultQueuePath(), now = Date.now(), ttlMs = CLAIM_TTL_MS } = {}) {
+  if (!id) return { ok: false, reason: "fleet-queue.claim: id is required" };
+  if (!claimant) return { ok: false, reason: "fleet-queue.claim: claimant is required" };
+  return withLock(queuePath, id, () => {
+    const state = deriveEffective(queuePath, { now, ttlMs }).get(id);
+    if (!state) return { ok: false, reason: `unknown id "${id}"` };
+    if (state.status === "completed") return { ok: false, reason: "already completed" };
+    if (state.status === "failed") return { ok: false, reason: "already failed" };
+    if (state.status === "claimed") return { ok: false, reason: `already claimed by ${state.claimant}` };
+    appendEvent(queuePath, { type: "claim", id, claimant, at: new Date(now).toISOString() });
+    return { ok: true, id, claimant, reclaimed: Boolean(state.stale) };
+  });
+}
+
+export function complete(id, result = null, { queuePath = defaultQueuePath(), now = Date.now() } = {}) {
+  if (!id) return { ok: false, reason: "fleet-queue.complete: id is required" };
+  return withLock(queuePath, id, () => {
+    const state = deriveEffective(queuePath, { now }).get(id);
+    if (!state) return { ok: false, reason: `unknown id "${id}"` };
+    if (state.status !== "claimed") return { ok: false, reason: `cannot complete from status "${state.status}" (must be claimed)` };
+    appendEvent(queuePath, { type: "complete", id, result, at: new Date(now).toISOString() });
+    return { ok: true, id };
+  });
+}
+
+export function fail(id, reason = null, { queuePath = defaultQueuePath(), now = Date.now() } = {}) {
+  if (!id) return { ok: false, reason: "fleet-queue.fail: id is required" };
+  return withLock(queuePath, id, () => {
+    const state = deriveEffective(queuePath, { now }).get(id);
+    if (!state) return { ok: false, reason: `unknown id "${id}"` };
+    if (state.status === "completed" || state.status === "failed") {
+      return { ok: false, reason: `cannot fail from terminal status "${state.status}"` };
+    }
+    appendEvent(queuePath, { type: "fail", id, reason, at: new Date(now).toISOString() });
+    return { ok: true, id };
+  });
+}
+
+export function list(lane = null, status = null, { queuePath = defaultQueuePath(), now = Date.now(), ttlMs = CLAIM_TTL_MS } = {}) {
+  const items = [...deriveEffective(queuePath, { now, ttlMs }).values()];
+  return items
+    .filter((s) => (lane ? s.lane === lane : true))
+    .filter((s) => (status ? s.status === status : true))
+    .sort((a, b) => a.enqueuedAt.localeCompare(b.enqueuedAt));
+}
+
+// ═══ RESOURCE GOVERNOR (HS-MUJ6) — implementation ══════════════════════════════════════════════
+// See the design-note block below for the full rationale, the stated liveness rule, and the
+// configuration contract. Everything in this section is additive; the queue API above is untouched.
+
+/** Governor tunables — DEFAULTS only. Never read these directly for behavior; go through
+ *  resolveGovernorConfig() so explicit overrides and HEYSPEX_GOVERNOR_* env vars apply
+ *  ([[HS-DEC-dev-factory-rules-are-configuration-not]]: every rule is a setting, never code). */
+export const GOVERNOR_DEFAULTS = Object.freeze({
+  /** Liveness rule input: a non-paused holder whose last liveness signal (grant, heartbeat,
+   *  resume, or idempotent re-request) is older than this is DEAD and reclaimable.
+   *  Default matches the queue's CLAIM_TTL_MS convention (15 min). */
+  leaseTtlMs: 15 * 60 * 1000,
+  /** Blocking acquire() poll interval — how quickly a waiter observes its wake-on-release. */
+  wakePollMs: 250,
+  /** Blocking acquire() gives up (and withdraws from the FIFO line) after this long. */
+  waitTimeoutMs: 10 * 60 * 1000,
+});
+
+const GOVERNOR_ENV_VARS = Object.freeze({
+  leaseTtlMs: "HEYSPEX_GOVERNOR_LEASE_TTL_MS",
+  wakePollMs: "HEYSPEX_GOVERNOR_WAKE_POLL_MS",
+  waitTimeoutMs: "HEYSPEX_GOVERNOR_WAIT_TIMEOUT_MS",
+});
+
+/** Layered config resolution: explicit overrides > HEYSPEX_GOVERNOR_* env > GOVERNOR_DEFAULTS.
+ *  Env is read at CALL time (not module load) so a process can be re-configured without a
+ *  restart and tests can exercise the env layer deterministically. */
+export function resolveGovernorConfig(overrides = {}) {
+  const out = {};
+  for (const key of Object.keys(GOVERNOR_DEFAULTS)) {
+    const explicit = overrides?.[key];
+    if (explicit !== undefined && explicit !== null && Number.isFinite(Number(explicit))) {
+      out[key] = Number(explicit);
+      continue;
+    }
+    const envRaw = process.env[GOVERNOR_ENV_VARS[key]];
+    const envNum = envRaw != null && String(envRaw).trim() !== "" ? Number(envRaw) : NaN;
+    out[key] = Number.isFinite(envNum) ? envNum : GOVERNOR_DEFAULTS[key];
+  }
+  return out;
+}
+
+/** THE stated liveness rule, in one place — stamped into every gov-reclaimed event so the audit
+ *  trail records not just THAT a lease was taken back, but under exactly which rule. */
+export function describeLivenessRule({ leaseTtlMs } = resolveGovernorConfig()) {
+  return (
+    `a holder is DEAD (lease reclaimable) iff it is NOT paused AND now - lastLivenessAt > ` +
+    `leaseTtlMs (${leaseTtlMs}ms); a PAUSED holder is never reclaimed by liveness — only ` +
+    `resume() or an administrator forceBreak() ends a pause`
+  );
+}
+
+// ── governor stores (the engine is store-agnostic; these two ship here) ────────────────────────
+
+/** Pure in-process store — for tests, and for the console dispatcher to mount server-side. */
+export function createMemoryGovernorStore() {
+  const byResource = new Map(); // resource -> event[]
+  return {
+    read(resource) {
+      return [...(byResource.get(resource) ?? [])];
+    },
+    append(record) {
+      if (!byResource.has(record.resource)) byResource.set(record.resource, []);
+      byResource.get(record.resource).push(record);
+    },
+    withLock(_resource, fn) {
+      return fn(); // single-process, synchronous — no cross-process exclusion needed
+    },
+  };
+}
+
+function defaultGovernorPath() {
+  const dir = join(findRepoRoot(), ".heyspex");
+  mkdirSync(dir, { recursive: true });
+  return process.env.HEYSPEX_RESOURCE_GOVERNOR_PATH || join(dir, "resource-governor.jsonl");
+}
+
+/** Resource keys ("git:repo") appear in lock FILENAMES — ':' is illegal on Windows, so the lock
+ *  key is sanitized. The event log itself keeps the exact resource string. */
+function sanitizeLockKey(resource) {
+  return String(resource).replace(/[^A-Za-z0-9._-]/g, "-");
+}
+
+/** Durable local store: append-only JSONL + the same per-key advisory lock the queue's claim()
+ *  uses — true cross-process mutual exclusion per resource on one filesystem. */
+export function createFileGovernorStore({ governorPath = defaultGovernorPath() } = {}) {
+  return {
+    read(resource) {
+      return readAllEvents(governorPath).filter((ev) => ev?.resource === resource);
+    },
+    append(record) {
+      appendEvent(governorPath, record);
+    },
+    withLock(resource, fn) {
+      const key = `gov.${sanitizeLockKey(resource)}`;
+      if (!acquireLock(governorPath, key)) {
+        return { ok: false, reason: `resource-governor: could not acquire lock for "${resource}" (held too long)` };
+      }
+      try {
+        return fn();
+      } finally {
+        releaseLock(governorPath, key);
+      }
+    },
+  };
+}
+
+// ── governor state derivation (event fold — same event-sourcing shape as the queue) ────────────
+
+/** Fold one resource's governor events (in log order) into { resource, holder, waiters }.
+ *  holder: { holderId, leaseId, grantedAt, livenessAt, paused } | null.
+ *  waiters: FIFO-ordered [{ holderId, requestId, enqueuedAt }]. Exported for the console
+ *  dispatcher's server-side wiring (later task) to reuse verbatim. */
+export function foldGovernorResource(events, resource) {
+  const st = { resource, holder: null, waiters: [] };
+  for (const ev of events) {
+    if (!ev || ev.resource !== resource) continue;
+    switch (ev.type) {
+      case "gov-granted":
+        st.holder = { holderId: ev.holderId, leaseId: ev.leaseId ?? null, grantedAt: ev.at, livenessAt: ev.at, paused: false };
+        st.waiters = st.waiters.filter((w) => w.holderId !== ev.holderId);
+        break;
+      case "gov-waiting":
+        if (st.holder?.holderId !== ev.holderId && !st.waiters.some((w) => w.holderId === ev.holderId)) {
+          st.waiters.push({ holderId: ev.holderId, requestId: ev.requestId ?? null, enqueuedAt: ev.at });
+        }
+        break;
+      case "gov-heartbeat":
+        if (st.holder?.holderId === ev.holderId) st.holder.livenessAt = ev.at;
+        break;
+      case "gov-paused":
+        if (st.holder?.holderId === ev.holderId) st.holder.paused = true;
+        break;
+      case "gov-resumed":
+        if (st.holder?.holderId === ev.holderId) {
+          st.holder.paused = false;
+          st.holder.livenessAt = ev.at; // resuming is itself a liveness signal
+        }
+        break;
+      case "gov-released":
+      case "gov-reclaimed":
+      case "gov-broken":
+        if (st.holder?.holderId === ev.holderId) st.holder = null;
+        break;
+      case "gov-abandoned":
+        st.waiters = st.waiters.filter((w) => w.requestId !== ev.requestId);
+        break;
+      default:
+        break; // unknown event kinds are ignored, never fatal (forward compatibility)
+    }
+  }
+  return st;
+}
+
+function governorGenId() {
+  return `lease_${Date.now().toString(36)}_${Math.random().toString(36).slice(2, 8)}`;
+}
+
+const defaultGovernorSleep = (ms) => new Promise((res) => setTimeout(res, ms));
+
+// ── the governor engine ────────────────────────────────────────────────────────────────────────
+
+/**
+ * createResourceGovernor({ store, now, genId, config }) -> governor
+ *
+ * Pure-logic engine over an injected event store. All mutations are read-decide-append critical
+ * sections under store.withLock(resource) — the same atomic-claim discipline as claim() above.
+ *
+ *   request(resource, holderId)                -> { ok, granted:true, leaseId, alreadyHeld }
+ *                                               | { ok, granted:false, position, holder }
+ *       Non-blocking. Grants when free (or when this holder is the FIFO head after a release/
+ *       reclaim); otherwise JOINS the FIFO line — never a refusal. Idempotent for the current
+ *       holder (re-request refreshes liveness: "held across multiple operations").
+ *       Runs dead-holder reclaim first, so any poller drives progress.
+ *   acquire(resource, holderId, { timeoutMs, pollMs, sleep }) -> Promise<same shapes>
+ *       Blocking convenience: request() then poll until granted or timeout (then withdraws
+ *       from the line and returns { ok:false, granted:false, reason }).
+ *   heartbeat(resource, holderId)              -> { ok } — refresh liveness while holding.
+ *   pause(resource, holderId) / resume(...)    -> { ok } — "alive but going quiet"; a paused
+ *       holder is exempt from liveness reclaim until resume()/forceBreak().
+ *   release(resource, holderId)                -> { ok, grantedTo } — explicit hand-back; the
+ *       FIFO head is granted in the same critical section (wake-on-release).
+ *   forceBreak(resource, { by, reason })       -> { ok, broken, grantedTo } — administrator's
+ *       escape hatch; `by` is REQUIRED so every break is attributably recorded.
+ *   withdraw(resource, holderId)               -> { ok } — leave the FIFO line voluntarily.
+ *   status(resource)                           -> read-only { resource, holder(+stale), waiters }.
+ */
+export function createResourceGovernor({
+  store = createMemoryGovernorStore(),
+  now = Date.now,
+  genId = governorGenId,
+  config = {},
+} = {}) {
+  const cfg = resolveGovernorConfig(config);
+  const iso = (ms) => new Date(ms).toISOString();
+  const fold = (resource) => foldGovernorResource(store.read(resource), resource);
+
+  /** The liveness rule, applied. Kept next to its documentation on purpose. */
+  function isDead(holder, nowMs) {
+    if (!holder) return false;
+    if (holder.paused) return false; // a live-but-paused holder is NEVER reclaimed
+    return nowMs - Date.parse(holder.livenessAt) > cfg.leaseTtlMs;
+  }
+
+  function grant(st, holderId, nowMs) {
+    const leaseId = genId();
+    store.append({ type: "gov-granted", resource: st.resource, holderId, leaseId, at: iso(nowMs) });
+    st.holder = { holderId, leaseId, grantedAt: iso(nowMs), livenessAt: iso(nowMs), paused: false };
+    st.waiters = st.waiters.filter((w) => w.holderId !== holderId);
+    return leaseId;
+  }
+
+  /** Reclaim a dead holder (recording the rule it died under), then hand a free resource to the
+   *  FIFO head. Called inside every mutating critical section. */
+  function reclaimAndPromote(st, nowMs) {
+    if (st.holder && isDead(st.holder, nowMs)) {
+      store.append({
+        type: "gov-reclaimed",
+        resource: st.resource,
+        holderId: st.holder.holderId,
+        leaseId: st.holder.leaseId,
+        rule: describeLivenessRule(cfg),
+        at: iso(nowMs),
+      });
+      st.holder = null;
+    }
+    if (!st.holder && st.waiters.length > 0) {
+      const head = st.waiters.shift();
+      grant(st, head.holderId, nowMs);
+    }
+  }
+
+  function request(resource, holderId) {
+    if (!resource) return { ok: false, reason: "resource-governor.request: resource is required" };
+    if (!holderId) return { ok: false, reason: "resource-governor.request: holderId is required" };
+    return store.withLock(resource, () => {
+      const nowMs = now();
+      const st = fold(resource);
+      const wasHolder = st.holder?.holderId === holderId;
+      reclaimAndPromote(st, nowMs);
+      if (st.holder?.holderId === holderId) {
+        // already held, or just promoted from the FIFO head — either way, refresh liveness
+        store.append({ type: "gov-heartbeat", resource, holderId, at: iso(nowMs) });
+        return { ok: true, granted: true, resource, holderId, leaseId: st.holder.leaseId, alreadyHeld: wasHolder };
+      }
+      if (!st.holder) {
+        // free with nobody ahead (promotion above would have taken the head) — grant directly
+        const leaseId = grant(st, holderId, nowMs);
+        return { ok: true, granted: true, resource, holderId, leaseId, alreadyHeld: false };
+      }
+      // held by someone else (live or paused) — wait in the FIFO line, never refuse
+      let pos = st.waiters.findIndex((w) => w.holderId === holderId);
+      if (pos === -1) {
+        store.append({ type: "gov-waiting", resource, holderId, requestId: genId(), at: iso(nowMs) });
+        pos = st.waiters.length; // 0-indexed slot of the new tail
+      }
+      return { ok: true, granted: false, resource, holderId, position: pos + 1, holder: st.holder.holderId };
+    });
+  }
+
+  async function acquire(resource, holderId, { timeoutMs = cfg.waitTimeoutMs, pollMs = cfg.wakePollMs, sleep = defaultGovernorSleep } = {}) {
+    const deadline = now() + timeoutMs;
+    for (;;) {
+      const r = request(resource, holderId);
+      if (!r.ok || r.granted) return r;
+      if (now() >= deadline) {
+        withdraw(resource, holderId); // leave the line — never linger as a ghost waiter
+        return {
+          ok: false,
+          granted: false,
+          resource,
+          holderId,
+          position: r.position,
+          reason: `resource-governor.acquire: timed out after ${timeoutMs}ms waiting for "${resource}" (held by ${r.holder}, position ${r.position})`,
+        };
+      }
+      await sleep(pollMs);
+    }
+  }
+
+  function heartbeat(resource, holderId) {
+    return store.withLock(resource, () => {
+      const nowMs = now();
+      const st = fold(resource);
+      if (st.holder?.holderId !== holderId) {
+        return { ok: false, reason: `resource-governor.heartbeat: "${holderId}" does not hold "${resource}" (holder: ${st.holder?.holderId ?? "none"})` };
+      }
+      store.append({ type: "gov-heartbeat", resource, holderId, at: iso(nowMs) });
+      return { ok: true, resource, holderId, livenessAt: iso(nowMs) };
+    });
+  }
+
+  function pause(resource, holderId) {
+    return store.withLock(resource, () => {
+      const nowMs = now();
+      const st = fold(resource);
+      if (st.holder?.holderId !== holderId) {
+        return { ok: false, reason: `resource-governor.pause: "${holderId}" does not hold "${resource}" (holder: ${st.holder?.holderId ?? "none"})` };
+      }
+      store.append({ type: "gov-paused", resource, holderId, at: iso(nowMs) });
+      return { ok: true, resource, holderId, paused: true };
+    });
+  }
+
+  function resume(resource, holderId) {
+    return store.withLock(resource, () => {
+      const nowMs = now();
+      const st = fold(resource);
+      if (st.holder?.holderId !== holderId) {
+        return { ok: false, reason: `resource-governor.resume: "${holderId}" does not hold "${resource}" (holder: ${st.holder?.holderId ?? "none"})` };
+      }
+      store.append({ type: "gov-resumed", resource, holderId, at: iso(nowMs) });
+      return { ok: true, resource, holderId, paused: false };
+    });
+  }
+
+  function release(resource, holderId) {
+    return store.withLock(resource, () => {
+      const nowMs = now();
+      const st = fold(resource);
+      if (st.holder?.holderId !== holderId) {
+        return { ok: false, reason: `resource-governor.release: "${holderId}" does not hold "${resource}" (holder: ${st.holder?.holderId ?? "none"})` };
+      }
+      store.append({ type: "gov-released", resource, holderId, at: iso(nowMs) });
+      st.holder = null;
+      reclaimAndPromote(st, nowMs); // wake-on-release: FIFO head granted in this same critical section
+      return { ok: true, resource, released: holderId, grantedTo: st.holder?.holderId ?? null };
+    });
+  }
+
+  function forceBreak(resource, { by, reason = null } = {}) {
+    if (!by) {
+      return { ok: false, reason: "resource-governor.forceBreak: `by` (the administrator identity) is required — every break is recorded" };
+    }
+    return store.withLock(resource, () => {
+      const nowMs = now();
+      const st = fold(resource);
+      const broken = st.holder?.holderId ?? null;
+      if (st.holder) {
+        store.append({ type: "gov-broken", resource, holderId: st.holder.holderId, leaseId: st.holder.leaseId, by, reason, at: iso(nowMs) });
+        st.holder = null;
+      }
+      reclaimAndPromote(st, nowMs);
+      return { ok: true, resource, broken, by, grantedTo: st.holder?.holderId ?? null };
+    });
+  }
+
+  function withdraw(resource, holderId) {
+    return store.withLock(resource, () => {
+      const st = fold(resource);
+      const w = st.waiters.find((x) => x.holderId === holderId);
+      if (!w) return { ok: false, reason: `resource-governor.withdraw: "${holderId}" is not waiting for "${resource}"` };
+      store.append({ type: "gov-abandoned", resource, holderId, requestId: w.requestId, at: iso(now()) });
+      return { ok: true, resource, holderId };
+    });
+  }
+
+  /** Read-only snapshot — never mutates (a dead holder is REPORTED stale here; reclaim happens
+   *  on the next mutating call, mirroring deriveEffective()'s non-destructive stale reporting). */
+  function status(resource) {
+    const nowMs = now();
+    const st = fold(resource);
+    return {
+      resource,
+      holder: st.holder ? { ...st.holder, stale: isDead(st.holder, nowMs) } : null,
+      waiters: st.waiters.map((w, i) => ({ ...w, position: i + 1 })),
+    };
+  }
+
+  return { config: cfg, request, acquire, heartbeat, pause, resume, release, forceBreak, withdraw, status };
+}
+
+/*
+ * ── RESOURCE GOVERNOR (HS-MUJ6, [[HS-AU7I-2U3R-EPIC-resource-governor-leases-and-wait]]) ───────
+ *
+ * The queue above is a WORK queue: a claim is per-task and terminal (complete/fail are the only
+ * exits). The governor below adds the missing WAIT-queue semantics for named, non-forkable
+ * resources ("git:repo", "service:spex-console", "machine:specops-gpu0", "mode:night-action", …):
+ *
+ *   one holder · a lease with liveness · a FIFO waiting line · an explicit release ·
+ *   an administrator's forced-break.
+ *
+ * It REUSES the atomic-claim discipline already proven above rather than rewriting it: every
+ * mutation is a read-decide-append critical section under the same per-key advisory lock
+ * (acquireLock/releaseLock), persisted to the same append-only, corrupt-tail-tolerant JSONL shape
+ * (appendEvent/readAllEvents) in its own ledger, `.heyspex/resource-governor.jsonl`
+ * (override: HEYSPEX_RESOURCE_GOVERNOR_PATH). The ledger doubles as the audit trail — a
+ * forced-break or a reclaim is an event line naming who/why/when, never an erased row.
+ *
+ * THE LIVENESS RULE (explicit, and stamped into every gov-reclaimed event):
+ *   a holder is DEAD (its lease reclaimable) iff it is NOT paused AND
+ *   now − lastLivenessAt > leaseTtlMs.
+ *   A PAUSED holder is NEVER reclaimed by liveness, no matter how old its last heartbeat —
+ *   pause() is the holder saying "alive, but about to go quiet" (a long tool call, lunch).
+ *   Only resume() or an administrator's forceBreak() ends a pause. forceBreak is the
+ *   deliberately-shipped-on-day-one escape hatch for the holder who paused and went home.
+ *
+ * CONFIGURATION, NOT CONSTANTS ([[HS-DEC-dev-factory-rules-are-configuration-not]]): every
+ * tunable is a config parameter with a default — resolveGovernorConfig() layers
+ * explicit overrides > HEYSPEX_GOVERNOR_* env vars > GOVERNOR_DEFAULTS. Nothing here is a bare
+ * constant an owner cannot change without editing code.
+ *
+ * PURITY / TESTABILITY: createResourceGovernor() takes an injectable store, clock (`now`),
+ * id generator (`genId`), and — for the blocking acquire() — an injectable `sleep`, so every
+ * semantic (FIFO order, reclaim-vs-paused, wake-on-release) is testable with a fake clock and
+ * no real waiting. Two stores ship here: createMemoryGovernorStore() (pure, in-process — also
+ * what the console dispatcher will mount server-side) and createFileGovernorStore() (this
+ * machine's durable JSONL mirror; cross-process on one filesystem via the advisory lock).
+ *
+ * WAKE-ON-RELEASE: release()/forceBreak() promote the FIFO head to holder IN THE SAME critical
+ * section — the next waiter is never left un-woken for a drain nobody triggers. In-process
+ * blocking waiters (acquire()) observe the grant on their next poll tick (wakePollMs); a waiter
+ * that crashed while holding its granted lease is itself reclaimed by the liveness rule, so a
+ * dead waiter can never wedge the line. Dead-holder reclaim also runs at the top of every
+ * request(), so any poller drives progress even when the holder died silently.
+ *
+ * ── DESIGN NOTE: how this generalizes fleet-wide, and the multi-machine port ───────────────────
+ *
+ * This module is deliberately domain-blind: `lane` is a free-form string, `payload` is opaque
+ * JSON, and nothing in enqueue/claim/complete/fail/list knows about research, Firecrawl, or
+ * Night Action. The first lane ("research-firecrawl") is a CONSUMER of this queue, not part of
+ * its design. Any other machine-gated resource in the fleet can stand up its own lane the same
+ * way, with zero changes to this file:
+ *   - build jobs waiting on a busy GPU box               -> lane "build-<machine>"
+ *   - breakdown/critic jobs waiting on the DeepSeek seat  -> lane "breakdown-deepseek"
+ *   - any single-tenant resource with a "one workload at a time" mode lease, mirroring
+ *     night-action-mode.mjs's Ollama-XOR-Firecrawl shape (mode lease refuses -> caller enqueues
+ *     instead of failing -> a drain step pops the lane once the resource frees up)
+ * The pattern each consumer follows is the same three-step shape night-action-mode.mjs
+ * demonstrates: (1) try the gated resource directly, (2) on refusal, enqueue() instead of
+ * returning a hard failure, (3) a --drain (or equivalent poll) on the resource-owning side pops
+ * nextPending()/list() once the resource is free and dispatches the work.
+ *
+ * MULTI-MACHINE PORT (done, HS-XOCK phase 4): this file-backed JSONL queue is correct for a
+ * SINGLE shared filesystem (one repo checkout, or machines that all see the same network share)
+ * — the per-id file lock only provides mutual exclusion within one filesystem. Machines with no
+ * shared disk now go through fleet-queue-neon.mjs instead — a Neon control-plane HTTP client
+ * against the console's bearer-authed `/api/fleet/queue*` routes (`fleet_tasks` table), with
+ * claim mutual-exclusion enforced by the dispatcher (`claimTaskById`) instead of a file lock.
+ * fleet-queue.mjs (the façade one level up) selects between this module and
+ * fleet-queue-neon.mjs at runtime — callers of the façade do not change; this file stays as the
+ * offline / no-dispatcher fallback and is never selected while an explicit `queuePath` is passed
+ * (tests keep pointing at a temp file unchanged).
+ */
+
+// ── self-test (local temp queue file — never touches the real .heyspex/fleet-queue.jsonl) ──────
+if (import.meta.url.endsWith("/" + (process.argv[1]?.split(/[\\/]/).pop() ?? "")) && process.argv.includes("--self-test")) {
+  (async () => {
+    const { tmpdir } = await import("node:os");
+    const { mkdtempSync, rmSync: rm } = await import("node:fs");
+    let ok = true;
+    const check = (n, c) => {
+      console.log(`  ${c ? "✓" : "✗"} ${n}`);
+      if (!c) ok = false;
+    };
+    console.log("== fleet-queue-file.mjs self-test ==\n");
+
+    const dir = mkdtempSync(join(tmpdir(), "hs-fleet-queue-selftest-"));
+    const qp = join(dir, "fleet-queue.jsonl");
+
+    // 1. enqueue: id + 1-indexed position, oldest-first within a lane
+    const a = enqueue({ lane: "research-firecrawl", payload: { q: "a" }, requestedBy: "agent-1", queuePath: qp, now: 1000 });
+    check("1 first enqueue gets position 1", a.position === 1 && typeof a.id === "string");
+    const b = enqueue({ lane: "research-firecrawl", payload: { q: "b" }, requestedBy: "agent-2", queuePath: qp, now: 2000 });
+    check("2 second enqueue in same lane gets position 2", b.position === 2);
+    const otherLane = enqueue({ lane: "build-gpu0", payload: { job: "x" }, queuePath: qp, now: 2500 });
+    check("3 a different lane starts its own position count", otherLane.position === 1);
+
+    // 2. nextPending returns oldest first, scoped to the lane
+    check("4 nextPending returns the oldest pending item in the lane", nextPending("research-firecrawl", { queuePath: qp, now: 3000 })?.id === a.id);
+    check("5 nextPending is lane-scoped", nextPending("build-gpu0", { queuePath: qp, now: 3000 })?.id === otherLane.id);
+
+    // 3. claim -> complete happy path
+    const claimA = claim(a.id, "claimant-1", { queuePath: qp, now: 4000 });
+    check("6 claim succeeds on a pending item", claimA.ok === true && claimA.reclaimed === false);
+    check("7 nextPending skips the now-claimed item", nextPending("research-firecrawl", { queuePath: qp, now: 4500 })?.id === b.id);
+    const doneA = complete(a.id, { savedTo: "specs/research/x.md" }, { queuePath: qp, now: 5000 });
+    check("8 complete succeeds on a claimed item", doneA.ok === true);
+    check("9 list reflects completed status + result", list("research-firecrawl", null, { queuePath: qp, now: 5500 }).find((i) => i.id === a.id)?.status === "completed");
+
+    // 4. double-claim rejection
+    const claimB1 = claim(b.id, "claimant-1", { queuePath: qp, now: 6000 });
+    check("10 first claim on b succeeds", claimB1.ok === true);
+    const claimB2 = claim(b.id, "claimant-2", { queuePath: qp, now: 6100 });
+    check("11 a second claim on the SAME still-live item is rejected", claimB2.ok === false && /already claimed/.test(claimB2.reason));
+
+    // 5. fail path
+    const failB = fail(b.id, "endpoint unreachable", { queuePath: qp, now: 7000 });
+    check("12 fail succeeds on a claimed item", failB.ok === true);
+    check("13 list reflects failed status + reason", list(null, "failed", { queuePath: qp, now: 7500 }).find((i) => i.id === b.id)?.failReason === "endpoint unreachable");
+    check("14 cannot claim a failed (terminal) item", claim(b.id, "claimant-3", { queuePath: qp, now: 7600 }).ok === false);
+    check("15 cannot complete an already-completed item", complete(a.id, {}, { queuePath: qp, now: 7700 }).ok === false);
+
+    // 6. crash-recovery semantics: a claimed-but-stale entry is reclaimable after the TTL
+    const c = enqueue({ lane: "research-firecrawl", payload: { q: "c" }, queuePath: qp, now: 10_000 });
+    const claimC = claim(c.id, "claimant-crashed", { queuePath: qp, now: 10_100, ttlMs: 1000 });
+    check("16 c gets claimed", claimC.ok === true);
+    const tooSoon = claim(c.id, "claimant-4", { queuePath: qp, now: 10_500, ttlMs: 1000 }); // 400ms later, still under 1000ms TTL
+    check("17 reclaim attempt before TTL expiry is rejected", tooSoon.ok === false);
+    const afterTtl = claim(c.id, "claimant-4", { queuePath: qp, now: 11_200, ttlMs: 1000 }); // 1100ms later, TTL exceeded
+    check("18 reclaim attempt after TTL expiry succeeds (stale claim reclaimed)", afterTtl.ok === true && afterTtl.reclaimed === true);
+    check("19 the ORIGINAL claim event is not erased — still visible in list()'s claimant history via the latest fold", list("research-firecrawl", "claimed", { queuePath: qp, now: 11_300, ttlMs: 1000 }).find((i) => i.id === c.id)?.claimant === "claimant-4");
+
+    // 7. unknown id handling
+    check("20 claim on an unknown id fails cleanly", claim("fq_does_not_exist", "x", { queuePath: qp }).ok === false);
+    check("21 complete on an unknown id fails cleanly", complete("fq_does_not_exist", {}, { queuePath: qp }).ok === false);
+
+    // 8. concurrent double-claim: two "processes" race claim() on the same id — only one wins
+    const d = enqueue({ lane: "research-firecrawl", payload: { q: "d" }, queuePath: qp, now: 12_000 });
+    const [raceA, raceB] = await Promise.all([
+      Promise.resolve().then(() => claim(d.id, "racer-A", { queuePath: qp, now: 12_100 })),
+      Promise.resolve().then(() => claim(d.id, "racer-B", { queuePath: qp, now: 12_100 })),
+    ]);
+    const winners = [raceA, raceB].filter((r) => r.ok);
+    check("22 concurrent claim() calls on the same id: exactly one wins", winners.length === 1);
+
+    // 9. append-only: the file only grows, corrupt trailing line tolerated
+    const beforeLines = readFileSync(qp, "utf8").split("\n").filter(Boolean).length;
+    const { appendFileSync } = await import("node:fs");
+    appendFileSync(qp, '{"type":"claim","id":"broken", partial-json');
+    check("23 a corrupt trailing line does not throw when reading state", (() => {
+      try {
+        list(null, null, { queuePath: qp });
+        return true;
+      } catch {
+        return false;
+      }
+    })());
+    check("24 file only grew (never rewritten) across the whole run", readFileSync(qp, "utf8").split("\n").filter(Boolean).length >= beforeLines);
+
+    rm(dir, { recursive: true, force: true });
+    console.log(`\n${ok ? "✓ self-test PASS" : "✗ self-test FAIL"}`);
+    process.exit(ok ? 0 : 1);
+  })();
+}