create-walle 0.10.2 → 0.10.3

package/template/coding-task-manager/lib/landing/queue.js

@@ -0,0 +1,998 @@
+'use strict';
+// WP-1 (docs/local-landing-queue-multi-session-integration-2026-10-26.html §5 D-1/D-2, §6.3,
+// §8 WP-1): job and queue state operations for the landing queue.
+//
+// All state lives under `<git-common-dir>/landing/` (WP-0 `state.js`), shared by every worktree of
+// the repository. `queue.json` is the single ordering authority — its `order` array holds the id of
+// every NON-TERMINAL job, sorted: the `in_progress` ids first in their existing relative order, then
+// the `queued` ids by priority rank descending (`normal` < `high` < `urgent`, §5 D-1) and, within a
+// level, by arrival (the id ascending, §5 D-3) — and each job's own `<id>.json` carries its state.
+// Every mutation of that pair happens inside `state.withMutex`, so two `land` processes can never
+// interleave a read-modify-write of the queue.
+//
+// There is no daemon and no scheduler (D-2): the runner is whichever `land` process holds the lease
+// in `lease.js`, and this module is the state machine it drives. No git, no spawn, no config here.
+
+const fs = require('fs');
+const path = require('path');
+const crypto = require('crypto');
+
+const state = require('./state');
+const lease = require('./lease');
+const { landingError } = require('./git');
+
+const JOB_STATES = Object.freeze(['queued', 'in_progress', 'landed', 'failed', 'blocked', 'canceled']);
+const TERMINAL_STATES = Object.freeze(['landed', 'failed', 'blocked', 'canceled']);
+const ACTIVE_STATES = Object.freeze(['queued', 'in_progress']);
+
+// D-1: three named levels, ascending rank. There is no numeric form and no `low`.
+const PRIORITIES = Object.freeze(['normal', 'high', 'urgent']);
+// D-7: a non-empty reason of at most this many characters is required for high and urgent.
+const PRIORITY_REASON_MAX = 200;
+
+// Sortable id (no deps): 10 Crockford-base32 characters of the submit timestamp (50 bits ≈ 35k
+// years of milliseconds) followed by 6 random characters. Lexicographic order of the id IS arrival
+// order, and two processes submitting in the same millisecond still cannot collide.
+const BASE32 = '0123456789abcdefghjkmnpqrstvwxyz';
+const ID_TIME_CHARS = 10;
+const ID_RANDOM_CHARS = 6;
+// OQ-3: the ETA stays `null` until this many gate durations have been recorded.
+const ETA_SAMPLES = 20;
+// OQ-3: the ETA is position × the MEDIAN of the last ETA_SAMPLES gate seconds.
+const ETA_WINDOW = 20;
+
+function nowIso(ms) {
+  return new Date(ms === undefined ? Date.now() : ms).toISOString();
+}
+
+function newJobId(nowMs) {
+  let value = Math.floor(nowMs === undefined ? Date.now() : nowMs);
+  let stamp = '';
+  for (let index = 0; index < ID_TIME_CHARS; index += 1) {
+    stamp = BASE32[value % 32] + stamp;
+    value = Math.floor(value / 32);
+  }
+  let suffix = '';
+  for (const byte of crypto.randomBytes(ID_RANDOM_CHARS)) suffix += BASE32[byte % 32];
+  return `${stamp}${suffix}`;
+}
+
+// WP-1 D-1 (§6): the batch id, the train id of a real batch (a singleton keeps its job id). A ULID:
+// 48 bits of timestamp then 80 bits of randomness, Crockford base32, so two batches built in the
+// same millisecond still cannot collide and the id sorts by creation time.
+function newBatchId(nowMs) {
+  let value = Math.floor(nowMs === undefined ? Date.now() : nowMs);
+  let stamp = '';
+  for (let index = 0; index < 10; index += 1) {
+    stamp = BASE32[value % 32] + stamp;
+    value = Math.floor(value / 32);
+  }
+  let suffix = '';
+  for (const byte of crypto.randomBytes(16)) suffix += BASE32[byte % 32];
+  return `b-${stamp}${suffix}`;
+}
+
+// --- paths (§6.3 layout, on top of state.js's SUBDIRS) ---------------------------------------
+
+function jobsDir(root) { return path.join(root, 'jobs'); }
+function jobFile(root, id) { return path.join(jobsDir(root), `${id}.json`); }
+function queueFile(root) { return path.join(root, 'queue.json'); }
+function metricsDir(root) { return path.join(root, 'metrics'); }
+function verdictsDir(root) { return path.join(root, 'verdicts'); }
+function logsDir(root) { return path.join(root, 'logs'); }
+function trainsDir(root) { return path.join(root, 'trains'); }
+
+// `landing/logs/` is not in state.js's SUBDIRS (WP-0 owns that list) and is created lazily here
+// because a job only needs a gate log once a gate actually runs.
+function ensureLogsDir(root) {
+  fs.mkdirSync(logsDir(root), { recursive: true });
+  return logsDir(root);
+}
+
+// --- queue.json ------------------------------------------------------------------------------
+
+function readQueue(root) {
+  const raw = state.readJsonSafe(queueFile(root));
+  if (!raw) return { contract: state.CONTRACT, order: [] };
+  state.assertContract(raw, { file: queueFile(root) });
+  const order = Array.isArray(raw.order) ? raw.order.filter((id) => typeof id === 'string') : [];
+  return { contract: state.CONTRACT, order };
+}
+
+function writeQueue(root, order) {
+  return state.writeJsonAtomic(queueFile(root), { contract: state.CONTRACT, order });
+}
+
+// --- jobs ------------------------------------------------------------------------------------
+
+function readJob(root, id) {
+  const raw = state.readJsonSafe(jobFile(root, id));
+  if (!raw) return null;
+  state.assertContract(raw, { file: jobFile(root, id) });
+  return raw;
+}
+
+function writeJob(root, job) {
+  return state.writeJsonAtomic(jobFile(root, job.id), job);
+}
+
+function listJobs(root) {
+  let names;
+  try {
+    names = fs.readdirSync(jobsDir(root));
+  } catch (error) {
+    /* expected: no job has ever been submitted in this repository, so landing/jobs/ does not
+       exist yet — an empty queue is the correct answer, never an error */
+    return [];
+  }
+  const jobs = [];
+  for (const name of names.filter((n) => n.endsWith('.json')).sort()) {
+    const job = readJob(root, name.slice(0, -'.json'.length));
+    if (job) jobs.push(job);
+  }
+  return jobs;
+}
+
+// One line per transition (§6.3). `gateSeconds` is what OQ-3's ETA reads; the doc's five fields
+// are all present.
+//
+// The `contract` default is the state contract, but an EVENT row (`refused`, `batch`) overrides it
+// with the metrics-schema version of §6/D-8 — a transition row and an event row are different
+// schemas in the same append-only file, and a reader can tell them apart by `contract`/`event`
+// without knowing which release wrote them.
+function appendMetric(root, entry) {
+  fs.mkdirSync(metricsDir(root), { recursive: true });
+  const file = path.join(metricsDir(root), `${nowIso().slice(0, 10)}.jsonl`);
+  fs.appendFileSync(file, `${JSON.stringify({ contract: state.CONTRACT, ...entry })}\n`);
+  return file;
+}
+
+// §6/D-8: the metrics-schema version an `event` row carries. A transition row stays at
+// `state.CONTRACT`; an event row is contract 2, which is what §7's metrics-shape invariant reads.
+const EVENT_CONTRACT = 2;
+
+function isTerminal(job) {
+  return Boolean(job) && TERMINAL_STATES.includes(job.state);
+}
+
+function isActive(job) {
+  return Boolean(job) && ACTIVE_STATES.includes(job.state);
+}
+
+// --- priority (§5 D-1, D-2, D-3, D-7) -----------------------------------------------------------
+
+// D-1: a job with no `priority`, or with a value outside the enum, is `normal` — every job written
+// before this change, and any job written by an older binary, keeps working unchanged.
+function priorityOf(job) {
+  const value = job && job.priority;
+  return PRIORITIES.includes(value) ? value : 'normal';
+}
+
+function priorityRank(job) {
+  return PRIORITIES.indexOf(priorityOf(job));
+}
+
+function jobFrom(jobsById, id) {
+  if (!jobsById) return null;
+  if (typeof jobsById.get === 'function') return jobsById.get(id) || null;
+  return jobsById[id] || null;
+}
+
+// D-2: pure. `jobsById` maps id -> job; an id with no job file and an id whose job is terminal are
+// both dropped (a terminal job is not in the queue). The result is the canonical order: the
+// `in_progress` ids first in their existing relative order, then the rest by rank descending, then
+// by id ascending (D-3 — the id is arrival-sortable, queue.js:28-30).
+function sortOrder(ids, jobsById) {
+  const running = [];
+  const queued = [];
+  for (const id of ids) {
+    const job = jobFrom(jobsById, id);
+    if (!job) continue;
+    if (TERMINAL_STATES.includes(job.state)) continue;
+    if (job.state === 'in_progress') running.push(id);
+    else queued.push(id);
+  }
+  queued.sort((a, b) => {
+    const byRank = priorityRank(jobFrom(jobsById, b)) - priorityRank(jobFrom(jobsById, a));
+    if (byRank !== 0) return byRank;
+    if (a === b) return 0;
+    return a < b ? -1 : 1;
+  });
+  return [...running, ...queued];
+}
+
+// The canonical order of the ids currently in `queue.json`, read from the job files themselves.
+function sortedQueueOrder(root, ids, overrides) {
+  const jobsById = {};
+  for (const id of ids) {
+    jobsById[id] = overrides && overrides[id] ? overrides[id] : readJob(root, id);
+  }
+  return sortOrder(ids, jobsById);
+}
+
+// Idempotency (§6.2): the same branch at the same tip that is already queued or in_progress is not
+// a second job.
+function activeJobFor(root, branch, tip) {
+  return listJobs(root).find((job) => job.branch === branch && job.tip === tip && isActive(job)) || null;
+}
+
+// D-11 / I-5: a second `land` process for the same CTM session is a refusal, but only while the
+// first one is actually alive — a session that died leaves its job reapable, not a lockout.
+//
+// A submit no longer drains in the foreground, so the submit process is usually already gone by the
+// time anyone asks. The process that actually represents the job is the RUNNER working it
+// (`runnerPid`, recorded by `land submit`); either pid being alive is what makes the session busy.
+//
+// D-7: the refusal is per (session, BRANCH), not per session — the I-5 race it guards (two resumed
+// sessions sharing one index) is per worktree+branch, so one session landing two DIFFERENT branches
+// from two worktrees is legitimate; only a second active job for the SAME branch collides.
+function duplicateSubmitter(root, session, pid, branch) {
+  if (!session) return null;
+  return listJobs(root).find((job) => {
+    if (!isActive(job)) return false;
+    if (!job.submittedBy || job.submittedBy.session !== session) return false;
+    if (job.branch !== branch) return false;
+    const owners = [job.submittedBy.pid, job.runnerPid]
+      .filter((value) => typeof value === 'number' && value !== pid);
+    return owners.some((value) => state.pidIsAlive(value));
+  }) || null;
+}
+
+function enqueue(root, job) {
+  return state.withMutex(root, () => {
+    state.ensureDirs(root);
+    writeJob(root, job);
+    const queued = readQueue(root).order;
+    // D-2: the sorted place of a new job is the canonical one — an older binary's plain append and
+    // this sort agree, because the end of the `queued` run IS a normal job's correct place.
+    const order = sortedQueueOrder(root, [...queued, job.id], { [job.id]: job });
+    writeQueue(root, order);
+    appendMetric(root, {
+      jobId: job.id,
+      from: null,
+      to: job.state,
+      at: job.timestamps && job.timestamps.queued ? job.timestamps.queued : nowIso(),
+      seconds: null,
+      gateSeconds: null,
+    });
+    return { position: order.indexOf(job.id) + 1, queued: order.length };
+  });
+}
+
+function positionOf(root, id) {
+  const index = readQueue(root).order.indexOf(id);
+  return index === -1 ? null : index + 1;
+}
+
+function updateJob(root, id, patch) {
+  return state.withMutex(root, () => {
+    const job = readJob(root, id);
+    if (!job) return null;
+    const next = { ...job, ...patch };
+    writeJob(root, next);
+    return next;
+  });
+}
+
+// Terminal transition: the job leaves the queue and one metrics line is appended. `metric` carries
+// the per-transition extras (`gateSeconds`) that the ETA needs.
+//
+// The body is mutex-FREE (`finishJobLocked`) so a batch can write N of them under ONE hold —
+// `state.withMutex` is not re-entrant, so `finishBatch` below must not call the public wrapper.
+function finishJob(root, id, patch, metric = {}) {
+  return state.withMutex(root, () => finishJobLocked(root, id, patch, metric));
+}
+
+function finishJobLocked(root, id, patch, metric = {}) {
+  const job = readJob(root, id);
+  const from = job ? job.state : null;
+  const next = { ...(job || { id }), ...patch, id };
+  writeJob(root, next);
+  writeQueue(root, readQueue(root).order.filter((queuedId) => queuedId !== id));
+  const startedAt = next.timestamps && next.timestamps.started ? Date.parse(next.timestamps.started) : NaN;
+  appendMetric(root, {
+    jobId: id,
+    from,
+    to: next.state,
+    at: nowIso(),
+    seconds: Number.isFinite(startedAt) ? Number(((Date.now() - startedAt) / 1000).toFixed(3)) : null,
+    gateSeconds: typeof metric.gateSeconds === 'number' ? metric.gateSeconds : null,
+    // WP-0 (§6.6/D-8): the terminal row carries the same `gate` object the job file does, so a
+    // later WP can measure every terminal transition from the metrics stream alone.
+    gate: metric.gate || null,
+  });
+  return next;
+}
+
+// WP-1 D-1/§6 (AC-1a): ONE terminal write for a whole batch. Every member lands with the same
+// `train`/`tree`/`base`/`batch`, and exactly ONE `{ event: 'batch' }` metrics row records the
+// verdict — a batch is one gate, so it must also be one row for the metrics readers to count.
+// `patch` is an object (spread onto every member) or `(member) => object`. `metric` is the batch's
+// ONE telemetry row; it is written under this same hold, so "the members landed" and "the row that
+// records the verdict is on disk" are one transition, exactly as `finishJobLocked` writes a job's
+// state and its row together. (A reader that does not take the mutex can still, in principle, catch
+// the instant between the last member's write and the row's; a reader that needs both waits for both.)
+function finishBatch(root, batchId, members, patch = {}, { gateSeconds = null, gate = null, metric = null } = {}) {
+  return state.withMutex(root, () => {
+    const results = [];
+    for (const member of members) {
+      const next = typeof patch === 'function' ? patch(member) : patch;
+      results.push(finishJobLocked(root, member.id, next, { gateSeconds, gate }));
+    }
+    if (metric) recordBatchMetric(root, { batchId, ...metric });
+    return results;
+  });
+}
+
+// WP-1 D-1 (§6): the batch telemetry row, written under the caller's hold (`finishBatch`) or on its
+// own (the FAIL path, which changes no job state). It carries `event`, so every state-transition
+// reader skips it (`metricSequence` filters on `to`; the ETA reader skips `event` rows explicitly).
+function recordBatchMetric(root, entry) {
+  return appendMetric(root, { contract: EVENT_CONTRACT, event: 'batch', ...entry, at: nowIso() });
+}
+
+// WP-1 D-2 (§6, AC-1b): the second half of a bisected batch. Every member goes back to `queued` —
+// NOT failed: the gate said the batch is bad, not which member is — carrying `batch.attempt + 1` so
+// the next bisection round is visible on the record. One metrics row per member, copying the
+// `requeuePreempted` shape with `reason: 'bisected'`. The ids sort by arrival (D-3), so the half is
+// ahead of everything submitted after it without any new ordering state.
+// `reason` names the metric row: 'bisected' (a FAIL halving) or 'withdrawn' (a member's cancel voided the run, D-4).
+function requeueHalf(root, members, { batchId = null, attempt = 1, batchMembers = null, reason = 'bisected' } = {}) {
+  return state.withMutex(root, () => {
+    const ids = members.map((member) => member.id);
+    const results = [];
+    for (const member of members) {
+      const job = readJob(root, member.id);
+      if (!job) continue;
+      const timestamps = { ...(job.timestamps || {}) };
+      delete timestamps.started;
+      const next = {
+        ...job,
+        state: 'queued',
+        timestamps,
+        batch: {
+          id: batchId || (job.batch && job.batch.id) || job.id,
+          members: (batchMembers || ids).slice(),
+          attempt,
+          index: ids.indexOf(member.id),
+        },
+      };
+      writeJob(root, next);
+      appendMetric(root, {
+        jobId: member.id,
+        from: 'in_progress',
+        to: 'queued',
+        at: nowIso(),
+        seconds: null,
+        gateSeconds: null,
+        reason,
+      });
+      results.push(next);
+    }
+    const order = readQueue(root).order;
+    const head = order.filter((queuedId) => !ids.includes(queuedId));
+    writeQueue(root, [...head, ...ids.filter((id) => order.includes(id))]);
+    return results;
+  });
+}
+
+// WP-0 D-8 (§6, §9 S-13, AC-0a): a submit REFUSAL is an event. `duplicate_submitter` and
+// `conflict_at_submit` are thrown before any job record exists, so until now the queue could not see
+// its own back-pressure at all (census §1). One row per refusal, in the day's metrics file, through
+// the same writer as every other row.
+function recordRefusal(root, { reason, session = null, branch = null } = {}) {
+  return appendMetric(root, {
+    contract: EVENT_CONTRACT,
+    event: 'refused',
+    reason: reason === undefined ? null : reason,
+    session: session === undefined ? null : session,
+    branch: branch === undefined ? null : branch,
+    // §6/D-8: the row is stamped like every other one, so a reader can order refusals against the
+    // transitions around them.
+    at: nowIso(),
+  });
+}
+
+// AC-0a/§6: how many refusals were recorded today, for `land status`'s `refused today: N` line. Pure
+// over the day's metrics file — a missing file is 0, never an error.
+function refusalsToday(root, day = nowIso().slice(0, 10)) {
+  const name = `${day}.jsonl`;
+  let raw;
+  try {
+    raw = fs.readFileSync(path.join(metricsDir(root), name), 'utf8');
+  } catch (error) {
+    /* expected: nothing has been refused (or nothing has happened at all) today */
+    return 0;
+  }
+  let count = 0;
+  for (const line of raw.split('\n')) {
+    if (!line) continue;
+    let entry;
+    try {
+      entry = JSON.parse(line);
+    } catch (error) {
+      /* expected: a torn final line from a process killed mid-append — one unparseable row never
+         invalidates the rows around it */
+      continue;
+    }
+    if (entry && entry.event === 'refused') count += 1;
+  }
+  return count;
+}
+
+// --- priority mutation (§5 D-1, D-7; §6.1) -----------------------------------------------------
+
+// D-7/D-4: one effective change = one `priorityHistory` entry + exactly one metrics line. The
+// metrics line deliberately carries `priorityFrom`/`priorityTo` and NEVER `from`/`to`, so the
+// state-transition readers (metricSequence, the ETA) skip it.
+//
+// Reason VALIDATION is the caller's job (a usage error, exit 2, whose message names the flag), so
+// nothing here re-formats it. There is no ownership check, matching `land cancel`.
+//
+// §6.1/D-7: the preempt field is the D-4 sentence's outcome, and asking for `urgent` really does
+// write the stop request of the running lower-ranked job (WP-3) — inside this same mutex hold, so no
+// runner checkpoint can read the queue between the rank change and the request.
+function setPriority(root, id, { priority, reason = null, by = null, via = null } = {}) {
+  return state.withMutex(root, () => {
+    const job = readJob(root, id);
+    if (!job) throw landingError('job_not_found', `no landing job ${JSON.stringify(id)} in ${root}`, { jobId: id, root });
+    if (TERMINAL_STATES.includes(job.state)) {
+      throw landingError('job_terminal', `job ${id} is already ${job.state}`, { jobId: id, state: job.state });
+    }
+    if (!PRIORITIES.includes(priority)) {
+      throw landingError('priority_invalid', `priority ${JSON.stringify(priority)} is not one of ${PRIORITIES.join(', ')}`, { jobId: id, priority });
+    }
+    const from = priorityOf(job);
+    const order = readQueue(root).order;
+    const at = nowIso();
+    // D-7: a no-op writes no history and no metric — asking for the level a job already has is not a
+    // change, and a log full of no-ops is not an audit trail.
+    if (from === priority) {
+      const index = order.indexOf(id);
+      return { job, changed: false, position: index === -1 ? null : index + 1, preempt: null };
+    }
+    const history = Array.isArray(job.priorityHistory) ? job.priorityHistory : [];
+    const next = { ...job, priority, priorityHistory: [...history, { from, to: priority, reason, at, by, via }] };
+    writeJob(root, next);
+    const jobsById = {};
+    for (const queuedId of order) jobsById[queuedId] = queuedId === id ? next : readJob(root, queuedId);
+    const sorted = sortOrder(order, jobsById);
+    writeQueue(root, sorted);
+    appendMetric(root, {
+      event: 'priority',
+      jobId: id,
+      priorityFrom: from,
+      priorityTo: priority,
+      reason,
+      by,
+      at,
+    });
+    const index = sorted.indexOf(id);
+    return {
+      job: next,
+      changed: true,
+      position: index === -1 ? null : index + 1,
+      preempt: priority === 'urgent' ? requestPreemptLocked(root, id, { by, reason }) : null,
+    };
+  });
+}
+
+// §6.1/D-4: `null` unless the requested level is `urgent`; then the D-4 sentence, decided and — when
+// it allows it — WRITTEN, all inside the caller's mutex hold. The full outcome set, in the order the
+// checks run:
+//   'not_running'        no `in_progress` job other than the bumped one — nothing to stop
+//   'not_lower'          the running job is itself `urgent`: urgent never stops urgent
+//   'already_preempted'  the running job already carries `preemptions >= 1` (D-4: at most once)
+//   'no_runner'          no lease record, or a dead one — nothing is working that job any more
+//   'runner_cannot_stop' a LIVE lease without the `stop` capability (D-10): a runner spawned from an
+//                        older `land.js` would land the job anyway, so nothing is written
+//   'already_requested'  the job already carries a stop request (cancel or preempt): a preempt never
+//                        replaces one, and never downgrades a cancel
+//   'requested'          the request was written
+//
+// Mutex-free on purpose: both callers run inside one `state.withMutex` hold (`setPriority`'s urgent
+// bump and a fresh urgent `cmdSubmit`), and `state.withMutex` is not re-entrant. Use
+// `requestPreemptFor` from anywhere else — one implementation, two callers, no duplicated decision.
+function requestPreemptLocked(root, urgentJobId, { by = null, reason = null } = {}) {
+  const order = readQueue(root).order;
+  let running = null;
+  for (const queuedId of order) {
+    if (queuedId === urgentJobId) continue;
+    const job = readJob(root, queuedId);
+    if (job && job.state === 'in_progress') {
+      running = job;
+      break;
+    }
+  }
+  if (!running) return 'not_running';
+  if (priorityOf(running) === 'urgent') return 'not_lower';
+  if ((Number.isFinite(running.preemptions) ? running.preemptions : 0) >= 1) return 'already_preempted';
+  const record = lease.read(root);
+  if (!record || lease.isDead(record)) return 'no_runner';
+  if (!lease.hasCap(record, 'stop')) return 'runner_cannot_stop';
+  if (running.stopRequest) return 'already_requested';
+  writeJob(root, {
+    ...running,
+    stopRequest: { kind: 'preempt', by, reason: reason || null, forJob: urgentJobId, at: nowIso() },
+  });
+  return 'requested';
+}
+
+// §6.1: the mutex-taking entry point (a fresh `--priority urgent` submit).
+function requestPreemptFor(root, urgentJobId, { by = null, reason = null } = {}) {
+  return state.withMutex(root, () => requestPreemptLocked(root, urgentJobId, { by, reason }));
+}
+
+// D-4: the runner's preempt transition, inside the mutex. The job keeps its id, so `sortOrder` puts
+// it back at the front of its own level (D-3 — an id is arrival-sortable), and the ONE metrics line
+// copies the `runner_reaped` shape with `reason: 'preempted'`.
+function requeuePreempted(root, id) {
+  return state.withMutex(root, () => {
+    const job = readJob(root, id);
+    if (!job) return null;
+    const request = job.stopRequest || null;
+    const timestamps = { ...(job.timestamps || {}) };
+    delete timestamps.started;
+    const next = {
+      ...job,
+      state: 'queued',
+      preemptions: (Number.isFinite(job.preemptions) ? job.preemptions : 0) + 1,
+      preemptedBy: { jobId: request ? request.forJob : null, at: nowIso() },
+      timestamps,
+    };
+    delete next.stopRequest;
+    writeJob(root, next);
+    const order = readQueue(root).order;
+    const ids = order.includes(id) ? order : [...order, id];
+    const jobsById = {};
+    for (const queuedId of ids) jobsById[queuedId] = queuedId === id ? next : readJob(root, queuedId);
+    writeQueue(root, sortOrder(ids, jobsById));
+    appendMetric(root, {
+      jobId: id,
+      from: 'in_progress',
+      to: 'queued',
+      at: nowIso(),
+      seconds: null,
+      gateSeconds: null,
+      reason: 'preempted',
+    });
+    return next;
+  });
+}
+
+// D-6 (§9 S-11, WP-2): the runner's base_broken transition, inside the mutex. The job is NOT
+// terminal — every file the gate named fails on the base commit too, so this is main's failure, not
+// the branch owner's, and re-gating the same tip would just burn another gate on the same broken
+// main. It goes back to `queued` in its own place (the id is arrival-sortable, so `sortOrder` puts
+// it where it was, and its priority is untouched) carrying `retryAfterTargetMoves`, which is what
+// `takeNextJob` holds it against until the target tip actually moves. The ONE metrics line copies
+// the `requeuePreempted` shape with `reason: 'base_broken'`.
+function requeueBaseBroken(root, id, patch = {}, metric = {}) {
+  return state.withMutex(root, () => {
+    const job = readJob(root, id);
+    if (!job) return null;
+    const next = { ...job, ...patch, id, state: 'queued', retryAfterTargetMoves: true };
+    writeJob(root, next);
+    const order = readQueue(root).order;
+    const ids = order.includes(id) ? order : [...order, id];
+    const jobsById = {};
+    for (const queuedId of ids) jobsById[queuedId] = queuedId === id ? next : readJob(root, queuedId);
+    writeQueue(root, sortOrder(ids, jobsById));
+    const startedAt = next.timestamps && next.timestamps.started ? Date.parse(next.timestamps.started) : NaN;
+    appendMetric(root, {
+      jobId: id,
+      from: 'in_progress',
+      to: 'queued',
+      at: nowIso(),
+      // `seconds` is the run that just ended, exactly as `finishJob` reports it, so the ETA reader
+      // sees one shape whether a job left the queue by failing or by being parked on a broken base.
+      seconds: typeof metric.seconds === 'number'
+        ? metric.seconds
+        : (Number.isFinite(startedAt) ? Number(((Date.now() - startedAt) / 1000).toFixed(3)) : null),
+      gateSeconds: typeof metric.gateSeconds === 'number' ? metric.gateSeconds : null,
+      reason: 'base_broken',
+    });
+    return next;
+  });
+}
+
+// D-6/D-7: a job whose base rerun found the failure on base as well is held until the target moves.
+// `triage.base` is the tip the rerun was measured against, so while the target still IS that tip the
+// job has nothing new to learn — it is not `takeNextJob`'s turn to run, it is main's turn to be
+// fixed. An unknown tip (`targetTip` undefined/null, e.g. a config this process cannot read) holds
+// nothing: the safe direction is always to run the job rather than to park it forever.
+function heldByTargetMove(job, targetTip) {
+  if (!job || job.retryAfterTargetMoves !== true || !targetTip) return false;
+  return Boolean(job.triage) && job.triage.base === targetTip;
+}
+
+// WP-1 D-1/§9 S-8: READ-ONLY peek at the runnable order — the ids `takeNextBatch` would consider,
+// in the order it would consider them. `drain` uses it to decide the window BEFORE anything is
+// marked `in_progress`: whether the head job must land alone (a proven/cached tip), and whether a
+// lone job should wait for company (`landing.batch.waitSeconds`). Writes nothing, takes no mutex.
+function peekRunnable(root, { targetTip, exclude = null } = {}) {
+  const excluded = exclude instanceof Set ? exclude : new Set(Array.isArray(exclude) ? exclude : []);
+  const order = readQueue(root).order;
+  const jobsById = new Map();
+  for (const id of order) jobsById.set(id, readJob(root, id));
+  const candidates = order.filter((id) => {
+    const job = jobsById.get(id);
+    if (!job || TERMINAL_STATES.includes(job.state)) return false;
+    // An `in_progress` job belongs to a runner; if that runner is dead, `takeNextBatch` reaps it back
+    // to `queued` and it is eligible again — the peek must not silently drop it, but it must also not
+    // claim the runner would gate it in this same tick. Treat it as eligible: the reap happens under
+    // the mutex immediately before the take.
+    if (excluded.has(id)) return false;
+    return !heldByTargetMove(job, targetTip);
+  });
+  return sortOrder(candidates, jobsById);
+}
+
+// --- the runner's queue operations (D-2) ------------------------------------------------------
+
+// D-5: the ONLY way another process asks the runner to stop a job — never by writing a terminal
+// state itself (the runner is the single writer of in-progress transitions). Inside the mutex, so a
+// request and the runner's own checkpoint read cannot interleave. Refuses `job_terminal`: a job that
+// already has an outcome has nothing left to stop.
+//
+// A pending `kind:'cancel'` is never DOWNGRADED to `'preempt'`: cancel is the stronger request, and
+// a preempt arriving behind it must not turn a "stop and never land" into a "stop and requeue".
+function requestStop(root, id, { kind, by = null, reason = null, forJob = null } = {}) {
+  return state.withMutex(root, () => {
+    if (kind !== 'cancel' && kind !== 'preempt') {
+      throw landingError('stop_kind_invalid', `stop kind ${JSON.stringify(kind)} is not one of cancel, preempt`, { jobId: id, kind });
+    }
+    const job = readJob(root, id);
+    if (!job) throw landingError('job_not_found', `no landing job ${JSON.stringify(id)} in ${root}`, { jobId: id, root });
+    if (TERMINAL_STATES.includes(job.state)) {
+      throw landingError('job_terminal', `job ${id} is already ${job.state}`, { jobId: id, state: job.state });
+    }
+    const existing = job.stopRequest;
+    const request = {
+      kind,
+      by,
+      reason: reason || null,
+      forJob: forJob || null,
+      at: nowIso(),
+    };
+    const stopRequest = existing && existing.kind === 'cancel' && kind === 'preempt' ? existing : request;
+    const next = { ...job, stopRequest };
+    writeJob(root, next);
+    return next;
+  });
+}
+
+// Take the head job. Two D-2 duties ride along, both safe because the caller already holds the
+// lease (so no other runner exists for this repository):
+//   * ids of terminal jobs are dropped from `queue.json`;
+//   * a job left in `in_progress` by a previous, dead runner goes back to the HEAD of the queue.
+//
+// D-2/§7: the remaining ids are re-sorted with `sortOrder` before `order[0]` is taken and the sorted
+// order is what gets written, so the invariant is SELF-HEALING — a hand edit, an older binary that
+// appended a normal job, or a crash cannot make this runner take a lower-ranked job first.
+//
+// `onEmpty` runs INSIDE the mutex when the queue is empty: the runner releases its lease there, so
+// a submitter's enqueue (also under the mutex) either lands before the check and is taken, or lands
+// after the release and finds no live runner and starts one. Releasing outside the mutex would leave
+// a window where a job is enqueued behind a runner that has already decided to exit.
+// WP-1 D-1 (§6, §9 S-1): a job is not RUNNABLE while the merge it will be gated on is already known
+// to conflict — the submit precheck wrote that preview, and re-gating a conflict the queue cannot
+// resolve burns a gate to reach a verdict `buildTrain` computes in milliseconds. A job with a CLEAR
+// preview (`null`) and a job that has never conflicted are both runnable.
+function hasConflictPaths(job) {
+  const preview = job && job.conflictPreview;
+  return Boolean(preview) && Array.isArray(preview.paths) && preview.paths.length > 0;
+}
+
+// WP-1 D-1 (§6, §9 S-1, AC-1a/AC-1b): take the batch. The HEAD of the runnable order is always
+// taken — that is the job `takeNextJob` has always taken, and a job the submitter queued with
+// `--queue-anyway` must still reach the runner's `blocked (conflict)` verdict rather than sit in the
+// queue forever. The remaining slots are filled, in the same order, with the runnable jobs that have
+// no conflict preview, up to `window`.
+//
+// The two D-2 duties of `takeNextJob` ride along unchanged (they are safe because the caller already
+// holds the lease): terminal ids are dropped from `queue.json`, and a job left `in_progress` by a
+// previous, dead runner goes back to the HEAD of the queue.
+//
+// `exclude` is the set of ids the caller has already decided not to include (D-7: a member whose stop
+// request arrived is dropped from the NEXT train build).
+function takeNextBatch(root, { onEmpty, targetTip, window: windowSize = 1, exclude = null, batchId = null, attempt = 1 } = {}) {
+  const limit = Number.isInteger(windowSize) && windowSize >= 1 ? windowSize : 1;
+  const excluded = exclude instanceof Set ? exclude : new Set(Array.isArray(exclude) ? exclude : []);
+  return state.withMutex(root, () => {
+    const raw = readQueue(root);
+    const jobsById = {};
+    const requeued = [];
+    const rest = [];
+    for (const id of raw.order) {
+      const job = readJob(root, id);
+      jobsById[id] = job;
+      if (!job) continue;
+      if (TERMINAL_STATES.includes(job.state)) continue;
+      if (job.state === 'in_progress') requeued.push(id);
+      else rest.push(id);
+    }
+    for (const id of requeued) {
+      const job = jobsById[id];
+      const timestamps = { ...(job.timestamps || {}) };
+      delete timestamps.started;
+      jobsById[id] = { ...job, state: 'queued', timestamps };
+      writeJob(root, jobsById[id]);
+      appendMetric(root, { jobId: id, from: 'in_progress', to: 'queued', at: nowIso(), seconds: null, gateSeconds: null, reason: 'runner_reaped' });
+    }
+    const order = [...requeued, ...sortOrder(rest, jobsById)];
+    // D-6/D-7 (WP-2): a job the runner parked as `base_broken` is not runnable while the target tip
+    // is still the tip its rerun measured — re-gating it would report main's own failure again as
+    // this branch's. It stays in `queue.json` (the whole non-terminal order is written either way),
+    // so it is still visible to `land status` and still holds its place; `drain` passes the tip it
+    // resolved this iteration. EVERY remaining job held is the same as an empty queue for the runner:
+    // there is nothing to do, so `onEmpty` releases the lease (a later `land submit` or the next
+    // target move starts it again).
+    const runnable = order.filter((queuedId) => !heldByTargetMove(jobsById[queuedId], targetTip) && !excluded.has(queuedId));
+    if (!runnable.length) {
+      writeQueue(root, order);
+      if (onEmpty) onEmpty();
+      return [];
+    }
+    const head = runnable[0];
+    const members = [head];
+    for (const id of runnable.slice(1)) {
+      if (members.length >= limit) break;
+      if (hasConflictPaths(jobsById[id])) continue;
+      members.push(id);
+    }
+    const id = batchId || (members.length > 1 ? newBatchId() : members[0]);
+    const taken = [];
+    for (let index = 0; index < members.length; index += 1) {
+      const memberId = members[index];
+      const job = readJob(root, memberId);
+      const from = job.state;
+      const next = { ...job, state: 'in_progress', timestamps: { ...(job.timestamps || {}), started: nowIso() } };
+      // Taking it is the whole point of the flag: the hold is spent, and the record must not claim a
+      // wait that is over (D-7). Written explicitly as `false`, never deleted, so a reader sees the
+      // transition rather than a missing key.
+      if (job.retryAfterTargetMoves === true) next.retryAfterTargetMoves = false;
+      // D-1 (§6 job record): the batch rides on every member — but ONLY on a real batch. A window of
+      // one is today's serial runner, and its job record (and therefore every line `land status`
+      // prints) must stay byte-identical to what it was.
+      if (members.length > 1) next.batch = { id, members: members.slice(), attempt, index };
+      writeJob(root, next);
+      appendMetric(root, { jobId: memberId, from, to: 'in_progress', at: nowIso(), seconds: null, gateSeconds: null });
+      taken.push(next);
+    }
+    writeQueue(root, order);
+    return taken;
+  });
+}
+
+// The serial entry point: one job, exactly as before this change. Every existing caller (and every
+// existing test) keeps its contract because a window of one never writes a `batch` field.
+function takeNextJob(root, opts = {}) {
+  return takeNextBatch(root, { ...opts, window: 1 })[0] || null;
+}
+
+// D-2: drop the lease of a runner that is provably gone. The in_progress job it left behind is
+// returned to the head by `takeNextJob`, which only ever runs while we hold the lease.
+function reapDeadRunner(root) {
+  const record = lease.read(root);
+  if (!record) return null;
+  if (!lease.isDead(record)) return null;
+  lease.reapIfDead(root);
+  return record;
+}
+
+// --- ETA (OQ-3) -------------------------------------------------------------------------------
+
+function median(values) {
+  if (!values.length) return null;
+  const sorted = [...values].sort((a, b) => a - b);
+  const middle = Math.floor(sorted.length / 2);
+  return sorted.length % 2 ? sorted[middle] : (sorted[middle - 1] + sorted[middle]) / 2;
+}
+
+// The last `limit` gate durations recorded for this repository, newest file first.
+function recentGateSeconds(root, limit = ETA_WINDOW) {
+  let names;
+  try {
+    names = fs.readdirSync(metricsDir(root));
+  } catch (error) {
+    /* expected: no job has completed yet, so landing/metrics/ does not exist — "no samples",
+       which the ETA caller renders as `null` (never as a guess) */
+    return [];
+  }
+  const values = [];
+  for (const name of names.filter((n) => n.endsWith('.jsonl')).sort().reverse()) {
+    let raw;
+    try {
+      raw = fs.readFileSync(path.join(metricsDir(root), name), 'utf8');
+    } catch (error) {
+      /* expected: a metrics file rotated away between readdir and read — skip it; older files
+         still provide the sample */
+      continue;
+    }
+    for (const line of raw.split('\n').filter(Boolean).reverse()) {
+      let entry;
+      try {
+        entry = JSON.parse(line);
+      } catch (error) {
+        /* expected: a torn final line from a process killed mid-append — a metrics file is
+           append-only evidence, and one unparseable line never invalidates the samples around it */
+        continue;
+      }
+      // WP-0/WP-1: an `event` row (`refused`, `priority`, `batch`) is NOT a state transition. The
+      // ETA reads transitions only, so the guard is explicit rather than a side effect of the row
+      // happening to lack `gateSeconds`.
+      if (entry && entry.event) continue;
+      if (typeof entry.gateSeconds === 'number' && Number.isFinite(entry.gateSeconds)) {
+        values.push(entry.gateSeconds);
+        if (values.length >= limit) return values;
+      }
+    }
+  }
+  return values;
+}
+
+// `eta_s` = position × median of the last 20 gate seconds, or null with fewer than 20 samples.
+function etaSeconds(root, position) {
+  if (!Number.isFinite(position) || position <= 0) return null;
+  const samples = recentGateSeconds(root, ETA_SAMPLES);
+  if (samples.length < ETA_SAMPLES) return null;
+  return Math.round(position * median(samples));
+}
+
+// --- read models ------------------------------------------------------------------------------
+
+function jobSummary(job) {
+  return {
+    id: job.id,
+    branch: job.branch,
+    tip: job.tip,
+    title: job.title,
+    state: job.state,
+    reason: job.reason === undefined ? null : job.reason,
+    priority: priorityOf(job),
+    priorityReason: priorityReasonOf(job),
+    preemptions: Number.isFinite(job.preemptions) ? job.preemptions : 0,
+    stopRequest: job.stopRequest === undefined ? null : job.stopRequest,
+    base: job.base === undefined ? null : job.base,
+    train: job.train === undefined ? null : job.train,
+    tree: job.tree === undefined ? null : job.tree,
+    verdict: job.verdict === undefined ? null : job.verdict,
+    // D-5 (design landing-throughput-and-big-file-decomposition-2026-10-29.html §6 job record): the
+    // conflict the queue measured for this job against the target tip, written by `land submit`
+    // (`--queue-anyway`) and re-written by the runner's post-landing sweep. `null` for a job that has
+    // never conflicted, and an explicit `null` (never `undefined`) so a JSON reader sees the key.
+    conflictPreview: job.conflictPreview === undefined ? null : job.conflictPreview,
+    // D-6/D-7 (WP-2): the base-rerun receipt (`{ base, files, failedOnBase, seconds }`, or null when
+    // the pass did not run) and the hold it implies. `land status` reads both: the first is the
+    // evidence for why a queued job is not this branch's fault, the second is why it is not running.
+    triage: job.triage === undefined ? null : job.triage,
+    retryAfterTargetMoves: job.retryAfterTargetMoves === true,
+    submittedBy: job.submittedBy || null,
+    timestamps: job.timestamps || {},
+    gateLog: job.gateLog === undefined ? null : job.gateLog,
+    // WP-1 D-1 (§6 job record): the batch this job is being gated in — `{ id, members, attempt,
+    // index }`, or an explicit `null` for the serial runner (a window of one writes no batch, so
+    // every pre-change job and every singleton row reads exactly as it did).
+    batch: job.batch === undefined ? null : job.batch,
+    // WP-0 (§6.6/D-8): the gate's self-description rides on every listing/`--json` row, so
+    // `land status` can print `gate <mode> <executed>/<selected>` without a second read.
+    gate: job.gate === undefined ? null : job.gate,
+  };
+}
+
+// The latest reason a human is meant to read: the last `priorityHistory` entry's reason, or null for
+// a job that has never been bumped (a normal job's audit trail is empty by D-7).
+function priorityReasonOf(job) {
+  const history = job && Array.isArray(job.priorityHistory) ? job.priorityHistory : [];
+  if (!history.length) return null;
+  const last = history[history.length - 1];
+  return last && last.reason !== undefined ? last.reason : null;
+}
+
+// D-8 revisit trigger: how many `normal` jobs (submitted within the last `days`) waited longer than
+// `thresholdMinutes` in the queue while a higher-ranked job started inside that same window. Pure over
+// the job files — no new metrics format, no clock beyond the timestamps already recorded.
+function priorityStarvationCount(root, { days = 14, thresholdMinutes = 180 } = {}) {
+  const since = Date.now() - days * 24 * 60 * 60 * 1000;
+  const thresholdMs = thresholdMinutes * 60 * 1000;
+  const all = listJobs(root);
+  let count = 0;
+  for (const job of all) {
+    if (priorityOf(job) !== 'normal') continue;
+    const queuedAt = job.timestamps && job.timestamps.queued ? Date.parse(job.timestamps.queued) : NaN;
+    const startedAt = job.timestamps && job.timestamps.started ? Date.parse(job.timestamps.started) : NaN;
+    if (!Number.isFinite(queuedAt) || !Number.isFinite(startedAt)) continue;
+    if (queuedAt < since) continue;
+    if (startedAt - queuedAt <= thresholdMs) continue;
+    const overtaken = all.some((other) => {
+      if (other.id === job.id) return false;
+      if (priorityRank(other) <= priorityRank(job)) return false;
+      const otherStarted = other.timestamps && other.timestamps.started ? Date.parse(other.timestamps.started) : NaN;
+      return Number.isFinite(otherStarted) && otherStarted >= queuedAt && otherStarted <= startedAt;
+    });
+    if (overtaken) count += 1;
+  }
+  return count;
+}
+
+function runnerSummary(root) {
+  const record = lease.read(root);
+  if (!record) return null;
+  return {
+    pid: record.pid,
+    jobId: record.jobId === undefined ? null : record.jobId,
+    host: record.host === undefined ? null : record.host,
+    startedAt: record.startedAt,
+    heartbeatAt: record.heartbeatAt,
+    alive: !lease.isDead(record),
+  };
+}
+
+function jobOrThrow(root, id) {
+  const job = readJob(root, id);
+  if (!job) throw landingError('job_not_found', `no landing job ${JSON.stringify(id)} in ${root}`, { jobId: id, root });
+  return job;
+}
+
+module.exports = {
+  JOB_STATES,
+  TERMINAL_STATES,
+  ACTIVE_STATES,
+  PRIORITIES,
+  PRIORITY_REASON_MAX,
+  BASE32,
+  ETA_SAMPLES,
+  ETA_WINDOW,
+  nowIso,
+  newJobId,
+  newBatchId,
+  jobsDir,
+  jobFile,
+  queueFile,
+  metricsDir,
+  verdictsDir,
+  logsDir,
+  trainsDir,
+  ensureLogsDir,
+  readQueue,
+  writeQueue,
+  readJob,
+  writeJob,
+  listJobs,
+  appendMetric,
+  isTerminal,
+  isActive,
+  priorityOf,
+  priorityRank,
+  sortOrder,
+  activeJobFor,
+  duplicateSubmitter,
+  enqueue,
+  setPriority,
+  requestStop,
+  requestPreemptFor,
+  requeuePreempted,
+  requeueBaseBroken,
+  heldByTargetMove,
+  priorityStarvationCount,
+  positionOf,
+  updateJob,
+  finishJob,
+  finishBatch,
+  recordBatchMetric,
+  recordRefusal,
+  refusalsToday,
+  requeueHalf,
+  hasConflictPaths,
+  peekRunnable,
+  takeNextJob,
+  takeNextBatch,
+  reapDeadRunner,
+  median,
+  recentGateSeconds,
+  etaSeconds,
+  jobSummary,
+  runnerSummary,
+  jobOrThrow,
+};