@bongos/core 1.19.875 → 1.19.877

package/modules/autonomy/cadence.js

@@ -0,0 +1,276 @@
+'use strict';
+
+// modules/autonomy/cadence.js — the runner has no terminal stop, only speeds
+// (task 1003904, ADR 0307 §5). Pure: no I/O, no clock, no timers. The caller
+// reports what happened and is told how fast to go and how long to wait.
+//
+// THE FLAW THIS FIXES, in the owner's words: "there is no loop to reboot once we
+// stop and hit the failures." The first draft of this supervisor exited on
+// anything that was not a completed piece of work. On a laptop that is fine —
+// someone sees the prompt come back. On an always-on machine nobody is watching,
+// a stopped daemon is indistinguishable from a working one, and stays that way
+// until a person happens to wonder. Silence has to mean "idle", never "dead".
+//
+// SO THERE ARE ONLY SPEEDS. `full` works task after task. Three straight failures
+// drop to `probing`: no new work, a growing wait, then exactly ONE task — a
+// deliberately cheap question, asked over and over, so the runner costs almost
+// nothing while broken and resumes the moment someone fixes the cause without
+// being told. One success returns it to `full`.
+//
+// WHY CLASSIFY AT ALL, when every class ends up backing off. Because the log is
+// read by a person at breakfast, and "the instance was unreachable for 40
+// minutes" and "your token expired six hours ago" want different actions from
+// them even though they want the same action from the machine. The classes are
+// also the honest answer to a question the naive design gets wrong:
+//
+//   · TRANSIENT (instance unreachable, grader outage, a rejected push) heals by
+//     waiting. Backing off IS the fix.
+//   · SYSTEMIC (expired token, full disk) will NOT heal by waiting. Backing off
+//     is not a fix, but cheap probing costs nothing while it waits for a human,
+//     and resumes by itself the moment one acts. It is called out separately so
+//     the digest can say "this one needs you".
+//   · UNWORKABLE_QUEUE — every task in the allowlist halts the matrix. This is
+//     NOT a runner failure and must never be reported as breakage: the machine is
+//     working perfectly and there is nothing for it to do. Reporting it as a
+//     failure is how a healthy runner gets "fixed" into a broken one.
+//   · QUIET — no capacity, nothing claimable, the fence closed. Not a failure at
+//     all, and deliberately does NOT trip the backoff: three quiet nights in a
+//     row must not leave the runner probing on a morning when work appears.
+//   · STOPPED — the server-side kill switch, the only true stop. Even this does
+//     not exit: the runner keeps asking, cheaply, so flipping the switch back on
+//     is enough to start it without touching the machine.
+//   · PROGRESS — work landed. Resets everything.
+
+// ---------------------------------------------------------------------------
+// Classification
+// ---------------------------------------------------------------------------
+
+const CLASSES = ['progress', 'quiet', 'stopped', 'transient', 'systemic', 'unworkable_queue'];
+
+// Only these advance the failure count toward `probing`.
+const COUNTS_AS_FAILURE = new Set(['transient', 'systemic']);
+
+// SYSTEMIC_MARKERS — substrings that mean waiting will not help.
+//
+// A heuristic over an error string, which is a thing to be uneasy about, so the
+// bound is deliberate: getting this WRONG in either direction costs only the
+// digest's wording, never the machine's behaviour. Transient and systemic back
+// off identically — the classes differ in what a person is told, not in what the
+// runner does. That is why a heuristic is acceptable here and would not be if it
+// gated an action.
+const SYSTEMIC_MARKERS = [
+  '401', 'unauthorized', 'token', 'expired', 'forbidden', '403',
+  'enospc', 'no space', 'disk full', 'quota', 'eacces', 'permission denied',
+  'not a git repository', 'command not found', 'enoent',
+];
+
+function looksSystemic(text) {
+  const s = String(text || '').toLowerCase();
+  return SYSTEMIC_MARKERS.some((m) => s.includes(m));
+}
+
+// classifyEvent — one run-log row becomes one class.
+//
+// The row shapes are autobongos-run.js's own `event`/`code` vocabulary; an event
+// this does not recognise is TRANSIENT, not quiet. An unknown failure that read
+// as "nothing to do" would let a broken runner look idle forever, which is the
+// exact failure this whole file exists against.
+function classifyEvent(row = {}) {
+  const event = String(row.event || '');
+  const code = String(row.code || '');
+  const reason = `${row.reason || ''} ${row.release_error || ''}`;
+
+  if (event === 'worked') {
+    // A worked task is progress only if the LEDGER agreed (task 1003903). A
+    // worker that claimed a ship the ledger never saw is not progress, and
+    // counting it as such would hold a broken runner at full speed indefinitely.
+    if (row.verified === true) return { class: 'progress', detail: 'a task shipped and the ledger confirms it' };
+    if (row.ledger_shape === 'undetermined') return { class: 'transient', detail: 'the ledger could not be read back' };
+    // It ran, it did not ship. Not breakage — a blocker was filed and the task is
+    // out of the pool — but not progress either. Quiet keeps the runner at full
+    // speed so it moves on to the next task, which is the right response.
+    return { class: 'quiet', detail: `the worker finished but the task came back ${row.ledger_shape || 'unshipped'}` };
+  }
+
+  if (event === 'dry_run') return { class: 'quiet', detail: 'dry run' };
+
+  if (event === 'hold') {
+    // Out of capacity, or the gauge could not be vouched for. Neither is a fault
+    // and neither is fixed by probing — the gauge already says when to come back.
+    return { class: 'quiet', detail: row.reason || 'no capacity in this window', sleepUntil: row.sleep_until ?? null };
+  }
+
+  if (event === 'fenced') {
+    if (code === 'kill_switch') return { class: 'stopped', detail: row.reason || 'an owner stopped the runner' };
+    if (code === 'grader_unavailable') return { class: 'transient', detail: 'the grader is unavailable, and it is the only review there is' };
+    if (code === 'fence_unreadable') return { class: 'transient', detail: 'the fence could not be read' };
+    // no_allowlist / goal_not_allowlisted / protected_path: the fence is working
+    // and there is nothing it permits right now. Configuration, not breakage.
+    return { class: 'quiet', detail: row.reason || 'the fence permits nothing right now' };
+  }
+
+  if (event === 'nothing_claimable') {
+    // THE DISTINCTION THAT MATTERS. An empty queue and a queue where every task
+    // halts look identical from here — both yield no task — and calling the
+    // second one "idle" is how a goal full of blocked work sits untouched for a
+    // week with the runner reporting itself healthy.
+    const skipped = Array.isArray(row.skipped) ? row.skipped : [];
+    if (skipped.length) {
+      return {
+        class: 'unworkable_queue',
+        detail: `${skipped.length} task${skipped.length === 1 ? '' : 's'} in the allowlist, every one of them halted by the matrix`,
+        skipped,
+      };
+    }
+    return { class: 'quiet', detail: 'nothing claimable' };
+  }
+
+  if (event === 'verify_halt') return { class: 'transient', detail: 'the ledger could not be read back after a worker' };
+
+  if (event === 'pick_failed' || event === 'worktree_failed' || event === 'claim_failed') {
+    return {
+      class: looksSystemic(reason) ? 'systemic' : 'transient',
+      detail: `${event}: ${String(row.reason || '').slice(0, 300)}`,
+    };
+  }
+
+  return { class: 'transient', detail: `unrecognised run event '${event || '(none)'}'` };
+}
+
+// ---------------------------------------------------------------------------
+// The state machine
+// ---------------------------------------------------------------------------
+
+const DEFAULTS = Object.freeze({
+  // Three straight failures, per ADR 0307 §5.
+  failuresBeforeProbing: 3,
+  // The growing wait, in seconds. Reasoned, not magic: a minute covers a service
+  // restart, half an hour covers a person noticing at breakfast, and the cap
+  // exists so a runner broken overnight still asks twice an hour rather than
+  // drifting into an interval nobody would call responsive.
+  backoffS: [60, 120, 300, 600, 1200, 1800],
+  // How long to idle when there is simply nothing to do. Short, because the cost
+  // of asking is one cheap request and the cost of missing new work is an hour of
+  // an always-on machine doing nothing.
+  idleS: 300,
+  // A closed fence is checked less often than an idle queue: flipping it is a
+  // deliberate human act, and nobody expects the runner back within the minute.
+  stoppedS: 600,
+  // An unworkable queue needs a person to unblock something. Same reasoning as a
+  // closed fence, and deliberately NOT the backoff ladder — it is not a failure
+  // and must not escalate as though it were.
+  unworkableS: 900,
+});
+
+const START = Object.freeze({ mode: 'full', consecutiveFailures: 0, backoffStep: 0 });
+
+// nextCadence — given where the runner is and what just happened, where does it
+// go and how long does it wait?
+//
+// Returns a WHOLE new state rather than mutating, so the caller can log both and
+// a test can assert a sequence. `waitS` is the wait BEFORE the next iteration;
+// `takeWork: false` is the probing promise that the runner claims nothing new.
+function nextCadence(state = START, classification = {}, cfg = {}) {
+  const c = { ...DEFAULTS, ...cfg };
+  const prev = { ...START, ...state };
+  const kind = CLASSES.includes(classification.class) ? classification.class : 'transient';
+
+  // PROGRESS resets everything. One shipped task is proof the whole path works.
+  if (kind === 'progress') {
+    return { mode: 'full', consecutiveFailures: 0, backoffStep: 0, waitS: 0, takeWork: true, why: 'a task shipped — full speed' };
+  }
+
+  // QUIET does not touch the failure count. Three quiet nights must not leave the
+  // runner probing on the morning work appears. It also does not RESET the count:
+  // a transient failure followed by an idle window is still one failure toward
+  // probing, and forgiving it there would let a runner that fails every other
+  // iteration never reach a backoff.
+  if (kind === 'quiet') {
+    const waitS = Number.isFinite(Number(classification.sleepUntilS)) ? Number(classification.sleepUntilS) : c.idleS;
+    return { ...prev, waitS, takeWork: prev.mode === 'full', why: classification.detail || 'nothing to do' };
+  }
+
+  if (kind === 'stopped') {
+    // Not a failure, and not an exit. The runner keeps asking so that flipping the
+    // switch back on is enough — nobody should have to log into the machine to
+    // restart something an owner turned off from a web page.
+    return { ...prev, waitS: c.stoppedS, takeWork: false, why: classification.detail || 'stopped by an owner' };
+  }
+
+  if (kind === 'unworkable_queue') {
+    // Explicitly NOT the failure ladder. The machine is healthy; the queue is not.
+    return { ...prev, waitS: c.unworkableS, takeWork: prev.mode === 'full', why: classification.detail || 'every task in the queue halts' };
+  }
+
+  // transient | systemic — the only two that count toward probing.
+  const consecutiveFailures = prev.consecutiveFailures + 1;
+  const mode = consecutiveFailures >= c.failuresBeforeProbing ? 'probing' : prev.mode;
+  const backoffStep = mode === 'probing' ? Math.min(prev.backoffStep + 1, c.backoffS.length) : 0;
+  const waitS = mode === 'probing' ? c.backoffS[Math.max(0, backoffStep - 1)] : 0;
+  return {
+    mode, consecutiveFailures, backoffStep, waitS,
+    // The probing promise: one task at a time, and only when the wait is over.
+    takeWork: mode === 'full',
+    probeOnly: mode === 'probing',
+    why: `${kind}: ${classification.detail || 'no detail'}`
+      + (mode === 'probing' ? ` — failure ${consecutiveFailures}, probing every ${waitS}s` : ` — failure ${consecutiveFailures} of ${c.failuresBeforeProbing}`),
+  };
+}
+
+// ---------------------------------------------------------------------------
+// Jumping the wait
+// ---------------------------------------------------------------------------
+
+// signalFingerprint — the cheap facts whose CHANGE means the world moved.
+//
+// Deliberately a string rather than a structured diff: the only question the
+// waiter asks is "is this different from last time", and a fingerprint answers it
+// without the caller having to know which of the three moved. What is in it is
+// what ADR 0307 §5 names: main moved, the allowlist changed, a blocker resolved
+// (which shows up as the claimable count changing).
+function signalFingerprint({ head = null, fenceUpdatedAt = null, claimableCount = null } = {}) {
+  // `-` is the ONLY spelling of "this probe could not answer", and each field
+  // has to reach it deliberately. `Number(null)` is 0 — finite — so a plain
+  // isFinite check turned an unread claimable count into the literal string
+  // "null" and made a blind probe look like a reading. Caught by the test that
+  // asserts a failed probe is not a signal.
+  const n = claimableCount === null || claimableCount === undefined || claimableCount === ''
+    ? null : Number(claimableCount);
+  return [
+    head ? String(head).slice(0, 12) : '-',
+    fenceUpdatedAt ? String(fenceUpdatedAt) : '-',
+    n !== null && Number.isFinite(n) ? String(n) : '-',
+  ].join('|');
+}
+
+// shouldJumpWait — has anything changed that makes waiting pointless?
+//
+// COMPARED FIELD BY FIELD, and only where BOTH sides know the answer. A whole-
+// string comparison looks right and is wrong in the case that matters: the probes
+// fail independently, so a flaky `git ls-remote` flips the fingerprint between
+// `abc|t1|3` and `-|t1|3` on alternate polls, and every one of those reads as a
+// change. That turns a thirty-minute backoff into a busy loop hammering the
+// instance it is backing off from — the precise opposite of the intent, and
+// worst exactly when things are already broken.
+//
+// So a `-` on EITHER side means "not evidence" for that field, and a comparison
+// with no usable field at all never jumps.
+function shouldJumpWait(before, after) {
+  if (typeof before !== 'string' || typeof after !== 'string') return false;
+  const a = before.split('|');
+  const b = after.split('|');
+  if (a.length !== b.length) return false;
+  let comparable = 0;
+  let changed = false;
+  for (let i = 0; i < a.length; i++) {
+    if (a[i] === '-' || b[i] === '-') continue;   // one side could not look
+    comparable += 1;
+    if (a[i] !== b[i]) changed = true;
+  }
+  return comparable > 0 && changed;
+}
+
+module.exports = {
+  CLASSES, COUNTS_AS_FAILURE, SYSTEMIC_MARKERS, DEFAULTS, START,
+  looksSystemic, classifyEvent, nextCadence, signalFingerprint, shouldJumpWait,
+};

package/modules/autonomy/db.js

@@ -114,9 +114,56 @@ async function disallowGoal(goalId) {
   return { removed: rowCount > 0 };
 }
 
+// ── the runner heartbeat (task 1003905) ──────────────────────────────────────
+
+// recordRunnerHeartbeat — upsert one host's check-in. ON CONFLICT so the normal
+// case is one write, and `host` is the key so a second runner appears as a second
+// row rather than silently overwriting the first.
+async function recordRunnerHeartbeat({ builderId, host, pid = null, startedAt = null, mode = null, consecutiveFailures = null, lastEvent = null, workingTaskId = null }) {
+  const { rows } = await pool.query(
+    `INSERT INTO autonomy_runner_heartbeat
+       (builder_id, host, pid, started_at, mode, consecutive_failures, last_event, working_task_id, last_seen_at, updated_at)
+     VALUES ($8, $1, $2, $3, $4, $5, $6, $7, now(), now())
+     ON CONFLICT (builder_id, host) DO UPDATE SET
+       pid = EXCLUDED.pid,
+       started_at = COALESCE(EXCLUDED.started_at, autonomy_runner_heartbeat.started_at),
+       mode = EXCLUDED.mode,
+       consecutive_failures = EXCLUDED.consecutive_failures,
+       last_event = EXCLUDED.last_event,
+       working_task_id = EXCLUDED.working_task_id,
+       last_seen_at = now(),
+       updated_at = now()
+     RETURNING host, pid, started_at, mode, consecutive_failures, last_event, working_task_id, last_seen_at`,
+    [String(host).toLowerCase().slice(0, 255), pid, startedAt, mode, consecutiveFailures, lastEvent, workingTaskId, builderId]
+  );
+  return rows[0] || null;
+}
+
+// readRunnerHeartbeats — every host, most recently heard from first. Returns an
+// EMPTY ARRAY when nothing has ever checked in, which the pure describeRunner
+// reads as `never_seen` — a different answer from "it died", and the reason the
+// caller must not paper over an empty read with a fabricated row.
+// SCOPED TO ONE BUILDER. A heartbeat row names a machine, its hostname and a pid;
+// that is the caller's own infrastructure and nobody else's business, and scoping
+// the read is also what keeps a stranger's forged-looking row out of the owner's
+// liveness view entirely (grader finding, task 1003905).
+async function readRunnerHeartbeats(builderId) {
+  const { rows } = await pool.query(
+    `SELECT host, pid, started_at, mode, consecutive_failures, last_event, working_task_id, last_seen_at
+       FROM autonomy_runner_heartbeat
+      WHERE builder_id = $1
+      ORDER BY last_seen_at DESC
+      LIMIT 20`,
+    [builderId]
+  );
+  return rows.map((r) => ({ ...r, working_task_id: r.working_task_id ? String(r.working_task_id) : null }));
+}
+
 module.exports = {
   recordAutonomyRun,
   lastAutonomyHead,
+  recordRunnerHeartbeat,
+  readRunnerHeartbeats,
   readFence,
   setFenceEnabled,
   allowGoal,

package/modules/autonomy/migrations/autonomy_002_runner_heartbeat.sql

@@ -0,0 +1,93 @@
+-- autonomy_002_runner_heartbeat.sql — a dead runner must be VISIBLE, not silent
+-- (task 1003905, ADR 0307 §5).
+--
+-- WHY THE SERVER HAS TO HOLD THIS. autonomy_001 gave the owner a switch the
+-- machine can read and cannot write. This is the same argument pointed the other
+-- way: the owner needs to see the machine, and a status written only on the
+-- machine is invisible on precisely the day it matters — the day the machine is
+-- off, unplugged, or wedged. The runner writes a local heartbeat file too
+-- (scripts/gds/autobongos-run.js, the guard against two runners fighting over one
+-- claim), but that file cannot be read from a web page in another country. This
+-- table can.
+--
+-- SILENCE IS THE SIGNAL, which is why the row records a TIME and not a state.
+-- A "status: running" column would be written by a healthy runner and then left
+-- untouched by a dead one, so it would read "running" forever — exactly the class
+-- of lie provisioning_006 was written to end ("status records what the runner DID,
+-- not what the instance actually does"). Only `last_seen_at` can go stale on its
+-- own, so only `last_seen_at` can tell the truth without being told.
+--
+-- NO ROW MEANS NEVER SEEN, and that is a third answer, not a missing one. A
+-- runner that has never checked in and a runner that checked in yesterday and
+-- stopped need different words from the hall — "not set up yet" versus "it died".
+-- Same distinction provisioning_006 draws between NULL and false.
+--
+-- KEYED BY (builder_id, host), AND THE BUILDER HALF IS A SECURITY BOUNDARY, not
+-- bookkeeping. With `host` alone as the key, any authenticated builder could POST
+-- a heartbeat naming the real runner's hostname and keep the liveness signal
+-- reading "alive" forever — defeating the exact property this table exists for,
+-- which is that a dead machine holding broad spawn permissions becomes VISIBLE.
+-- An ownership check in the handler would also close it, and would be one `if`
+-- away from being removed by a later edit that looked reasonable. Putting the
+-- caller in the KEY makes the forgery structurally impossible instead: a stranger
+-- writing "winbox" writes THEIR OWN row and cannot touch the owner's. (Found by
+-- the grader's security worker on task 1003905.)
+--
+-- A singleton was the other option and is worse for a second reason: one row per
+-- host means a second runner shows up as a second row instead of silently
+-- overwriting the first, which is the failure a singleton discovers at the worst
+-- possible moment.
+--
+-- Module migrations are ADDITIVE and NAMESPACED (ADR 0083 §Decision #5): every
+-- object is autonomy_-prefixed, there is no down-migration, and the filename is
+-- autonomy_-prefixed (ADR 0108 §5). Idempotent: safe to re-run.
+
+BEGIN;
+
+CREATE TABLE IF NOT EXISTS autonomy_runner_heartbeat (
+  -- WHO is reporting. Half the primary key — see the header: this is what makes a
+  -- forged heartbeat impossible rather than merely checked for.
+  builder_id           bigint NOT NULL,
+  -- The machine's own name for itself (os.hostname()). Lowercased by the writer
+  -- so one host cannot appear twice under two spellings.
+  host                 text NOT NULL,
+  pid                  integer,
+  -- THE ONLY LOAD-BEARING COLUMN. Everything else is colour for the hall.
+  last_seen_at         timestamptz NOT NULL DEFAULT now(),
+  -- When this process started, so the hall can say "up 3 days" and a restart loop
+  -- is visible as a start time that keeps moving.
+  started_at           timestamptz,
+  -- The cadence, verbatim from modules/autonomy/cadence.js: 'full' | 'probing'.
+  -- Free text rather than an enum: the vocabulary lives in that file, and a CHECK
+  -- here would mean a schema migration every time it gains a word.
+  mode                 text,
+  consecutive_failures integer,
+  -- The last run-log event, so "last seen 4 minutes ago" can be followed by WHAT
+  -- it was doing. A runner quiet because it is mid-worker and one quiet because it
+  -- is wedged look identical without this.
+  last_event           text,
+  -- The task a worker is building, when there is one. A long gap is EXPECTED while
+  -- a worker runs (they may take an hour and a half), and without this the hall
+  -- would cry dead every time the runner did its job.
+  working_task_id      bigint,
+  updated_at           timestamptz NOT NULL DEFAULT now(),
+  PRIMARY KEY (builder_id, host)
+);
+
+-- ON RE-RUNNING THIS AGAINST A TABLE THAT ALREADY EXISTS IN A DIFFERENT SHAPE:
+-- it would not be corrected, and that is deliberate rather than overlooked. An
+-- earlier cut of this same file keyed the table on `host` alone, and the obvious
+-- repair -- detect the old key, DROP CONSTRAINT, re-add -- is DESTRUCTIVE DDL,
+-- which a module migration may not use (ADR 0083 section Decision #5, enforced by
+-- fitness.js: "disabling a module stops applying, it never drops"). It was
+-- written, it was caught by that gate, and the gate is right: a module that can
+-- drop things on its way in is a module that can take a host's data with it.
+--
+-- The reason it is safe to leave is narrower than "nobody will hit it". This
+-- table is NEW in an UNRELEASED task -- no published core version contains it, so
+-- no instance can have applied any shape of it. The only databases that ever saw
+-- the host-only version were throwaway probes on the author's machine, since
+-- deleted. If that ever stops being true, the fix is a NEW additive migration,
+-- not an edit here. (Grader finding, task 1003905.)
+
+COMMIT;

package/modules/autonomy/routes/autonomy.js

@@ -28,6 +28,7 @@ const auth = api;
 // neither core internals nor a sibling module, so importing it here is allowed.
 const autonomyGate = require('../../../scripts/gds/autonomy-gate');
 const db = require('../db');
+const runnerHealth = require('../runner-health');
 // The doorway's namespaced logger and strict body validator. Raw console.* is
 // ratcheted repo-wide (fitness.js console_call_count) and unstructured besides.
 const log = api.logger('autonomy');
@@ -143,5 +144,73 @@ module.exports = function buildAutonomyRouter() {
     }
   });
 
+  // ── the runner heartbeat (task 1003905) ────────────────────────────────────
+  //
+  // A dead runner must be VISIBLE, not silent. The runner also writes a heartbeat
+  // FILE on its own host — that one stops two runners fighting over a claim — but
+  // a file on the machine is unreadable on exactly the day it matters, the day
+  // the machine is off. This pair is the half the owner can see from a web page.
+
+  // The runner's check-in, once per loop. ANY-BUILDER is safe because the row is
+  // keyed by the CALLER: a stranger reporting "winbox" writes their own row and
+  // cannot touch the owner's, so the liveness signal cannot be forged into reading
+  // alive. Keeping the bar low matters — the failure mode of a refused heartbeat
+  // is a healthy runner reported dead.
+  router.post('/autonomy/heartbeat', auth.requireBuilder, async (req, res) => {
+    if (validateOrRespond(req, res, {
+      host: { required: true, type: 'string', maxLength: 255, minLength: 1 },
+      pid: { type: 'number' },
+      started_at: { type: 'string', maxLength: 64 },
+      mode: { type: 'string', maxLength: 32 },
+      consecutive_failures: { type: 'number' },
+      last_event: { type: 'string', maxLength: 64 },
+      working_task_id: { type: 'number' },
+    })) return;
+    const b = req.body || {};
+    try {
+      const row = await db.recordRunnerHeartbeat({
+        // From the SESSION, never the body. A caller cannot name someone else's
+        // runner, because the row they write is keyed by who they are.
+        builderId: req.builder.id,
+        host: b.host,
+        pid: Number.isInteger(b.pid) ? b.pid : null,
+        startedAt: b.started_at || null,
+        mode: b.mode || null,
+        consecutiveFailures: Number.isInteger(b.consecutive_failures) ? b.consecutive_failures : null,
+        lastEvent: b.last_event || null,
+        workingTaskId: Number.isInteger(b.working_task_id) ? b.working_task_id : null,
+      });
+      res.json({ ok: true, heartbeat: row });
+    } catch (err) {
+      // FAIL SOFT, and deliberately unlike the fence's GET. A heartbeat that
+      // cannot be written is a monitoring gap; turning it into an error the runner
+      // has to handle would let the watching mechanism take down the thing it
+      // watches. The gap shows up on its own — as silence, which is the signal.
+      log.error('[gds] POST /autonomy/heartbeat', err);
+      res.fail('heartbeat_write_failed', { status: 500, message: 'internal error' });
+    }
+  });
+
+  // What the hall renders: the CALLER'S OWN runners, with the pure verdict attached
+  // so the page and any future alert cannot disagree about what "dead" means. A
+  // row names a machine, its hostname and a pid — the caller's infrastructure, and
+  // nobody else's business.
+  router.get('/autonomy/runners', auth.requireBuilder, async (req, res) => {
+    try {
+      const rows = await db.readRunnerHeartbeats(req.builder.id);
+      const nowMs = Date.now();
+      res.json({
+        // An EMPTY list is the honest answer for "nobody has ever checked in" —
+        // never_seen is a third state, not a missing one, so no row is fabricated
+        // to stand in for it.
+        runners: rows.map((r) => ({ ...r, health: runnerHealth.describeRunner(r, { nowMs }) })),
+        none: runnerHealth.describeRunner(null, { nowMs }),
+      });
+    } catch (err) {
+      log.error('[gds] GET /autonomy/runners', err);
+      res.fail('runners_unreadable', { status: 503, message: 'could not read the runner heartbeats' });
+    }
+  });
+
   return router;
 };

package/modules/autonomy/runner-health.js

@@ -0,0 +1,97 @@
+'use strict';
+
+// modules/autonomy/runner-health.js — is the runner alive? (task 1003905.)
+// Pure: no I/O, no clock of its own. `now` is injected.
+//
+// THE QUESTION THIS ANSWERS IS ASYMMETRIC, and that is the whole design. A
+// running machine can say so; a dead one cannot say anything. So the signal is
+// not a status the runner writes — a dead runner leaves "running" on the screen
+// forever — but the AGE of the last thing it said. Silence is the only report a
+// dead process can file, so silence is what this reads.
+//
+// FOUR STATES, because three of them are different kinds of quiet:
+//   · never_seen — no row at all. Not a failure: nobody has set the host up yet,
+//     and telling an owner their runner "died" before it ever ran is how a status
+//     light trains people to ignore it.
+//   · alive      — heard from within the window.
+//   · late       — quiet longer than a loop, but not longer than a WORKER. The
+//     runner legitimately goes quiet for up to the worker ceiling while doing
+//     exactly what it is for, and crying dead every time it works is worse than
+//     saying nothing.
+//   · dead       — quiet longer than any legitimate explanation.
+
+// Reasoned from the runner's own numbers, not picked. The loop beats at the top of
+// every pass and again on each poll inside a wait (every 60s), so a healthy runner
+// that is merely idle or backing off is never quiet for more than about a minute.
+const ALIVE_S = 15 * 60;
+// The worker wall-clock ceiling is 90 minutes (DEFAULT_WORKER_TIMEOUT_MS), and a
+// worker holds the loop for its whole run. Two hours is that ceiling plus room for
+// the ship, the grade and the verify that follow it.
+const LATE_S = 2 * 60 * 60;
+
+function ageS(lastSeenAt, nowMs) {
+  const t = Date.parse(lastSeenAt || '');
+  if (!Number.isFinite(t) || !Number.isFinite(nowMs)) return null;
+  return Math.max(0, Math.round((nowMs - t) / 1000));
+}
+
+// humanAge — the same shape strand-watch.js uses, so two surfaces describing a
+// duration do not describe it two ways.
+function humanAge(s) {
+  if (!Number.isFinite(s)) return 'unknown';
+  if (s < 90) return `${s}s`;
+  if (s < 5400) return `${Math.round(s / 60)}m`;
+  if (s < 172800) return `${Math.round(s / 3600)}h`;
+  return `${Math.round(s / 86400)}d`;
+}
+
+// describeRunner — one heartbeat row becomes one sentence the hall can print.
+//
+// `heartbeat` is null/undefined for "no row", which is never_seen. An UNPARSEABLE
+// last_seen_at is `dead`, not never_seen: a row that exists but cannot be read is
+// a runner we heard from once and cannot account for now, and the fail direction
+// on a liveness check is to raise the alarm rather than to explain it away.
+function describeRunner(heartbeat, { nowMs = Date.now(), aliveS = ALIVE_S, lateS = LATE_S } = {}) {
+  if (!heartbeat || typeof heartbeat !== 'object') {
+    return {
+      state: 'never_seen', ok: true, ageS: null, age: null,
+      detail: 'no runner has ever checked in from this instance',
+    };
+  }
+
+  const age = ageS(heartbeat.last_seen_at, nowMs);
+  if (age === null) {
+    return { state: 'dead', ok: false, ageS: null, age: 'unknown', detail: 'the runner’s last check-in cannot be read' };
+  }
+
+  const host = typeof heartbeat.host === 'string' ? heartbeat.host : null;
+  const mode = heartbeat.mode === 'probing' ? 'probing' : (heartbeat.mode === 'full' ? 'full' : null);
+  const working = heartbeat.working_task_id ? String(heartbeat.working_task_id) : null;
+  const where = host ? ` on ${host}` : '';
+
+  if (age <= aliveS) {
+    return {
+      state: 'alive', ok: true, ageS: age, age: humanAge(age), host, mode, workingTaskId: working,
+      detail: working
+        ? `working task ${working}${where}`
+        : `${mode === 'probing' ? 'probing after failures' : 'running'}${where}`,
+    };
+  }
+
+  if (age <= lateS) {
+    return {
+      state: 'late', ok: true, ageS: age, age: humanAge(age), host, mode, workingTaskId: working,
+      // Deliberately NOT an alarm. This is what a runner doing its job looks like.
+      detail: working
+        ? `quiet for ${humanAge(age)} — expected while task ${working} builds`
+        : `quiet for ${humanAge(age)} — longer than a loop, still within a worker's run`,
+    };
+  }
+
+  return {
+    state: 'dead', ok: false, ageS: age, age: humanAge(age), host, mode, workingTaskId: working,
+    detail: `no check-in for ${humanAge(age)}${where} — longer than any worker takes, so the runner is not running`,
+  };
+}
+
+module.exports = { ALIVE_S, LATE_S, ageS, humanAge, describeRunner };

package/modules/dev-box/app/src/vendor/bongos-client.cjs

@@ -180,8 +180,12 @@ function createClient(opts = {}) {
       postAutonomyFenceGoals: (args) => request("POST", "/autonomy/fence/goals", { hasBody: true }, args),
       // DELETE /autonomy/fence/goals/{goalId} — rank: archon — DELETE /autonomy/fence/goals/:goalId
       deleteAutonomyFenceGoalsGoalId: (args) => request("DELETE", "/autonomy/fence/goals/{goalId}", { hasBody: false }, args),
+      // POST /autonomy/heartbeat — rank: any-builder — POST /autonomy/heartbeat
+      postAutonomyHeartbeat: (args) => request("POST", "/autonomy/heartbeat", { hasBody: true }, args),
       // GET /autonomy/precheck — rank: metic+archon — GET /autonomy/precheck
       getAutonomyPrecheck: (args) => request("GET", "/autonomy/precheck", { hasBody: false }, args),
+      // GET /autonomy/runners — rank: any-builder — GET /autonomy/runners
+      getAutonomyRunners: (args) => request("GET", "/autonomy/runners", { hasBody: false }, args),
     },
     "backup": {
       // GET /backup/status — rank: metic+archon — GET /backup/status

package/modules/hall-ui/public/gate.html

@@ -52,6 +52,9 @@
           <section class="scroll" id="fence-panel" aria-labelledby="fence-h" hidden>
             <h2 class="scroll__h" id="fence-h">Autobongos runner</h2>
             <p class="scroll__lede">The unattended runner works only the goals listed here, and only while it is on. This is the switch — it lives on the server, so the machine can read it and cannot change it.</p>
+            <!-- Liveness first (task 1003905): "is it running at all" is the
+                 question before "is it allowed to run". -->
+            <div id="fence-runner"><span class="kit-skel"></span></div>
             <div id="fence-state"><span class="kit-skel"></span></div>
             <div id="fence-goals"></div>
           </section>
@@ -78,6 +81,6 @@
 <script src="/builders/palette.js" defer></script>
 <script src="/builders/dom-utils.js?v=2026-06-16-gate"></script>
 <script src="/builders/hall-kit.js"></script>
-<script src="/builders/gate.js?v=2026-09-19-fence"></script>
+<script src="/builders/gate.js?v=2026-09-19-runner"></script>
 </body>
 </html>