@bongos/core 1.19.876 → 1.19.878

package/modules/autonomy/cadence.js

@@ -0,0 +1,276 @@
+'use strict';
+
+// modules/autonomy/cadence.js — the runner has no terminal stop, only speeds
+// (task 1003904, ADR 0307 §5). Pure: no I/O, no clock, no timers. The caller
+// reports what happened and is told how fast to go and how long to wait.
+//
+// THE FLAW THIS FIXES, in the owner's words: "there is no loop to reboot once we
+// stop and hit the failures." The first draft of this supervisor exited on
+// anything that was not a completed piece of work. On a laptop that is fine —
+// someone sees the prompt come back. On an always-on machine nobody is watching,
+// a stopped daemon is indistinguishable from a working one, and stays that way
+// until a person happens to wonder. Silence has to mean "idle", never "dead".
+//
+// SO THERE ARE ONLY SPEEDS. `full` works task after task. Three straight failures
+// drop to `probing`: no new work, a growing wait, then exactly ONE task — a
+// deliberately cheap question, asked over and over, so the runner costs almost
+// nothing while broken and resumes the moment someone fixes the cause without
+// being told. One success returns it to `full`.
+//
+// WHY CLASSIFY AT ALL, when every class ends up backing off. Because the log is
+// read by a person at breakfast, and "the instance was unreachable for 40
+// minutes" and "your token expired six hours ago" want different actions from
+// them even though they want the same action from the machine. The classes are
+// also the honest answer to a question the naive design gets wrong:
+//
+//   · TRANSIENT (instance unreachable, grader outage, a rejected push) heals by
+//     waiting. Backing off IS the fix.
+//   · SYSTEMIC (expired token, full disk) will NOT heal by waiting. Backing off
+//     is not a fix, but cheap probing costs nothing while it waits for a human,
+//     and resumes by itself the moment one acts. It is called out separately so
+//     the digest can say "this one needs you".
+//   · UNWORKABLE_QUEUE — every task in the allowlist halts the matrix. This is
+//     NOT a runner failure and must never be reported as breakage: the machine is
+//     working perfectly and there is nothing for it to do. Reporting it as a
+//     failure is how a healthy runner gets "fixed" into a broken one.
+//   · QUIET — no capacity, nothing claimable, the fence closed. Not a failure at
+//     all, and deliberately does NOT trip the backoff: three quiet nights in a
+//     row must not leave the runner probing on a morning when work appears.
+//   · STOPPED — the server-side kill switch, the only true stop. Even this does
+//     not exit: the runner keeps asking, cheaply, so flipping the switch back on
+//     is enough to start it without touching the machine.
+//   · PROGRESS — work landed. Resets everything.
+
+// ---------------------------------------------------------------------------
+// Classification
+// ---------------------------------------------------------------------------
+
+const CLASSES = ['progress', 'quiet', 'stopped', 'transient', 'systemic', 'unworkable_queue'];
+
+// Only these advance the failure count toward `probing`.
+const COUNTS_AS_FAILURE = new Set(['transient', 'systemic']);
+
+// SYSTEMIC_MARKERS — substrings that mean waiting will not help.
+//
+// A heuristic over an error string, which is a thing to be uneasy about, so the
+// bound is deliberate: getting this WRONG in either direction costs only the
+// digest's wording, never the machine's behaviour. Transient and systemic back
+// off identically — the classes differ in what a person is told, not in what the
+// runner does. That is why a heuristic is acceptable here and would not be if it
+// gated an action.
+const SYSTEMIC_MARKERS = [
+  '401', 'unauthorized', 'token', 'expired', 'forbidden', '403',
+  'enospc', 'no space', 'disk full', 'quota', 'eacces', 'permission denied',
+  'not a git repository', 'command not found', 'enoent',
+];
+
+function looksSystemic(text) {
+  const s = String(text || '').toLowerCase();
+  return SYSTEMIC_MARKERS.some((m) => s.includes(m));
+}
+
+// classifyEvent — one run-log row becomes one class.
+//
+// The row shapes are autobongos-run.js's own `event`/`code` vocabulary; an event
+// this does not recognise is TRANSIENT, not quiet. An unknown failure that read
+// as "nothing to do" would let a broken runner look idle forever, which is the
+// exact failure this whole file exists against.
+function classifyEvent(row = {}) {
+  const event = String(row.event || '');
+  const code = String(row.code || '');
+  const reason = `${row.reason || ''} ${row.release_error || ''}`;
+
+  if (event === 'worked') {
+    // A worked task is progress only if the LEDGER agreed (task 1003903). A
+    // worker that claimed a ship the ledger never saw is not progress, and
+    // counting it as such would hold a broken runner at full speed indefinitely.
+    if (row.verified === true) return { class: 'progress', detail: 'a task shipped and the ledger confirms it' };
+    if (row.ledger_shape === 'undetermined') return { class: 'transient', detail: 'the ledger could not be read back' };
+    // It ran, it did not ship. Not breakage — a blocker was filed and the task is
+    // out of the pool — but not progress either. Quiet keeps the runner at full
+    // speed so it moves on to the next task, which is the right response.
+    return { class: 'quiet', detail: `the worker finished but the task came back ${row.ledger_shape || 'unshipped'}` };
+  }
+
+  if (event === 'dry_run') return { class: 'quiet', detail: 'dry run' };
+
+  if (event === 'hold') {
+    // Out of capacity, or the gauge could not be vouched for. Neither is a fault
+    // and neither is fixed by probing — the gauge already says when to come back.
+    return { class: 'quiet', detail: row.reason || 'no capacity in this window', sleepUntil: row.sleep_until ?? null };
+  }
+
+  if (event === 'fenced') {
+    if (code === 'kill_switch') return { class: 'stopped', detail: row.reason || 'an owner stopped the runner' };
+    if (code === 'grader_unavailable') return { class: 'transient', detail: 'the grader is unavailable, and it is the only review there is' };
+    if (code === 'fence_unreadable') return { class: 'transient', detail: 'the fence could not be read' };
+    // no_allowlist / goal_not_allowlisted / protected_path: the fence is working
+    // and there is nothing it permits right now. Configuration, not breakage.
+    return { class: 'quiet', detail: row.reason || 'the fence permits nothing right now' };
+  }
+
+  if (event === 'nothing_claimable') {
+    // THE DISTINCTION THAT MATTERS. An empty queue and a queue where every task
+    // halts look identical from here — both yield no task — and calling the
+    // second one "idle" is how a goal full of blocked work sits untouched for a
+    // week with the runner reporting itself healthy.
+    const skipped = Array.isArray(row.skipped) ? row.skipped : [];
+    if (skipped.length) {
+      return {
+        class: 'unworkable_queue',
+        detail: `${skipped.length} task${skipped.length === 1 ? '' : 's'} in the allowlist, every one of them halted by the matrix`,
+        skipped,
+      };
+    }
+    return { class: 'quiet', detail: 'nothing claimable' };
+  }
+
+  if (event === 'verify_halt') return { class: 'transient', detail: 'the ledger could not be read back after a worker' };
+
+  if (event === 'pick_failed' || event === 'worktree_failed' || event === 'claim_failed') {
+    return {
+      class: looksSystemic(reason) ? 'systemic' : 'transient',
+      detail: `${event}: ${String(row.reason || '').slice(0, 300)}`,
+    };
+  }
+
+  return { class: 'transient', detail: `unrecognised run event '${event || '(none)'}'` };
+}
+
+// ---------------------------------------------------------------------------
+// The state machine
+// ---------------------------------------------------------------------------
+
+const DEFAULTS = Object.freeze({
+  // Three straight failures, per ADR 0307 §5.
+  failuresBeforeProbing: 3,
+  // The growing wait, in seconds. Reasoned, not magic: a minute covers a service
+  // restart, half an hour covers a person noticing at breakfast, and the cap
+  // exists so a runner broken overnight still asks twice an hour rather than
+  // drifting into an interval nobody would call responsive.
+  backoffS: [60, 120, 300, 600, 1200, 1800],
+  // How long to idle when there is simply nothing to do. Short, because the cost
+  // of asking is one cheap request and the cost of missing new work is an hour of
+  // an always-on machine doing nothing.
+  idleS: 300,
+  // A closed fence is checked less often than an idle queue: flipping it is a
+  // deliberate human act, and nobody expects the runner back within the minute.
+  stoppedS: 600,
+  // An unworkable queue needs a person to unblock something. Same reasoning as a
+  // closed fence, and deliberately NOT the backoff ladder — it is not a failure
+  // and must not escalate as though it were.
+  unworkableS: 900,
+});
+
+const START = Object.freeze({ mode: 'full', consecutiveFailures: 0, backoffStep: 0 });
+
+// nextCadence — given where the runner is and what just happened, where does it
+// go and how long does it wait?
+//
+// Returns a WHOLE new state rather than mutating, so the caller can log both and
+// a test can assert a sequence. `waitS` is the wait BEFORE the next iteration;
+// `takeWork: false` is the probing promise that the runner claims nothing new.
+function nextCadence(state = START, classification = {}, cfg = {}) {
+  const c = { ...DEFAULTS, ...cfg };
+  const prev = { ...START, ...state };
+  const kind = CLASSES.includes(classification.class) ? classification.class : 'transient';
+
+  // PROGRESS resets everything. One shipped task is proof the whole path works.
+  if (kind === 'progress') {
+    return { mode: 'full', consecutiveFailures: 0, backoffStep: 0, waitS: 0, takeWork: true, why: 'a task shipped — full speed' };
+  }
+
+  // QUIET does not touch the failure count. Three quiet nights must not leave the
+  // runner probing on the morning work appears. It also does not RESET the count:
+  // a transient failure followed by an idle window is still one failure toward
+  // probing, and forgiving it there would let a runner that fails every other
+  // iteration never reach a backoff.
+  if (kind === 'quiet') {
+    const waitS = Number.isFinite(Number(classification.sleepUntilS)) ? Number(classification.sleepUntilS) : c.idleS;
+    return { ...prev, waitS, takeWork: prev.mode === 'full', why: classification.detail || 'nothing to do' };
+  }
+
+  if (kind === 'stopped') {
+    // Not a failure, and not an exit. The runner keeps asking so that flipping the
+    // switch back on is enough — nobody should have to log into the machine to
+    // restart something an owner turned off from a web page.
+    return { ...prev, waitS: c.stoppedS, takeWork: false, why: classification.detail || 'stopped by an owner' };
+  }
+
+  if (kind === 'unworkable_queue') {
+    // Explicitly NOT the failure ladder. The machine is healthy; the queue is not.
+    return { ...prev, waitS: c.unworkableS, takeWork: prev.mode === 'full', why: classification.detail || 'every task in the queue halts' };
+  }
+
+  // transient | systemic — the only two that count toward probing.
+  const consecutiveFailures = prev.consecutiveFailures + 1;
+  const mode = consecutiveFailures >= c.failuresBeforeProbing ? 'probing' : prev.mode;
+  const backoffStep = mode === 'probing' ? Math.min(prev.backoffStep + 1, c.backoffS.length) : 0;
+  const waitS = mode === 'probing' ? c.backoffS[Math.max(0, backoffStep - 1)] : 0;
+  return {
+    mode, consecutiveFailures, backoffStep, waitS,
+    // The probing promise: one task at a time, and only when the wait is over.
+    takeWork: mode === 'full',
+    probeOnly: mode === 'probing',
+    why: `${kind}: ${classification.detail || 'no detail'}`
+      + (mode === 'probing' ? ` — failure ${consecutiveFailures}, probing every ${waitS}s` : ` — failure ${consecutiveFailures} of ${c.failuresBeforeProbing}`),
+  };
+}
+
+// ---------------------------------------------------------------------------
+// Jumping the wait
+// ---------------------------------------------------------------------------
+
+// signalFingerprint — the cheap facts whose CHANGE means the world moved.
+//
+// Deliberately a string rather than a structured diff: the only question the
+// waiter asks is "is this different from last time", and a fingerprint answers it
+// without the caller having to know which of the three moved. What is in it is
+// what ADR 0307 §5 names: main moved, the allowlist changed, a blocker resolved
+// (which shows up as the claimable count changing).
+function signalFingerprint({ head = null, fenceUpdatedAt = null, claimableCount = null } = {}) {
+  // `-` is the ONLY spelling of "this probe could not answer", and each field
+  // has to reach it deliberately. `Number(null)` is 0 — finite — so a plain
+  // isFinite check turned an unread claimable count into the literal string
+  // "null" and made a blind probe look like a reading. Caught by the test that
+  // asserts a failed probe is not a signal.
+  const n = claimableCount === null || claimableCount === undefined || claimableCount === ''
+    ? null : Number(claimableCount);
+  return [
+    head ? String(head).slice(0, 12) : '-',
+    fenceUpdatedAt ? String(fenceUpdatedAt) : '-',
+    n !== null && Number.isFinite(n) ? String(n) : '-',
+  ].join('|');
+}
+
+// shouldJumpWait — has anything changed that makes waiting pointless?
+//
+// COMPARED FIELD BY FIELD, and only where BOTH sides know the answer. A whole-
+// string comparison looks right and is wrong in the case that matters: the probes
+// fail independently, so a flaky `git ls-remote` flips the fingerprint between
+// `abc|t1|3` and `-|t1|3` on alternate polls, and every one of those reads as a
+// change. That turns a thirty-minute backoff into a busy loop hammering the
+// instance it is backing off from — the precise opposite of the intent, and
+// worst exactly when things are already broken.
+//
+// So a `-` on EITHER side means "not evidence" for that field, and a comparison
+// with no usable field at all never jumps.
+function shouldJumpWait(before, after) {
+  if (typeof before !== 'string' || typeof after !== 'string') return false;
+  const a = before.split('|');
+  const b = after.split('|');
+  if (a.length !== b.length) return false;
+  let comparable = 0;
+  let changed = false;
+  for (let i = 0; i < a.length; i++) {
+    if (a[i] === '-' || b[i] === '-') continue;   // one side could not look
+    comparable += 1;
+    if (a[i] !== b[i]) changed = true;
+  }
+  return comparable > 0 && changed;
+}
+
+module.exports = {
+  CLASSES, COUNTS_AS_FAILURE, SYSTEMIC_MARKERS, DEFAULTS, START,
+  looksSystemic, classifyEvent, nextCadence, signalFingerprint, shouldJumpWait,
+};

package/modules/autonomy/db.js

@@ -114,9 +114,56 @@ async function disallowGoal(goalId) {
   return { removed: rowCount > 0 };
 }
 
+// ── the runner heartbeat (task 1003905) ──────────────────────────────────────
+
+// recordRunnerHeartbeat — upsert one host's check-in. ON CONFLICT so the normal
+// case is one write, and `host` is the key so a second runner appears as a second
+// row rather than silently overwriting the first.
+async function recordRunnerHeartbeat({ builderId, host, pid = null, startedAt = null, mode = null, consecutiveFailures = null, lastEvent = null, workingTaskId = null }) {
+  const { rows } = await pool.query(
+    `INSERT INTO autonomy_runner_heartbeat
+       (builder_id, host, pid, started_at, mode, consecutive_failures, last_event, working_task_id, last_seen_at, updated_at)
+     VALUES ($8, $1, $2, $3, $4, $5, $6, $7, now(), now())
+     ON CONFLICT (builder_id, host) DO UPDATE SET
+       pid = EXCLUDED.pid,
+       started_at = COALESCE(EXCLUDED.started_at, autonomy_runner_heartbeat.started_at),
+       mode = EXCLUDED.mode,
+       consecutive_failures = EXCLUDED.consecutive_failures,
+       last_event = EXCLUDED.last_event,
+       working_task_id = EXCLUDED.working_task_id,
+       last_seen_at = now(),
+       updated_at = now()
+     RETURNING host, pid, started_at, mode, consecutive_failures, last_event, working_task_id, last_seen_at`,
+    [String(host).toLowerCase().slice(0, 255), pid, startedAt, mode, consecutiveFailures, lastEvent, workingTaskId, builderId]
+  );
+  return rows[0] || null;
+}
+
+// readRunnerHeartbeats — every host, most recently heard from first. Returns an
+// EMPTY ARRAY when nothing has ever checked in, which the pure describeRunner
+// reads as `never_seen` — a different answer from "it died", and the reason the
+// caller must not paper over an empty read with a fabricated row.
+// SCOPED TO ONE BUILDER. A heartbeat row names a machine, its hostname and a pid;
+// that is the caller's own infrastructure and nobody else's business, and scoping
+// the read is also what keeps a stranger's forged-looking row out of the owner's
+// liveness view entirely (grader finding, task 1003905).
+async function readRunnerHeartbeats(builderId) {
+  const { rows } = await pool.query(
+    `SELECT host, pid, started_at, mode, consecutive_failures, last_event, working_task_id, last_seen_at
+       FROM autonomy_runner_heartbeat
+      WHERE builder_id = $1
+      ORDER BY last_seen_at DESC
+      LIMIT 20`,
+    [builderId]
+  );
+  return rows.map((r) => ({ ...r, working_task_id: r.working_task_id ? String(r.working_task_id) : null }));
+}
+
 module.exports = {
   recordAutonomyRun,
   lastAutonomyHead,
+  recordRunnerHeartbeat,
+  readRunnerHeartbeats,
   readFence,
   setFenceEnabled,
   allowGoal,