@bongos/core 1.20.32 → 1.20.34

package/scripts/gds/autobongos-run.js

@@ -40,7 +40,7 @@ const os = require('node:os');
 const seq = require('./sequence.js');
 const gauge = require('./autonomy-gauge.js');
 const {
-  buildWorkerArgs, buildWorkerPrompt, classifyRun, DEFAULT_MODEL, DEFAULT_WORKER_TIMEOUT_MS,
+  buildWorkerArgs, buildWorkerPrompt, classifyRun, usageLimitHit, DEFAULT_MODEL, DEFAULT_WORKER_TIMEOUT_MS,
 } = require('./autobongos-loop.js');
 const { verifyShip, failureReason } = require('./autobongos-verify.js');
 const fence = require('../../modules/autonomy/fence.js');
@@ -95,55 +95,6 @@ function logPath() {
   try { return require('../../src/instance-config.js').configPath('autobongos-runs.jsonl'); }
   catch (_) { return path.join(REPO_ROOT, 'autobongos-runs.jsonl'); }
 }
-// countWorkedSince — how many tasks this runner has worked since an epoch-second
-// mark. Reads the run log, because that is the only record that survives a
-// restart, and the cap it feeds has to hold across one (a runner that forgets
-// its count on every crash has no cap at all).
-//
-// BOUNDED ON BOTH AXES, because this runs every iteration of a loop that never
-// exits and the log only grows. It reads the TAIL rather than the file, and it
-// scans BACKWARDS and stops at the first row older than the mark — the rows that
-// can possibly count are the newest ones, so the common case touches a handful
-// of lines whatever the log's size.
-//
-// A torn or missing log counts as zero rather than refusing: the fence and the
-// gauge are the gates, this is a bound on top of them, and a bound that bricks
-// the runner when its own log is unreadable is worse than one that occasionally
-// allows an extra task.
-const WORKED_SCAN_TAIL_BYTES = 1024 * 1024;
-const NEWLINE = String.fromCharCode(10);
-
-function countWorkedSince(sinceEpochS) {
-  if (!Number.isFinite(sinceEpochS) || sinceEpochS <= 0) return 0;
-  let text;
-  try {
-    const p = logPath();
-    const { size } = fs.statSync(p);
-    const from = Math.max(0, size - WORKED_SCAN_TAIL_BYTES);
-    const fd = fs.openSync(p, 'r');
-    try {
-      const buf = Buffer.alloc(Math.min(size, WORKED_SCAN_TAIL_BYTES));
-      fs.readSync(fd, buf, 0, buf.length, from);
-      text = buf.toString('utf8');
-    } finally { fs.closeSync(fd); }
-    // A tail read can land mid-line; drop the first partial one.
-    if (from > 0) text = text.slice(text.indexOf(NEWLINE) + 1);
-  } catch (_) { return 0; }
-
-  const lines = text.split(NEWLINE);
-  let n = 0;
-  for (let i = lines.length - 1; i >= 0; i--) {
-    const line = lines[i];
-    if (!line.trim()) continue;
-    let row;
-    try { row = JSON.parse(line); } catch (_) { continue; }   // a torn line costs one row, not the count
-    const at = Date.parse(row.at);
-    if (!Number.isFinite(at)) continue;
-    if (at / 1000 < sinceEpochS) break;   // the log is append-ordered: nothing older can count
-    if (row.event === 'worked') n += 1;
-  }
-  return n;
-}
 
 function record(entry) {
   const row = { at: new Date().toISOString(), ...entry };
@@ -189,7 +140,12 @@ async function pickTask(goals, deps = {}) {
 
     const candidates = Array.from(byId.values()).filter((t) => !seq.TERMINAL.has(t.status));
     const { order } = seq.topoOrder(candidates);
-    const { next, skipped } = seq.pickNext(order, { rank, claimableIds });
+    // Tasks this process set aside after a refused claim (task 1004396) drop out
+    // AFTER ordering, so a dependent of one is still halted by its own unmet
+    // dependency rather than promoted by the task's absence.
+    const exclude = deps.exclude || null;
+    const open = exclude && exclude.size ? order.filter((t) => !exclude.has(String(t.id))) : order;
+    const { next, skipped } = seq.pickNext(open, { rank, claimableIds });
     if (next) return { task: next, goalId, skipped };
     for (const sk of skipped) allSkipped.push({ ...sk, goal_id: goalId });
   }
@@ -369,6 +325,80 @@ async function readFence(deps = {}) {
   return { raw, graderBypassed };
 }
 
+// ── what one process remembers between iterations (task 1004396) ────────────
+//
+// Owner ruling, 2026-09-30: a refused claim must not put the runner into waiting
+// by default. It sets THAT task aside and tries the next one. It waits only when
+// there is a block and nothing else it can claim.
+//
+// The two refusals are told apart by claim.js's exit code, because the message is
+// prose and the first cut of this read prose. The run that found this burned
+// ~30 attempts on task 1004254, whose refusal quoted "Your work changes the
+// tools…" — which reads like a per-task protected-path verdict and was in fact
+// the plain-words line of a QUEUE-WIDE gate about a different, stranded task.
+// Setting 1004254 aside would have moved the runner on to the next task, which
+// the same gate refuses too.
+//
+// QUEUE_GATED_EXIT is pinned against claim.js's own value in a test rather than
+// required from it: requiring the CLI would load its whole dependency graph into
+// a process that runs for weeks, to read one integer.
+const QUEUE_GATED_EXIT = 4;
+// How long a task refused ON ITS OWN is left alone. Not forever: most per-task
+// refusals clear (a dependency ships, a claim is released, a blocker resolves),
+// and a process that remembered them for its whole life would shelve work it can
+// do. Half an hour is several wakes at full speed, so a refused head of the queue
+// costs one attempt per half hour instead of one per wake.
+const CLAIM_SET_ASIDE_MS = 30 * 60 * 1000;
+// A bound on claim attempts in ONE pass, so a queue of refusals cannot turn one
+// iteration into dozens of worktree creations.
+const MAX_CLAIM_ATTEMPTS = 5;
+// A limit message with no machine-readable reset (the CLI's human forms name a
+// clock time in a zone we would have to guess). Half an hour is short against a
+// five-hour window and long against a spawn that is cut off in two seconds, so a
+// guess that is wrong costs one wasted spawn per half hour, not one per wake.
+const LIMIT_UNKNOWN_RESET_MS = 30 * 60 * 1000;
+
+function newRunnerState() {
+  return { setAside: new Map(), queueGate: null, limitUntil: null };
+}
+const RUNNER_STATE = newRunnerState();
+
+// The stranded task ids a queue-gated refusal names ("  - task 777  Title"). An
+// empty list is a legal answer: the gate is still a gate, the runner simply has
+// nothing to poll and tries one claim per wake instead.
+function owedTaskIds(text) {
+  const ids = [];
+  const re = /^\s*-\s*task\s+(\d+)/gm;
+  let m;
+  while ((m = re.exec(String(text || ''))) !== null) ids.push(m[1]);
+  return ids;
+}
+
+// Does the gate still hold? Asked of the LEDGER, one read per stranded task, and
+// never by attempting a claim: a claim attempt per wake is the loop this replaces.
+// An unreadable task keeps the wait — guessing "cleared" would restart the
+// attempts on a read failure, the opposite of backing off.
+async function queueGateHolds(queueGate, deps = {}) {
+  if (!queueGate.owed.length) return false;
+  const api = deps.api && deps.api.tasks ? deps.api : await (require('./cli-lib').cliClient)();
+  for (const id of queueGate.owed) {
+    let r = null;
+    try { r = await api.tasks.getTasksId({ id: Number(id) }); } catch (_) { return true; }
+    if (!r || !r.ok || !r.data || !r.data.task) return true;
+    if (r.data.task.status === 'confirmed') return true;
+  }
+  return false;
+}
+
+function queueGatedRow(queueGate, now) {
+  const on = queueGate.owed.length ? queueGate.owed.map((id) => `task ${id}`).join(', ') : 'a stranded confirmed task';
+  return {
+    event: 'queue_gated', waiting_on: queueGate.owed,
+    waited_s: Math.max(0, Math.round((now - queueGate.since) / 1000)),
+    reason: `every claim is refused until ${on} lands — waiting, not claiming (${queueGate.reason})`,
+  };
+}
+
 async function iteration(opts, deps = {}) {
   const runCmd = deps.run || run;
   const spawnWork = deps.spawnWorker || spawnWorker;
@@ -400,88 +430,116 @@ async function iteration(opts, deps = {}) {
     return log({ event: 'hold', go: false, unknown: !!g.decision.unknown, reason: g.decision.reason, sleep_until: g.decision.sleepUntil ?? null });
   }
 
-  // 1b. A DERIVED gauge buys a BOUNDED amount of work (task 1004178).
-  //
-  // `derived` means the last real reading's window had already reset, so the
-  // baseline is a retired observation rather than a measurement. That is sound
-  // for the FIVE-hour window, which turns over during a night. It does nothing
-  // for the SEVEN-day one, which does not — so the weekly pace check, the thing
-  // the module's notes call what actually binds a heavy week, is simply absent
-  // on a derived reading. A stale weekly figure cannot substitute: usage only
-  // grows, so an under-reported value makes the check more permissive, and that
-  // is the wrong direction to be wrong in.
-  //
-  // So the bound is a task count, not a percentage: on one derived window, do
-  // AUTOBONGOS_DERIVED_TASK_CAP tasks and then hold until a real reading exists.
-  // A terminal session writes one; the owner opening a terminal in the morning
-  // is what lifts it. Default 2 — each worker is wall-clock bounded at 90
-  // minutes, so two of them is well inside a five-hour window even at worst, and
-  // the cap only has to be small enough that being wrong is survivable.
-  if (g.decision.derived) {
-    const cap = Number(process.env.AUTOBONGOS_DERIVED_TASK_CAP ?? 2);
-    const since = Number(g.decision.derivedSince) || 0;
-    const done = countWorkedSince(since);
-    if (Number.isFinite(cap) && cap >= 0 && done >= cap) {
+  // 1b. The usage limit, once HIT (task 1004454). Owner ruling 2026-09-30: no
+  // fixed cap on how much work a reading buys — the gauge above is read before
+  // every task and a derived reading proceeds — and hitting the limit is
+  // acceptable, so long as the runner then waits for the reset and resumes by
+  // itself. A worker cut off by the limit records when it resets; until then no
+  // task is taken, because every worker spawned would be cut off the same way.
+  const state = deps.state || RUNNER_STATE;
+  const now = deps.now ? deps.now() : Date.now();
+  if (state.limitUntil) {
+    if (now < state.limitUntil) {
       return log({
-        event: 'hold', go: false, unknown: false, derived: true, worked_since_derived: done,
-        reason: `derived gauge: ${done} task(s) already done since the window reset and the cap is ${cap} — holding for a real reading (a terminal session writes one)`,
-        sleep_until: null,
+        event: 'hold', go: false, unknown: false, usage_limit: true,
+        reason: 'the subscription usage limit was hit — sleeping to its reset, then resuming by itself',
+        sleep_until: Math.round(state.limitUntil / 1000),
       });
     }
+    log({ event: 'usage_limit_reset', was_until: new Date(state.limitUntil).toISOString() });
+    state.limitUntil = null;
   }
 
-  // 2. Pick — from the ALLOWLISTED goals, never from what the command line asked
-  // for. gate.goals is already the intersection.
-  const picked = await pick(gate.goals, deps.pickDeps || {});
-  if (picked.error) return log({ event: 'pick_failed', reason: picked.error });
-  if (picked.none) {
-    return log({
-      event: 'nothing_claimable', goals: gate.goals,
-      // Carried so the cadence can tell an EMPTY queue from one where every task
-      // halts — the reasons are the matrix's own codes, safe to print.
-      skipped: (picked.skipped || []).map((sk) => ({ id: sk.id, goal_id: sk.goal_id, reasons: (sk.reasons || []).map((r) => r.code) })),
-    });
-  }
-  const task = picked.task;
-
-  // 2b. The per-TASK half of the fence. sequence.js's halt matrix already refuses
-  // a protected-path task, and this re-asks anyway — not from distrust of the
-  // matrix, but because the matrix is a shared component with its own callers and
-  // the fence must not be a property of whoever happens to be picking. A gate that
-  // only holds while a collaborator keeps calling it is not a gate.
-  //
-  // Both halves read the task's DECLARED touches[], which is a claim, not a fact —
-  // see the header of modules/autonomy/fence.js for why that is acceptable as the
-  // OUTER of three layers and would not be as the only one.
-  const taskGate = fence.decideTask({
-    task, goals: gate.goals, protectedHits: matchProtected(task.touches || []),
-  });
-  if (!taskGate.go) {
-    return log({ event: 'fenced', code: taskGate.code, reason: taskGate.reason, task_id: task.id, goal_id: picked.goalId, globs: taskGate.globs || [] });
+  // 1c. A QUEUE-WIDE gate from an earlier pass (task 1004396). While the
+  // stranded task it named is still `confirmed`, every claim would be refused, so
+  // none is attempted. The wait wakes early when main moves (waitOrJump), which is
+  // exactly what the strand landing does.
+  if (state.queueGate) {
+    if (await queueGateHolds(state.queueGate, deps)) return log(queueGatedRow(state.queueGate, now));
+    log({ event: 'queue_gate_cleared', waited_on: state.queueGate.owed, waited_s: Math.max(0, Math.round((now - state.queueGate.since) / 1000)) });
+    state.queueGate = null;
   }
+  for (const [id, until] of state.setAside) if (until <= now) state.setAside.delete(id);
+
+  // 2–4, up to MAX_CLAIM_ATTEMPTS times: pick, fence, tree, claim. A task refused
+  // on its own is set aside and the NEXT one is tried in the same pass.
+  let task = null; let picked = null; let wtName = null; let wtPath = null;
+  let lastRefusal = null;
+  for (let attempt = 0; attempt < MAX_CLAIM_ATTEMPTS && !task; attempt++) {
+    const exclude = new Set(state.setAside.keys());
+    // 2. Pick — from the ALLOWLISTED goals, never from what the command line asked
+    // for. gate.goals is already the intersection.
+    const got = await pick(gate.goals, { ...(deps.pickDeps || {}), exclude });
+    if (got.error) return log({ event: 'pick_failed', reason: got.error });
+    // A picker that hands back a set-aside task is treated as having none left:
+    // the exclusion is this function's contract, not the picker's courtesy.
+    if (got.none || exclude.has(String(got.task.id))) {
+      // Everything left was refused in THIS pass: return the last refusal, not
+      // "nothing claimable". Every claim failing at once is also what an outage
+      // looks like, and it must still reach the failure ladder (task 1004179).
+      if (lastRefusal) return lastRefusal;
+      return log({
+        event: 'nothing_claimable', goals: gate.goals,
+        // Carried so the cadence can tell an EMPTY queue from one where every task
+        // halts — the reasons are the matrix's own codes, safe to print.
+        skipped: (got.skipped || []).map((sk) => ({ id: sk.id, goal_id: sk.goal_id, reasons: (sk.reasons || []).map((r) => r.code) })),
+        set_aside: Array.from(state.setAside, ([id, until]) => ({ id, until: new Date(until).toISOString() })),
+      });
+    }
+    const candidate = got.task;
+
+    // 2b. The per-TASK half of the fence. sequence.js's halt matrix already refuses
+    // a protected-path task, and this re-asks anyway — not from distrust of the
+    // matrix, but because the matrix is a shared component with its own callers and
+    // the fence must not be a property of whoever happens to be picking. A gate that
+    // only holds while a collaborator keeps calling it is not a gate.
+    //
+    // Both halves read the task's DECLARED touches[], which is a claim, not a fact —
+    // see the header of modules/autonomy/fence.js for why that is acceptable as the
+    // OUTER of three layers and would not be as the only one.
+    const taskGate = fence.decideTask({
+      task: candidate, goals: gate.goals, protectedHits: matchProtected(candidate.touches || []),
+    });
+    if (!taskGate.go) {
+      return log({ event: 'fenced', code: taskGate.code, reason: taskGate.reason, task_id: candidate.id, goal_id: got.goalId, globs: taskGate.globs || [] });
+    }
 
-  if (opts.dryRun) return log({ event: 'dry_run', task_id: task.id, title: task.title, goal_id: picked.goalId });
+    if (opts.dryRun) return log({ event: 'dry_run', task_id: candidate.id, title: candidate.title, goal_id: got.goalId });
 
-  // 3. The task's own working tree.
-  const wtName = deps.worktreeName ? deps.worktreeName(task.id) : worktreeName(task.id);
-  const made = await runCmd(process.execPath, [path.join('scripts', 'gds', 'worktree.js'), 'add', wtName, '--base', 'origin/main']);
-  if (!made.ok) {
-    return log({ event: 'worktree_failed', task_id: task.id, worktree: wtName, reason: failureReason(made, 'worktree.js') });
-  }
-  const wtPath = path.join(REPO_ROOT, '.claude', 'worktrees', wtName);
-
-  // 4. Claim FROM that tree, so claim.js records it as the claim's tree and the
-  // ship lock later agrees with where the work actually is.
-  const claimed = await runCmd(process.execPath, [path.join(REPO_ROOT, 'scripts', 'gds', 'claim.js'), String(task.id)], { cwd: wtPath });
-  if (!claimed.ok) {
-    await runCmd(process.execPath, [path.join('scripts', 'gds', 'worktree.js'), 'remove', wtName]);
-    return log({ event: 'claim_failed', task_id: task.id, worktree: wtName, reason: failureReason(claimed, 'claim.js') });
+    // 3. The task's own working tree.
+    const name = deps.worktreeName ? deps.worktreeName(candidate.id) : worktreeName(candidate.id);
+    const made = await runCmd(process.execPath, [path.join('scripts', 'gds', 'worktree.js'), 'add', name, '--base', 'origin/main']);
+    if (!made.ok) {
+      return log({ event: 'worktree_failed', task_id: candidate.id, worktree: name, reason: failureReason(made, 'worktree.js') });
+    }
+    const treePath = path.join(REPO_ROOT, '.claude', 'worktrees', name);
+
+    // 4. Claim FROM that tree, so claim.js records it as the claim's tree and the
+    // ship lock later agrees with where the work actually is.
+    const claimed = await runCmd(process.execPath, [path.join(REPO_ROOT, 'scripts', 'gds', 'claim.js'), String(candidate.id)], { cwd: treePath });
+    if (!claimed.ok) {
+      await runCmd(process.execPath, [path.join('scripts', 'gds', 'worktree.js'), 'remove', name]);
+      const reason = failureReason(claimed, 'claim.js');
+      if (claimed.code === QUEUE_GATED_EXIT) {
+        state.queueGate = { owed: owedTaskIds(`${claimed.stderr || ''}\n${claimed.stdout || ''}`), since: now, reason };
+        return log({ ...queueGatedRow(state.queueGate, now), task_id: candidate.id, worktree: name });
+      }
+      state.setAside.set(String(candidate.id), now + CLAIM_SET_ASIDE_MS);
+      lastRefusal = log({ event: 'claim_failed', task_id: candidate.id, worktree: name, reason, set_aside_s: CLAIM_SET_ASIDE_MS / 1000 });
+      continue;
+    }
+    task = candidate; picked = got; wtName = name; wtPath = treePath;
   }
+  if (!task) return lastRefusal;
 
   // 5. Work, in that tree.
   const started = Date.now();
   const res = await spawnWork({ task, model: opts.model, timeoutMs: opts.timeoutMs, cwd: wtPath, deps });
   const verdict = classifyRun(res);
+  // Cut off by the usage limit? Recorded here so the NEXT iteration sleeps to the
+  // reset instead of spawning a worker that will be cut off the same way.
+  const limit = verdict.outcome === 'claims_shipped' ? null : usageLimitHit({ envelope: res.envelope, stderr: res.stderr });
+  if (limit) state.limitUntil = limit.resetAt ? limit.resetAt * 1000 : now + LIMIT_UNKNOWN_RESET_MS;
 
   // 6. VERIFY FROM THE LEDGER (task 1003903). Read the task back from Bongos and
   // let IT say what happened — for every outcome, not only a claimed ship. The
@@ -513,6 +571,7 @@ async function iteration(opts, deps = {}) {
     session_id: res.sessionId || null, duration_s: Math.round((Date.now() - started) / 1000),
     cost_usd: verdict.costUsd ?? null,
     output_truncated: !!res.truncated,
+    ...(limit ? { usage_limit: { reset_at: limit.resetAt } } : {}),
     undetermined_decisions: verdict.verdict ? verdict.verdict.undetermined_decisions : null,
     // What the LEDGER says, kept separate from what the worker said, so the run
     // log can be read afterwards without having to trust either one alone.
@@ -912,6 +971,15 @@ async function forever(opts, deps = {}) {
       // corpse.
       await beat({ mode: state.mode, last_event: row.event, working_task_id: Number(row.task_id) || undefined });
       const seen = cadence.classifyEvent(row);
+      // A hold that names its reset SECOND waits until that second (task 1004454).
+      // classifyEvent carries it as an absolute epoch; the cadence has no clock of
+      // its own, so the conversion to a wait lives here beside the one clock read.
+      // Without it every hold waited the fixed idle and the "wake at the reset"
+      // the goal asks for was only ever as accurate as the poll.
+      if (Number.isFinite(Number(seen.sleepUntil)) && seen.sleepUntil !== null) {
+        const nowS = Math.floor((deps.now ? deps.now() : Date.now()) / 1000);
+        seen.sleepUntilS = Math.max(0, Number(seen.sleepUntil) - nowS);
+      }
       state = cadence.nextCadence(state, seen, opts.cadence);
       log({ event: 'cadence', mode: state.mode, consecutive_failures: state.consecutiveFailures, wait_s: state.waitS, class: seen.class, why: state.why });
       // A burst continues only while work is actually landing. Anything else ends
@@ -972,7 +1040,8 @@ if (require.main === module) {
 
 module.exports = {
   readArgv, pickTask, iteration, record, logPath, spawnWorker, workerEnv, WORKER_ENV_ALLOW, resolveWorkerBin,
-  worktreeName, readFence, reconcileLeftoverClaims, pollSignals, waitOrJump, forever, CLAIM_PREFIX, countWorkedSince, codeDrifted, loadedHead, UPGRADE_EXIT_CODE, MIN_UPTIME_BEFORE_UPGRADE_MS,
+  worktreeName, readFence, reconcileLeftoverClaims, pollSignals, waitOrJump, forever, CLAIM_PREFIX, codeDrifted, loadedHead, UPGRADE_EXIT_CODE, MIN_UPTIME_BEFORE_UPGRADE_MS,
   heartbeatPath, writeHeartbeat, readHeartbeat, pidAlive, anotherRunnerIsAlive, HEARTBEAT_STALE_MS,
   isRunnerTree, RUNNER_TREE_RE, postHeartbeat, RUNNER_STARTED_AT, failureReason,
+  newRunnerState, LIMIT_UNKNOWN_RESET_MS, QUEUE_GATED_EXIT, CLAIM_SET_ASIDE_MS, MAX_CLAIM_ATTEMPTS, owedTaskIds,
 };

package/scripts/gds/claim.js

@@ -282,6 +282,14 @@ function claimErrorInfo(r) {
 // control characters that repaint a terminal, and a runaway one can bury the
 // guidance it was meant to add. Clip to one line, strip C0/C1 controls, cap length.
 const MAX_LISTED_TASKS = 20;
+
+// QUEUE_GATED_EXIT — the one refusal that is about the QUEUE, not the task
+// (task 1004396). REBASE_REQUIRED refuses every claim until a stranded confirmed
+// task lands, so an unattended caller that reads it as "this task is refused"
+// sets aside task after task for a fault none of them has. Every other refusal
+// keeps exit 1. The autobongos runner is the consumer and pins the same value
+// in a test.
+const QUEUE_GATED_EXIT = 4;
 function clip(s, max = 160) {
   if (s == null) return '';
   // eslint-disable-next-line no-control-regex
@@ -370,7 +378,7 @@ function claimFailureGuidance({ code, message, d }, r, taskId, ctx) {
       if (t.reason) lines.push(`      flagged: ${clip(t.reason)}`);
     }
     lines.push(`  ${clip(d.hint, 600) || 'Clear it with /strand-fix N, or rebase the branch and re-ship — or /builder-release it to give up on one.'}`);
-    return { lines, exit: 1 };
+    return { lines, exit: QUEUE_GATED_EXIT };
   }
   if (code === 'ALREADY_CLAIMED') {
     lines.push(`Task #${taskId} was just claimed by another session. Try a different task.`);
@@ -771,4 +779,4 @@ if (require.main === module) {
   });
 }
 
-module.exports = { formatClaimFailure, claimErrorInfo, goalMembershipOfferPlan, materializeModeSkills, printRoleLine, worktreeDetectionNote };
+module.exports = { formatClaimFailure, claimErrorInfo, goalMembershipOfferPlan, materializeModeSkills, printRoleLine, worktreeDetectionNote, QUEUE_GATED_EXIT };

package/scripts/gds/move-escalation.js

@@ -0,0 +1,141 @@
+'use strict';
+//
+// scripts/gds/move-escalation.js — a core move nobody can heal reaches a person: one blocker
+// and one Discord notice per failed move, closed by itself when a later move lands (task
+// 1004449, goal 1000090 deploy self-heal).
+//
+// WHY. After task 1004448 a failed /deploy move ends one of three ways: it waits and retries
+// (transient), it is fixed and retried (known_wedge), or it is retired. A retired
+// needs_decision move is the owner's own rule refusing it, and the deploy page says so. But a
+// move retired as `unknown` — an unlisted failure, a fix that did not hold, and every
+// rollback that FAILED (the project may be down) — only wrote a line on the intent, which
+// nobody reads until they open the page. That is exactly the failure that needs a person.
+//
+// EDGE-TRIGGERED, WITHOUT NEW STATE. The blocker's (source, source_ref) is unique, and
+// source_ref names the INTENT: `core-move:<instance id>:<intent id>`. An intent retires once,
+// so a failed move files one blocker however many retries led up to it, and a re-run of the
+// same retire files nothing. The Discord notice is posted only when the insert actually
+// created the row, so it is exactly as edge-triggered as the blocker.
+//
+// CLOSED BY SUCCESS. A later core move on the same project that lands resolves every open
+// blocker this file filed for that project, with a note saying which move fixed it, and
+// posts one all-clear. A person can still resolve one by hand; this never reopens it.
+//
+// The Discord copy names a project only when it is public on the platform's map (the rule
+// liveness-notify.js keeps), and never carries upgrade output — that stays in the blocker,
+// inside the trust boundary. Writes go through the runner's own db handle (the hub
+// database, where the blockers table lives), not modules/ideas/blockers.js's shared pool,
+// which this CLI process never opens; the INSERT is that file's createBlocker, verbatim.
+
+const { execFile } = require('child_process');
+const { REPO_ROOT } = require('./provision-config.js');
+
+const BLOCKER_SOURCE = 'provisioning';
+const REF_PREFIX = 'core-move';
+const DETAIL_MAX = 1500;
+
+/** Does this retired move need a person? PURE. */
+function needsPerson({ intent, failure, terminal }) {
+  if (!intent || intent.action !== 'core-upgrade') return false;
+  return !!terminal || !!(failure && (failure.class === 'unknown' || failure.reason === 'rollback_failed'));
+}
+
+function sourceRef(inst, intent) { return `${REF_PREFIX}:${inst.id}:${intent.id}`; }
+
+function projectName(inst, provisioning) {
+  try { return provisioning.effectiveSettings(inst).visibility === 'public' ? inst.slug : null; } catch { return null; }
+}
+
+/** The blocker a failed move files. PURE. */
+function blockerFor({ inst, intent, failure, error }) {
+  const reason = (failure && failure.reason) || 'unknown';
+  const down = reason === 'rollback_failed';
+  const to = intent.target_version || '(no target)';
+  const detail = String(error || '').slice(0, DETAIL_MAX).replace(/```/g, "'''");
+  return {
+    title: down
+      ? `${inst.slug}: the core move to ${to} failed AND its rollback did not finish — the project may be down`
+      : `${inst.slug}: the core move to ${to} failed and the runner cannot fix it`,
+    bodyMd: [
+      `**Project:** ${inst.slug} (instance ${inst.id}) · **target:** ${to} · **reason:** \`${reason}\` · **move:** intent ${intent.id}`,
+      '',
+      down
+        ? 'The move changed the project, failed, and the automatic rollback also failed. It may be down or on a half-installed core. Check it before anything else.'
+        : 'The runner retired this move: it is not a known snag it can fix, or its one automatic fix did not work. Nothing is being retried.',
+      '',
+      'What the move reported:',
+      '```',
+      detail || '(nothing)',
+      '```',
+      '',
+      'This closes by itself when a later core move on this project succeeds.',
+    ].join('\n'),
+    source: BLOCKER_SOURCE,
+    sourceRef: sourceRef(inst, intent),
+  };
+}
+
+/** The owner-facing Discord line. `name` null → described, never identified. PURE. */
+function noticeText({ kind, name, to, down }) {
+  const who = name ? `**${name}**` : 'A private project';
+  if (kind === 'fixed') return `✅ ${who} took its core update${to ? ` to ${to}` : ''}; the earlier failed update is closed.`;
+  return down
+    ? `🚨 ${who}: a core update failed and could not be undone. The project may be down. A blocker is open with the details.`
+    : `⚠️ ${who}: a core update${to ? ` to ${to}` : ''} failed in a way the platform cannot fix by itself. A blocker is open with the details. Nothing is being retried.`;
+}
+
+// Fire-and-forget, the liveness-notify.js postNotice shape: CHAT_DRY_RUN=1 short-circuits and
+// a Discord hiccup never stalls the drain.
+function postNotice(text, { log = console.log } = {}) {
+  if (!text) return;
+  if (process.env.CHAT_DRY_RUN === '1') { log(`[core-move] [broadcast dry-run] would post: ${text}`); return; }
+  try {
+    execFile('node', ['scripts/discord/post.js', text], { cwd: REPO_ROOT, timeout: 10_000 }, (err) => {
+      if (err) log(`[core-move] notice broadcast failed (non-blocking): ${err.message}`);
+    });
+  } catch (e) { log(`[core-move] notice broadcast failed (non-blocking): ${(e && e.message) || e}`); }
+}
+
+/** After a move is retired: file its blocker and notice, once. Never throws. */
+async function afterRetire({ intent, inst, result, deps }) {
+  const failure = result && result.failure;
+  if (!deps.apply || !needsPerson({ intent, failure, terminal: result && result.terminal })) return { filed: false };
+  const b = blockerFor({ inst, intent, failure, error: result.error });
+  try {
+    const { rows } = await deps.db.query(
+      `INSERT INTO blockers (title, body_md, source, source_ref)
+       VALUES ($1, $2, $3, $4)
+       ON CONFLICT (source, source_ref) WHERE source IS NOT NULL AND source_ref IS NOT NULL
+         DO NOTHING
+       RETURNING id`, [b.title, b.bodyMd, b.source, b.sourceRef]);
+    if (!rows || !rows[0]) return { filed: false };
+    const down = (failure && failure.reason) === 'rollback_failed';
+    (deps.postNotice || postNotice)(noticeText({ kind: 'failed', name: projectName(inst, deps.provisioning), to: intent.target_version, down }), { log: deps.log });
+    (deps.log || (() => {}))(`  ⚑ filed blocker ${rows[0].id} for ${inst.slug}'s failed move to ${intent.target_version}`);
+    return { filed: true, blockerId: rows[0].id };
+  } catch (e) {
+    (deps.log || (() => {}))(`  ⚠ could not file a blocker for ${inst.slug}'s failed move: ${(e && e.message) || e}`);
+    return { filed: false };
+  }
+}
+
+/** After a move LANDS: close every open blocker this file filed for the project. Never throws. */
+async function afterSuccess({ intent, inst, result, deps }) {
+  if (!deps.apply || !intent || intent.action !== 'core-upgrade' || !result || result.ok === false || result.noop) return { closed: 0 };
+  const to = result.served || intent.target_version;
+  try {
+    const { rows } = await deps.db.query(
+      `UPDATE blockers SET status = 'resolved', resolved_at = now(), resolution_note = $3
+        WHERE status = 'open' AND source = $1 AND source_ref LIKE $2
+        RETURNING id`,
+      [BLOCKER_SOURCE, `${REF_PREFIX}:${inst.id}:%`, `closed automatically: a later core move to ${to} succeeded (intent ${intent.id})`]);
+    const n = (rows || []).length;
+    if (n) (deps.postNotice || postNotice)(noticeText({ kind: 'fixed', name: projectName(inst, deps.provisioning), to }), { log: deps.log });
+    return { closed: n };
+  } catch (e) {
+    (deps.log || (() => {}))(`  ⚠ could not close ${inst.slug}'s core-move blockers: ${(e && e.message) || e}`);
+    return { closed: 0 };
+  }
+}
+
+module.exports = { BLOCKER_SOURCE, REF_PREFIX, needsPerson, sourceRef, blockerFor, noticeText, postNotice, afterRetire, afterSuccess };

package/scripts/gds/provision.js

@@ -1209,12 +1209,13 @@ async function cmdRunIntents(deps) {
           // can say why the work stopped.
           await provisioning.setInstanceStatus(db, intent.instance_id, inst.status, { error_note: result.error || 'op failed', error_note_action: intent.action }).catch(() => {});
           if (result.failure) await recordFailureClass(db, intent.id, result.failure); // WHAT KIND of failure, beside the sentence (task 1004446)
+          await require('./move-escalation.js').afterRetire({ intent, inst, result, deps }); // nobody can heal it: one blocker + one notice per move (task 1004449)
         } else {
           await db.query(`UPDATE provisioning_intents SET state='pending', last_error=$2, not_before = now() + ($3 * interval '1 millisecond'), updated_at=now() WHERE id=$1`, [intent.id, result.error || 'op failed', next.delayMs]);
           if (result.failure) await recordFailureClass(db, intent.id, result.failure);
         }
         errors++;
-      } else { await provisioning.resolveIntent(db, intent.id, 'done', (result && result.doneNote) || null); ok++; } // done WITH a note: it landed, and this is what it could not finish (task 1004065)
+      } else { await provisioning.resolveIntent(db, intent.id, 'done', (result && result.doneNote) || null); ok++; await require('./move-escalation.js').afterSuccess({ intent, inst, result, deps }); } // done WITH a note: it landed, and this is what it could not finish (task 1004065)
     } catch (e) {
       await provisioning.resolveIntent(db, intent.id, 'error', e.message).catch(() => {});
       // Surface the failure on the instance too, so a stuck instance is visible.

package/src/module-api.js

@@ -75,7 +75,7 @@ const { responsibilityFor, ROLE_RESPONSIBILITIES } = require('./role-responsibil
 // MAJOR (see allowBoxScope below): passes the request through untouched.
 function deprecatedNoopMiddleware(_req, _res, next) { next(); }
 
-const CORE_VERSION = '1.20.32'; // CI auto-patch carrier (ADR 0161); changelog: docs/module-api-changelog.md
+const CORE_VERSION = '1.20.34'; // CI auto-patch carrier (ADR 0161); changelog: docs/module-api-changelog.md
 
 // A namespaced logger so a module's log lines are attributable + consistent.
 // Usage: const log = api.logger('discord'); log.info('mounted');

package/tests/autobongos_cadence.mjs

@@ -40,6 +40,15 @@ const walk = (classes, cfg) => {
 
 // ── classification ──────────────────────────────────────────────────────────────
 
+test('a queue-wide gate is a wait, not a failure (task 1004396)', () => {
+  // The strand behind it is somebody's unlanded ship, not this machine breaking.
+  // Escalating it would slow the runner down for a fault it cannot fix, and the
+  // wait already wakes early when main moves, which is what the strand landing does.
+  const seen = classifyEvent({ event: 'queue_gated', waiting_on: ['777'], reason: 'x' });
+  assert.equal(seen.class, 'quiet');
+  assert.match(seen.detail, /777/);
+});
+
 test('a worked task counts as progress only when the LEDGER agreed', () => {
   assert.equal(classifyEvent({ event: 'worked', verified: true }).class, 'progress');
   // A worker's claim the ledger never saw is not progress; counting it would hold
@@ -284,6 +293,33 @@ function loopHarness({ rows, maxLoops = 6 }) {
   return { deps, waits: emitted, recorded, opts: { goals: [], maxTasks: 3, maxLoops } };
 }
 
+// ── the usage limit: a wait to a known second, not a failure (task 1004454) ──
+
+test('a worker cut off by the usage limit is quiet, not a failed launch', () => {
+  // It exits in seconds having spent nothing — exactly workerNeverRan()'s shape —
+  // but the machine is fine: the window is spent. Escalating it would put the
+  // runner into probing for a limit the owner has said is acceptable to hit.
+  const seen = classifyEvent({ event: 'worked', outcome: 'worker_failed', duration_s: 2, cost_usd: null, reason: 'x', usage_limit: { reset_at: 1790300000 } });
+  assert.equal(seen.class, 'quiet');
+  assert.equal(seen.sleepUntil, 1790300000);
+});
+
+test('a hold with a reset time makes the loop wait until THAT second, not a fixed idle', async () => {
+  const nowS = 1_790_290_000;
+  const h = loopHarness({ rows: [{ event: 'hold', go: false, reason: 'five-hour burn at the ceiling', sleep_until: nowS + 3600 }], maxLoops: 1 });
+  h.deps.now = () => nowS * 1000;
+  await runner.forever(h.opts, h.deps);
+  assert.equal(h.waits[0].waitS, 3600, 'the wake is the reset second the gauge named');
+});
+
+test('a hold whose reset has already passed waits no time at all', async () => {
+  const nowS = 1_790_290_000;
+  const h = loopHarness({ rows: [{ event: 'hold', go: false, reason: 'x', sleep_until: nowS - 5 }], maxLoops: 1 });
+  h.deps.now = () => nowS * 1000;
+  await runner.forever(h.opts, h.deps);
+  assert.equal(h.waits[0].waitS, 0);
+});
+
 test('PULLING THE NETWORK: the loop backs off and never returns', async () => {
   const h = loopHarness({ rows: [{ event: 'pick_failed', reason: 'fetch failed ECONNREFUSED' }] });
   await runner.forever(h.opts, h.deps);