@bongos/core 1.20.31 → 1.20.33

package/scripts/gds/move-escalation.js

@@ -0,0 +1,141 @@
+'use strict';
+//
+// scripts/gds/move-escalation.js — a core move nobody can heal reaches a person: one blocker
+// and one Discord notice per failed move, closed by itself when a later move lands (task
+// 1004449, goal 1000090 deploy self-heal).
+//
+// WHY. After task 1004448 a failed /deploy move ends one of three ways: it waits and retries
+// (transient), it is fixed and retried (known_wedge), or it is retired. A retired
+// needs_decision move is the owner's own rule refusing it, and the deploy page says so. But a
+// move retired as `unknown` — an unlisted failure, a fix that did not hold, and every
+// rollback that FAILED (the project may be down) — only wrote a line on the intent, which
+// nobody reads until they open the page. That is exactly the failure that needs a person.
+//
+// EDGE-TRIGGERED, WITHOUT NEW STATE. The blocker's (source, source_ref) is unique, and
+// source_ref names the INTENT: `core-move:<instance id>:<intent id>`. An intent retires once,
+// so a failed move files one blocker however many retries led up to it, and a re-run of the
+// same retire files nothing. The Discord notice is posted only when the insert actually
+// created the row, so it is exactly as edge-triggered as the blocker.
+//
+// CLOSED BY SUCCESS. A later core move on the same project that lands resolves every open
+// blocker this file filed for that project, with a note saying which move fixed it, and
+// posts one all-clear. A person can still resolve one by hand; this never reopens it.
+//
+// The Discord copy names a project only when it is public on the platform's map (the rule
+// liveness-notify.js keeps), and never carries upgrade output — that stays in the blocker,
+// inside the trust boundary. Writes go through the runner's own db handle (the hub
+// database, where the blockers table lives), not modules/ideas/blockers.js's shared pool,
+// which this CLI process never opens; the INSERT is that file's createBlocker, verbatim.
+
+const { execFile } = require('child_process');
+const { REPO_ROOT } = require('./provision-config.js');
+
+const BLOCKER_SOURCE = 'provisioning';
+const REF_PREFIX = 'core-move';
+const DETAIL_MAX = 1500;
+
+/** Does this retired move need a person? PURE. */
+function needsPerson({ intent, failure, terminal }) {
+  if (!intent || intent.action !== 'core-upgrade') return false;
+  return !!terminal || !!(failure && (failure.class === 'unknown' || failure.reason === 'rollback_failed'));
+}
+
+function sourceRef(inst, intent) { return `${REF_PREFIX}:${inst.id}:${intent.id}`; }
+
+function projectName(inst, provisioning) {
+  try { return provisioning.effectiveSettings(inst).visibility === 'public' ? inst.slug : null; } catch { return null; }
+}
+
+/** The blocker a failed move files. PURE. */
+function blockerFor({ inst, intent, failure, error }) {
+  const reason = (failure && failure.reason) || 'unknown';
+  const down = reason === 'rollback_failed';
+  const to = intent.target_version || '(no target)';
+  const detail = String(error || '').slice(0, DETAIL_MAX).replace(/```/g, "'''");
+  return {
+    title: down
+      ? `${inst.slug}: the core move to ${to} failed AND its rollback did not finish — the project may be down`
+      : `${inst.slug}: the core move to ${to} failed and the runner cannot fix it`,
+    bodyMd: [
+      `**Project:** ${inst.slug} (instance ${inst.id}) · **target:** ${to} · **reason:** \`${reason}\` · **move:** intent ${intent.id}`,
+      '',
+      down
+        ? 'The move changed the project, failed, and the automatic rollback also failed. It may be down or on a half-installed core. Check it before anything else.'
+        : 'The runner retired this move: it is not a known snag it can fix, or its one automatic fix did not work. Nothing is being retried.',
+      '',
+      'What the move reported:',
+      '```',
+      detail || '(nothing)',
+      '```',
+      '',
+      'This closes by itself when a later core move on this project succeeds.',
+    ].join('\n'),
+    source: BLOCKER_SOURCE,
+    sourceRef: sourceRef(inst, intent),
+  };
+}
+
+/** The owner-facing Discord line. `name` null → described, never identified. PURE. */
+function noticeText({ kind, name, to, down }) {
+  const who = name ? `**${name}**` : 'A private project';
+  if (kind === 'fixed') return `✅ ${who} took its core update${to ? ` to ${to}` : ''}; the earlier failed update is closed.`;
+  return down
+    ? `🚨 ${who}: a core update failed and could not be undone. The project may be down. A blocker is open with the details.`
+    : `⚠️ ${who}: a core update${to ? ` to ${to}` : ''} failed in a way the platform cannot fix by itself. A blocker is open with the details. Nothing is being retried.`;
+}
+
+// Fire-and-forget, the liveness-notify.js postNotice shape: CHAT_DRY_RUN=1 short-circuits and
+// a Discord hiccup never stalls the drain.
+function postNotice(text, { log = console.log } = {}) {
+  if (!text) return;
+  if (process.env.CHAT_DRY_RUN === '1') { log(`[core-move] [broadcast dry-run] would post: ${text}`); return; }
+  try {
+    execFile('node', ['scripts/discord/post.js', text], { cwd: REPO_ROOT, timeout: 10_000 }, (err) => {
+      if (err) log(`[core-move] notice broadcast failed (non-blocking): ${err.message}`);
+    });
+  } catch (e) { log(`[core-move] notice broadcast failed (non-blocking): ${(e && e.message) || e}`); }
+}
+
+/** After a move is retired: file its blocker and notice, once. Never throws. */
+async function afterRetire({ intent, inst, result, deps }) {
+  const failure = result && result.failure;
+  if (!deps.apply || !needsPerson({ intent, failure, terminal: result && result.terminal })) return { filed: false };
+  const b = blockerFor({ inst, intent, failure, error: result.error });
+  try {
+    const { rows } = await deps.db.query(
+      `INSERT INTO blockers (title, body_md, source, source_ref)
+       VALUES ($1, $2, $3, $4)
+       ON CONFLICT (source, source_ref) WHERE source IS NOT NULL AND source_ref IS NOT NULL
+         DO NOTHING
+       RETURNING id`, [b.title, b.bodyMd, b.source, b.sourceRef]);
+    if (!rows || !rows[0]) return { filed: false };
+    const down = (failure && failure.reason) === 'rollback_failed';
+    (deps.postNotice || postNotice)(noticeText({ kind: 'failed', name: projectName(inst, deps.provisioning), to: intent.target_version, down }), { log: deps.log });
+    (deps.log || (() => {}))(`  ⚑ filed blocker ${rows[0].id} for ${inst.slug}'s failed move to ${intent.target_version}`);
+    return { filed: true, blockerId: rows[0].id };
+  } catch (e) {
+    (deps.log || (() => {}))(`  ⚠ could not file a blocker for ${inst.slug}'s failed move: ${(e && e.message) || e}`);
+    return { filed: false };
+  }
+}
+
+/** After a move LANDS: close every open blocker this file filed for the project. Never throws. */
+async function afterSuccess({ intent, inst, result, deps }) {
+  if (!deps.apply || !intent || intent.action !== 'core-upgrade' || !result || result.ok === false || result.noop) return { closed: 0 };
+  const to = result.served || intent.target_version;
+  try {
+    const { rows } = await deps.db.query(
+      `UPDATE blockers SET status = 'resolved', resolved_at = now(), resolution_note = $3
+        WHERE status = 'open' AND source = $1 AND source_ref LIKE $2
+        RETURNING id`,
+      [BLOCKER_SOURCE, `${REF_PREFIX}:${inst.id}:%`, `closed automatically: a later core move to ${to} succeeded (intent ${intent.id})`]);
+    const n = (rows || []).length;
+    if (n) (deps.postNotice || postNotice)(noticeText({ kind: 'fixed', name: projectName(inst, deps.provisioning), to }), { log: deps.log });
+    return { closed: n };
+  } catch (e) {
+    (deps.log || (() => {}))(`  ⚠ could not close ${inst.slug}'s core-move blockers: ${(e && e.message) || e}`);
+    return { closed: 0 };
+  }
+}
+
+module.exports = { BLOCKER_SOURCE, REF_PREFIX, needsPerson, sourceRef, blockerFor, noticeText, postNotice, afterRetire, afterSuccess };

package/scripts/gds/provision-core-upgrade.js

@@ -241,6 +241,16 @@ async function coreUpgradeInstance(inst, deps, intent) {
     log(`  already serving core ${to} — nothing to move`);
     return { ok: true, noop: true };
   }
+  // ---- 2b. is the database current for the code it runs NOW (task 1004448) -------------
+  // A database behind its own code is already failing every page that touches the missing
+  // tables (the 2026-07-04 shape), and moving the code on top of it tangles a catch-up
+  // migration into a pin move. So it is refused as a known snag, and wedge-remedy.js
+  // migrates that database and tries the move once more. An answer it cannot read proceeds.
+  if (apply) {
+    const probe = boxExec(versionCmd(inst), { allowFail: true });
+    const pending = parseSchemaPending(probe && !probe.softFailed ? (probe.stdout || '') : null);
+    if (pending > 0) return failed('schema_pending', `the project's database is ${pending} migration(s) behind the code it is running now — it has to catch up before the core moves`);
+  }
 
   // ---- 3. drive the proven path -----------------------------------------------------
   // Exactly the invocation docs/recipes/core-release-pipeline.md gate 3 documents, plus

package/scripts/gds/provision.js

@@ -1196,6 +1196,9 @@ async function cmdRunIntents(deps) {
         // WAIT OR RETIRE (task 1004447, intent-retry.js): a transient failure waits ~15m/1h/6h; a
         // classified non-transient one is retired at once (retrying cannot change its answer);
         // an unclassified one keeps its budget but never re-runs in the same tick.
+        const fix = await require('./wedge-remedy.js').remedyAfterFailure({ intent, inst, result, deps }); // a KNOWN snag: fix it, try once more (task 1004448)
+        if (fix.repended) { errors++; continue; }
+        Object.assign(result, { failure: fix.failure, error: fix.error });
         const next = retry.retryDecision({ attempts: intent.attempts, failure: result.failure, terminal: result.terminal, maxAttempts: MAX_INTENT_ATTEMPTS });
         if (next.action === 'retire') { // terminal: a leg that must not be retried unattended (the Render leg's key is already gone — task 1004183)
           await provisioning.resolveIntent(db, intent.id, 'error', result.error || 'op failed');
@@ -1206,12 +1209,13 @@ async function cmdRunIntents(deps) {
           // can say why the work stopped.
           await provisioning.setInstanceStatus(db, intent.instance_id, inst.status, { error_note: result.error || 'op failed', error_note_action: intent.action }).catch(() => {});
           if (result.failure) await recordFailureClass(db, intent.id, result.failure); // WHAT KIND of failure, beside the sentence (task 1004446)
+          await require('./move-escalation.js').afterRetire({ intent, inst, result, deps }); // nobody can heal it: one blocker + one notice per move (task 1004449)
         } else {
           await db.query(`UPDATE provisioning_intents SET state='pending', last_error=$2, not_before = now() + ($3 * interval '1 millisecond'), updated_at=now() WHERE id=$1`, [intent.id, result.error || 'op failed', next.delayMs]);
           if (result.failure) await recordFailureClass(db, intent.id, result.failure);
         }
         errors++;
-      } else { await provisioning.resolveIntent(db, intent.id, 'done', (result && result.doneNote) || null); ok++; } // done WITH a note: it landed, and this is what it could not finish (task 1004065)
+      } else { await provisioning.resolveIntent(db, intent.id, 'done', (result && result.doneNote) || null); ok++; await require('./move-escalation.js').afterSuccess({ intent, inst, result, deps }); } // done WITH a note: it landed, and this is what it could not finish (task 1004065)
     } catch (e) {
       await provisioning.resolveIntent(db, intent.id, 'error', e.message).catch(() => {});
       // Surface the failure on the instance too, so a stuck instance is visible.

package/scripts/gds/upgrade-outcome.js

@@ -19,7 +19,7 @@
 //
 // THE FOUR CLASSES the rest of the self-heal chain keys on:
 //   transient       — try again later and it may simply work (registry blip, not published yet)
-//   known_wedge     — a known snag with a known fix the runner may apply itself (dirty tree, full disk)
+//   known_wedge     — a known snag with a known fix the runner applies itself, once (wedge-remedy.js)
 //   needs_decision  — a rule the owner set is refusing it; only the owner can change that
 //   unknown         — anything else, and every rollback that failed; a person should look
 //
@@ -42,6 +42,7 @@ const REASON_CLASS = Object.freeze({
   bad_slug: 'unknown',
   installed_unreadable: 'unknown',
   served_mismatch: 'unknown',
+  schema_pending: 'known_wedge', // the database is behind the code it runs NOW (task 1004448)
   // upgrade.js refusals, before the pin moves
   dirty_tree: 'known_wedge',
   downgrade: 'needs_decision',
@@ -59,6 +60,9 @@ const REASON_CLASS = Object.freeze({
   migrate: 'unknown',
   restart: 'unknown',
   health: 'unknown',
+  // the runner's one automatic fix did not clear it, or itself failed (task 1004448)
+  remedy_did_not_hold: 'unknown',
+  remedy_failed: 'unknown',
   // the one that must never be retried unattended
   rollback_failed: 'unknown',
 });

package/scripts/gds/wedge-remedy.js

@@ -0,0 +1,157 @@
+'use strict';
+//
+// scripts/gds/wedge-remedy.js — a core move that failed on a KNOWN snag fixes the snag and
+// tries once more (task 1004448, goal 1000090 deploy self-heal).
+//
+// WHY. upgrade-outcome.js names three failures that are not a fault in the release and not a
+// decision the owner has to make, but a mess on the box with one known fix — the fixes the
+// owner has been doing over SSH (docs/recipes/upgrading-the-core.md, Gotchas):
+//   dirty_tree      — the project's checkout has uncommitted changes to tracked files (the
+//                     upgrade's own rewrites, left behind by an earlier move). Fix: commit them
+//                     inside the project.
+//   disk_full       — the move ran out of disk. Fix: prune this project's OWN nightly
+//                     pg_dumps down to the newest few (the only thing on the box this runner
+//                     knows is safe to delete — they are copies, and the newest are kept).
+//   schema_pending  — the project's database is behind the code it is running NOW. Fix:
+//                     migrate that database, with its own PGDATABASE, before the code moves.
+// intent-retry.js retires every known_wedge at once, so on its own the failure only waited
+// for a person. This file runs the fix, records it as an event on the project, and re-pends
+// the move for ONE more try.
+//
+// AT MOST ONCE PER FAILURE. The reason a fix was applied for is stored on the intent
+// (provisioning_intents.remedied_reason, migration provisioning_032). If the retry fails on
+// the SAME reason, the fix did not hold: the move is retired as `unknown` /
+// remedy_did_not_hold, which is the class that reaches a person — never a second fix, never
+// a loop. A fix that itself fails is retired the same way (remedy_failed). A different
+// known reason on the retry gets its own one fix (a disk full after a dirty tree is a new
+// snag, not the same one twice); the attempt budget still bounds the whole run.
+//
+// 'tagged-not-published' needs nothing here: it is `not_published`, a transient failure that
+// intent-retry.js already waits out.
+//
+// planRemedy is PURE (the command, never run). remedyAfterFailure does the I/O through the
+// runner's own exec and db, so every fix is testable with an injected exec.
+
+const { CONFIG } = require('./provision-config.js');
+const { dbName, selfInstanceRoot, standaloneRoot } = require('./provision-repo.js');
+const { migratePlan, CORE_MIGRATE_SCRIPT } = require('./upgrade-migrate.js');
+const { isValidSlug } = require('../../modules/provisioning/provisioning.js');
+const outcome = require('./upgrade-outcome.js');
+const { DEFAULT_RETRY_DELAY_MS } = require('./intent-retry.js');
+
+const REMEDY_REASONS = Object.freeze(['dirty_tree', 'disk_full', 'schema_pending']);
+// The newest dumps a disk-full prune always leaves behind: a week of nightlies. The nightly
+// job's own retention is 30 days; this only ever cuts below that when the disk is full.
+const DUMP_KEEP_FLOOR = 7;
+const REMEDY_EVENT = 'core-upgrade-remedy';
+// A value that reaches a shell string below. Narrower than what Postgres or a path allow, on
+// purpose: anything outside it is refused, never quoted.
+const SHELL_SAFE_RE = /^[A-Za-z0-9._/-]{1,200}$/;
+const DB_NAME_RE = /^[A-Za-z0-9_-]{1,63}$/;
+const ACCOUNT_RE = /^[a-z_][a-z0-9_-]{0,31}$/;
+
+/**
+ * The fix for one known snag, as a command to run on the box. PURE.
+ * @param {object} inst   the provisioning_instances row
+ * @param {string} reason the failure's reason code
+ * @param {object} [o]    { privileged, runAs, instanceDir, selfRoot, backupDir, migrate }
+ * @returns {{ reason, cmd, cwd, env, describe } | { refused: string } | null}  null: no known fix
+ */
+function planRemedy(inst, reason, o = {}) {
+  if (!REMEDY_REASONS.includes(reason)) return null;
+  if (!inst || !isValidSlug(inst.slug)) return { refused: 'the project name is not a valid slug' };
+  const dir = o.instanceDir || (inst.hosting_shape === 'cloud-host' ? standaloneRoot(inst) : (o.selfRoot || selfInstanceRoot)());
+  if (!SHELL_SAFE_RE.test(dir)) return { refused: 'the project folder is not a path this runner will put in a command' };
+  const db = dbName(inst);
+  if (!DB_NAME_RE.test(db)) return { refused: 'the project database name is not one this runner will put in a command' };
+
+  if (reason === 'dirty_tree') {
+    // TRACKED files only (`add -u`): the snag is the upgrade's own rewrites of tracked files.
+    // A new untracked file is not swept into the owner's history on a guess — if that is the
+    // dirt, the retry fails the same way and a person looks. Runs as the runner, which owns
+    // the checkout (pushUpgradePin's shape), with hooks off and a runner identity so the
+    // commit says who made it.
+    const g = `git -C ${dir} -c core.hooksPath=/dev/null -c user.name=bongos-runner -c user.email=runner@bongos.invalid`;
+    return {
+      reason, cwd: null, env: null,
+      cmd: `${g} add -u && ${g} commit -m "chore: commit changes left in the working tree (automatic fix before a core move)"`,
+      describe: `committed the uncommitted changes in ${dir} so the core move could run`,
+    };
+  }
+  if (reason === 'disk_full') {
+    const backupDir = o.backupDir || CONFIG.backupDir;
+    if (!SHELL_SAFE_RE.test(backupDir)) return { refused: 'the backup folder is not a path this runner will put in a command' };
+    // EXACTLY db-backup-nightly.sh's name, so this never touches a sibling project's dumps
+    // (`test` must not prune `test-nk`'s — task 1003948). The timestamp sorts by name, so
+    // `sort | head -n -K` drops all but the newest K. Nothing is deleted when K or fewer exist.
+    const D = '[0-9]';
+    const glob = `${db}-${D}${D}${D}${D}-${D}${D}-${D}${D}T${D}${D}${D}${D}${D}${D}Z.sql.gz`;
+    return {
+      reason, cwd: null, env: null,
+      cmd: `find ${backupDir} -maxdepth 1 -type f -name '${glob}' | sort | head -n -${DUMP_KEEP_FLOOR} | xargs -r rm -f --`,
+      describe: `pruned ${db}'s old database backups in ${backupDir}, keeping the newest ${DUMP_KEEP_FLOOR}`,
+    };
+  }
+  // schema_pending — the project's OWN migrate command (a declared script wins, else the
+  // core's migrate.sh: upgrade-migrate.js), aimed at its OWN database. migrate.sh defaults to
+  // production when PGDATABASE is unset, so it is always set, never inherited.
+  const plan = (o.migrate || migratePlan)(dir);
+  const run = plan.via === 'npm-script' ? 'npm run migrate' : `bash ${CORE_MIGRATE_SCRIPT}`;
+  const initCwd = plan.via === 'npm-script' ? '' : `INIT_CWD=${dir} `;
+  if (o.privileged) {
+    if (!ACCOUNT_RE.test(String(o.runAs || ''))) return { refused: 'no account to run the migration as' };
+    return { reason, cwd: dir, env: null, cmd: `sudo -u ${o.runAs} env PGDATABASE=${db} ${initCwd}${run}`, describe: `migrated ${db} to match the code it is running` };
+  }
+  return { reason, cwd: dir, env: { PGDATABASE: db, ...(initCwd ? { INIT_CWD: dir } : {}) }, cmd: run, describe: `migrated ${db} to match the code it is running` };
+}
+
+// The account the project runs as, probed the way the move itself probes (task 1004128).
+function runAsFor(inst, deps) {
+  try { return require('./provision-core-upgrade.js').resolveRunAs(inst, deps).runAs; } catch { return null; }
+}
+
+/**
+ * After a core move failed: apply the known fix, record it, and re-pend the move once.
+ *
+ * Returns `{ repended: true }` when the move will be tried again, otherwise
+ * `{ repended: false, failure, error }` — the failure the drain loop should RETIRE with,
+ * which is the original one when there is no fix for it, and `unknown` when a fix was
+ * already spent on this reason or the fix itself failed.
+ */
+async function remedyAfterFailure({ intent, inst, result, deps }) {
+  const failure = result && result.failure;
+  const reason = failure && failure.reason;
+  const none = { repended: false, failure, error: result && result.error };
+  if (!intent || intent.action !== 'core-upgrade' || !failure || failure.class !== 'known_wedge') return none;
+  const { db, provisioning, log = () => {}, apply } = deps;
+  if (!apply) return none;
+  if (intent.remedied_reason === reason) {
+    return {
+      repended: false, failure: outcome.classifyFailure('remedy_did_not_hold'),
+      error: `${result.error || 'core move failed'} — the automatic fix for this (${reason}) was already tried once and did not clear it, so a person needs to look`,
+    };
+  }
+  const boxExec = inst.hosting_shape === 'cloud-host' ? (deps.controlExec || deps.exec) : deps.exec;
+  const plan = planRemedy(inst, reason, { privileged: deps.privileged, runAs: reason === 'schema_pending' && deps.privileged ? runAsFor(inst, deps) : null, selfRoot: deps.selfInstanceRoot, migrate: deps.migratePlan });
+  if (!plan) return none;
+  const giveUp = async (why) => {
+    await provisioning.recordEvent(db, { instanceId: inst.id, ownerBuilderId: inst.owner_builder_id, event: REMEDY_EVENT, detail: `could not fix ${reason} automatically: ${why}`, actor: 'runner' }).catch(() => {});
+    return { repended: false, failure: outcome.classifyFailure('remedy_failed'), error: `${result.error || 'core move failed'} — the automatic fix for ${reason} did not work (${why}), so a person needs to look` };
+  };
+  if (plan.refused) return giveUp(plan.refused);
+  log(`  [remedy] ${reason}: ${plan.cmd}`);
+  const r = boxExec(plan.cmd, { allowFail: true, ...(plan.cwd ? { cwd: plan.cwd } : {}), ...(plan.env ? { env: plan.env } : {}) });
+  if (!r || r.softFailed === true || r.ok === false) {
+    const said = String((r && (r.stderr || r.stdout || r.error)) || 'the command failed').trim().split('\n').pop().slice(0, 160);
+    return giveUp(said);
+  }
+  await provisioning.recordEvent(db, { instanceId: inst.id, ownerBuilderId: inst.owner_builder_id, event: REMEDY_EVENT, detail: `${plan.describe}; trying the move to ${intent.target_version} once more`, actor: 'runner' }).catch(() => {});
+  await db.query(
+    `UPDATE provisioning_intents SET state='pending', last_error=$2, remedied_reason=$3, failure_class=$4, failure_reason=$5,
+            not_before = now() + ($6 * interval '1 millisecond'), updated_at=now() WHERE id=$1`,
+    [intent.id, result.error || 'core move failed', reason, failure.class, failure.reason, DEFAULT_RETRY_DELAY_MS]);
+  log(`  ✓ fixed ${reason} on ${inst.slug}; the move to ${intent.target_version} will be tried once more`);
+  return { repended: true };
+}
+
+module.exports = { REMEDY_REASONS, DUMP_KEEP_FLOOR, REMEDY_EVENT, planRemedy, remedyAfterFailure };

package/src/module-api.js

@@ -75,7 +75,7 @@ const { responsibilityFor, ROLE_RESPONSIBILITIES } = require('./role-responsibil
 // MAJOR (see allowBoxScope below): passes the request through untouched.
 function deprecatedNoopMiddleware(_req, _res, next) { next(); }
 
-const CORE_VERSION = '1.20.31'; // CI auto-patch carrier (ADR 0161); changelog: docs/module-api-changelog.md
+const CORE_VERSION = '1.20.33'; // CI auto-patch carrier (ADR 0161); changelog: docs/module-api-changelog.md
 
 // A namespaced logger so a module's log lines are attributable + consistent.
 // Usage: const log = api.logger('discord'); log.info('mounted');

package/tests/autobongos_cadence.mjs

@@ -40,6 +40,15 @@ const walk = (classes, cfg) => {
 
 // ── classification ──────────────────────────────────────────────────────────────
 
+test('a queue-wide gate is a wait, not a failure (task 1004396)', () => {
+  // The strand behind it is somebody's unlanded ship, not this machine breaking.
+  // Escalating it would slow the runner down for a fault it cannot fix, and the
+  // wait already wakes early when main moves, which is what the strand landing does.
+  const seen = classifyEvent({ event: 'queue_gated', waiting_on: ['777'], reason: 'x' });
+  assert.equal(seen.class, 'quiet');
+  assert.match(seen.detail, /777/);
+});
+
 test('a worked task counts as progress only when the LEDGER agreed', () => {
   assert.equal(classifyEvent({ event: 'worked', verified: true }).class, 'progress');
   // A worker's claim the ledger never saw is not progress; counting it would hold

package/tests/autobongos_loop.mjs

@@ -237,6 +237,12 @@ test('an unreadable claimable feed does not halt everything', async () => {
   assert.equal(got.task.id, 9001, 'the not_claimable reason stays quiet when the feed cannot be read');
 });
 
+test('pickTask passes over a task the runner has set aside (task 1004396)', async () => {
+  const api = fakeApi({ tasks: [ready({ id: 9001 }), ready({ id: 9002 })], claimable: [{ id: 9001 }, { id: 9002 }] });
+  const got = await runner.pickTask([1000119], { api, rank: 'archon', exclude: new Set(['9001']) });
+  assert.equal(got.task.id, 9002, 'a refused task must not stay at the head of the order');
+});
+
 test('a goal whose tasks cannot be read is reported, not silently skipped', async () => {
   const api = fakeApi({ tasks: [], claimable: [] });
   api.tasks.getTasks = async () => { throw new Error('boom'); };
@@ -273,7 +279,11 @@ function harness(over = {}) {
     // The LEDGER, faked. iteration() now reads the task back from Bongos after
     // every worker (task 1003903) instead of believing the worker's own verdict,
     // so a test that does not say what the ledger holds is not describing a run.
-    api: {},
+    api: over.api || {},
+    // Per-test runner memory (task 1004396): refused tasks and a queue gate
+    // outlive one iteration, so a shared module-level copy would leak between tests.
+    state: over.state || runner.newRunnerState(),
+    now: over.now,
     verifyDeps: {
       task: over.ledger !== undefined ? over.ledger : { id: 4242, status: 'shipped', updated_at: new Date().toISOString() },
       probeArtifact: over.probeArtifact || (async () => ({ checked: true, onMain: true, sha: 'a'.repeat(40), subject: 'x (task 4242)' })),
@@ -422,6 +432,137 @@ test('failureReason: stdout is used when stderr is empty, whitespace is collapse
   assert.equal(runner.failureReason({ code: 1, stderr: 'a'.repeat(400) }, 'x').length, 400);
 });
 
+// ── task 1004396: a refusal is not a reason to stop, unless it refuses everything ──
+//
+// Owner ruling (2026-09-30): a refused claim must not put the runner into waiting
+// by default. It sets that task aside and tries the next one. It waits only when
+// there is a block AND nothing else it can claim — which is exactly the shape of
+// the queue-wide gate (a stranded confirmed task refuses EVERY claim).
+
+const taskA = { ...aTask, id: 4242 };
+const taskB = { ...aTask, id: 4343, title: 'The next thing' };
+// A picker that honours the exclusion set, the way the real pickTask now does.
+const pickAorB = async (_goals, pd = {}) => {
+  const ex = pd.exclude || new Set();
+  for (const t of [taskA, taskB]) if (!ex.has(String(t.id))) return { task: t, goalId: 1000119, skipped: [] };
+  return { none: true, skipped: [] };
+};
+const scriptOf = (args) => String(args[0]).split(/[\\/]/).pop();
+const gatedCard = [
+  'Your queue is gated — 1 confirmed task(s) are waiting on your rebase:',
+  '  - task 777  A stranded thing',
+  '      flagged: strand:branch_modifies_executed_code',
+].join(String.fromCharCode(10));
+
+test('a task refused on its own is set aside and the NEXT task is worked in the same pass', async () => {
+  const claims = [];
+  const h = harness({
+    pickTask: pickAorB,
+    ledger: { id: 4343, status: 'shipped', updated_at: new Date().toISOString() },
+    run: async (bin, args) => {
+      if (scriptOf(args) === 'claim.js') {
+        claims.push(args[1]);
+        return args[1] === '4242' ? { ok: false, code: 1, stdout: '', stderr: 'DEPS_NOT_SHIPPED' } : { ok: true, code: 0, stdout: '', stderr: '' };
+      }
+      return { ok: true, code: 0, stdout: '', stderr: '' };
+    },
+  });
+  const row = await runner.iteration(opts, h.deps);
+  assert.deepEqual(claims, ['4242', '4343'], 'the refusal must not end the pass');
+  assert.equal(row.event, 'worked');
+  assert.equal(row.task_id, 4343);
+  const refused = h.rows.find((r) => r.event === 'claim_failed');
+  assert.equal(refused.task_id, 4242, 'the refusal is still logged, with its reason');
+  assert.ok(refused.set_aside_s > 0, 'and says the task is set aside');
+});
+
+test('a set-aside task is not re-tried on the next wake, and comes back once the set-aside expires', async () => {
+  let t = 1_000_000;
+  const state = runner.newRunnerState();
+  const claims = [];
+  const mk = () => harness({
+    state, now: () => t, pickTask: pickAorB,
+    run: async (bin, args) => {
+      if (scriptOf(args) === 'claim.js') { claims.push(args[1]); return { ok: false, code: 1, stdout: '', stderr: 'nope' }; }
+      return { ok: true, code: 0, stdout: '', stderr: '' };
+    },
+  });
+  await runner.iteration(opts, mk().deps);
+  assert.deepEqual(claims, ['4242', '4343']);
+  const second = await runner.iteration(opts, mk().deps);
+  assert.deepEqual(claims, ['4242', '4343'], 'no claim is re-attempted while both are set aside');
+  assert.equal(second.event, 'nothing_claimable');
+  assert.deepEqual(second.set_aside.map((s) => s.id).sort(), ['4242', '4343']);
+  t += runner.CLAIM_SET_ASIDE_MS + 1;
+  await runner.iteration(opts, mk().deps);
+  assert.deepEqual(claims.slice(2), ['4242', '4343'], 'after the set-aside they are tried again');
+});
+
+test('when EVERY candidate is refused, the last failure is returned so a real outage still escalates', async () => {
+  const h = harness({
+    pickTask: pickAorB,
+    run: async (bin, args) => (scriptOf(args) === 'claim.js'
+      ? { ok: false, code: 1, stdout: '', stderr: 'fetch failed' }
+      : { ok: true, code: 0, stdout: '', stderr: '' }),
+  });
+  const row = await runner.iteration(opts, h.deps);
+  assert.equal(row.event, 'claim_failed', 'returning "nothing claimable" here would hide a network outage as idle');
+  assert.equal(h.rows.filter((r) => r.event === 'claim_failed').length, 2);
+});
+
+test('a queue-wide gate WAITS, names the stranded task, and makes no claim until it lands', async () => {
+  const state = runner.newRunnerState();
+  let status = 'confirmed';
+  let gated = true;
+  const claims = [];
+  const api = { tasks: { getTasksId: async ({ id }) => ({ ok: true, data: { task: { id, status } } }) } };
+  const mk = () => harness({
+    state, api, pickTask: pickAorB,
+    run: async (bin, args) => {
+      if (scriptOf(args) === 'claim.js') {
+        claims.push(args[1]);
+        return gated ? { ok: false, code: runner.QUEUE_GATED_EXIT, stdout: '', stderr: gatedCard } : { ok: true, code: 0, stdout: '', stderr: '' };
+      }
+      return { ok: true, code: 0, stdout: '', stderr: '' };
+    },
+  });
+
+  const first = await runner.iteration(opts, mk().deps);
+  assert.equal(first.event, 'queue_gated');
+  assert.deepEqual(first.waiting_on, ['777'], 'the wait names what it is waiting on');
+  assert.deepEqual(claims, ['4242'], 'a gate refuses every claim, so trying the next task is pointless');
+
+  const second = await runner.iteration(opts, mk().deps);
+  assert.equal(second.event, 'queue_gated');
+  assert.deepEqual(claims, ['4242'], 'no claim is attempted while the strand is still confirmed');
+
+  status = 'shipped'; gated = false;
+  const third = await runner.iteration(opts, mk().deps);
+  assert.equal(third.event, 'worked', 'once the strand lands the runner resumes by itself');
+  assert.deepEqual(claims, ['4242', '4242']);
+  assert.equal(state.queueGate, null, 'the gate is forgotten');
+});
+
+test('a gate whose stranded task cannot be read back keeps waiting rather than guessing', async () => {
+  const state = runner.newRunnerState();
+  state.queueGate = { owed: ['777'], since: Date.now(), reason: 'x' };
+  let claimed = false;
+  const h = harness({
+    state, pickTask: pickAorB,
+    api: { tasks: { getTasksId: async () => ({ ok: false, status: 502 }) } },
+    run: async (bin, args) => { if (scriptOf(args) === 'claim.js') claimed = true; return { ok: true, code: 0, stdout: '', stderr: '' }; },
+  });
+  const row = await runner.iteration(opts, h.deps);
+  assert.equal(row.event, 'queue_gated');
+  assert.equal(claimed, false);
+});
+
+test('the runner and claim.js agree on the queue-gated exit code', () => {
+  const claim = require('../scripts/gds/claim.js');
+  assert.equal(runner.QUEUE_GATED_EXIT, claim.QUEUE_GATED_EXIT);
+  assert.notEqual(runner.QUEUE_GATED_EXIT, 1, 'it must differ from the generic refusal, or the runner cannot tell them apart');
+});
+
 test('a worker that cannot finish frees the claim; one that really shipped does not', async () => {
   // Which of these happens is now decided by the LEDGER, not by the worker's own
   // verdict: a stuck worker leaves the task still `active`, a finished one leaves

package/tests/autonomy_digest.mjs

@@ -97,6 +97,16 @@ test('shipped is the LEDGER’s answer, never the worker’s', () => {
   assert.deepEqual(d.onHold.ranButDidNotShip.map((r) => r.shape), ['still_held', 'shipped_no_artifact']);
 });
 
+test('a queue-gate wait is a hold that names the stranded task, not an unrecognised event (task 1004396)', () => {
+  const d = buildDigest([
+    { at: at(0), event: 'queue_gated', waiting_on: ['777'], reason: 'every claim is refused until task 777 lands — waiting, not claiming (x)' },
+    { at: at(5), event: 'queue_gate_cleared', waited_on: ['777'], waited_s: 300 },
+  ], { nowEpochS: NOW_S });
+  assert.equal(d.runner.unrecognised, 0);
+  assert.equal(d.onHold.held.length, 1);
+  assert.match(d.onHold.held[0].reason, /task 777/);
+});
+
 test('an unworkable queue is reported as a queue problem, not as breakage', () => {
   const d = buildDigest([{ at: at(0), event: 'nothing_claimable', skipped: [{ id: 9, reasons: ['needs_migration'] }] }], { nowEpochS: NOW_S });
   assert.equal(d.onHold.unworkable.length, 1);

package/tests/claim_error_surface.mjs

@@ -231,6 +231,13 @@ t('REBASE_REQUIRED: names the owed tasks and relays the server hint', () => {
     'must not fall through to the bare code echo');
 });
 
+t('REBASE_REQUIRED exits with its OWN code, so an unattended caller can tell a queue gate from a task refusal (task 1004396)', () => {
+  const { QUEUE_GATED_EXIT } = require('../scripts/gds/claim.js');
+  assert.equal(formatClaimFailure(envelope('REBASE_REQUIRED', {}), 5).exit, QUEUE_GATED_EXIT);
+  assert.notEqual(QUEUE_GATED_EXIT, 1);
+  assert.equal(formatClaimFailure(envelope('ALREADY_CLAIMED', {}), 5).exit, 1, 'a per-task refusal keeps exit 1');
+});
+
 t('REBASE_REQUIRED: still guides when the server sends no hint and no task list', () => {
   const out = text(formatClaimFailure(envelope('REBASE_REQUIRED', {}), 5));
   assert.match(out, /gated/i, 'must say the queue is gated');

package/tests/cli_exit_no_abort.mjs

@@ -348,7 +348,12 @@ test('land-watch.js exits 0 after a REAL request — the teardown that aborted 3
 // throwaway session, so no real API is touched and the developer's own config is
 // never read or written. `--here` because CI may check out the MAIN checkout,
 // where the worktree guard would refuse at exit 2 before any request is made.
-test('claim.js exits 1 without aborting on a REFUSED claim — measured 3/3 aborting before', async () => {
+// The stub refuses with REBASE_REQUIRED, the queue-wide gate, which exits with its
+// own code since task 1004396 (QUEUE_GATED_EXIT) so an unattended caller can tell a
+// gate from a task refusal. The abort this guards against shows as 127 either way.
+const { QUEUE_GATED_EXIT } = createRequire(import.meta.url)('../scripts/gds/claim.js');
+
+test('claim.js exits with its refusal code without aborting on a REFUSED claim — measured 3/3 aborting before', async () => {
   const { proc, port } = await startStub({ claimStatus: 409 });
   try {
     const home = makeHome({ apiBase: `http://127.0.0.1:${port}` });
@@ -361,7 +366,7 @@ test('claim.js exits 1 without aborting on a REFUSED claim — measured 3/3 abor
       });
       assert.notEqual(r.signal, 'SIGTERM', `run ${i + 1} timed out — draining left a handle armed`);
       assert.doesNotMatch(r.stderr || '', ABORT_RE, `run ${i + 1} aborted natively:\n${r.stderr}`);
-      assert.equal(r.status, 1, `a refused claim must exit 1, got ${r.status} (127 = the abort); stderr: ${r.stderr}`);
+      assert.equal(r.status, QUEUE_GATED_EXIT, `a queue-gated claim must exit ${QUEUE_GATED_EXIT}, got ${r.status} (127 = the abort); stderr: ${r.stderr}`);
       // End-to-end proof that the new REBASE_REQUIRED rendering reaches a terminal,
       // not just the unit test: the stub refuses with that exact envelope.
       //