@bongos/core 1.20.31 → 1.20.33

package/tests/move_escalation.mjs

@@ -0,0 +1,191 @@
+// tests/move_escalation.mjs — a core move nobody can heal files ONE blocker and ONE notice,
+// and a later move that lands closes it (task 1004449, goal 1000090 deploy self-heal).
+//
+// WHAT IS AT RISK. Two opposite failures: a person never hears about a move that left a
+// project down, or hears about it on every runner tick until they mute the channel. The
+// blockers table here is a fake that enforces the real (source, source_ref) uniqueness, and
+// the drain loop is the real one, so "one per failed move" is measured across retries and
+// re-runs rather than assumed.
+//
+// Run:  node tests/move_escalation.mjs
+
+import { strict as assert } from 'node:assert';
+import { createRequire } from 'node:module';
+
+process.env.PROVISION_APP_USER = process.env.PROVISION_APP_USER || 'lars'; // appUser() refuses a Windows account name
+const require = createRequire(import.meta.url);
+const E = require('../scripts/gds/move-escalation.js');
+const P = require('../scripts/gds/provision.js');
+
+let passed = 0;
+let failed = 0;
+const t = async (name, fn) => {
+  try { await fn(); passed += 1; console.log(`  PASS  ${name}`); }
+  catch (e) { failed += 1; console.log(`  FAIL  ${name}\n        ${e.message}`); }
+};
+
+const inst = (over = {}) => ({ id: 7, slug: 'cloudbongos', db_name: 'cloudbongos', port: 3002, status: 'active', hosting_shape: 'control-plane', owner_builder_id: 1, ...over });
+const unknownFail = { ok: false, error: 'core upgrade to 1.20.31 failed and was rolled back: health check failed after restart', failure: { class: 'unknown', reason: 'health' } };
+
+// A blockers table with the real partial unique index on (source, source_ref).
+function blockersTable() {
+  const rows = []; let next = 500;
+  const query = async (sql, params) => {
+    if (/INSERT INTO blockers/.test(sql)) {
+      const [title, body_md, source, source_ref] = params;
+      if (rows.some((r) => r.source === source && r.source_ref === source_ref)) return { rows: [] };
+      const r = { id: next++, title, body_md, source, source_ref, status: 'open' }; rows.push(r); return { rows: [{ id: r.id }] };
+    }
+    if (/UPDATE blockers/.test(sql)) {
+      const [source, like, note] = params; const prefix = like.replace(/%$/, '');
+      const hit = rows.filter((r) => r.status === 'open' && r.source === source && r.source_ref.startsWith(prefix));
+      for (const r of hit) { r.status = 'resolved'; r.resolution_note = note; }
+      return { rows: hit.map((r) => ({ id: r.id })) };
+    }
+    return { rows: [] };
+  };
+  return { rows, query };
+}
+
+console.log('\nwho needs a person (needsPerson):');
+
+await t('unknown and rollback_failed escalate; every other class does not', () => {
+  const i = { action: 'core-upgrade' };
+  assert.equal(E.needsPerson({ intent: i, failure: { class: 'unknown', reason: 'health' } }), true);
+  assert.equal(E.needsPerson({ intent: i, failure: { class: 'unknown', reason: 'rollback_failed' }, terminal: true }), true);
+  assert.equal(E.needsPerson({ intent: i, failure: { class: 'unknown', reason: 'remedy_did_not_hold' } }), true, 'a fix that did not hold (task 1004448)');
+  for (const c of ['transient', 'known_wedge', 'needs_decision']) assert.equal(E.needsPerson({ intent: i, failure: { class: c, reason: 'x' } }), false, c);
+  assert.equal(E.needsPerson({ intent: { action: 'restart' }, failure: { class: 'unknown', reason: 'x' } }), false, 'core moves only');
+});
+
+console.log('\nwhat is said (blockerFor, noticeText):');
+
+await t('the blocker names project, target, reason and what the move reported', () => {
+  const b = E.blockerFor({ inst: inst(), intent: { id: 11, target_version: '1.20.31' }, failure: unknownFail.failure, error: unknownFail.error });
+  assert.equal(b.source, 'provisioning');
+  assert.equal(b.sourceRef, 'core-move:7:11');
+  assert.match(b.title, /cloudbongos: the core move to 1\.20\.31 failed/);
+  assert.match(b.bodyMd, /`health`/);
+  assert.match(b.bodyMd, /health check failed after restart/);
+  assert.match(b.bodyMd, /closes by itself/);
+});
+
+await t('a failed rollback says the project may be down', () => {
+  const b = E.blockerFor({ inst: inst(), intent: { id: 11, target_version: '1.20.31' }, failure: { class: 'unknown', reason: 'rollback_failed' }, error: 'x' });
+  assert.match(b.title, /may be down/);
+  assert.match(E.noticeText({ kind: 'failed', name: 'x', down: true }), /may be down/);
+});
+
+await t('reported output cannot break out of its code block', () => {
+  const b = E.blockerFor({ inst: inst(), intent: { id: 1, target_version: '1' }, failure: unknownFail.failure, error: 'a```\n# injected' });
+  assert.equal((b.bodyMd.match(/```/g) || []).length, 2);
+});
+
+await t('Discord names only a PUBLIC project, and never carries the upgrade output', () => {
+  assert.match(E.noticeText({ kind: 'failed', name: null, to: '1.20.31' }), /^⚠️ A private project/);
+  assert.match(E.noticeText({ kind: 'failed', name: 'mercury', to: '1.20.31' }), /\*\*mercury\*\*/);
+  assert.doesNotMatch(E.noticeText({ kind: 'failed', name: 'm', to: '1' }), /health/);
+});
+
+console.log('\nonce per failed move, closed by success (afterRetire / afterSuccess):');
+
+function deps(table, { visibility = 'private' } = {}) {
+  const posts = [];
+  return { posts, d: { apply: true, log: () => {}, db: { query: table.query }, postNotice: (text) => posts.push(text), provisioning: { effectiveSettings: () => ({ visibility }) } } };
+}
+
+await t('a retired unknown move files one blocker and one notice', async () => {
+  const table = blockersTable(); const { posts, d } = deps(table);
+  const r = await E.afterRetire({ intent: { id: 11, action: 'core-upgrade', target_version: '1.20.31' }, inst: inst(), result: unknownFail, deps: d });
+  assert.equal(r.filed, true);
+  assert.equal(table.rows.length, 1);
+  assert.equal(posts.length, 1);
+});
+
+await t('the same move retiring again files nothing and posts nothing', async () => {
+  const table = blockersTable(); const { posts, d } = deps(table);
+  const i = { id: 11, action: 'core-upgrade', target_version: '1.20.31' };
+  await E.afterRetire({ intent: i, inst: inst(), result: unknownFail, deps: d });
+  const again = await E.afterRetire({ intent: i, inst: inst(), result: unknownFail, deps: d });
+  assert.equal(again.filed, false);
+  assert.equal(table.rows.length, 1);
+  assert.equal(posts.length, 1, 'edge-triggered: the notice follows the insert');
+});
+
+await t('a later move that lands closes this project\'s blockers — and only this project\'s', async () => {
+  const table = blockersTable(); const { posts, d } = deps(table, { visibility: 'public' });
+  await E.afterRetire({ intent: { id: 11, action: 'core-upgrade', target_version: '1.20.31' }, inst: inst(), result: unknownFail, deps: d });
+  await E.afterRetire({ intent: { id: 12, action: 'core-upgrade', target_version: '1.20.31' }, inst: inst({ id: 70, slug: 'other' }), result: unknownFail, deps: d });
+  const r = await E.afterSuccess({ intent: { id: 13, action: 'core-upgrade', target_version: '1.20.32' }, inst: inst(), result: { ok: true, served: '1.20.32' }, deps: d });
+  assert.equal(r.closed, 1);
+  assert.deepEqual(table.rows.map((x) => x.status), ['resolved', 'open'], 'instance 70 is not instance 7 — a LIKE prefix must not over-match');
+  assert.match(table.rows[0].resolution_note, /later core move to 1\.20\.32 succeeded \(intent 13\)/);
+  assert.match(posts[posts.length - 1], /^✅ \*\*cloudbongos\*\* took its core update to 1\.20\.32/);
+});
+
+await t('a no-op or a failed result closes nothing; nothing open posts no all-clear', async () => {
+  const table = blockersTable(); const { posts, d } = deps(table);
+  await E.afterRetire({ intent: { id: 11, action: 'core-upgrade', target_version: '1' }, inst: inst(), result: unknownFail, deps: d });
+  assert.equal((await E.afterSuccess({ intent: { id: 13, action: 'core-upgrade' }, inst: inst(), result: { ok: true, noop: true }, deps: d })).closed, 0);
+  assert.equal((await E.afterSuccess({ intent: { id: 13, action: 'core-upgrade' }, inst: inst(), result: { ok: false }, deps: d })).closed, 0);
+  const empty = blockersTable(); const e2 = deps(empty);
+  await E.afterSuccess({ intent: { id: 13, action: 'core-upgrade' }, inst: inst(), result: { ok: true, served: '1' }, deps: e2.d });
+  assert.equal(e2.posts.length, 0);
+});
+
+await t('a database error never throws into the drain', async () => {
+  const { d } = deps({ query: async () => { throw new Error('boom'); } });
+  assert.deepEqual(await E.afterRetire({ intent: { id: 1, action: 'core-upgrade' }, inst: inst(), result: unknownFail, deps: d }), { filed: false });
+  assert.deepEqual(await E.afterSuccess({ intent: { id: 1, action: 'core-upgrade' }, inst: inst(), result: { ok: true, served: '1' }, deps: d }), { closed: 0 });
+});
+
+console.log('\nthe drain loop, end to end:');
+
+// One move that fails `health` (rolled back, unknown), then — a later, separate move — lands.
+async function drain({ table, row, upgradeFails }) {
+  const posts = []; const resolved = [];
+  const deps = {
+    apply: true, privileged: false, log: () => {}, sleep: async () => {},
+    postNotice: (text) => posts.push(text),
+    db: { query: async (sql, params) => {
+      if (/blockers/.test(sql)) return table.query(sql, params);
+      if (/SET state='pending'/.test(sql)) { row.state = 'pending'; row.not_before = new Date(Date.now() + params[params.length - 1]); }
+      return { rows: [] };
+    } },
+    provisioning: {
+      claimNextIntent: async () => { if (row.state !== 'pending' || (row.not_before && row.not_before > new Date())) return null; row.state = 'running'; row.attempts += 1; return { ...row }; },
+      getInstanceById: async () => inst(),
+      resolveIntent: async (_db, id, state) => { resolved.push(state); row.state = state; },
+      setInstanceStatus: async () => {}, recordEvent: async () => {}, clearErrorNote: async () => {},
+      effectiveSettings: () => ({ visibility: 'private' }),
+      claimNextOAuthManifest: async () => null, resolveOAuthManifest: async () => {},
+    },
+    exec: (cmd) => {
+      if (/upgrade\.js/.test(cmd) && upgradeFails) { const e = new Error('Command failed'); e.stderr = '✖ health check failed after restart\nBONGOS_UPGRADE_OUTCOME {"stage":"rolled_back","reason":"health"}\n'; throw e; }
+      return { ok: true, stdout: /\/version/.test(cmd) ? JSON.stringify({ coreVersion: upgradeFails ? '1.20.30' : row.target_version, schemaPending: 0 }) : '' };
+    },
+    writeFile: () => ({ ok: true }),
+    readInstalledCoreVersion: () => '1.20.30', selfInstanceRoot: () => '/srv/cb',
+    updateChannel: { DEFAULT_CHANNEL: 'patch', normalizeChannel: (c) => c || 'patch', channelAllows: () => true, listAvailableVersions: () => ({ ok: true, versions: ['1.20.30', '1.20.31', '1.20.32'] }) },
+    checkDnsTokenScope: async () => ({ reachable: true, zone: 'z' }), checkCaddyWiring: () => null,
+  };
+  await P.cmdRunIntents(deps);
+  return { posts, resolved };
+}
+
+await t('a move that fails unknown files ONE blocker; a later move that lands closes it', async () => {
+  const table = blockersTable();
+  const first = await drain({ table, upgradeFails: true, row: { id: 11, instance_id: 7, action: 'core-upgrade', target_version: '1.20.31', attempts: 0, state: 'pending' } });
+  assert.deepEqual(first.resolved, ['error'], 'retired at once — unknown is not retried');
+  assert.equal(table.rows.length, 1);
+  assert.equal(table.rows[0].source_ref, 'core-move:7:11');
+  assert.equal(first.posts.length, 1);
+  const later = await drain({ table, upgradeFails: false, row: { id: 12, instance_id: 7, action: 'core-upgrade', target_version: '1.20.32', attempts: 0, state: 'pending' } });
+  assert.deepEqual(later.resolved, ['done']);
+  assert.equal(table.rows[0].status, 'resolved');
+  assert.equal(later.posts.length, 1);
+  assert.match(later.posts[0], /^✅ A private project took its core update to 1\.20\.32/);
+});
+
+console.log(`\nmove_escalation: ${passed} passed, ${failed} failed`);
+if (failed) process.exit(1);

package/tests/plain_cards.mjs

@@ -18,7 +18,7 @@ import * as shipLand from '../scripts/gds/ship-land.js';
 
 const require = createRequire(import.meta.url);
 const { JARGON_TERMS } = require('../modules/sessions/accessibility.js');
-const { formatClaimFailure } = require('../scripts/gds/claim.js');
+const { formatClaimFailure, QUEUE_GATED_EXIT } = require('../scripts/gds/claim.js');
 const { formatGradeOutcome } = require('../scripts/gds/ship-grade-steps.js');
 
 let passed = 0;
@@ -130,7 +130,7 @@ test('a card interpolates nothing — it is static text, so it adds no injection
 test('a refused claim prints the engineer text first and the card after it', () => {
   const r = { status: 409, data: { error: { code: 'REBASE_REQUIRED', message: 'x', details: { rebaseTasks: [{ id: 7, reason: 'strand:pr_not_landed' }] } } } };
   const { lines, exit } = formatClaimFailure(r, 42);
-  assert.equal(exit, 1);
+  assert.equal(exit, QUEUE_GATED_EXIT, 'the queue-wide gate keeps its own exit code (task 1004396)');
   assert.match(lines[0], /queue is gated/);
   const at = lines.findIndex((l) => l.includes(PLAIN_MARKER));
   assert.ok(at > 0, 'card follows the existing guidance');

package/tests/wedge_remedy.mjs

@@ -0,0 +1,268 @@
+// tests/wedge_remedy.mjs — a core move that failed on a KNOWN snag fixes the snag and tries
+// once more, and never twice (task 1004448, goal 1000090 deploy self-heal).
+//
+// WHAT IS AT RISK. Each fix runs a command on a live project's box: a commit in the owner's
+// checkout, a delete among database backups, a migration against a database. The cases pin
+// WHAT each command is (and what it refuses to build), that it runs at most once per
+// failure, and that a fix which did not hold reaches a person instead of looping. The
+// disk-full prune is also run for real against a temp folder, because a `find` pattern is
+// the kind of thing that reads right and deletes a sibling project's dumps.
+//
+// Run:  node tests/wedge_remedy.mjs
+
+import { strict as assert } from 'node:assert';
+import { createRequire } from 'node:module';
+import { spawnSync } from 'node:child_process';
+import fs from 'node:fs';
+import os from 'node:os';
+import path from 'node:path';
+
+process.env.PROVISION_APP_USER = process.env.PROVISION_APP_USER || 'lars'; // appUser() refuses a Windows account name
+const require = createRequire(import.meta.url);
+const W = require('../scripts/gds/wedge-remedy.js');
+const R = require('../scripts/gds/provision-core-upgrade.js');
+const P = require('../scripts/gds/provision.js');
+const { REASON_CLASS } = require('../scripts/gds/upgrade-outcome.js');
+const { DEFAULT_RETRY_DELAY_MS } = require('../scripts/gds/intent-retry.js');
+
+let passed = 0;
+let failed = 0;
+const t = async (name, fn) => {
+  try { await fn(); passed += 1; console.log(`  PASS  ${name}`); }
+  catch (e) { failed += 1; console.log(`  FAIL  ${name}\n        ${e.message}`); }
+};
+
+const inst = (over = {}) => ({ id: 7, slug: 'charter', db_name: 'charter', port: 3010, status: 'active', hosting_shape: 'co-tenant', owner_builder_id: 1, ...over });
+const intent = (over = {}) => ({ id: 99, action: 'core-upgrade', target_version: '1.20.31', attempts: 1, remedied_reason: null, ...over });
+const wedge = (reason) => ({ ok: false, error: `core upgrade to 1.20.31 was refused (${reason})`, failure: { class: 'known_wedge', reason } });
+const selfRoot = () => '/srv/charter';
+
+function harness({ execFails = false } = {}) {
+  const ran = []; const events = []; const queries = [];
+  const deps = {
+    apply: true, privileged: false, log: () => {}, selfInstanceRoot: selfRoot,
+    exec: (cmd, opts) => { ran.push({ cmd, ...opts }); return execFails ? { ok: false, softFailed: true, stderr: 'fatal: nope' } : { ok: true, stdout: '' }; },
+    db: { query: async (sql, params) => { queries.push({ sql, params }); return { rows: [] }; } },
+    provisioning: { recordEvent: async (_db, e) => { events.push(e); } },
+    migratePlan: () => ({ via: 'npm-script' }),
+  };
+  return { deps, ran, events, queries };
+}
+
+console.log('\nthe fix for each known snag (planRemedy):');
+
+await t('every reason with a fix is a known_wedge in the classifier — and back', () => {
+  assert.deepEqual([...W.REMEDY_REASONS].sort(), ['dirty_tree', 'disk_full', 'schema_pending']);
+  for (const r of W.REMEDY_REASONS) assert.equal(REASON_CLASS[r], 'known_wedge', r);
+  for (const [r, c] of Object.entries(REASON_CLASS)) if (c === 'known_wedge') assert.ok(W.REMEDY_REASONS.includes(r), `${r} is known_wedge but has no fix — it would be retired with nothing done`);
+  assert.equal(REASON_CLASS.remedy_did_not_hold, 'unknown', 'a fix that did not hold reaches a person');
+  assert.equal(REASON_CLASS.remedy_failed, 'unknown');
+});
+
+await t('dirty tree: commits TRACKED changes inside the project, hooks off, as the runner', () => {
+  const p = W.planRemedy(inst(), 'dirty_tree', { selfRoot });
+  assert.match(p.cmd, /^git -C \/srv\/charter -c core\.hooksPath=\/dev\/null .* add -u && git -C \/srv\/charter .* commit -m /);
+  assert.doesNotMatch(p.cmd, /add -A|add \./, 'an untracked file is never swept into the owner\'s history on a guess');
+  assert.doesNotMatch(p.cmd, /sudo/, 'the runner owns the checkout (pushUpgradePin\'s shape)');
+});
+
+await t('a cloud-host project is fixed in ITS folder, not the control plane\'s', () => {
+  const p = W.planRemedy(inst({ hosting_shape: 'cloud-host' }), 'dirty_tree', { selfRoot });
+  assert.match(p.cmd, /git -C \/\S+\/charter /);
+  assert.doesNotMatch(p.cmd, /\/srv\/charter/);
+});
+
+await t('disk full: prunes only THIS database\'s dumps, keeping the newest week', () => {
+  const p = W.planRemedy(inst(), 'disk_full', { selfRoot, backupDir: '/var/backups/gds' });
+  assert.match(p.cmd, /^find \/var\/backups\/gds -maxdepth 1 -type f -name 'charter-\[0-9\]/);
+  assert.match(p.cmd, new RegExp(`head -n -${W.DUMP_KEEP_FLOOR} \\| xargs -r rm -f --$`));
+  assert.equal(W.DUMP_KEEP_FLOOR, 7);
+});
+
+await t('schema pending: migrates the project\'s OWN database, PGDATABASE always set', () => {
+  const plain = W.planRemedy(inst(), 'schema_pending', { selfRoot, migrate: () => ({ via: 'npm-script' }) });
+  assert.equal(plain.cmd, 'npm run migrate');
+  assert.equal(plain.cwd, '/srv/charter');
+  assert.deepEqual(plain.env, { PGDATABASE: 'charter' });
+  const adopted = W.planRemedy(inst(), 'schema_pending', { selfRoot, migrate: () => ({ via: 'core-script' }) });
+  assert.equal(adopted.cmd, 'bash node_modules/@bongos/core/scripts/migrate.sh');
+  assert.deepEqual(adopted.env, { PGDATABASE: 'charter', INIT_CWD: '/srv/charter' });
+  const priv = W.planRemedy(inst(), 'schema_pending', { selfRoot, privileged: true, runAs: 'bongos-charter', migrate: () => ({ via: 'npm-script' }) });
+  assert.equal(priv.cmd, 'sudo -u bongos-charter env PGDATABASE=charter npm run migrate', 'sudo resets env, so PGDATABASE rides the command');
+  assert.ok(W.planRemedy(inst(), 'schema_pending', { selfRoot, privileged: true, runAs: null }).refused);
+});
+
+await t('no command is built from a name that is not shell-safe', () => {
+  assert.ok(W.planRemedy(inst({ slug: 'a;rm -rf' }), 'dirty_tree', { selfRoot }).refused);
+  assert.ok(W.planRemedy(inst({ db_name: 'x; drop' }), 'disk_full', { selfRoot }).refused);
+  assert.ok(W.planRemedy(inst(), 'dirty_tree', { selfRoot: () => '/srv/a b' }).refused);
+  assert.ok(W.planRemedy(inst(), 'disk_full', { selfRoot, backupDir: '/tmp/$(x)' }).refused);
+});
+
+await t('a reason with no fix gets no command', () => {
+  for (const r of ['artist_gate', 'downgrade', 'health', 'not_published', 'nope']) assert.equal(W.planRemedy(inst(), r, { selfRoot }), null, r);
+});
+
+console.log('\nthe prune, run for real:');
+
+const bash = spawnSync('bash', ['-c', 'echo ok'], { encoding: 'utf8' });
+await t('keeps the newest 7 of this project\'s dumps and never touches a sibling\'s', () => {
+  if (bash.status !== 0) { console.log('        (skipped: no bash here)'); return; }
+  const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'wedge-prune-'));
+  try {
+    const mine = []; for (let d = 1; d <= 10; d++) mine.push(`charter-2026-09-${String(d).padStart(2, '0')}T030000Z.sql.gz`);
+    const others = ['charter-nk-2026-09-01T030000Z.sql.gz', 'test-2026-09-01T030000Z.sql.gz', 'charter-notes.txt'];
+    for (const f of [...mine, ...others]) fs.writeFileSync(path.join(dir, f), 'x');
+    const p = W.planRemedy(inst(), 'disk_full', { selfRoot, backupDir: '.' }); // run IN the folder: a temp path can hold a `~`
+    const r = spawnSync('bash', ['-c', p.cmd], { encoding: 'utf8', cwd: dir });
+    assert.equal(r.status, 0, r.stderr);
+    const left = fs.readdirSync(dir).sort();
+    assert.deepEqual(left, [...others, ...mine.slice(3)].sort());
+    // Nothing is deleted when the floor is not exceeded.
+    const again = spawnSync('bash', ['-c', p.cmd], { encoding: 'utf8', cwd: dir });
+    assert.equal(again.status, 0);
+    assert.equal(fs.readdirSync(dir).length, left.length);
+  } finally { fs.rmSync(dir, { recursive: true, force: true }); }
+});
+
+console.log('\nonce per failure (remedyAfterFailure):');
+
+for (const reason of W.REMEDY_REASONS) {
+  await t(`${reason}: runs the fix, records an event, re-pends the move with the reason stored`, async () => {
+    const h = harness();
+    const r = await W.remedyAfterFailure({ intent: intent(), inst: inst(), result: wedge(reason), deps: h.deps });
+    assert.deepEqual(r, { repended: true });
+    assert.equal(h.ran.length, 1);
+    assert.equal(h.events.length, 1);
+    assert.equal(h.events[0].event, W.REMEDY_EVENT);
+    assert.match(h.events[0].detail, /trying the move to 1\.20\.31 once more/);
+    const up = h.queries.find((q) => /SET state='pending'/.test(q.sql));
+    assert.ok(up, 're-pended');
+    assert.match(up.sql, /remedied_reason=\$3/);
+    assert.deepEqual(up.params, [99, wedge(reason).error, reason, 'known_wedge', reason, DEFAULT_RETRY_DELAY_MS]);
+  });
+}
+
+await t('the same reason again after a fix: no second fix — retired as unknown for a person', async () => {
+  const h = harness();
+  const r = await W.remedyAfterFailure({ intent: intent({ remedied_reason: 'dirty_tree' }), inst: inst(), result: wedge('dirty_tree'), deps: h.deps });
+  assert.equal(r.repended, false);
+  assert.deepEqual(r.failure, { class: 'unknown', reason: 'remedy_did_not_hold' });
+  assert.match(r.error, /already tried once/);
+  assert.equal(h.ran.length, 0, 'nothing ran');
+  assert.equal(h.queries.length, 0);
+});
+
+await t('a DIFFERENT known snag on the retry gets its own one fix', async () => {
+  const h = harness();
+  const r = await W.remedyAfterFailure({ intent: intent({ remedied_reason: 'dirty_tree' }), inst: inst(), result: wedge('disk_full'), deps: h.deps });
+  assert.equal(r.repended, true);
+});
+
+await t('a fix that fails is recorded and retired as unknown — never re-pended', async () => {
+  const h = harness({ execFails: true });
+  const r = await W.remedyAfterFailure({ intent: intent(), inst: inst(), result: wedge('dirty_tree'), deps: h.deps });
+  assert.equal(r.repended, false);
+  assert.deepEqual(r.failure, { class: 'unknown', reason: 'remedy_failed' });
+  assert.match(r.error, /did not work \(fatal: nope\)/);
+  assert.equal(h.events.length, 1);
+  assert.match(h.events[0].detail, /could not fix dirty_tree/);
+  assert.equal(h.queries.length, 0);
+});
+
+await t('leaves everything else alone: other classes, other actions, and a dry run', async () => {
+  for (const [i, res] of [[intent(), { ok: false, error: 'x', failure: { class: 'transient', reason: 'not_published' } }],
+    [intent(), { ok: false, error: 'x', failure: { class: 'needs_decision', reason: 'artist_gate' } }],
+    [intent(), { ok: false, error: 'x' }],
+    [intent({ action: 'restart' }), wedge('dirty_tree')]]) {
+    const h = harness();
+    const r = await W.remedyAfterFailure({ intent: i, inst: inst(), result: res, deps: h.deps });
+    assert.equal(r.repended, false);
+    assert.deepEqual(r.failure, res.failure);
+    assert.equal(h.ran.length, 0);
+  }
+  const h = harness(); h.deps.apply = false;
+  assert.equal((await W.remedyAfterFailure({ intent: intent(), inst: inst(), result: wedge('dirty_tree'), deps: h.deps })).repended, false);
+  assert.equal(h.ran.length, 0);
+});
+
+console.log('\nthe move refuses on a database behind its code:');
+
+await t('schemaPending > 0 on the box refuses as schema_pending BEFORE the upgrade runs', async () => {
+  const ran = [];
+  const deps = {
+    apply: true, privileged: false, log: () => {}, sleep: async () => {},
+    exec: (cmd) => { ran.push(cmd); return { ok: true, stdout: /\/version/.test(cmd) ? JSON.stringify({ coreVersion: '1.20.30', schemaPending: 2 }) : '' }; },
+    db: { query: async () => ({ rows: [] }) }, provisioning: { recordEvent: async () => {} },
+    readInstalledCoreVersion: () => '1.20.30', selfInstanceRoot: () => '/srv/cb',
+    updateChannel: { DEFAULT_CHANNEL: 'patch', normalizeChannel: (c) => c || 'patch', channelAllows: () => true, listAvailableVersions: () => ({ ok: true, versions: ['1.20.30', '1.20.31'] }) },
+  };
+  const r = await R.coreUpgradeInstance(inst({ hosting_shape: 'control-plane', slug: 'cloudbongos' }), deps, intent());
+  assert.equal(r.ok, false);
+  assert.deepEqual(r.failure, { class: 'known_wedge', reason: 'schema_pending' });
+  assert.match(r.error, /2 migration\(s\) behind/);
+  assert.ok(!ran.some((c) => /upgrade\.js/.test(c)), 'the pin never moved');
+});
+
+console.log('\nthe drain loop, end to end:');
+
+// A stateful fake queue, the shape tests/provision.mjs uses: claimNextIntent honours
+// not_before, so "fixed, then tried once more on a LATER tick" is measured.
+async function drain({ row, reply }) {
+  const ran = []; const events = []; const resolved = []; let runs = 0;
+  const deps = {
+    apply: true, privileged: false, log: () => {}, sleep: async () => {},
+    db: { query: async (sql, params) => {
+      if (/SET state='pending'/.test(sql)) { row.state = 'pending'; row.not_before = new Date(Date.now() + params[params.length - 1]); if (/remedied_reason/.test(sql)) row.remedied_reason = params[2]; }
+      if (/SET failure_class/.test(sql)) { row.failure_class = params[1]; row.failure_reason = params[2]; }
+      return { rows: [] };
+    } },
+    provisioning: {
+      claimNextIntent: async () => {
+        if (row.state !== 'pending' || (row.not_before && row.not_before > new Date())) return null;
+        row.state = 'running'; row.attempts += 1; runs += 1; return { ...row };
+      },
+      getInstanceById: async () => inst({ hosting_shape: 'control-plane', slug: 'cloudbongos', db_name: 'cloudbongos' }),
+      resolveIntent: async (_db, id, state, note) => { resolved.push({ id, state, note }); row.state = state; },
+      setInstanceStatus: async () => {}, recordEvent: async (_db, e) => { events.push(e); },
+      claimNextOAuthManifest: async () => null, resolveOAuthManifest: async () => {},
+    },
+    exec: (cmd, opts) => { ran.push(cmd); return reply(cmd, opts); },
+    writeFile: () => ({ ok: true }),
+    readInstalledCoreVersion: () => '1.20.30', selfInstanceRoot: () => '/srv/cb',
+    updateChannel: { DEFAULT_CHANNEL: 'patch', normalizeChannel: (c) => c || 'patch', channelAllows: () => true, listAvailableVersions: () => ({ ok: true, versions: ['1.20.30', '1.20.31'] }) },
+    checkDnsTokenScope: async () => ({ reachable: true, zone: 'z' }), checkCaddyWiring: () => null,
+  };
+  await P.cmdRunIntents(deps);
+  return { ran, events, resolved, runs, row };
+}
+const dirtyUpgrade = (cmd) => {
+  if (/upgrade\.js/.test(cmd)) { const e = new Error('Command failed'); e.stderr = '✖ working tree not clean — commit/stash first\nBONGOS_UPGRADE_OUTCOME {"stage":"refused","reason":"dirty_tree"}\n'; throw e; }
+  return { ok: true, stdout: /\/version/.test(cmd) ? JSON.stringify({ coreVersion: '1.20.30', schemaPending: 0 }) : '' };
+};
+
+await t('a dirty tree is committed, the move waits for the next tick, and is NOT retired', async () => {
+  const row = { id: 11, instance_id: 7, action: 'core-upgrade', target_version: '1.20.31', attempts: 0, state: 'pending', not_before: null, remedied_reason: null };
+  const d = await drain({ row, reply: dirtyUpgrade });
+  assert.equal(d.runs, 1, 'the retry is not run in the same tick');
+  assert.equal(d.resolved.length, 0, 'not retired');
+  assert.equal(d.row.state, 'pending');
+  assert.equal(d.row.remedied_reason, 'dirty_tree');
+  assert.ok(d.ran.some((c) => /add -u && .* commit/.test(c)), 'the commit ran');
+  assert.ok(d.events.some((e) => e.event === W.REMEDY_EVENT));
+});
+
+await t('the retry fails dirty AGAIN: retired as unknown/remedy_did_not_hold, no second commit', async () => {
+  const row = { id: 11, instance_id: 7, action: 'core-upgrade', target_version: '1.20.31', attempts: 1, state: 'pending', not_before: null, remedied_reason: 'dirty_tree' };
+  const d = await drain({ row, reply: dirtyUpgrade });
+  assert.equal(d.runs, 1);
+  assert.equal(d.resolved.length, 1);
+  assert.equal(d.resolved[0].state, 'error');
+  assert.match(d.resolved[0].note, /already tried once/);
+  assert.equal(d.row.failure_class, 'unknown');
+  assert.equal(d.row.failure_reason, 'remedy_did_not_hold');
+  assert.ok(!d.ran.some((c) => /commit -m/.test(c)), 'no second fix');
+});
+
+console.log(`\nwedge_remedy: ${passed} passed, ${failed} failed`);
+if (failed) process.exit(1);