@bongos/core 1.20.31 → 1.20.32

package/.bongos-core.json

@@ -2,22 +2,22 @@
   "artifact": "bongos-core",
   "manifest_schema": 1,
   "generator": "scripts/gds/package-core.js",
-  "core_version": "1.20.31",
-  "core_contract": "1.20.31",
-  "source_commit": "7d9880eac50e580874638a8cca10da6606c18228",
+  "core_version": "1.20.32",
+  "core_contract": "1.20.32",
+  "source_commit": "22743b01c499a1cc3bd285e9b25d6efe937aa8a7",
   "source_ref": "HEAD",
-  "built_at": "2026-09-30T21:24:24.438Z",
+  "built_at": "2026-09-30T21:45:12.402Z",
   "redaction": {
     "model": "docs-redacted+functional-verbatim",
     "docs_redacted": 562,
     "agent_docs_stubbed": 25,
-    "functional_verbatim": 2666,
+    "functional_verbatim": 2669,
     "rules": 3,
     "gate_literals": 3,
     "gate": "passed"
   },
-  "file_count": 3254,
-  "tree_sha256": "88a5431f2eb9f9248c57014423f4881035e2d1b9a6cee88740a6800ab933ba38",
+  "file_count": 3257,
+  "tree_sha256": "7de0ec4831b06324127256eef3791a481ef07b33654cd0ca8b2cc058ec0e14ef",
   "files": [
     {
       "path": ".claude/skills/ask-for-help/SKILL.md",
@@ -2802,7 +2802,7 @@
     {
       "path": "docs/module-api-changelog.md",
       "mode": "0000644",
-      "sha256": "fd78a2ab7d52f77fbd7181a64af68b8cf4c45c25693bf7dbf31e1b9bba69ada7"
+      "sha256": "bd97221b57256af19ec7b09b897cce082e54fca07e69500ccf485feba106317f"
     },
     {
       "path": "docs/modules-contract.md",
@@ -3057,7 +3057,7 @@
     {
       "path": "docs/recipes/upgrading-the-core.md",
       "mode": "0000644",
-      "sha256": "169e4aedf245478230027ad6a202aa7dd2ce35f019d489d2ea4c0735647d0fe6"
+      "sha256": "6d44cfdcd7fa92d3b6c6514af69356d6e986d5443219251cd091685918e4e1f2"
     },
     {
       "path": "docs/recipes/windows-builders.md",
@@ -7469,6 +7469,11 @@
       "mode": "0000644",
       "sha256": "a93d8946cbe4631b2af64b2e3638dca188981032a08f5d29d63e4d0adb22d235"
     },
+    {
+      "path": "modules/provisioning/migrations/provisioning_032_intent_remedied_reason.sql",
+      "mode": "0000644",
+      "sha256": "73794821e3beca0129a435747e89dd5755b26f054dca0d8ca0b421392abf7127"
+    },
     {
       "path": "modules/provisioning/module.json",
       "mode": "0000644",
@@ -8987,12 +8992,12 @@
     {
       "path": "package-lock.json",
       "mode": "0000644",
-      "sha256": "892ad66ccaa864ee07860cd8578f134a72f8cc48625f728f186eaa0f994ac8ad"
+      "sha256": "cd508994cbd9d44ef721787e0eb7c803bfeda98459587d53ba726a36cff2581e"
     },
     {
       "path": "package.json",
       "mode": "0000644",
-      "sha256": "fdebee1582c5673a44b97471f05814ab6bc8e850958ca3131e4a356f3a663d90"
+      "sha256": "5c53762b7665282aeca491b037cb2c02ee8c6def6c00018262596235d02bbb64"
     },
     {
       "path": "public-docs/index.html",
@@ -9012,7 +9017,7 @@
     {
       "path": "release-notes.json",
       "mode": "0000644",
-      "sha256": "4398d89d420cf704f3030db8752e1f9e67321e43e49b0765e23329c26160385e"
+      "sha256": "1f7d261d95f2fbaf1d0e438e9c3ad7f13773c5b56d349263815d5be03ebab401"
     },
     {
       "path": "scripts/bongos-mcp.js",
@@ -9977,7 +9982,7 @@
     {
       "path": "scripts/gds/provision-core-upgrade.js",
       "mode": "0000644",
-      "sha256": "58599968201f98723a531c19f77985c0b26b9afa6d6448ae4803074497a6f2ad"
+      "sha256": "53904eb7a91f7f0c7c80c3caaceed29d6317652d6830e1f575198b0ba9f5158f"
     },
     {
       "path": "scripts/gds/provision-disconnect.js",
@@ -10032,7 +10037,7 @@
     {
       "path": "scripts/gds/provision.js",
       "mode": "0000644",
-      "sha256": "4da6314b41905bcffbc51abe127911512743ad14f8130b4c6a25c23600a3eb7f"
+      "sha256": "dd972cc62ed8282af5f1239a3b728761fb06a414120cac8a39b38838e8301f7d"
     },
     {
       "path": "scripts/gds/publish-credential-check.js",
@@ -10647,7 +10652,7 @@
     {
       "path": "scripts/gds/upgrade-outcome.js",
       "mode": "0000644",
-      "sha256": "a4f05b5e1042cf3d8c7096ef413b47fd039a02b78f2e4dbefb5bd002973784cb"
+      "sha256": "3bd19ba68a09a6739379bdd626bcf5b93addccdf1cd8774e20857dc9a66686fb"
     },
     {
       "path": "scripts/gds/upgrade.js",
@@ -10679,6 +10684,11 @@
       "mode": "0000644",
       "sha256": "cf034e9c4c5887dde47951272dba1e31b9b94a212640181de1cc3086e225b357"
     },
+    {
+      "path": "scripts/gds/wedge-remedy.js",
+      "mode": "0000644",
+      "sha256": "abe6ecedc808f3f57ef2dc5e9f953792ae26f39625425952a8ce529dbf6de576"
+    },
     {
       "path": "scripts/gds/worktree-claim-guard.js",
       "mode": "0000644",
@@ -11142,7 +11152,7 @@
     {
       "path": "src/module-api.js",
       "mode": "0000644",
-      "sha256": "8acd38044812dee3022427fad27e12cc7f9d81a90c1a0d921e860135a30f9e43"
+      "sha256": "a1f03380a933a2c9fdcd63f55f0152bf7d36101135d171c729f7cd79c9ef0ec4"
     },
     {
       "path": "src/module-loader/catalog.js",
@@ -16229,6 +16239,11 @@
       "mode": "0000644",
       "sha256": "456b20334164a710f59116d6a6c4b85090dc0e65668284d0a736e6ee1df5dea6"
     },
+    {
+      "path": "tests/wedge_remedy.mjs",
+      "mode": "0000644",
+      "sha256": "9810e6b2cf5ec77fad5af12b195a3ff4919143b146ff43f106c67c4090c61c25"
+    },
     {
       "path": "tests/wizard_born_first_board.mjs",
       "mode": "0000644",

package/docs/module-api-changelog.md

@@ -2719,5 +2719,7 @@ is load-bearing: the script throws rather than guess if it is missing, and
         landed since 1.20.29 with no explicit bump. run 36777226204. (task 1002620)
   1.20.31 — CI auto-patch (publish-on-merge, ADR 0161): carrier for merges
         landed since 1.20.30 with no explicit bump. run 36779142919. (task 1002620)
+  1.20.32 — CI auto-patch (publish-on-merge, ADR 0161): carrier for merges
+        landed since 1.20.31 with no explicit bump. run 36781343271. (task 1002620)
 ---------------------------------------------------------------------------
 ```

package/docs/recipes/upgrading-the-core.md

@@ -179,6 +179,7 @@ running a go-live.
 
 ## Gotchas
 
+- **The Deploy door fixes three known snags by itself, once** (task 1004448, `scripts/gds/wedge-remedy.js`). When a core move through `/deploy` fails on a **dirty tree** (it commits the tracked changes inside the project), a **full disk** (it prunes that project's own nightly dumps down to the newest 7) or a **database behind its code** (it migrates that database with its own `PGDATABASE` before the code moves), the runner applies the fix, records a `core-upgrade-remedy` event on the project, and tries the move once more on the next tick. If the retry fails for the same reason, or the fix itself fails, the move is retired as `unknown` (`remedy_did_not_hold` / `remedy_failed`) for a person — never a second fix. The manual fixes below still apply to the unattended nightly sweep, which does not go through the door.
 - **`--registry` needs no credential at all** (task 1003878). `@bongos/core` is **public** on npm (task 1003872), so `bongos upgrade --registry` resolves it from the default registry like any other dependency — no token, no login, and no instance `.npmrc`. An install failure on this path is therefore a *real* failure (network, registry outage, or a version that was never published), never a missing credential.
   *History, so the symptom is recognisable on an old instance:* the core used to publish private, and the instance `.npmrc` read `_authToken=<redacted> npm gives that per-project key precedence, so with the env var unset it expanded EMPTY and **masked a perfectly valid literal token in `~/.npmrc`** — npm went anonymous, the private core 404'd, and the error blamed a credential the box actually held. Task 1002754 worked around it by resolving the token in-process; task 1003878 deleted the whole mechanism along with the `.npmrc` that caused it. An instance still carrying a committed `.npmrc` from a pre-1003878 scaffold can safely delete it.
 - **A green `publish` run is not evidence of a publish.** The publish-on-merge lane succeeds on its no-op paths too (cascade terminator, armed-gate). Read the run's **job summary**, which now states the verdict explicitly: `PUBLISHED @bongos/core@<v>` vs `NO-OP — nothing was published this run`.

package/modules/provisioning/migrations/provisioning_032_intent_remedied_reason.sql

@@ -0,0 +1,19 @@
+-- provisioning_032_intent_remedied_reason.sql — which known snag the runner already fixed
+-- for a core move (task 1004448, goal 1000090 deploy self-heal).
+--
+-- WHY. A core move that fails on a known snag (a dirty checkout, a full disk, a database
+-- behind its code) now has the fix applied by the runner and is tried once more
+-- (scripts/gds/wedge-remedy.js). "Once" has to survive between runner ticks, so the reason
+-- the fix was applied for is kept on the intent. A retry that fails on the SAME reason is
+-- retired for a person instead of fixed again — never a loop.
+--
+-- remedied_reason: a reason code from scripts/gds/upgrade-outcome.js (fixed vocabulary,
+-- never builder text). NULL = no fix has been applied for this intent.
+--
+-- Additive and namespaced (ADR 0083): no down-migration, idempotent — safe to re-run.
+
+BEGIN;
+
+ALTER TABLE provisioning_intents ADD COLUMN IF NOT EXISTS remedied_reason text;
+
+COMMIT;

package/package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "@bongos/core",
-  "version": "1.20.31",
+  "version": "1.20.32",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "@bongos/core",
-      "version": "1.20.31",
+      "version": "1.20.32",
       "license": "AGPL-3.0-or-later",
       "dependencies": {
         "express": "^4.21.2",

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@bongos/core",
-  "version": "1.20.31",
+  "version": "1.20.32",
   "description": "Cloud Bongos — the AI-first build platform core (GDS + platform surfaces + module system), installed as a versioned dependency (ADR 0108).",
   "license": "AGPL-3.0-or-later",
   "main": "src/platform-server.js",

package/release-notes.json

@@ -8258,5 +8258,11 @@
       "id": "1004447",
       "text": "A software update that fails for a temporary reason now waits and tries again later — after 15 minutes, then an hour, then six hours — instead of hammering the same failing step three times in a row. Failures that retrying can"
     }
+  ],
+  "1.20.32": [
+    {
+      "id": "1004448",
+      "text": "When an update to a project gets stuck on one of three known problems (leftover file changes, a full disk, or a database that is behind), the system now fixes it by itself and tries again once. If that doesn't work, it stops a"
+    }
   ]
 }

package/scripts/gds/provision-core-upgrade.js

@@ -241,6 +241,16 @@ async function coreUpgradeInstance(inst, deps, intent) {
     log(`  already serving core ${to} — nothing to move`);
     return { ok: true, noop: true };
   }
+  // ---- 2b. is the database current for the code it runs NOW (task 1004448) -------------
+  // A database behind its own code is already failing every page that touches the missing
+  // tables (the 2026-07-04 shape), and moving the code on top of it tangles a catch-up
+  // migration into a pin move. So it is refused as a known snag, and wedge-remedy.js
+  // migrates that database and tries the move once more. An answer it cannot read proceeds.
+  if (apply) {
+    const probe = boxExec(versionCmd(inst), { allowFail: true });
+    const pending = parseSchemaPending(probe && !probe.softFailed ? (probe.stdout || '') : null);
+    if (pending > 0) return failed('schema_pending', `the project's database is ${pending} migration(s) behind the code it is running now — it has to catch up before the core moves`);
+  }
 
   // ---- 3. drive the proven path -----------------------------------------------------
   // Exactly the invocation docs/recipes/core-release-pipeline.md gate 3 documents, plus

package/scripts/gds/provision.js

@@ -1196,6 +1196,9 @@ async function cmdRunIntents(deps) {
         // WAIT OR RETIRE (task 1004447, intent-retry.js): a transient failure waits ~15m/1h/6h; a
         // classified non-transient one is retired at once (retrying cannot change its answer);
         // an unclassified one keeps its budget but never re-runs in the same tick.
+        const fix = await require('./wedge-remedy.js').remedyAfterFailure({ intent, inst, result, deps }); // a KNOWN snag: fix it, try once more (task 1004448)
+        if (fix.repended) { errors++; continue; }
+        Object.assign(result, { failure: fix.failure, error: fix.error });
         const next = retry.retryDecision({ attempts: intent.attempts, failure: result.failure, terminal: result.terminal, maxAttempts: MAX_INTENT_ATTEMPTS });
         if (next.action === 'retire') { // terminal: a leg that must not be retried unattended (the Render leg's key is already gone — task 1004183)
           await provisioning.resolveIntent(db, intent.id, 'error', result.error || 'op failed');

package/scripts/gds/upgrade-outcome.js

@@ -19,7 +19,7 @@
 //
 // THE FOUR CLASSES the rest of the self-heal chain keys on:
 //   transient       — try again later and it may simply work (registry blip, not published yet)
-//   known_wedge     — a known snag with a known fix the runner may apply itself (dirty tree, full disk)
+//   known_wedge     — a known snag with a known fix the runner applies itself, once (wedge-remedy.js)
 //   needs_decision  — a rule the owner set is refusing it; only the owner can change that
 //   unknown         — anything else, and every rollback that failed; a person should look
 //
@@ -42,6 +42,7 @@ const REASON_CLASS = Object.freeze({
   bad_slug: 'unknown',
   installed_unreadable: 'unknown',
   served_mismatch: 'unknown',
+  schema_pending: 'known_wedge', // the database is behind the code it runs NOW (task 1004448)
   // upgrade.js refusals, before the pin moves
   dirty_tree: 'known_wedge',
   downgrade: 'needs_decision',
@@ -59,6 +60,9 @@ const REASON_CLASS = Object.freeze({
   migrate: 'unknown',
   restart: 'unknown',
   health: 'unknown',
+  // the runner's one automatic fix did not clear it, or itself failed (task 1004448)
+  remedy_did_not_hold: 'unknown',
+  remedy_failed: 'unknown',
   // the one that must never be retried unattended
   rollback_failed: 'unknown',
 });

package/scripts/gds/wedge-remedy.js

@@ -0,0 +1,157 @@
+'use strict';
+//
+// scripts/gds/wedge-remedy.js — a core move that failed on a KNOWN snag fixes the snag and
+// tries once more (task 1004448, goal 1000090 deploy self-heal).
+//
+// WHY. upgrade-outcome.js names three failures that are not a fault in the release and not a
+// decision the owner has to make, but a mess on the box with one known fix — the fixes the
+// owner has been doing over SSH (docs/recipes/upgrading-the-core.md, Gotchas):
+//   dirty_tree      — the project's checkout has uncommitted changes to tracked files (the
+//                     upgrade's own rewrites, left behind by an earlier move). Fix: commit them
+//                     inside the project.
+//   disk_full       — the move ran out of disk. Fix: prune this project's OWN nightly
+//                     pg_dumps down to the newest few (the only thing on the box this runner
+//                     knows is safe to delete — they are copies, and the newest are kept).
+//   schema_pending  — the project's database is behind the code it is running NOW. Fix:
+//                     migrate that database, with its own PGDATABASE, before the code moves.
+// intent-retry.js retires every known_wedge at once, so on its own the failure only waited
+// for a person. This file runs the fix, records it as an event on the project, and re-pends
+// the move for ONE more try.
+//
+// AT MOST ONCE PER FAILURE. The reason a fix was applied for is stored on the intent
+// (provisioning_intents.remedied_reason, migration provisioning_032). If the retry fails on
+// the SAME reason, the fix did not hold: the move is retired as `unknown` /
+// remedy_did_not_hold, which is the class that reaches a person — never a second fix, never
+// a loop. A fix that itself fails is retired the same way (remedy_failed). A different
+// known reason on the retry gets its own one fix (a disk full after a dirty tree is a new
+// snag, not the same one twice); the attempt budget still bounds the whole run.
+//
+// 'tagged-not-published' needs nothing here: it is `not_published`, a transient failure that
+// intent-retry.js already waits out.
+//
+// planRemedy is PURE (the command, never run). remedyAfterFailure does the I/O through the
+// runner's own exec and db, so every fix is testable with an injected exec.
+
+const { CONFIG } = require('./provision-config.js');
+const { dbName, selfInstanceRoot, standaloneRoot } = require('./provision-repo.js');
+const { migratePlan, CORE_MIGRATE_SCRIPT } = require('./upgrade-migrate.js');
+const { isValidSlug } = require('../../modules/provisioning/provisioning.js');
+const outcome = require('./upgrade-outcome.js');
+const { DEFAULT_RETRY_DELAY_MS } = require('./intent-retry.js');
+
+const REMEDY_REASONS = Object.freeze(['dirty_tree', 'disk_full', 'schema_pending']);
+// The newest dumps a disk-full prune always leaves behind: a week of nightlies. The nightly
+// job's own retention is 30 days; this only ever cuts below that when the disk is full.
+const DUMP_KEEP_FLOOR = 7;
+const REMEDY_EVENT = 'core-upgrade-remedy';
+// A value that reaches a shell string below. Narrower than what Postgres or a path allow, on
+// purpose: anything outside it is refused, never quoted.
+const SHELL_SAFE_RE = /^[A-Za-z0-9._/-]{1,200}$/;
+const DB_NAME_RE = /^[A-Za-z0-9_-]{1,63}$/;
+const ACCOUNT_RE = /^[a-z_][a-z0-9_-]{0,31}$/;
+
+/**
+ * The fix for one known snag, as a command to run on the box. PURE.
+ * @param {object} inst   the provisioning_instances row
+ * @param {string} reason the failure's reason code
+ * @param {object} [o]    { privileged, runAs, instanceDir, selfRoot, backupDir, migrate }
+ * @returns {{ reason, cmd, cwd, env, describe } | { refused: string } | null}  null: no known fix
+ */
+function planRemedy(inst, reason, o = {}) {
+  if (!REMEDY_REASONS.includes(reason)) return null;
+  if (!inst || !isValidSlug(inst.slug)) return { refused: 'the project name is not a valid slug' };
+  const dir = o.instanceDir || (inst.hosting_shape === 'cloud-host' ? standaloneRoot(inst) : (o.selfRoot || selfInstanceRoot)());
+  if (!SHELL_SAFE_RE.test(dir)) return { refused: 'the project folder is not a path this runner will put in a command' };
+  const db = dbName(inst);
+  if (!DB_NAME_RE.test(db)) return { refused: 'the project database name is not one this runner will put in a command' };
+
+  if (reason === 'dirty_tree') {
+    // TRACKED files only (`add -u`): the snag is the upgrade's own rewrites of tracked files.
+    // A new untracked file is not swept into the owner's history on a guess — if that is the
+    // dirt, the retry fails the same way and a person looks. Runs as the runner, which owns
+    // the checkout (pushUpgradePin's shape), with hooks off and a runner identity so the
+    // commit says who made it.
+    const g = `git -C ${dir} -c core.hooksPath=/dev/null -c user.name=bongos-runner -c user.email=runner@bongos.invalid`;
+    return {
+      reason, cwd: null, env: null,
+      cmd: `${g} add -u && ${g} commit -m "chore: commit changes left in the working tree (automatic fix before a core move)"`,
+      describe: `committed the uncommitted changes in ${dir} so the core move could run`,
+    };
+  }
+  if (reason === 'disk_full') {
+    const backupDir = o.backupDir || CONFIG.backupDir;
+    if (!SHELL_SAFE_RE.test(backupDir)) return { refused: 'the backup folder is not a path this runner will put in a command' };
+    // EXACTLY db-backup-nightly.sh's name, so this never touches a sibling project's dumps
+    // (`test` must not prune `test-nk`'s — task 1003948). The timestamp sorts by name, so
+    // `sort | head -n -K` drops all but the newest K. Nothing is deleted when K or fewer exist.
+    const D = '[0-9]';
+    const glob = `${db}-${D}${D}${D}${D}-${D}${D}-${D}${D}T${D}${D}${D}${D}${D}${D}Z.sql.gz`;
+    return {
+      reason, cwd: null, env: null,
+      cmd: `find ${backupDir} -maxdepth 1 -type f -name '${glob}' | sort | head -n -${DUMP_KEEP_FLOOR} | xargs -r rm -f --`,
+      describe: `pruned ${db}'s old database backups in ${backupDir}, keeping the newest ${DUMP_KEEP_FLOOR}`,
+    };
+  }
+  // schema_pending — the project's OWN migrate command (a declared script wins, else the
+  // core's migrate.sh: upgrade-migrate.js), aimed at its OWN database. migrate.sh defaults to
+  // production when PGDATABASE is unset, so it is always set, never inherited.
+  const plan = (o.migrate || migratePlan)(dir);
+  const run = plan.via === 'npm-script' ? 'npm run migrate' : `bash ${CORE_MIGRATE_SCRIPT}`;
+  const initCwd = plan.via === 'npm-script' ? '' : `INIT_CWD=${dir} `;
+  if (o.privileged) {
+    if (!ACCOUNT_RE.test(String(o.runAs || ''))) return { refused: 'no account to run the migration as' };
+    return { reason, cwd: dir, env: null, cmd: `sudo -u ${o.runAs} env PGDATABASE=${db} ${initCwd}${run}`, describe: `migrated ${db} to match the code it is running` };
+  }
+  return { reason, cwd: dir, env: { PGDATABASE: db, ...(initCwd ? { INIT_CWD: dir } : {}) }, cmd: run, describe: `migrated ${db} to match the code it is running` };
+}
+
+// The account the project runs as, probed the way the move itself probes (task 1004128).
+function runAsFor(inst, deps) {
+  try { return require('./provision-core-upgrade.js').resolveRunAs(inst, deps).runAs; } catch { return null; }
+}
+
+/**
+ * After a core move failed: apply the known fix, record it, and re-pend the move once.
+ *
+ * Returns `{ repended: true }` when the move will be tried again, otherwise
+ * `{ repended: false, failure, error }` — the failure the drain loop should RETIRE with,
+ * which is the original one when there is no fix for it, and `unknown` when a fix was
+ * already spent on this reason or the fix itself failed.
+ */
+async function remedyAfterFailure({ intent, inst, result, deps }) {
+  const failure = result && result.failure;
+  const reason = failure && failure.reason;
+  const none = { repended: false, failure, error: result && result.error };
+  if (!intent || intent.action !== 'core-upgrade' || !failure || failure.class !== 'known_wedge') return none;
+  const { db, provisioning, log = () => {}, apply } = deps;
+  if (!apply) return none;
+  if (intent.remedied_reason === reason) {
+    return {
+      repended: false, failure: outcome.classifyFailure('remedy_did_not_hold'),
+      error: `${result.error || 'core move failed'} — the automatic fix for this (${reason}) was already tried once and did not clear it, so a person needs to look`,
+    };
+  }
+  const boxExec = inst.hosting_shape === 'cloud-host' ? (deps.controlExec || deps.exec) : deps.exec;
+  const plan = planRemedy(inst, reason, { privileged: deps.privileged, runAs: reason === 'schema_pending' && deps.privileged ? runAsFor(inst, deps) : null, selfRoot: deps.selfInstanceRoot, migrate: deps.migratePlan });
+  if (!plan) return none;
+  const giveUp = async (why) => {
+    await provisioning.recordEvent(db, { instanceId: inst.id, ownerBuilderId: inst.owner_builder_id, event: REMEDY_EVENT, detail: `could not fix ${reason} automatically: ${why}`, actor: 'runner' }).catch(() => {});
+    return { repended: false, failure: outcome.classifyFailure('remedy_failed'), error: `${result.error || 'core move failed'} — the automatic fix for ${reason} did not work (${why}), so a person needs to look` };
+  };
+  if (plan.refused) return giveUp(plan.refused);
+  log(`  [remedy] ${reason}: ${plan.cmd}`);
+  const r = boxExec(plan.cmd, { allowFail: true, ...(plan.cwd ? { cwd: plan.cwd } : {}), ...(plan.env ? { env: plan.env } : {}) });
+  if (!r || r.softFailed === true || r.ok === false) {
+    const said = String((r && (r.stderr || r.stdout || r.error)) || 'the command failed').trim().split('\n').pop().slice(0, 160);
+    return giveUp(said);
+  }
+  await provisioning.recordEvent(db, { instanceId: inst.id, ownerBuilderId: inst.owner_builder_id, event: REMEDY_EVENT, detail: `${plan.describe}; trying the move to ${intent.target_version} once more`, actor: 'runner' }).catch(() => {});
+  await db.query(
+    `UPDATE provisioning_intents SET state='pending', last_error=$2, remedied_reason=$3, failure_class=$4, failure_reason=$5,
+            not_before = now() + ($6 * interval '1 millisecond'), updated_at=now() WHERE id=$1`,
+    [intent.id, result.error || 'core move failed', reason, failure.class, failure.reason, DEFAULT_RETRY_DELAY_MS]);
+  log(`  ✓ fixed ${reason} on ${inst.slug}; the move to ${intent.target_version} will be tried once more`);
+  return { repended: true };
+}
+
+module.exports = { REMEDY_REASONS, DUMP_KEEP_FLOOR, REMEDY_EVENT, planRemedy, remedyAfterFailure };

package/src/module-api.js

@@ -75,7 +75,7 @@ const { responsibilityFor, ROLE_RESPONSIBILITIES } = require('./role-responsibil
 // MAJOR (see allowBoxScope below): passes the request through untouched.
 function deprecatedNoopMiddleware(_req, _res, next) { next(); }
 
-const CORE_VERSION = '1.20.31'; // CI auto-patch carrier (ADR 0161); changelog: docs/module-api-changelog.md
+const CORE_VERSION = '1.20.32'; // CI auto-patch carrier (ADR 0161); changelog: docs/module-api-changelog.md
 
 // A namespaced logger so a module's log lines are attributable + consistent.
 // Usage: const log = api.logger('discord'); log.info('mounted');

package/tests/wedge_remedy.mjs

@@ -0,0 +1,268 @@
+// tests/wedge_remedy.mjs — a core move that failed on a KNOWN snag fixes the snag and tries
+// once more, and never twice (task 1004448, goal 1000090 deploy self-heal).
+//
+// WHAT IS AT RISK. Each fix runs a command on a live project's box: a commit in the owner's
+// checkout, a delete among database backups, a migration against a database. The cases pin
+// WHAT each command is (and what it refuses to build), that it runs at most once per
+// failure, and that a fix which did not hold reaches a person instead of looping. The
+// disk-full prune is also run for real against a temp folder, because a `find` pattern is
+// the kind of thing that reads right and deletes a sibling project's dumps.
+//
+// Run:  node tests/wedge_remedy.mjs
+
+import { strict as assert } from 'node:assert';
+import { createRequire } from 'node:module';
+import { spawnSync } from 'node:child_process';
+import fs from 'node:fs';
+import os from 'node:os';
+import path from 'node:path';
+
+process.env.PROVISION_APP_USER = process.env.PROVISION_APP_USER || 'lars'; // appUser() refuses a Windows account name
+const require = createRequire(import.meta.url);
+const W = require('../scripts/gds/wedge-remedy.js');
+const R = require('../scripts/gds/provision-core-upgrade.js');
+const P = require('../scripts/gds/provision.js');
+const { REASON_CLASS } = require('../scripts/gds/upgrade-outcome.js');
+const { DEFAULT_RETRY_DELAY_MS } = require('../scripts/gds/intent-retry.js');
+
+let passed = 0;
+let failed = 0;
+const t = async (name, fn) => {
+  try { await fn(); passed += 1; console.log(`  PASS  ${name}`); }
+  catch (e) { failed += 1; console.log(`  FAIL  ${name}\n        ${e.message}`); }
+};
+
+const inst = (over = {}) => ({ id: 7, slug: 'charter', db_name: 'charter', port: 3010, status: 'active', hosting_shape: 'co-tenant', owner_builder_id: 1, ...over });
+const intent = (over = {}) => ({ id: 99, action: 'core-upgrade', target_version: '1.20.31', attempts: 1, remedied_reason: null, ...over });
+const wedge = (reason) => ({ ok: false, error: `core upgrade to 1.20.31 was refused (${reason})`, failure: { class: 'known_wedge', reason } });
+const selfRoot = () => '/srv/charter';
+
+function harness({ execFails = false } = {}) {
+  const ran = []; const events = []; const queries = [];
+  const deps = {
+    apply: true, privileged: false, log: () => {}, selfInstanceRoot: selfRoot,
+    exec: (cmd, opts) => { ran.push({ cmd, ...opts }); return execFails ? { ok: false, softFailed: true, stderr: 'fatal: nope' } : { ok: true, stdout: '' }; },
+    db: { query: async (sql, params) => { queries.push({ sql, params }); return { rows: [] }; } },
+    provisioning: { recordEvent: async (_db, e) => { events.push(e); } },
+    migratePlan: () => ({ via: 'npm-script' }),
+  };
+  return { deps, ran, events, queries };
+}
+
+console.log('\nthe fix for each known snag (planRemedy):');
+
+await t('every reason with a fix is a known_wedge in the classifier — and back', () => {
+  assert.deepEqual([...W.REMEDY_REASONS].sort(), ['dirty_tree', 'disk_full', 'schema_pending']);
+  for (const r of W.REMEDY_REASONS) assert.equal(REASON_CLASS[r], 'known_wedge', r);
+  for (const [r, c] of Object.entries(REASON_CLASS)) if (c === 'known_wedge') assert.ok(W.REMEDY_REASONS.includes(r), `${r} is known_wedge but has no fix — it would be retired with nothing done`);
+  assert.equal(REASON_CLASS.remedy_did_not_hold, 'unknown', 'a fix that did not hold reaches a person');
+  assert.equal(REASON_CLASS.remedy_failed, 'unknown');
+});
+
+await t('dirty tree: commits TRACKED changes inside the project, hooks off, as the runner', () => {
+  const p = W.planRemedy(inst(), 'dirty_tree', { selfRoot });
+  assert.match(p.cmd, /^git -C \/srv\/charter -c core\.hooksPath=\/dev\/null .* add -u && git -C \/srv\/charter .* commit -m /);
+  assert.doesNotMatch(p.cmd, /add -A|add \./, 'an untracked file is never swept into the owner\'s history on a guess');
+  assert.doesNotMatch(p.cmd, /sudo/, 'the runner owns the checkout (pushUpgradePin\'s shape)');
+});
+
+await t('a cloud-host project is fixed in ITS folder, not the control plane\'s', () => {
+  const p = W.planRemedy(inst({ hosting_shape: 'cloud-host' }), 'dirty_tree', { selfRoot });
+  assert.match(p.cmd, /git -C \/\S+\/charter /);
+  assert.doesNotMatch(p.cmd, /\/srv\/charter/);
+});
+
+await t('disk full: prunes only THIS database\'s dumps, keeping the newest week', () => {
+  const p = W.planRemedy(inst(), 'disk_full', { selfRoot, backupDir: '/var/backups/gds' });
+  assert.match(p.cmd, /^find \/var\/backups\/gds -maxdepth 1 -type f -name 'charter-\[0-9\]/);
+  assert.match(p.cmd, new RegExp(`head -n -${W.DUMP_KEEP_FLOOR} \\| xargs -r rm -f --$`));
+  assert.equal(W.DUMP_KEEP_FLOOR, 7);
+});
+
+await t('schema pending: migrates the project\'s OWN database, PGDATABASE always set', () => {
+  const plain = W.planRemedy(inst(), 'schema_pending', { selfRoot, migrate: () => ({ via: 'npm-script' }) });
+  assert.equal(plain.cmd, 'npm run migrate');
+  assert.equal(plain.cwd, '/srv/charter');
+  assert.deepEqual(plain.env, { PGDATABASE: 'charter' });
+  const adopted = W.planRemedy(inst(), 'schema_pending', { selfRoot, migrate: () => ({ via: 'core-script' }) });
+  assert.equal(adopted.cmd, 'bash node_modules/@bongos/core/scripts/migrate.sh');
+  assert.deepEqual(adopted.env, { PGDATABASE: 'charter', INIT_CWD: '/srv/charter' });
+  const priv = W.planRemedy(inst(), 'schema_pending', { selfRoot, privileged: true, runAs: 'bongos-charter', migrate: () => ({ via: 'npm-script' }) });
+  assert.equal(priv.cmd, 'sudo -u bongos-charter env PGDATABASE=charter npm run migrate', 'sudo resets env, so PGDATABASE rides the command');
+  assert.ok(W.planRemedy(inst(), 'schema_pending', { selfRoot, privileged: true, runAs: null }).refused);
+});
+
+await t('no command is built from a name that is not shell-safe', () => {
+  assert.ok(W.planRemedy(inst({ slug: 'a;rm -rf' }), 'dirty_tree', { selfRoot }).refused);
+  assert.ok(W.planRemedy(inst({ db_name: 'x; drop' }), 'disk_full', { selfRoot }).refused);
+  assert.ok(W.planRemedy(inst(), 'dirty_tree', { selfRoot: () => '/srv/a b' }).refused);
+  assert.ok(W.planRemedy(inst(), 'disk_full', { selfRoot, backupDir: '/tmp/$(x)' }).refused);
+});
+
+await t('a reason with no fix gets no command', () => {
+  for (const r of ['artist_gate', 'downgrade', 'health', 'not_published', 'nope']) assert.equal(W.planRemedy(inst(), r, { selfRoot }), null, r);
+});
+
+console.log('\nthe prune, run for real:');
+
+const bash = spawnSync('bash', ['-c', 'echo ok'], { encoding: 'utf8' });
+await t('keeps the newest 7 of this project\'s dumps and never touches a sibling\'s', () => {
+  if (bash.status !== 0) { console.log('        (skipped: no bash here)'); return; }
+  const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'wedge-prune-'));
+  try {
+    const mine = []; for (let d = 1; d <= 10; d++) mine.push(`charter-2026-09-${String(d).padStart(2, '0')}T030000Z.sql.gz`);
+    const others = ['charter-nk-2026-09-01T030000Z.sql.gz', 'test-2026-09-01T030000Z.sql.gz', 'charter-notes.txt'];
+    for (const f of [...mine, ...others]) fs.writeFileSync(path.join(dir, f), 'x');
+    const p = W.planRemedy(inst(), 'disk_full', { selfRoot, backupDir: '.' }); // run IN the folder: a temp path can hold a `~`
+    const r = spawnSync('bash', ['-c', p.cmd], { encoding: 'utf8', cwd: dir });
+    assert.equal(r.status, 0, r.stderr);
+    const left = fs.readdirSync(dir).sort();
+    assert.deepEqual(left, [...others, ...mine.slice(3)].sort());
+    // Nothing is deleted when the floor is not exceeded.
+    const again = spawnSync('bash', ['-c', p.cmd], { encoding: 'utf8', cwd: dir });
+    assert.equal(again.status, 0);
+    assert.equal(fs.readdirSync(dir).length, left.length);
+  } finally { fs.rmSync(dir, { recursive: true, force: true }); }
+});
+
+console.log('\nonce per failure (remedyAfterFailure):');
+
+for (const reason of W.REMEDY_REASONS) {
+  await t(`${reason}: runs the fix, records an event, re-pends the move with the reason stored`, async () => {
+    const h = harness();
+    const r = await W.remedyAfterFailure({ intent: intent(), inst: inst(), result: wedge(reason), deps: h.deps });
+    assert.deepEqual(r, { repended: true });
+    assert.equal(h.ran.length, 1);
+    assert.equal(h.events.length, 1);
+    assert.equal(h.events[0].event, W.REMEDY_EVENT);
+    assert.match(h.events[0].detail, /trying the move to 1\.20\.31 once more/);
+    const up = h.queries.find((q) => /SET state='pending'/.test(q.sql));
+    assert.ok(up, 're-pended');
+    assert.match(up.sql, /remedied_reason=\$3/);
+    assert.deepEqual(up.params, [99, wedge(reason).error, reason, 'known_wedge', reason, DEFAULT_RETRY_DELAY_MS]);
+  });
+}
+
+await t('the same reason again after a fix: no second fix — retired as unknown for a person', async () => {
+  const h = harness();
+  const r = await W.remedyAfterFailure({ intent: intent({ remedied_reason: 'dirty_tree' }), inst: inst(), result: wedge('dirty_tree'), deps: h.deps });
+  assert.equal(r.repended, false);
+  assert.deepEqual(r.failure, { class: 'unknown', reason: 'remedy_did_not_hold' });
+  assert.match(r.error, /already tried once/);
+  assert.equal(h.ran.length, 0, 'nothing ran');
+  assert.equal(h.queries.length, 0);
+});
+
+await t('a DIFFERENT known snag on the retry gets its own one fix', async () => {
+  const h = harness();
+  const r = await W.remedyAfterFailure({ intent: intent({ remedied_reason: 'dirty_tree' }), inst: inst(), result: wedge('disk_full'), deps: h.deps });
+  assert.equal(r.repended, true);
+});
+
+await t('a fix that fails is recorded and retired as unknown — never re-pended', async () => {
+  const h = harness({ execFails: true });
+  const r = await W.remedyAfterFailure({ intent: intent(), inst: inst(), result: wedge('dirty_tree'), deps: h.deps });
+  assert.equal(r.repended, false);
+  assert.deepEqual(r.failure, { class: 'unknown', reason: 'remedy_failed' });
+  assert.match(r.error, /did not work \(fatal: nope\)/);
+  assert.equal(h.events.length, 1);
+  assert.match(h.events[0].detail, /could not fix dirty_tree/);
+  assert.equal(h.queries.length, 0);
+});
+
+await t('leaves everything else alone: other classes, other actions, and a dry run', async () => {
+  for (const [i, res] of [[intent(), { ok: false, error: 'x', failure: { class: 'transient', reason: 'not_published' } }],
+    [intent(), { ok: false, error: 'x', failure: { class: 'needs_decision', reason: 'artist_gate' } }],
+    [intent(), { ok: false, error: 'x' }],
+    [intent({ action: 'restart' }), wedge('dirty_tree')]]) {
+    const h = harness();
+    const r = await W.remedyAfterFailure({ intent: i, inst: inst(), result: res, deps: h.deps });
+    assert.equal(r.repended, false);
+    assert.deepEqual(r.failure, res.failure);
+    assert.equal(h.ran.length, 0);
+  }
+  const h = harness(); h.deps.apply = false;
+  assert.equal((await W.remedyAfterFailure({ intent: intent(), inst: inst(), result: wedge('dirty_tree'), deps: h.deps })).repended, false);
+  assert.equal(h.ran.length, 0);
+});
+
+console.log('\nthe move refuses on a database behind its code:');
+
+await t('schemaPending > 0 on the box refuses as schema_pending BEFORE the upgrade runs', async () => {
+  const ran = [];
+  const deps = {
+    apply: true, privileged: false, log: () => {}, sleep: async () => {},
+    exec: (cmd) => { ran.push(cmd); return { ok: true, stdout: /\/version/.test(cmd) ? JSON.stringify({ coreVersion: '1.20.30', schemaPending: 2 }) : '' }; },
+    db: { query: async () => ({ rows: [] }) }, provisioning: { recordEvent: async () => {} },
+    readInstalledCoreVersion: () => '1.20.30', selfInstanceRoot: () => '/srv/cb',
+    updateChannel: { DEFAULT_CHANNEL: 'patch', normalizeChannel: (c) => c || 'patch', channelAllows: () => true, listAvailableVersions: () => ({ ok: true, versions: ['1.20.30', '1.20.31'] }) },
+  };
+  const r = await R.coreUpgradeInstance(inst({ hosting_shape: 'control-plane', slug: 'cloudbongos' }), deps, intent());
+  assert.equal(r.ok, false);
+  assert.deepEqual(r.failure, { class: 'known_wedge', reason: 'schema_pending' });
+  assert.match(r.error, /2 migration\(s\) behind/);
+  assert.ok(!ran.some((c) => /upgrade\.js/.test(c)), 'the pin never moved');
+});
+
+console.log('\nthe drain loop, end to end:');
+
+// A stateful fake queue, the shape tests/provision.mjs uses: claimNextIntent honours
+// not_before, so "fixed, then tried once more on a LATER tick" is measured.
+async function drain({ row, reply }) {
+  const ran = []; const events = []; const resolved = []; let runs = 0;
+  const deps = {
+    apply: true, privileged: false, log: () => {}, sleep: async () => {},
+    db: { query: async (sql, params) => {
+      if (/SET state='pending'/.test(sql)) { row.state = 'pending'; row.not_before = new Date(Date.now() + params[params.length - 1]); if (/remedied_reason/.test(sql)) row.remedied_reason = params[2]; }
+      if (/SET failure_class/.test(sql)) { row.failure_class = params[1]; row.failure_reason = params[2]; }
+      return { rows: [] };
+    } },
+    provisioning: {
+      claimNextIntent: async () => {
+        if (row.state !== 'pending' || (row.not_before && row.not_before > new Date())) return null;
+        row.state = 'running'; row.attempts += 1; runs += 1; return { ...row };
+      },
+      getInstanceById: async () => inst({ hosting_shape: 'control-plane', slug: 'cloudbongos', db_name: 'cloudbongos' }),
+      resolveIntent: async (_db, id, state, note) => { resolved.push({ id, state, note }); row.state = state; },
+      setInstanceStatus: async () => {}, recordEvent: async (_db, e) => { events.push(e); },
+      claimNextOAuthManifest: async () => null, resolveOAuthManifest: async () => {},
+    },
+    exec: (cmd, opts) => { ran.push(cmd); return reply(cmd, opts); },
+    writeFile: () => ({ ok: true }),
+    readInstalledCoreVersion: () => '1.20.30', selfInstanceRoot: () => '/srv/cb',
+    updateChannel: { DEFAULT_CHANNEL: 'patch', normalizeChannel: (c) => c || 'patch', channelAllows: () => true, listAvailableVersions: () => ({ ok: true, versions: ['1.20.30', '1.20.31'] }) },
+    checkDnsTokenScope: async () => ({ reachable: true, zone: 'z' }), checkCaddyWiring: () => null,
+  };
+  await P.cmdRunIntents(deps);
+  return { ran, events, resolved, runs, row };
+}
+const dirtyUpgrade = (cmd) => {
+  if (/upgrade\.js/.test(cmd)) { const e = new Error('Command failed'); e.stderr = '✖ working tree not clean — commit/stash first\nBONGOS_UPGRADE_OUTCOME {"stage":"refused","reason":"dirty_tree"}\n'; throw e; }
+  return { ok: true, stdout: /\/version/.test(cmd) ? JSON.stringify({ coreVersion: '1.20.30', schemaPending: 0 }) : '' };
+};
+
+await t('a dirty tree is committed, the move waits for the next tick, and is NOT retired', async () => {
+  const row = { id: 11, instance_id: 7, action: 'core-upgrade', target_version: '1.20.31', attempts: 0, state: 'pending', not_before: null, remedied_reason: null };
+  const d = await drain({ row, reply: dirtyUpgrade });
+  assert.equal(d.runs, 1, 'the retry is not run in the same tick');
+  assert.equal(d.resolved.length, 0, 'not retired');
+  assert.equal(d.row.state, 'pending');
+  assert.equal(d.row.remedied_reason, 'dirty_tree');
+  assert.ok(d.ran.some((c) => /add -u && .* commit/.test(c)), 'the commit ran');
+  assert.ok(d.events.some((e) => e.event === W.REMEDY_EVENT));
+});
+
+await t('the retry fails dirty AGAIN: retired as unknown/remedy_did_not_hold, no second commit', async () => {
+  const row = { id: 11, instance_id: 7, action: 'core-upgrade', target_version: '1.20.31', attempts: 1, state: 'pending', not_before: null, remedied_reason: 'dirty_tree' };
+  const d = await drain({ row, reply: dirtyUpgrade });
+  assert.equal(d.runs, 1);
+  assert.equal(d.resolved.length, 1);
+  assert.equal(d.resolved[0].state, 'error');
+  assert.match(d.resolved[0].note, /already tried once/);
+  assert.equal(d.row.failure_class, 'unknown');
+  assert.equal(d.row.failure_reason, 'remedy_did_not_hold');
+  assert.ok(!d.ran.some((c) => /commit -m/.test(c)), 'no second fix');
+});
+
+console.log(`\nwedge_remedy: ${passed} passed, ${failed} failed`);
+if (failed) process.exit(1);