@bongos/core 1.20.30 → 1.20.32

package/.bongos-core.json

@@ -2,22 +2,22 @@
   "artifact": "bongos-core",
   "manifest_schema": 1,
   "generator": "scripts/gds/package-core.js",
-  "core_version": "1.20.30",
-  "core_contract": "1.20.30",
-  "source_commit": "b56a05d974ae5208394507078971bf8e22f415f3",
+  "core_version": "1.20.32",
+  "core_contract": "1.20.32",
+  "source_commit": "22743b01c499a1cc3bd285e9b25d6efe937aa8a7",
   "source_ref": "HEAD",
-  "built_at": "2026-09-30T21:06:55.806Z",
+  "built_at": "2026-09-30T21:45:12.402Z",
   "redaction": {
     "model": "docs-redacted+functional-verbatim",
     "docs_redacted": 562,
     "agent_docs_stubbed": 25,
-    "functional_verbatim": 2663,
+    "functional_verbatim": 2669,
     "rules": 3,
     "gate_literals": 3,
     "gate": "passed"
   },
-  "file_count": 3251,
-  "tree_sha256": "3b45f4c67302ad870157396c60f693712b46bcdc86a9236e964c914418657c35",
+  "file_count": 3257,
+  "tree_sha256": "7de0ec4831b06324127256eef3791a481ef07b33654cd0ca8b2cc058ec0e14ef",
   "files": [
     {
       "path": ".claude/skills/ask-for-help/SKILL.md",
@@ -2802,7 +2802,7 @@
     {
       "path": "docs/module-api-changelog.md",
       "mode": "0000644",
-      "sha256": "fca494d7f518bc42742fde6c5ba2e45cfbf0605d5a7a79024bba190ec4c3c1da"
+      "sha256": "bd97221b57256af19ec7b09b897cce082e54fca07e69500ccf485feba106317f"
     },
     {
       "path": "docs/modules-contract.md",
@@ -3057,7 +3057,7 @@
     {
       "path": "docs/recipes/upgrading-the-core.md",
       "mode": "0000644",
-      "sha256": "169e4aedf245478230027ad6a202aa7dd2ce35f019d489d2ea4c0735647d0fe6"
+      "sha256": "6d44cfdcd7fa92d3b6c6514af69356d6e986d5443219251cd091685918e4e1f2"
     },
     {
       "path": "docs/recipes/windows-builders.md",
@@ -7282,7 +7282,7 @@
     {
       "path": "modules/provisioning/core-upgrade.js",
       "mode": "0000644",
-      "sha256": "7604ea4c137c5c1cedca41329cfd6fce49458d454f2e956944f7fb8d0c4ab10a"
+      "sha256": "5467ab673dfad5969dec0029188d0be18ea82732a26f99bca720de6680e1d618"
     },
     {
       "path": "modules/provisioning/cost-ledger.js",
@@ -7464,6 +7464,16 @@
       "mode": "0000644",
       "sha256": "7287181b4b42d3e391cd97e19ffa5fbc76026e81ad6f54648eb12a69dfea55a1"
     },
+    {
+      "path": "modules/provisioning/migrations/provisioning_031_intent_not_before.sql",
+      "mode": "0000644",
+      "sha256": "a93d8946cbe4631b2af64b2e3638dca188981032a08f5d29d63e4d0adb22d235"
+    },
+    {
+      "path": "modules/provisioning/migrations/provisioning_032_intent_remedied_reason.sql",
+      "mode": "0000644",
+      "sha256": "73794821e3beca0129a435747e89dd5755b26f054dca0d8ca0b421392abf7127"
+    },
     {
       "path": "modules/provisioning/module.json",
       "mode": "0000644",
@@ -7512,7 +7522,7 @@
     {
       "path": "modules/provisioning/provisioning.js",
       "mode": "0000644",
-      "sha256": "f2dd673e30cf17abbfbadd3c0142616a1c40b3b680c865b57db8330933816405"
+      "sha256": "fb4f09b621654eebe595dcbca305773ea11ed063506a6e4116391e43566a540d"
     },
     {
       "path": "modules/provisioning/public-refusal.js",
@@ -8982,12 +8992,12 @@
     {
       "path": "package-lock.json",
       "mode": "0000644",
-      "sha256": "872b4c6ad6eacb09cbcd5698c892fc798ce6e632b505304b5665c7f04ea99325"
+      "sha256": "cd508994cbd9d44ef721787e0eb7c803bfeda98459587d53ba726a36cff2581e"
     },
     {
       "path": "package.json",
       "mode": "0000644",
-      "sha256": "6c29a554265b6c3e6fa7f29fbf9b334c3a73e30b240b393498b89304876b5495"
+      "sha256": "5c53762b7665282aeca491b037cb2c02ee8c6def6c00018262596235d02bbb64"
     },
     {
       "path": "public-docs/index.html",
@@ -9007,7 +9017,7 @@
     {
       "path": "release-notes.json",
       "mode": "0000644",
-      "sha256": "1ce998432070110f9c3a9b128145bf8d44c032aa2ff55437207ecaf5efe8943a"
+      "sha256": "1f7d261d95f2fbaf1d0e438e9c3ad7f13773c5b56d349263815d5be03ebab401"
     },
     {
       "path": "scripts/bongos-mcp.js",
@@ -9739,6 +9749,11 @@
       "mode": "0000644",
       "sha256": "7b2402543c3b1eadbafc14907e6dc6689b434ac65fb9c709115dd81a92eb13eb"
     },
+    {
+      "path": "scripts/gds/intent-retry.js",
+      "mode": "0000644",
+      "sha256": "056622ecd8327527f4fb98b37ce8af90778d42c816455431ed3dd2f76800a75b"
+    },
     {
       "path": "scripts/gds/key-card.js",
       "mode": "0000644",
@@ -9967,7 +9982,7 @@
     {
       "path": "scripts/gds/provision-core-upgrade.js",
       "mode": "0000644",
-      "sha256": "58599968201f98723a531c19f77985c0b26b9afa6d6448ae4803074497a6f2ad"
+      "sha256": "53904eb7a91f7f0c7c80c3caaceed29d6317652d6830e1f575198b0ba9f5158f"
     },
     {
       "path": "scripts/gds/provision-disconnect.js",
@@ -10022,7 +10037,7 @@
     {
       "path": "scripts/gds/provision.js",
       "mode": "0000644",
-      "sha256": "0f7a9cd81a78ef3ed30a40e9f048f8905b6a88e99a930f7cbdcdcd5c1661f71d"
+      "sha256": "dd972cc62ed8282af5f1239a3b728761fb06a414120cac8a39b38838e8301f7d"
     },
     {
       "path": "scripts/gds/publish-credential-check.js",
@@ -10637,7 +10652,7 @@
     {
       "path": "scripts/gds/upgrade-outcome.js",
       "mode": "0000644",
-      "sha256": "a4f05b5e1042cf3d8c7096ef413b47fd039a02b78f2e4dbefb5bd002973784cb"
+      "sha256": "3bd19ba68a09a6739379bdd626bcf5b93addccdf1cd8774e20857dc9a66686fb"
     },
     {
       "path": "scripts/gds/upgrade.js",
@@ -10669,6 +10684,11 @@
       "mode": "0000644",
       "sha256": "cf034e9c4c5887dde47951272dba1e31b9b94a212640181de1cc3086e225b357"
     },
+    {
+      "path": "scripts/gds/wedge-remedy.js",
+      "mode": "0000644",
+      "sha256": "abe6ecedc808f3f57ef2dc5e9f953792ae26f39625425952a8ce529dbf6de576"
+    },
     {
       "path": "scripts/gds/worktree-claim-guard.js",
       "mode": "0000644",
@@ -11132,7 +11152,7 @@
     {
       "path": "src/module-api.js",
       "mode": "0000644",
-      "sha256": "4002317dabfa7084c1b541e2a6a605197ce8c1e45df90f81bfa2ef186b0d6776"
+      "sha256": "a1f03380a933a2c9fdcd63f55f0152bf7d36101135d171c729f7cd79c9ef0ec4"
     },
     {
       "path": "src/module-loader/catalog.js",
@@ -12077,7 +12097,7 @@
     {
       "path": "tests/core_upgrade_door.mjs",
       "mode": "0000644",
-      "sha256": "24f41ed750fc24003e31cd1e551256162119016ab299a94dda5e4c4fe030ff69"
+      "sha256": "8c8b03b8a3bb83b0c733726f3aa1256b6af5ac61c76e6af465fe2bc4f792ea6b"
     },
     {
       "path": "tests/core_upgrade_owner_door.mjs",
@@ -13744,6 +13764,11 @@
       "mode": "0000644",
       "sha256": "9e69603c05fae6c39e2db7336d6569327f17365ff46ef05b1074943dc34a49e7"
     },
+    {
+      "path": "tests/intent_retry.mjs",
+      "mode": "0000644",
+      "sha256": "750745cf866dcf1012f2677881738aa64ae2b8ceaed078ab688612a8b0d2208c"
+    },
     {
       "path": "tests/interaction_posture.mjs",
       "mode": "0000644",
@@ -14702,7 +14727,7 @@
     {
       "path": "tests/provision.mjs",
       "mode": "0000644",
-      "sha256": "6b4405a5a4e0263f321a49e04d25cad87a042d5d9a296ba65ff3d14a1941deb0"
+      "sha256": "46b2235ee186ba304c18f5cf3002a9e5311f7bda11fa230bbada8a7a43bec3cd"
     },
     {
       "path": "tests/provision_account_identity.mjs",
@@ -16214,6 +16239,11 @@
       "mode": "0000644",
       "sha256": "456b20334164a710f59116d6a6c4b85090dc0e65668284d0a736e6ee1df5dea6"
     },
+    {
+      "path": "tests/wedge_remedy.mjs",
+      "mode": "0000644",
+      "sha256": "9810e6b2cf5ec77fad5af12b195a3ff4919143b146ff43f106c67c4090c61c25"
+    },
     {
       "path": "tests/wizard_born_first_board.mjs",
       "mode": "0000644",

package/docs/module-api-changelog.md

@@ -2717,5 +2717,9 @@ is load-bearing: the script throws rather than guess if it is missing, and
         landed since 1.20.28 with no explicit bump. run 36775277990. (task 1002620)
   1.20.30 — CI auto-patch (publish-on-merge, ADR 0161): carrier for merges
         landed since 1.20.29 with no explicit bump. run 36777226204. (task 1002620)
+  1.20.31 — CI auto-patch (publish-on-merge, ADR 0161): carrier for merges
+        landed since 1.20.30 with no explicit bump. run 36779142919. (task 1002620)
+  1.20.32 — CI auto-patch (publish-on-merge, ADR 0161): carrier for merges
+        landed since 1.20.31 with no explicit bump. run 36781343271. (task 1002620)
 ---------------------------------------------------------------------------
 ```

package/docs/recipes/upgrading-the-core.md

@@ -179,6 +179,7 @@ running a go-live.
 
 ## Gotchas
 
+- **The Deploy door fixes three known snags by itself, once** (task 1004448, `scripts/gds/wedge-remedy.js`). When a core move through `/deploy` fails on a **dirty tree** (it commits the tracked changes inside the project), a **full disk** (it prunes that project's own nightly dumps down to the newest 7) or a **database behind its code** (it migrates that database with its own `PGDATABASE` before the code moves), the runner applies the fix, records a `core-upgrade-remedy` event on the project, and tries the move once more on the next tick. If the retry fails for the same reason, or the fix itself fails, the move is retired as `unknown` (`remedy_did_not_hold` / `remedy_failed`) for a person — never a second fix. The manual fixes below still apply to the unattended nightly sweep, which does not go through the door.
 - **`--registry` needs no credential at all** (task 1003878). `@bongos/core` is **public** on npm (task 1003872), so `bongos upgrade --registry` resolves it from the default registry like any other dependency — no token, no login, and no instance `.npmrc`. An install failure on this path is therefore a *real* failure (network, registry outage, or a version that was never published), never a missing credential.
   *History, so the symptom is recognisable on an old instance:* the core used to publish private, and the instance `.npmrc` read `_authToken=<redacted> npm gives that per-project key precedence, so with the env var unset it expanded EMPTY and **masked a perfectly valid literal token in `~/.npmrc`** — npm went anonymous, the private core 404'd, and the error blamed a credential the box actually held. Task 1002754 worked around it by resolving the token in-process; task 1003878 deleted the whole mechanism along with the `.npmrc` that caused it. An instance still carrying a committed `.npmrc` from a pre-1003878 scaffold can safely delete it.
 - **A green `publish` run is not evidence of a publish.** The publish-on-merge lane succeeds on its no-op paths too (cascade terminator, armed-gate). Read the run's **job summary**, which now states the verdict explicitly: `PUBLISHED @bongos/core@<v>` vs `NO-OP — nothing was published this run`.

package/modules/provisioning/core-upgrade.js

@@ -128,6 +128,16 @@ function coreUpgradeFault(inst, to, live = null) {
 async function enqueueCoreUpgrade(provisioning, db, instanceId, to, requestedBy, action = 'core-upgrade') {
   const { intent, created } = await provisioning.enqueueIntent(db, instanceId, action, requestedBy, to);
   if (created) return { intent, created: true };
+  // THE SAME MOVE, WAITING OUT A RETRY (task 1004447). A transient failure re-pends the
+  // intent with a not_before up to six hours away, and it keeps the single open slot all
+  // that time — so without this, the owner pressing Move again was told "a run is pending,
+  // try again later" about the very move they were asking for. Same action, same target,
+  // still pending: bring its next attempt forward instead. Anything else stays a conflict.
+  if (intent && intent.state === 'pending' && intent.action === action && String(intent.target_version || '') === String(to)
+    && intent.not_before && new Date(intent.not_before).getTime() > Date.now()) {
+    await db.query("UPDATE provisioning_intents SET not_before = NULL, updated_at = now() WHERE id = $1 AND state = 'pending'", [intent.id]);
+    return { intent: { ...intent, not_before: null }, created: false, retryNow: true };
+  }
   return { intent, created: false, conflict: 'open_intent' };
 }
 
@@ -336,7 +346,9 @@ function previewFromSnapshot(inst, snap, openIntent, lastMove) {
     // and a stale pass is worse than no answer because it reads as a guarantee.
     preflight: preflightForOffer(pf, served, recommended),
     preflight_at: pf && recommended && pf.target === recommended ? (snap.preflight_at || null) : null,
-    open_intent: openIntent ? { action: openIntent.action, state: openIntent.state, target_version: openIntent.target_version || null } : null,
+    // not_before: a failed move waiting out its retry backoff (task 1004447) — the page can
+    // say "trying again at …" instead of an unexplained "pending" for up to six hours.
+    open_intent: openIntent ? { action: openIntent.action, state: openIntent.state, target_version: openIntent.target_version || null, not_before: openIntent.not_before || null, last_error: openIntent.not_before && openIntent.last_error ? String(openIntent.last_error).slice(0, 600) : null } : null,
     // THE LAST MOVE, WHATEVER BECAME OF IT (task 1004145).
     //
     // `open_intent` is `state IN ('pending','running')` — by the index's own definition.

package/modules/provisioning/migrations/provisioning_031_intent_not_before.sql

@@ -0,0 +1,19 @@
+-- provisioning_031_intent_not_before.sql — a failed intent waits before its next attempt
+-- (task 1004447, goal 1000090 deploy self-heal).
+--
+-- WHY. A failed intent was re-pended with no delay, and the drain loop claims the oldest
+-- pending row — so one runner tick ran the same failing move up to MAX_INTENT_ATTEMPTS
+-- times back to back, restarting the service each time a pin had moved, and then gave up.
+-- A registry blip or a release still publishing needs minutes, not milliseconds.
+--
+-- not_before: the earliest time the runner may claim this row again. NULL = now (every row
+-- written before this column, and every fresh request). claimNextIntent skips a row whose
+-- not_before is still in the future; scripts/gds/provision.js sets it on each re-pend.
+--
+-- Additive and namespaced (ADR 0083): no down-migration, idempotent — safe to re-run.
+
+BEGIN;
+
+ALTER TABLE provisioning_intents ADD COLUMN IF NOT EXISTS not_before timestamptz;
+
+COMMIT;

package/modules/provisioning/migrations/provisioning_032_intent_remedied_reason.sql

@@ -0,0 +1,19 @@
+-- provisioning_032_intent_remedied_reason.sql — which known snag the runner already fixed
+-- for a core move (task 1004448, goal 1000090 deploy self-heal).
+--
+-- WHY. A core move that fails on a known snag (a dirty checkout, a full disk, a database
+-- behind its code) now has the fix applied by the runner and is tried once more
+-- (scripts/gds/wedge-remedy.js). "Once" has to survive between runner ticks, so the reason
+-- the fix was applied for is kept on the intent. A retry that fails on the SAME reason is
+-- retired for a person instead of fixed again — never a loop.
+--
+-- remedied_reason: a reason code from scripts/gds/upgrade-outcome.js (fixed vocabulary,
+-- never builder text). NULL = no fix has been applied for this intent.
+--
+-- Additive and namespaced (ADR 0083): no down-migration, idempotent — safe to re-run.
+
+BEGIN;
+
+ALTER TABLE provisioning_intents ADD COLUMN IF NOT EXISTS remedied_reason text;
+
+COMMIT;

package/modules/provisioning/provisioning.js

@@ -1254,7 +1254,7 @@ async function claimNextIntent(db) {
         SET state = 'running', attempts = attempts + 1, updated_at = now()
       WHERE id = (
         SELECT id FROM provisioning_intents
-         WHERE state = 'pending'
+         WHERE state = 'pending' AND (not_before IS NULL OR not_before <= now()) -- a re-pended failure waits out its backoff (task 1004447)
          ORDER BY created_at
          FOR UPDATE SKIP LOCKED
          LIMIT 1

package/package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "@bongos/core",
-  "version": "1.20.30",
+  "version": "1.20.32",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "@bongos/core",
-      "version": "1.20.30",
+      "version": "1.20.32",
       "license": "AGPL-3.0-or-later",
       "dependencies": {
         "express": "^4.21.2",

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@bongos/core",
-  "version": "1.20.30",
+  "version": "1.20.32",
   "description": "Cloud Bongos — the AI-first build platform core (GDS + platform surfaces + module system), installed as a versioned dependency (ADR 0108).",
   "license": "AGPL-3.0-or-later",
   "main": "src/platform-server.js",

package/release-notes.json

@@ -8252,5 +8252,17 @@
       "id": "1004446",
       "text": "When a software update fails, the system now records what kind of failure it was — a temporary glitch, a known snag, a rule only the owner can change, or something unknown — and tells the truth about it instead of always sayin"
     }
+  ],
+  "1.20.31": [
+    {
+      "id": "1004447",
+      "text": "A software update that fails for a temporary reason now waits and tries again later — after 15 minutes, then an hour, then six hours — instead of hammering the same failing step three times in a row. Failures that retrying can"
+    }
+  ],
+  "1.20.32": [
+    {
+      "id": "1004448",
+      "text": "When an update to a project gets stuck on one of three known problems (leftover file changes, a full disk, or a database that is behind), the system now fixes it by itself and tries again once. If that doesn't work, it stops a"
+    }
   ]
 }

package/scripts/gds/intent-retry.js

@@ -0,0 +1,52 @@
+'use strict';
+//
+// scripts/gds/intent-retry.js — when a failed provisioning intent is tried again, and when it
+// is retired instead (task 1004447, goal 1000090 deploy self-heal).
+//
+// WHY. The drain loop re-pended a failure with no delay and claims the oldest pending row, so
+// one tick ran the same failing move up to MAX_INTENT_ATTEMPTS times back to back — seconds
+// apart, restarting the service after every pin move — and then gave up. That is the wrong
+// answer for each kind of failure upgrade-outcome.js now names:
+//   transient       — the registry blipped, the release is still publishing: WAIT, then retry
+//                     (about 15 minutes, 1 hour, 6 hours), which is what a person would do.
+//   known_wedge, needs_decision, unknown — trying the same thing again cannot change the
+//                     answer, so the intent is RETIRED at once and the failure left standing
+//                     for the next step (a fix, the owner, a person) to act on.
+// A failure with no class (every action other than a core move) keeps its old attempt budget,
+// but each retry now waits DEFAULT_RETRY_DELAY_MS — at least the next tick, never the same one.
+//
+// PURE: no I/O. The drain loop (scripts/gds/provision.js cmdRunIntents) asks and obeys.
+
+const MINUTE = 60 * 1000;
+const TRANSIENT_BACKOFF_MS = Object.freeze([15 * MINUTE, 60 * MINUTE, 6 * 60 * MINUTE]);
+const DEFAULT_RETRY_DELAY_MS = MINUTE;
+
+/**
+ * What to do with an intent whose run just failed. PURE.
+ * @param {object} p
+ * @param {number} p.attempts     the attempt that just ran (claimNextIntent increments first)
+ * @param {object} [p.failure]    `{ class, reason }` from the result, when classified
+ * @param {boolean} [p.terminal]  the leg said it must not be retried unattended
+ * @param {number} p.maxAttempts  the runner's budget for unclassified failures
+ * @returns {{ action: 'retire' } | { action: 'retry', delayMs: number }}
+ */
+function retryDecision({ attempts, failure, terminal, maxAttempts }) {
+  if (terminal) return { action: 'retire' };
+  if (failure && failure.class) {
+    if (failure.class !== 'transient') return { action: 'retire' };
+    const i = Math.max(0, (attempts || 1) - 1);
+    return i < TRANSIENT_BACKOFF_MS.length ? { action: 'retry', delayMs: TRANSIENT_BACKOFF_MS[i] } : { action: 'retire' };
+  }
+  return (attempts || 0) >= maxAttempts ? { action: 'retire' } : { action: 'retry', delayMs: DEFAULT_RETRY_DELAY_MS };
+}
+
+/**
+ * The attempt ceiling checked BEFORE an intent runs. A row waiting out a transient backoff
+ * has one attempt per backoff step plus the first, so the pre-run guard must not retire it
+ * on the default budget before its last scheduled try. PURE.
+ */
+function maxAttemptsFor(intent, maxAttempts) {
+  return intent && intent.failure_class === 'transient' ? Math.max(maxAttempts, TRANSIENT_BACKOFF_MS.length + 1) : maxAttempts;
+}
+
+module.exports = { TRANSIENT_BACKOFF_MS, DEFAULT_RETRY_DELAY_MS, retryDecision, maxAttemptsFor };

package/scripts/gds/provision-core-upgrade.js

@@ -241,6 +241,16 @@ async function coreUpgradeInstance(inst, deps, intent) {
     log(`  already serving core ${to} — nothing to move`);
     return { ok: true, noop: true };
   }
+  // ---- 2b. is the database current for the code it runs NOW (task 1004448) -------------
+  // A database behind its own code is already failing every page that touches the missing
+  // tables (the 2026-07-04 shape), and moving the code on top of it tangles a catch-up
+  // migration into a pin move. So it is refused as a known snag, and wedge-remedy.js
+  // migrates that database and tries the move once more. An answer it cannot read proceeds.
+  if (apply) {
+    const probe = boxExec(versionCmd(inst), { allowFail: true });
+    const pending = parseSchemaPending(probe && !probe.softFailed ? (probe.stdout || '') : null);
+    if (pending > 0) return failed('schema_pending', `the project's database is ${pending} migration(s) behind the code it is running now — it has to catch up before the core moves`);
+  }
 
   // ---- 3. drive the proven path -----------------------------------------------------
   // Exactly the invocation docs/recipes/core-release-pipeline.md gate 3 documents, plus

package/scripts/gds/provision.js

@@ -46,6 +46,7 @@ const path = require('node:path');
 const cp = require('node:child_process');
 const crypto = require('node:crypto');
 
+const retry = require('./intent-retry.js'); // when a failed intent is tried again (task 1004447)
 const { CONFIG, MANIFEST_UA, softFailResult, MANIFEST_VERIFY_INTERVAL_MS, MANIFEST_VERIFY_TRIES, MAX_INTENT_ATTEMPTS, REPO_ROOT, coreVersionSafe, hasFlag, loadDeps, oauthSecret, provisionerBotEmail } = require('./provision-config.js');
 const { accountPreflight, appUser, alreadyScaffolded, buildCorePinRefreshCommit, coreCheckoutHasGit, dbCreateCmd, dbName, dbRoleCmd, dbRoleCmdShown, dbRoleGrantsCmd, ensureInstanceUserCmd, generateDbPassword, grantInstanceRepoReadCmd, instanceDbRole, instanceStateDir, instanceUser, deployKeyPath, deployKeyTitle, ensurePrivateRepoAccess, installedCorePackDir, installedCoreTarball, instanceInitSpec, instanceRepoRemote, migrateCmd, onboardMode, ownerLoginOf, parseTargetRef, planCorePinRefresh, refreshStandaloneCorePin, resolveOwnerGithubToken, resolveVendorableCoreTarball, safeVersionLabel, scaffoldStandaloneRepo, seedFirstVersionCmd, standaloneInstallCmd, standaloneMigrateCmd, standalonePullCmd, standaloneRegenDocsCmd, standaloneRoot } = require('./provision-repo.js');
 const { backupScriptPath, backupService, backupServicePath, backupTimer, backupTimerPath, backupUnitName, instanceManifestCmd, originEnvVarsFor, serviceUnit, serviceUnitPath, settingsConsumed, settingsEnvVarsFor, renamedEnvCarry, upsertEnvVars, webEnvBody, webEnvPath } = require('./provision-units.js');
@@ -1171,8 +1172,8 @@ async function cmdRunIntents(deps) {
     const intent = await provisioning.claimNextIntent(db);
     if (!intent) break;
     ran++;
-    if (intent.attempts > MAX_INTENT_ATTEMPTS) {
-      await provisioning.resolveIntent(db, intent.id, 'error', `exceeded ${MAX_INTENT_ATTEMPTS} attempts`);
+    if (intent.attempts > retry.maxAttemptsFor(intent, MAX_INTENT_ATTEMPTS)) { // a transient core move gets one try per backoff step (task 1004447)
+      await provisioning.resolveIntent(db, intent.id, 'error', `exceeded ${retry.maxAttemptsFor(intent, MAX_INTENT_ATTEMPTS)} attempts`);
       errors++; log(`  ✖ intent #${intent.id} retired (too many attempts)`); continue;
     }
     try {
@@ -1192,7 +1193,14 @@ async function cmdRunIntents(deps) {
       else if (intent.action === 'repo-private' || intent.action === 'render-standup') result = intent.action === 'repo-private' ? await require('./provision-repo-private.js').repoPrivateInstance(inst, deps) : await require('./provision-render.js').renderStandupInstance(inst, deps, intent); // wire the deploy key, THEN flip (task 1004193) · spend the borrowed Render key once (task 1004183); required here — this file is at the size ratchet
       else result = { ok: false, error: `unknown_action '${intent.action}' — runner core ${coreVersionSafe() || '?'} does not know it (runner too old?); refusing to guess` };
       if (result && result.ok === false) {
-        if (intent.attempts >= MAX_INTENT_ATTEMPTS || result.terminal) { // terminal: a leg that must not be retried unattended (the Render leg's key is already gone — task 1004183)
+        // WAIT OR RETIRE (task 1004447, intent-retry.js): a transient failure waits ~15m/1h/6h; a
+        // classified non-transient one is retired at once (retrying cannot change its answer);
+        // an unclassified one keeps its budget but never re-runs in the same tick.
+        const fix = await require('./wedge-remedy.js').remedyAfterFailure({ intent, inst, result, deps }); // a KNOWN snag: fix it, try once more (task 1004448)
+        if (fix.repended) { errors++; continue; }
+        Object.assign(result, { failure: fix.failure, error: fix.error });
+        const next = retry.retryDecision({ attempts: intent.attempts, failure: result.failure, terminal: result.terminal, maxAttempts: MAX_INTENT_ATTEMPTS });
+        if (next.action === 'retire') { // terminal: a leg that must not be retried unattended (the Render leg's key is already gone — task 1004183)
           await provisioning.resolveIntent(db, intent.id, 'error', result.error || 'op failed');
           // A retired intent stops silently otherwise (F19): record the reason
           // on the instance WITHOUT changing its lifecycle status — passing the
@@ -1202,7 +1210,7 @@ async function cmdRunIntents(deps) {
           await provisioning.setInstanceStatus(db, intent.instance_id, inst.status, { error_note: result.error || 'op failed', error_note_action: intent.action }).catch(() => {});
           if (result.failure) await recordFailureClass(db, intent.id, result.failure); // WHAT KIND of failure, beside the sentence (task 1004446)
         } else {
-          await db.query(`UPDATE provisioning_intents SET state='pending', last_error=$2, updated_at=now() WHERE id=$1`, [intent.id, result.error || 'op failed']);
+          await db.query(`UPDATE provisioning_intents SET state='pending', last_error=$2, not_before = now() + ($3 * interval '1 millisecond'), updated_at=now() WHERE id=$1`, [intent.id, result.error || 'op failed', next.delayMs]);
           if (result.failure) await recordFailureClass(db, intent.id, result.failure);
         }
         errors++;

package/scripts/gds/upgrade-outcome.js

@@ -19,7 +19,7 @@
 //
 // THE FOUR CLASSES the rest of the self-heal chain keys on:
 //   transient       — try again later and it may simply work (registry blip, not published yet)
-//   known_wedge     — a known snag with a known fix the runner may apply itself (dirty tree, full disk)
+//   known_wedge     — a known snag with a known fix the runner applies itself, once (wedge-remedy.js)
 //   needs_decision  — a rule the owner set is refusing it; only the owner can change that
 //   unknown         — anything else, and every rollback that failed; a person should look
 //
@@ -42,6 +42,7 @@ const REASON_CLASS = Object.freeze({
   bad_slug: 'unknown',
   installed_unreadable: 'unknown',
   served_mismatch: 'unknown',
+  schema_pending: 'known_wedge', // the database is behind the code it runs NOW (task 1004448)
   // upgrade.js refusals, before the pin moves
   dirty_tree: 'known_wedge',
   downgrade: 'needs_decision',
@@ -59,6 +60,9 @@ const REASON_CLASS = Object.freeze({
   migrate: 'unknown',
   restart: 'unknown',
   health: 'unknown',
+  // the runner's one automatic fix did not clear it, or itself failed (task 1004448)
+  remedy_did_not_hold: 'unknown',
+  remedy_failed: 'unknown',
   // the one that must never be retried unattended
   rollback_failed: 'unknown',
 });

package/scripts/gds/wedge-remedy.js

@@ -0,0 +1,157 @@
+'use strict';
+//
+// scripts/gds/wedge-remedy.js — a core move that failed on a KNOWN snag fixes the snag and
+// tries once more (task 1004448, goal 1000090 deploy self-heal).
+//
+// WHY. upgrade-outcome.js names three failures that are not a fault in the release and not a
+// decision the owner has to make, but a mess on the box with one known fix — the fixes the
+// owner has been doing over SSH (docs/recipes/upgrading-the-core.md, Gotchas):
+//   dirty_tree      — the project's checkout has uncommitted changes to tracked files (the
+//                     upgrade's own rewrites, left behind by an earlier move). Fix: commit them
+//                     inside the project.
+//   disk_full       — the move ran out of disk. Fix: prune this project's OWN nightly
+//                     pg_dumps down to the newest few (the only thing on the box this runner
+//                     knows is safe to delete — they are copies, and the newest are kept).
+//   schema_pending  — the project's database is behind the code it is running NOW. Fix:
+//                     migrate that database, with its own PGDATABASE, before the code moves.
+// intent-retry.js retires every known_wedge at once, so on its own the failure only waited
+// for a person. This file runs the fix, records it as an event on the project, and re-pends
+// the move for ONE more try.
+//
+// AT MOST ONCE PER FAILURE. The reason a fix was applied for is stored on the intent
+// (provisioning_intents.remedied_reason, migration provisioning_032). If the retry fails on
+// the SAME reason, the fix did not hold: the move is retired as `unknown` /
+// remedy_did_not_hold, which is the class that reaches a person — never a second fix, never
+// a loop. A fix that itself fails is retired the same way (remedy_failed). A different
+// known reason on the retry gets its own one fix (a disk full after a dirty tree is a new
+// snag, not the same one twice); the attempt budget still bounds the whole run.
+//
+// 'tagged-not-published' needs nothing here: it is `not_published`, a transient failure that
+// intent-retry.js already waits out.
+//
+// planRemedy is PURE (the command, never run). remedyAfterFailure does the I/O through the
+// runner's own exec and db, so every fix is testable with an injected exec.
+
+const { CONFIG } = require('./provision-config.js');
+const { dbName, selfInstanceRoot, standaloneRoot } = require('./provision-repo.js');
+const { migratePlan, CORE_MIGRATE_SCRIPT } = require('./upgrade-migrate.js');
+const { isValidSlug } = require('../../modules/provisioning/provisioning.js');
+const outcome = require('./upgrade-outcome.js');
+const { DEFAULT_RETRY_DELAY_MS } = require('./intent-retry.js');
+
+const REMEDY_REASONS = Object.freeze(['dirty_tree', 'disk_full', 'schema_pending']);
+// The newest dumps a disk-full prune always leaves behind: a week of nightlies. The nightly
+// job's own retention is 30 days; this only ever cuts below that when the disk is full.
+const DUMP_KEEP_FLOOR = 7;
+const REMEDY_EVENT = 'core-upgrade-remedy';
+// A value that reaches a shell string below. Narrower than what Postgres or a path allow, on
+// purpose: anything outside it is refused, never quoted.
+const SHELL_SAFE_RE = /^[A-Za-z0-9._/-]{1,200}$/;
+const DB_NAME_RE = /^[A-Za-z0-9_-]{1,63}$/;
+const ACCOUNT_RE = /^[a-z_][a-z0-9_-]{0,31}$/;
+
+/**
+ * The fix for one known snag, as a command to run on the box. PURE.
+ * @param {object} inst   the provisioning_instances row
+ * @param {string} reason the failure's reason code
+ * @param {object} [o]    { privileged, runAs, instanceDir, selfRoot, backupDir, migrate }
+ * @returns {{ reason, cmd, cwd, env, describe } | { refused: string } | null}  null: no known fix
+ */
+function planRemedy(inst, reason, o = {}) {
+  if (!REMEDY_REASONS.includes(reason)) return null;
+  if (!inst || !isValidSlug(inst.slug)) return { refused: 'the project name is not a valid slug' };
+  const dir = o.instanceDir || (inst.hosting_shape === 'cloud-host' ? standaloneRoot(inst) : (o.selfRoot || selfInstanceRoot)());
+  if (!SHELL_SAFE_RE.test(dir)) return { refused: 'the project folder is not a path this runner will put in a command' };
+  const db = dbName(inst);
+  if (!DB_NAME_RE.test(db)) return { refused: 'the project database name is not one this runner will put in a command' };
+
+  if (reason === 'dirty_tree') {
+    // TRACKED files only (`add -u`): the snag is the upgrade's own rewrites of tracked files.
+    // A new untracked file is not swept into the owner's history on a guess — if that is the
+    // dirt, the retry fails the same way and a person looks. Runs as the runner, which owns
+    // the checkout (pushUpgradePin's shape), with hooks off and a runner identity so the
+    // commit says who made it.
+    const g = `git -C ${dir} -c core.hooksPath=/dev/null -c user.name=bongos-runner -c user.email=runner@bongos.invalid`;
+    return {
+      reason, cwd: null, env: null,
+      cmd: `${g} add -u && ${g} commit -m "chore: commit changes left in the working tree (automatic fix before a core move)"`,
+      describe: `committed the uncommitted changes in ${dir} so the core move could run`,
+    };
+  }
+  if (reason === 'disk_full') {
+    const backupDir = o.backupDir || CONFIG.backupDir;
+    if (!SHELL_SAFE_RE.test(backupDir)) return { refused: 'the backup folder is not a path this runner will put in a command' };
+    // EXACTLY db-backup-nightly.sh's name, so this never touches a sibling project's dumps
+    // (`test` must not prune `test-nk`'s — task 1003948). The timestamp sorts by name, so
+    // `sort | head -n -K` drops all but the newest K. Nothing is deleted when K or fewer exist.
+    const D = '[0-9]';
+    const glob = `${db}-${D}${D}${D}${D}-${D}${D}-${D}${D}T${D}${D}${D}${D}${D}${D}Z.sql.gz`;
+    return {
+      reason, cwd: null, env: null,
+      cmd: `find ${backupDir} -maxdepth 1 -type f -name '${glob}' | sort | head -n -${DUMP_KEEP_FLOOR} | xargs -r rm -f --`,
+      describe: `pruned ${db}'s old database backups in ${backupDir}, keeping the newest ${DUMP_KEEP_FLOOR}`,
+    };
+  }
+  // schema_pending — the project's OWN migrate command (a declared script wins, else the
+  // core's migrate.sh: upgrade-migrate.js), aimed at its OWN database. migrate.sh defaults to
+  // production when PGDATABASE is unset, so it is always set, never inherited.
+  const plan = (o.migrate || migratePlan)(dir);
+  const run = plan.via === 'npm-script' ? 'npm run migrate' : `bash ${CORE_MIGRATE_SCRIPT}`;
+  const initCwd = plan.via === 'npm-script' ? '' : `INIT_CWD=${dir} `;
+  if (o.privileged) {
+    if (!ACCOUNT_RE.test(String(o.runAs || ''))) return { refused: 'no account to run the migration as' };
+    return { reason, cwd: dir, env: null, cmd: `sudo -u ${o.runAs} env PGDATABASE=${db} ${initCwd}${run}`, describe: `migrated ${db} to match the code it is running` };
+  }
+  return { reason, cwd: dir, env: { PGDATABASE: db, ...(initCwd ? { INIT_CWD: dir } : {}) }, cmd: run, describe: `migrated ${db} to match the code it is running` };
+}
+
+// The account the project runs as, probed the way the move itself probes (task 1004128).
+function runAsFor(inst, deps) {
+  try { return require('./provision-core-upgrade.js').resolveRunAs(inst, deps).runAs; } catch { return null; }
+}
+
+/**
+ * After a core move failed: apply the known fix, record it, and re-pend the move once.
+ *
+ * Returns `{ repended: true }` when the move will be tried again, otherwise
+ * `{ repended: false, failure, error }` — the failure the drain loop should RETIRE with,
+ * which is the original one when there is no fix for it, and `unknown` when a fix was
+ * already spent on this reason or the fix itself failed.
+ */
+async function remedyAfterFailure({ intent, inst, result, deps }) {
+  const failure = result && result.failure;
+  const reason = failure && failure.reason;
+  const none = { repended: false, failure, error: result && result.error };
+  if (!intent || intent.action !== 'core-upgrade' || !failure || failure.class !== 'known_wedge') return none;
+  const { db, provisioning, log = () => {}, apply } = deps;
+  if (!apply) return none;
+  if (intent.remedied_reason === reason) {
+    return {
+      repended: false, failure: outcome.classifyFailure('remedy_did_not_hold'),
+      error: `${result.error || 'core move failed'} — the automatic fix for this (${reason}) was already tried once and did not clear it, so a person needs to look`,
+    };
+  }
+  const boxExec = inst.hosting_shape === 'cloud-host' ? (deps.controlExec || deps.exec) : deps.exec;
+  const plan = planRemedy(inst, reason, { privileged: deps.privileged, runAs: reason === 'schema_pending' && deps.privileged ? runAsFor(inst, deps) : null, selfRoot: deps.selfInstanceRoot, migrate: deps.migratePlan });
+  if (!plan) return none;
+  const giveUp = async (why) => {
+    await provisioning.recordEvent(db, { instanceId: inst.id, ownerBuilderId: inst.owner_builder_id, event: REMEDY_EVENT, detail: `could not fix ${reason} automatically: ${why}`, actor: 'runner' }).catch(() => {});
+    return { repended: false, failure: outcome.classifyFailure('remedy_failed'), error: `${result.error || 'core move failed'} — the automatic fix for ${reason} did not work (${why}), so a person needs to look` };
+  };
+  if (plan.refused) return giveUp(plan.refused);
+  log(`  [remedy] ${reason}: ${plan.cmd}`);
+  const r = boxExec(plan.cmd, { allowFail: true, ...(plan.cwd ? { cwd: plan.cwd } : {}), ...(plan.env ? { env: plan.env } : {}) });
+  if (!r || r.softFailed === true || r.ok === false) {
+    const said = String((r && (r.stderr || r.stdout || r.error)) || 'the command failed').trim().split('\n').pop().slice(0, 160);
+    return giveUp(said);
+  }
+  await provisioning.recordEvent(db, { instanceId: inst.id, ownerBuilderId: inst.owner_builder_id, event: REMEDY_EVENT, detail: `${plan.describe}; trying the move to ${intent.target_version} once more`, actor: 'runner' }).catch(() => {});
+  await db.query(
+    `UPDATE provisioning_intents SET state='pending', last_error=$2, remedied_reason=$3, failure_class=$4, failure_reason=$5,
+            not_before = now() + ($6 * interval '1 millisecond'), updated_at=now() WHERE id=$1`,
+    [intent.id, result.error || 'core move failed', reason, failure.class, failure.reason, DEFAULT_RETRY_DELAY_MS]);
+  log(`  ✓ fixed ${reason} on ${inst.slug}; the move to ${intent.target_version} will be tried once more`);
+  return { repended: true };
+}
+
+module.exports = { REMEDY_REASONS, DUMP_KEEP_FLOOR, REMEDY_EVENT, planRemedy, remedyAfterFailure };

package/src/module-api.js

@@ -75,7 +75,7 @@ const { responsibilityFor, ROLE_RESPONSIBILITIES } = require('./role-responsibil
 // MAJOR (see allowBoxScope below): passes the request through untouched.
 function deprecatedNoopMiddleware(_req, _res, next) { next(); }
 
-const CORE_VERSION = '1.20.30'; // CI auto-patch carrier (ADR 0161); changelog: docs/module-api-changelog.md
+const CORE_VERSION = '1.20.32'; // CI auto-patch carrier (ADR 0161); changelog: docs/module-api-changelog.md
 
 // A namespaced logger so a module's log lines are attributable + consistent.
 // Usage: const log = api.logger('discord'); log.info('mounted');

package/tests/core_upgrade_door.mjs

@@ -431,7 +431,7 @@ await t('schema_pending passes through as a count, and UNKNOWN stays null rather
 await t('an open intent is surfaced with its target, so the page can say what is already running', () => {
   const p = CU.previewFromSnapshot(previewInst, { served_version: '1.19.758', available: [], recommended: null },
     { action: 'core-upgrade', state: 'running', target_version: '1.19.762' });
-  assert.deepEqual(p.open_intent, { action: 'core-upgrade', state: 'running', target_version: '1.19.762' });
+  assert.deepEqual(p.open_intent, { action: 'core-upgrade', state: 'running', target_version: '1.19.762', not_before: null, last_error: null });
 });
 
 console.log('\nthe deploy page is gated on the same atom as its routes:');
@@ -973,5 +973,39 @@ await t('a failed move carries its class to the page; a move that worked carries
   assert.equal(CU.previewFromSnapshot(ACTIVE, null, null, old).last_move.failure, null, 'a row from before the columns');
 });
 
+console.log('\na move waiting out its retry (task 1004447):');
+
+const waiting = (over = {}) => ({ id: 5, action: 'core-upgrade', state: 'pending', target_version: '1.20.1', not_before: new Date(Date.now() + 3600e3).toISOString(), ...over });
+const enqueueAgainst = async (open, to = '1.20.1') => {
+  const writes = [];
+  const provisioning = { enqueueIntent: async () => ({ intent: open, created: false }) };
+  const db = { query: async (sql, params) => { writes.push({ sql, params }); return { rows: [] }; } };
+  return { r: await CU.enqueueCoreUpgrade(provisioning, db, 7, to, 'api:archon:42'), writes };
+};
+
+await t('pressing Move for the SAME waiting move brings its retry forward instead of refusing', async () => {
+  const { r, writes } = await enqueueAgainst(waiting());
+  assert.equal(r.retryNow, true);
+  assert.ok(!r.conflict);
+  assert.match(writes[0].sql, /SET not_before = NULL/);
+  assert.deepEqual(writes[0].params, [5]);
+});
+
+await t('a DIFFERENT target, a running move, or one not waiting is still a conflict', async () => {
+  for (const open of [waiting({ target_version: '1.20.2' }), waiting({ state: 'running' }), waiting({ not_before: null }), waiting({ action: 'restart' })]) {
+    const { r, writes } = await enqueueAgainst(open);
+    assert.equal(r.conflict, 'open_intent');
+    assert.equal(writes.length, 0);
+  }
+});
+
+await t('the preview says when a waiting move will try again, and why it is waiting', () => {
+  const p = CU.previewFromSnapshot(ACTIVE, null, waiting({ last_error: 'core 1.20.1 is not published yet' }), null);
+  assert.ok(p.open_intent.not_before);
+  assert.match(p.open_intent.last_error, /not published yet/);
+  assert.equal(CU.previewFromSnapshot(ACTIVE, null, waiting({ not_before: null, last_error: 'x' }), null).open_intent.last_error, null,
+    'an ordinary pending intent shows no error');
+});
+
 console.log(`\ncore_upgrade_door: ${passed} passed, ${failed} failed`);
 if (failed) process.exit(1);

package/tests/intent_retry.mjs

@@ -0,0 +1,57 @@
+// tests/intent_retry.mjs — a failed intent waits, or is retired, by what kind of failure it
+// was (task 1004447, goal 1000090 deploy self-heal).
+//
+// WHAT IS AT RISK. One runner tick used to run the same failing core move up to three times
+// back to back, restarting the service each time, then give up. These cases pin the policy
+// that replaced it; tests/provision.mjs drives the drain loop with it.
+//
+// Run:  node tests/intent_retry.mjs
+
+import { strict as assert } from 'node:assert';
+import { retryDecision, maxAttemptsFor, TRANSIENT_BACKOFF_MS, DEFAULT_RETRY_DELAY_MS } from '../scripts/gds/intent-retry.js';
+
+let passed = 0;
+let failed = 0;
+const t = async (name, fn) => {
+  try { await fn(); passed += 1; console.log(`  PASS  ${name}`); }
+  catch (e) { failed += 1; console.log(`  FAIL  ${name}\n        ${e.message}`); }
+};
+const MIN = 60 * 1000;
+const transient = { class: 'transient', reason: 'not_published' };
+
+await t('a transient failure waits about 15 minutes, then 1 hour, then 6 hours', () => {
+  assert.deepEqual([...TRANSIENT_BACKOFF_MS], [15 * MIN, 60 * MIN, 360 * MIN]);
+  for (const [attempts, ms] of [[1, 15 * MIN], [2, 60 * MIN], [3, 360 * MIN]]) {
+    assert.deepEqual(retryDecision({ attempts, failure: transient, maxAttempts: 3 }), { action: 'retry', delayMs: ms });
+  }
+});
+
+await t('and is retired after its last scheduled try', () => {
+  assert.deepEqual(retryDecision({ attempts: 4, failure: transient, maxAttempts: 3 }), { action: 'retire' });
+});
+
+await t('every NON-transient class is retired at once — retrying cannot change the answer', () => {
+  for (const c of ['known_wedge', 'needs_decision', 'unknown']) {
+    assert.deepEqual(retryDecision({ attempts: 1, failure: { class: c, reason: 'x' }, maxAttempts: 3 }), { action: 'retire' }, c);
+  }
+});
+
+await t('terminal always retires, even a transient class', () => {
+  assert.deepEqual(retryDecision({ attempts: 1, failure: transient, terminal: true, maxAttempts: 3 }), { action: 'retire' });
+});
+
+await t('an UNCLASSIFIED failure keeps its budget, but never retries in the same tick', () => {
+  assert.deepEqual(retryDecision({ attempts: 1, maxAttempts: 3 }), { action: 'retry', delayMs: DEFAULT_RETRY_DELAY_MS });
+  assert.ok(DEFAULT_RETRY_DELAY_MS >= MIN, 'at least a minute: the runner ticks about every two');
+  assert.deepEqual(retryDecision({ attempts: 3, maxAttempts: 3 }), { action: 'retire' });
+});
+
+await t('the pre-run guard gives a transient row room for its last scheduled try', () => {
+  assert.equal(maxAttemptsFor({ failure_class: 'transient' }, 3), 4);
+  assert.equal(maxAttemptsFor({ failure_class: 'unknown' }, 3), 3);
+  assert.equal(maxAttemptsFor({}, 3), 3);
+  assert.equal(maxAttemptsFor({ failure_class: 'transient' }, 9), 9, 'never lowers a configured budget');
+});
+
+console.log(`\nintent_retry: ${passed} passed, ${failed} failed`);
+if (failed) process.exit(1);

package/tests/provision.mjs

@@ -2564,38 +2564,63 @@ await ta('F19: a NON-final failure re-pends the intent and writes nothing on the
   assert.equal(statusWrites.length, 0, 'no instance write until the intent actually retires');
 });
 
-// A failed core move stores WHAT KIND of failure beside its sentence (task 1004446).
-const classifiedRun = async (attempts) => {
-  let claims = 0; const resolved = []; const classWrites = [];
+// A failed core move stores WHAT KIND of failure beside its sentence (task 1004446), and
+// waits or retires by that kind (task 1004447). The queue here is STATEFUL — a fake
+// claimNextIntent that honours not_before the way the real query does — so "one tick runs a
+// failing move at most once" is measured, not assumed.
+const queueRun = async ({ attempts = 0, transient = false } = {}) => {
+  const row = { id: 11, instance_id: 18, action: 'core-upgrade', target_version: '1.20.1', attempts, state: 'pending', not_before: null, failure_class: attempts && transient ? 'transient' : null }; // a re-tried row carries the class its last attempt wrote
+  let runs = 0; const resolved = []; const classWrites = []; const repends = [];
   const deps = {
     apply: true, log: () => {},
-    db: { query: async (sql, params) => { if (/SET failure_class/.test(sql)) classWrites.push(params); return { rows: [] }; } },
+    db: { query: async (sql, params) => {
+      if (/SET failure_class/.test(sql)) { classWrites.push(params); row.failure_class = params[1]; }
+      if (/SET state='pending'/.test(sql)) { repends.push(params); row.state = 'pending'; row.not_before = new Date(Date.now() + params[2]); }
+      return { rows: [] };
+    } },
     provisioning: {
-      claimNextIntent: async () => (claims++ === 0
-        ? { id: 11, instance_id: 18, action: 'core-upgrade', target_version: '1.20.1', attempts } : null),
-      // 'dedicated' is a shape the runner cannot move — a classified refusal, no exec needed.
-      getInstanceById: async () => ({ ...activeStandalone, hosting_shape: 'dedicated' }),
-      resolveIntent: async (_db, id, state) => { resolved.push({ id, state }); },
+      claimNextIntent: async () => {
+        if (row.state !== 'pending' || (row.not_before && row.not_before > new Date())) return null;
+        row.state = 'running'; row.attempts += 1; runs += 1; return { ...row };
+      },
+      // 'dedicated' is a shape the runner cannot move (needs_decision); the control plane with
+      // an unreadable registry is the transient case. Neither needs an exec.
+      getInstanceById: async () => (transient ? { ...activeStandalone, hosting_shape: 'control-plane' } : { ...activeStandalone, hosting_shape: 'dedicated' }),
+      resolveIntent: async (_db, id, state) => { resolved.push({ id, state }); row.state = state; },
       setInstanceStatus: async () => {}, recordEvent: async () => {},
       claimNextOAuthManifest: async () => null, resolveOAuthManifest: async () => {},
     },
     exec: () => ({ ok: true }), writeFile: () => ({ ok: true }),
+    readInstalledCoreVersion: () => '1.19.1081', selfInstanceRoot: () => '/srv/cb',
+    updateChannel: { DEFAULT_CHANNEL: 'patch', normalizeChannel: (c) => c || 'patch', channelAllows: () => true, listAvailableVersions: () => ({ ok: false, versions: [] }) },
     checkDnsTokenScope: async () => ({ reachable: true, zone: 'z' }), checkCaddyWiring: () => null,
   };
   await P.cmdRunIntents(deps);
-  return { resolved, classWrites };
+  return { runs, resolved, classWrites, repends, row };
 };
 
-await ta('a re-pended core move records its failure class and reason', async () => {
-  const { resolved, classWrites } = await classifiedRun(1);
-  assert.equal(resolved.length, 0);
+await ta('a needs-decision core move is RETIRED at once with its class — retrying cannot change it', async () => {
+  const { runs, resolved, classWrites } = await queueRun();
+  assert.equal(runs, 1);
+  assert.equal(resolved[0].state, 'error');
   assert.deepEqual(classWrites, [[11, 'needs_decision', 'shape_not_automated']]);
 });
 
-await ta('a RETIRED core move records its failure class and reason too', async () => {
-  const { resolved, classWrites } = await classifiedRun(3); // = MAX_INTENT_ATTEMPTS: the last real attempt, retired after it runs
+await ta('a TRANSIENT core move runs ONCE per tick and waits ~15 minutes before its next try', async () => {
+  const { runs, resolved, repends, classWrites } = await queueRun({ transient: true });
+  assert.equal(runs, 1, 'the same failing move is not re-run inside one tick');
+  assert.equal(resolved.length, 0, 'not retired — it waits');
+  assert.equal(repends.length, 1);
+  assert.equal(repends[0][2], 15 * 60 * 1000, 'not_before = now + 15 minutes');
+  assert.deepEqual(classWrites, [[11, 'transient', 'registry_unreadable']]);
+});
+
+await ta('a transient move on its LAST scheduled try (the 6-hour one) is retired after it', async () => {
+  const { runs, resolved } = await queueRun({ transient: true, attempts: 3 });
+  // attempt 4 RUNS (the pre-run guard allows it for a transient row) and is then retired.
+  assert.equal(runs, 1, 'the 6-hour try actually ran — not retired unrun by the pre-run guard');
+  assert.equal(resolved.length, 1);
   assert.equal(resolved[0].state, 'error');
-  assert.deepEqual(classWrites, [[11, 'needs_decision', 'shape_not_automated']]);
 });
 
 console.log('\nreview-panel hardening (task 1002720 — the loopback/latch/birth-note seams):');

package/tests/wedge_remedy.mjs

@@ -0,0 +1,268 @@
+// tests/wedge_remedy.mjs — a core move that failed on a KNOWN snag fixes the snag and tries
+// once more, and never twice (task 1004448, goal 1000090 deploy self-heal).
+//
+// WHAT IS AT RISK. Each fix runs a command on a live project's box: a commit in the owner's
+// checkout, a delete among database backups, a migration against a database. The cases pin
+// WHAT each command is (and what it refuses to build), that it runs at most once per
+// failure, and that a fix which did not hold reaches a person instead of looping. The
+// disk-full prune is also run for real against a temp folder, because a `find` pattern is
+// the kind of thing that reads right and deletes a sibling project's dumps.
+//
+// Run:  node tests/wedge_remedy.mjs
+
+import { strict as assert } from 'node:assert';
+import { createRequire } from 'node:module';
+import { spawnSync } from 'node:child_process';
+import fs from 'node:fs';
+import os from 'node:os';
+import path from 'node:path';
+
+process.env.PROVISION_APP_USER = process.env.PROVISION_APP_USER || 'lars'; // appUser() refuses a Windows account name
+const require = createRequire(import.meta.url);
+const W = require('../scripts/gds/wedge-remedy.js');
+const R = require('../scripts/gds/provision-core-upgrade.js');
+const P = require('../scripts/gds/provision.js');
+const { REASON_CLASS } = require('../scripts/gds/upgrade-outcome.js');
+const { DEFAULT_RETRY_DELAY_MS } = require('../scripts/gds/intent-retry.js');
+
+let passed = 0;
+let failed = 0;
+const t = async (name, fn) => {
+  try { await fn(); passed += 1; console.log(`  PASS  ${name}`); }
+  catch (e) { failed += 1; console.log(`  FAIL  ${name}\n        ${e.message}`); }
+};
+
+const inst = (over = {}) => ({ id: 7, slug: 'charter', db_name: 'charter', port: 3010, status: 'active', hosting_shape: 'co-tenant', owner_builder_id: 1, ...over });
+const intent = (over = {}) => ({ id: 99, action: 'core-upgrade', target_version: '1.20.31', attempts: 1, remedied_reason: null, ...over });
+const wedge = (reason) => ({ ok: false, error: `core upgrade to 1.20.31 was refused (${reason})`, failure: { class: 'known_wedge', reason } });
+const selfRoot = () => '/srv/charter';
+
+function harness({ execFails = false } = {}) {
+  const ran = []; const events = []; const queries = [];
+  const deps = {
+    apply: true, privileged: false, log: () => {}, selfInstanceRoot: selfRoot,
+    exec: (cmd, opts) => { ran.push({ cmd, ...opts }); return execFails ? { ok: false, softFailed: true, stderr: 'fatal: nope' } : { ok: true, stdout: '' }; },
+    db: { query: async (sql, params) => { queries.push({ sql, params }); return { rows: [] }; } },
+    provisioning: { recordEvent: async (_db, e) => { events.push(e); } },
+    migratePlan: () => ({ via: 'npm-script' }),
+  };
+  return { deps, ran, events, queries };
+}
+
+console.log('\nthe fix for each known snag (planRemedy):');
+
+await t('every reason with a fix is a known_wedge in the classifier — and back', () => {
+  assert.deepEqual([...W.REMEDY_REASONS].sort(), ['dirty_tree', 'disk_full', 'schema_pending']);
+  for (const r of W.REMEDY_REASONS) assert.equal(REASON_CLASS[r], 'known_wedge', r);
+  for (const [r, c] of Object.entries(REASON_CLASS)) if (c === 'known_wedge') assert.ok(W.REMEDY_REASONS.includes(r), `${r} is known_wedge but has no fix — it would be retired with nothing done`);
+  assert.equal(REASON_CLASS.remedy_did_not_hold, 'unknown', 'a fix that did not hold reaches a person');
+  assert.equal(REASON_CLASS.remedy_failed, 'unknown');
+});
+
+await t('dirty tree: commits TRACKED changes inside the project, hooks off, as the runner', () => {
+  const p = W.planRemedy(inst(), 'dirty_tree', { selfRoot });
+  assert.match(p.cmd, /^git -C \/srv\/charter -c core\.hooksPath=\/dev\/null .* add -u && git -C \/srv\/charter .* commit -m /);
+  assert.doesNotMatch(p.cmd, /add -A|add \./, 'an untracked file is never swept into the owner\'s history on a guess');
+  assert.doesNotMatch(p.cmd, /sudo/, 'the runner owns the checkout (pushUpgradePin\'s shape)');
+});
+
+await t('a cloud-host project is fixed in ITS folder, not the control plane\'s', () => {
+  const p = W.planRemedy(inst({ hosting_shape: 'cloud-host' }), 'dirty_tree', { selfRoot });
+  assert.match(p.cmd, /git -C \/\S+\/charter /);
+  assert.doesNotMatch(p.cmd, /\/srv\/charter/);
+});
+
+await t('disk full: prunes only THIS database\'s dumps, keeping the newest week', () => {
+  const p = W.planRemedy(inst(), 'disk_full', { selfRoot, backupDir: '/var/backups/gds' });
+  assert.match(p.cmd, /^find \/var\/backups\/gds -maxdepth 1 -type f -name 'charter-\[0-9\]/);
+  assert.match(p.cmd, new RegExp(`head -n -${W.DUMP_KEEP_FLOOR} \\| xargs -r rm -f --$`));
+  assert.equal(W.DUMP_KEEP_FLOOR, 7);
+});
+
+await t('schema pending: migrates the project\'s OWN database, PGDATABASE always set', () => {
+  const plain = W.planRemedy(inst(), 'schema_pending', { selfRoot, migrate: () => ({ via: 'npm-script' }) });
+  assert.equal(plain.cmd, 'npm run migrate');
+  assert.equal(plain.cwd, '/srv/charter');
+  assert.deepEqual(plain.env, { PGDATABASE: 'charter' });
+  const adopted = W.planRemedy(inst(), 'schema_pending', { selfRoot, migrate: () => ({ via: 'core-script' }) });
+  assert.equal(adopted.cmd, 'bash node_modules/@bongos/core/scripts/migrate.sh');
+  assert.deepEqual(adopted.env, { PGDATABASE: 'charter', INIT_CWD: '/srv/charter' });
+  const priv = W.planRemedy(inst(), 'schema_pending', { selfRoot, privileged: true, runAs: 'bongos-charter', migrate: () => ({ via: 'npm-script' }) });
+  assert.equal(priv.cmd, 'sudo -u bongos-charter env PGDATABASE=charter npm run migrate', 'sudo resets env, so PGDATABASE rides the command');
+  assert.ok(W.planRemedy(inst(), 'schema_pending', { selfRoot, privileged: true, runAs: null }).refused);
+});
+
+await t('no command is built from a name that is not shell-safe', () => {
+  assert.ok(W.planRemedy(inst({ slug: 'a;rm -rf' }), 'dirty_tree', { selfRoot }).refused);
+  assert.ok(W.planRemedy(inst({ db_name: 'x; drop' }), 'disk_full', { selfRoot }).refused);
+  assert.ok(W.planRemedy(inst(), 'dirty_tree', { selfRoot: () => '/srv/a b' }).refused);
+  assert.ok(W.planRemedy(inst(), 'disk_full', { selfRoot, backupDir: '/tmp/$(x)' }).refused);
+});
+
+await t('a reason with no fix gets no command', () => {
+  for (const r of ['artist_gate', 'downgrade', 'health', 'not_published', 'nope']) assert.equal(W.planRemedy(inst(), r, { selfRoot }), null, r);
+});
+
+console.log('\nthe prune, run for real:');
+
+const bash = spawnSync('bash', ['-c', 'echo ok'], { encoding: 'utf8' });
+await t('keeps the newest 7 of this project\'s dumps and never touches a sibling\'s', () => {
+  if (bash.status !== 0) { console.log('        (skipped: no bash here)'); return; }
+  const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'wedge-prune-'));
+  try {
+    const mine = []; for (let d = 1; d <= 10; d++) mine.push(`charter-2026-09-${String(d).padStart(2, '0')}T030000Z.sql.gz`);
+    const others = ['charter-nk-2026-09-01T030000Z.sql.gz', 'test-2026-09-01T030000Z.sql.gz', 'charter-notes.txt'];
+    for (const f of [...mine, ...others]) fs.writeFileSync(path.join(dir, f), 'x');
+    const p = W.planRemedy(inst(), 'disk_full', { selfRoot, backupDir: '.' }); // run IN the folder: a temp path can hold a `~`
+    const r = spawnSync('bash', ['-c', p.cmd], { encoding: 'utf8', cwd: dir });
+    assert.equal(r.status, 0, r.stderr);
+    const left = fs.readdirSync(dir).sort();
+    assert.deepEqual(left, [...others, ...mine.slice(3)].sort());
+    // Nothing is deleted when the floor is not exceeded.
+    const again = spawnSync('bash', ['-c', p.cmd], { encoding: 'utf8', cwd: dir });
+    assert.equal(again.status, 0);
+    assert.equal(fs.readdirSync(dir).length, left.length);
+  } finally { fs.rmSync(dir, { recursive: true, force: true }); }
+});
+
+console.log('\nonce per failure (remedyAfterFailure):');
+
+for (const reason of W.REMEDY_REASONS) {
+  await t(`${reason}: runs the fix, records an event, re-pends the move with the reason stored`, async () => {
+    const h = harness();
+    const r = await W.remedyAfterFailure({ intent: intent(), inst: inst(), result: wedge(reason), deps: h.deps });
+    assert.deepEqual(r, { repended: true });
+    assert.equal(h.ran.length, 1);
+    assert.equal(h.events.length, 1);
+    assert.equal(h.events[0].event, W.REMEDY_EVENT);
+    assert.match(h.events[0].detail, /trying the move to 1\.20\.31 once more/);
+    const up = h.queries.find((q) => /SET state='pending'/.test(q.sql));
+    assert.ok(up, 're-pended');
+    assert.match(up.sql, /remedied_reason=\$3/);
+    assert.deepEqual(up.params, [99, wedge(reason).error, reason, 'known_wedge', reason, DEFAULT_RETRY_DELAY_MS]);
+  });
+}
+
+await t('the same reason again after a fix: no second fix — retired as unknown for a person', async () => {
+  const h = harness();
+  const r = await W.remedyAfterFailure({ intent: intent({ remedied_reason: 'dirty_tree' }), inst: inst(), result: wedge('dirty_tree'), deps: h.deps });
+  assert.equal(r.repended, false);
+  assert.deepEqual(r.failure, { class: 'unknown', reason: 'remedy_did_not_hold' });
+  assert.match(r.error, /already tried once/);
+  assert.equal(h.ran.length, 0, 'nothing ran');
+  assert.equal(h.queries.length, 0);
+});
+
+await t('a DIFFERENT known snag on the retry gets its own one fix', async () => {
+  const h = harness();
+  const r = await W.remedyAfterFailure({ intent: intent({ remedied_reason: 'dirty_tree' }), inst: inst(), result: wedge('disk_full'), deps: h.deps });
+  assert.equal(r.repended, true);
+});
+
+await t('a fix that fails is recorded and retired as unknown — never re-pended', async () => {
+  const h = harness({ execFails: true });
+  const r = await W.remedyAfterFailure({ intent: intent(), inst: inst(), result: wedge('dirty_tree'), deps: h.deps });
+  assert.equal(r.repended, false);
+  assert.deepEqual(r.failure, { class: 'unknown', reason: 'remedy_failed' });
+  assert.match(r.error, /did not work \(fatal: nope\)/);
+  assert.equal(h.events.length, 1);
+  assert.match(h.events[0].detail, /could not fix dirty_tree/);
+  assert.equal(h.queries.length, 0);
+});
+
+await t('leaves everything else alone: other classes, other actions, and a dry run', async () => {
+  for (const [i, res] of [[intent(), { ok: false, error: 'x', failure: { class: 'transient', reason: 'not_published' } }],
+    [intent(), { ok: false, error: 'x', failure: { class: 'needs_decision', reason: 'artist_gate' } }],
+    [intent(), { ok: false, error: 'x' }],
+    [intent({ action: 'restart' }), wedge('dirty_tree')]]) {
+    const h = harness();
+    const r = await W.remedyAfterFailure({ intent: i, inst: inst(), result: res, deps: h.deps });
+    assert.equal(r.repended, false);
+    assert.deepEqual(r.failure, res.failure);
+    assert.equal(h.ran.length, 0);
+  }
+  const h = harness(); h.deps.apply = false;
+  assert.equal((await W.remedyAfterFailure({ intent: intent(), inst: inst(), result: wedge('dirty_tree'), deps: h.deps })).repended, false);
+  assert.equal(h.ran.length, 0);
+});
+
+console.log('\nthe move refuses on a database behind its code:');
+
+await t('schemaPending > 0 on the box refuses as schema_pending BEFORE the upgrade runs', async () => {
+  const ran = [];
+  const deps = {
+    apply: true, privileged: false, log: () => {}, sleep: async () => {},
+    exec: (cmd) => { ran.push(cmd); return { ok: true, stdout: /\/version/.test(cmd) ? JSON.stringify({ coreVersion: '1.20.30', schemaPending: 2 }) : '' }; },
+    db: { query: async () => ({ rows: [] }) }, provisioning: { recordEvent: async () => {} },
+    readInstalledCoreVersion: () => '1.20.30', selfInstanceRoot: () => '/srv/cb',
+    updateChannel: { DEFAULT_CHANNEL: 'patch', normalizeChannel: (c) => c || 'patch', channelAllows: () => true, listAvailableVersions: () => ({ ok: true, versions: ['1.20.30', '1.20.31'] }) },
+  };
+  const r = await R.coreUpgradeInstance(inst({ hosting_shape: 'control-plane', slug: 'cloudbongos' }), deps, intent());
+  assert.equal(r.ok, false);
+  assert.deepEqual(r.failure, { class: 'known_wedge', reason: 'schema_pending' });
+  assert.match(r.error, /2 migration\(s\) behind/);
+  assert.ok(!ran.some((c) => /upgrade\.js/.test(c)), 'the pin never moved');
+});
+
+console.log('\nthe drain loop, end to end:');
+
+// A stateful fake queue, the shape tests/provision.mjs uses: claimNextIntent honours
+// not_before, so "fixed, then tried once more on a LATER tick" is measured.
+async function drain({ row, reply }) {
+  const ran = []; const events = []; const resolved = []; let runs = 0;
+  const deps = {
+    apply: true, privileged: false, log: () => {}, sleep: async () => {},
+    db: { query: async (sql, params) => {
+      if (/SET state='pending'/.test(sql)) { row.state = 'pending'; row.not_before = new Date(Date.now() + params[params.length - 1]); if (/remedied_reason/.test(sql)) row.remedied_reason = params[2]; }
+      if (/SET failure_class/.test(sql)) { row.failure_class = params[1]; row.failure_reason = params[2]; }
+      return { rows: [] };
+    } },
+    provisioning: {
+      claimNextIntent: async () => {
+        if (row.state !== 'pending' || (row.not_before && row.not_before > new Date())) return null;
+        row.state = 'running'; row.attempts += 1; runs += 1; return { ...row };
+      },
+      getInstanceById: async () => inst({ hosting_shape: 'control-plane', slug: 'cloudbongos', db_name: 'cloudbongos' }),
+      resolveIntent: async (_db, id, state, note) => { resolved.push({ id, state, note }); row.state = state; },
+      setInstanceStatus: async () => {}, recordEvent: async (_db, e) => { events.push(e); },
+      claimNextOAuthManifest: async () => null, resolveOAuthManifest: async () => {},
+    },
+    exec: (cmd, opts) => { ran.push(cmd); return reply(cmd, opts); },
+    writeFile: () => ({ ok: true }),
+    readInstalledCoreVersion: () => '1.20.30', selfInstanceRoot: () => '/srv/cb',
+    updateChannel: { DEFAULT_CHANNEL: 'patch', normalizeChannel: (c) => c || 'patch', channelAllows: () => true, listAvailableVersions: () => ({ ok: true, versions: ['1.20.30', '1.20.31'] }) },
+    checkDnsTokenScope: async () => ({ reachable: true, zone: 'z' }), checkCaddyWiring: () => null,
+  };
+  await P.cmdRunIntents(deps);
+  return { ran, events, resolved, runs, row };
+}
+const dirtyUpgrade = (cmd) => {
+  if (/upgrade\.js/.test(cmd)) { const e = new Error('Command failed'); e.stderr = '✖ working tree not clean — commit/stash first\nBONGOS_UPGRADE_OUTCOME {"stage":"refused","reason":"dirty_tree"}\n'; throw e; }
+  return { ok: true, stdout: /\/version/.test(cmd) ? JSON.stringify({ coreVersion: '1.20.30', schemaPending: 0 }) : '' };
+};
+
+await t('a dirty tree is committed, the move waits for the next tick, and is NOT retired', async () => {
+  const row = { id: 11, instance_id: 7, action: 'core-upgrade', target_version: '1.20.31', attempts: 0, state: 'pending', not_before: null, remedied_reason: null };
+  const d = await drain({ row, reply: dirtyUpgrade });
+  assert.equal(d.runs, 1, 'the retry is not run in the same tick');
+  assert.equal(d.resolved.length, 0, 'not retired');
+  assert.equal(d.row.state, 'pending');
+  assert.equal(d.row.remedied_reason, 'dirty_tree');
+  assert.ok(d.ran.some((c) => /add -u && .* commit/.test(c)), 'the commit ran');
+  assert.ok(d.events.some((e) => e.event === W.REMEDY_EVENT));
+});
+
+await t('the retry fails dirty AGAIN: retired as unknown/remedy_did_not_hold, no second commit', async () => {
+  const row = { id: 11, instance_id: 7, action: 'core-upgrade', target_version: '1.20.31', attempts: 1, state: 'pending', not_before: null, remedied_reason: 'dirty_tree' };
+  const d = await drain({ row, reply: dirtyUpgrade });
+  assert.equal(d.runs, 1);
+  assert.equal(d.resolved.length, 1);
+  assert.equal(d.resolved[0].state, 'error');
+  assert.match(d.resolved[0].note, /already tried once/);
+  assert.equal(d.row.failure_class, 'unknown');
+  assert.equal(d.row.failure_reason, 'remedy_did_not_hold');
+  assert.ok(!d.ran.some((c) => /commit -m/.test(c)), 'no second fix');
+});
+
+console.log(`\nwedge_remedy: ${passed} passed, ${failed} failed`);
+if (failed) process.exit(1);