@bongos/core 1.20.29 → 1.20.30
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.bongos-core.json +34 -19
- package/docs/module-api-changelog.md +2 -0
- package/modules/provisioning/core-upgrade.js +5 -0
- package/modules/provisioning/migrations/provisioning_030_intent_failure_class.sql +28 -0
- package/package-lock.json +2 -2
- package/package.json +1 -1
- package/release-notes.json +6 -0
- package/scripts/gds/provision-core-upgrade.js +23 -9
- package/scripts/gds/provision.js +11 -0
- package/scripts/gds/upgrade-outcome.js +155 -0
- package/scripts/gds/upgrade.js +1 -1
- package/src/module-api.js +1 -1
- package/tests/core_upgrade_door.mjs +9 -0
- package/tests/core_upgrade_runner.mjs +66 -0
- package/tests/provision.mjs +34 -0
- package/tests/upgrade_outcome.mjs +129 -0
package/.bongos-core.json
CHANGED
|
@@ -2,22 +2,22 @@
|
|
|
2
2
|
"artifact": "bongos-core",
|
|
3
3
|
"manifest_schema": 1,
|
|
4
4
|
"generator": "scripts/gds/package-core.js",
|
|
5
|
-
"core_version": "1.20.
|
|
6
|
-
"core_contract": "1.20.
|
|
7
|
-
"source_commit": "
|
|
5
|
+
"core_version": "1.20.30",
|
|
6
|
+
"core_contract": "1.20.30",
|
|
7
|
+
"source_commit": "b56a05d974ae5208394507078971bf8e22f415f3",
|
|
8
8
|
"source_ref": "HEAD",
|
|
9
|
-
"built_at": "2026-09-
|
|
9
|
+
"built_at": "2026-09-30T21:06:55.806Z",
|
|
10
10
|
"redaction": {
|
|
11
11
|
"model": "docs-redacted+functional-verbatim",
|
|
12
12
|
"docs_redacted": 562,
|
|
13
13
|
"agent_docs_stubbed": 25,
|
|
14
|
-
"functional_verbatim":
|
|
14
|
+
"functional_verbatim": 2663,
|
|
15
15
|
"rules": 3,
|
|
16
16
|
"gate_literals": 3,
|
|
17
17
|
"gate": "passed"
|
|
18
18
|
},
|
|
19
|
-
"file_count":
|
|
20
|
-
"tree_sha256": "
|
|
19
|
+
"file_count": 3251,
|
|
20
|
+
"tree_sha256": "3b45f4c67302ad870157396c60f693712b46bcdc86a9236e964c914418657c35",
|
|
21
21
|
"files": [
|
|
22
22
|
{
|
|
23
23
|
"path": ".claude/skills/ask-for-help/SKILL.md",
|
|
@@ -2802,7 +2802,7 @@
|
|
|
2802
2802
|
{
|
|
2803
2803
|
"path": "docs/module-api-changelog.md",
|
|
2804
2804
|
"mode": "0000644",
|
|
2805
|
-
"sha256": "
|
|
2805
|
+
"sha256": "fca494d7f518bc42742fde6c5ba2e45cfbf0605d5a7a79024bba190ec4c3c1da"
|
|
2806
2806
|
},
|
|
2807
2807
|
{
|
|
2808
2808
|
"path": "docs/modules-contract.md",
|
|
@@ -7282,7 +7282,7 @@
|
|
|
7282
7282
|
{
|
|
7283
7283
|
"path": "modules/provisioning/core-upgrade.js",
|
|
7284
7284
|
"mode": "0000644",
|
|
7285
|
-
"sha256": "
|
|
7285
|
+
"sha256": "7604ea4c137c5c1cedca41329cfd6fce49458d454f2e956944f7fb8d0c4ab10a"
|
|
7286
7286
|
},
|
|
7287
7287
|
{
|
|
7288
7288
|
"path": "modules/provisioning/cost-ledger.js",
|
|
@@ -7459,6 +7459,11 @@
|
|
|
7459
7459
|
"mode": "0000644",
|
|
7460
7460
|
"sha256": "b667ec5165e611bd6f0d86b09465c5d9e632de853becce139bac49f5a88e5c1e"
|
|
7461
7461
|
},
|
|
7462
|
+
{
|
|
7463
|
+
"path": "modules/provisioning/migrations/provisioning_030_intent_failure_class.sql",
|
|
7464
|
+
"mode": "0000644",
|
|
7465
|
+
"sha256": "7287181b4b42d3e391cd97e19ffa5fbc76026e81ad6f54648eb12a69dfea55a1"
|
|
7466
|
+
},
|
|
7462
7467
|
{
|
|
7463
7468
|
"path": "modules/provisioning/module.json",
|
|
7464
7469
|
"mode": "0000644",
|
|
@@ -8977,12 +8982,12 @@
|
|
|
8977
8982
|
{
|
|
8978
8983
|
"path": "package-lock.json",
|
|
8979
8984
|
"mode": "0000644",
|
|
8980
|
-
"sha256": "
|
|
8985
|
+
"sha256": "872b4c6ad6eacb09cbcd5698c892fc798ce6e632b505304b5665c7f04ea99325"
|
|
8981
8986
|
},
|
|
8982
8987
|
{
|
|
8983
8988
|
"path": "package.json",
|
|
8984
8989
|
"mode": "0000644",
|
|
8985
|
-
"sha256": "
|
|
8990
|
+
"sha256": "6c29a554265b6c3e6fa7f29fbf9b334c3a73e30b240b393498b89304876b5495"
|
|
8986
8991
|
},
|
|
8987
8992
|
{
|
|
8988
8993
|
"path": "public-docs/index.html",
|
|
@@ -9002,7 +9007,7 @@
|
|
|
9002
9007
|
{
|
|
9003
9008
|
"path": "release-notes.json",
|
|
9004
9009
|
"mode": "0000644",
|
|
9005
|
-
"sha256": "
|
|
9010
|
+
"sha256": "1ce998432070110f9c3a9b128145bf8d44c032aa2ff55437207ecaf5efe8943a"
|
|
9006
9011
|
},
|
|
9007
9012
|
{
|
|
9008
9013
|
"path": "scripts/bongos-mcp.js",
|
|
@@ -9962,7 +9967,7 @@
|
|
|
9962
9967
|
{
|
|
9963
9968
|
"path": "scripts/gds/provision-core-upgrade.js",
|
|
9964
9969
|
"mode": "0000644",
|
|
9965
|
-
"sha256": "
|
|
9970
|
+
"sha256": "58599968201f98723a531c19f77985c0b26b9afa6d6448ae4803074497a6f2ad"
|
|
9966
9971
|
},
|
|
9967
9972
|
{
|
|
9968
9973
|
"path": "scripts/gds/provision-disconnect.js",
|
|
@@ -10017,7 +10022,7 @@
|
|
|
10017
10022
|
{
|
|
10018
10023
|
"path": "scripts/gds/provision.js",
|
|
10019
10024
|
"mode": "0000644",
|
|
10020
|
-
"sha256": "
|
|
10025
|
+
"sha256": "0f7a9cd81a78ef3ed30a40e9f048f8905b6a88e99a930f7cbdcdcd5c1661f71d"
|
|
10021
10026
|
},
|
|
10022
10027
|
{
|
|
10023
10028
|
"path": "scripts/gds/publish-credential-check.js",
|
|
@@ -10629,10 +10634,15 @@
|
|
|
10629
10634
|
"mode": "0000644",
|
|
10630
10635
|
"sha256": "1f0a8d52622328e29bd3bdbe0402f680f425480e37e29ee42ceee907368710af"
|
|
10631
10636
|
},
|
|
10637
|
+
{
|
|
10638
|
+
"path": "scripts/gds/upgrade-outcome.js",
|
|
10639
|
+
"mode": "0000644",
|
|
10640
|
+
"sha256": "a4f05b5e1042cf3d8c7096ef413b47fd039a02b78f2e4dbefb5bd002973784cb"
|
|
10641
|
+
},
|
|
10632
10642
|
{
|
|
10633
10643
|
"path": "scripts/gds/upgrade.js",
|
|
10634
10644
|
"mode": "0000644",
|
|
10635
|
-
"sha256": "
|
|
10645
|
+
"sha256": "399c2225543fc0d37505bca9cd96e33cf5598aa1f364a458d5b5efb3a588ddb3"
|
|
10636
10646
|
},
|
|
10637
10647
|
{
|
|
10638
10648
|
"path": "scripts/gds/validate-design.js",
|
|
@@ -11122,7 +11132,7 @@
|
|
|
11122
11132
|
{
|
|
11123
11133
|
"path": "src/module-api.js",
|
|
11124
11134
|
"mode": "0000644",
|
|
11125
|
-
"sha256": "
|
|
11135
|
+
"sha256": "4002317dabfa7084c1b541e2a6a605197ce8c1e45df90f81bfa2ef186b0d6776"
|
|
11126
11136
|
},
|
|
11127
11137
|
{
|
|
11128
11138
|
"path": "src/module-loader/catalog.js",
|
|
@@ -12067,7 +12077,7 @@
|
|
|
12067
12077
|
{
|
|
12068
12078
|
"path": "tests/core_upgrade_door.mjs",
|
|
12069
12079
|
"mode": "0000644",
|
|
12070
|
-
"sha256": "
|
|
12080
|
+
"sha256": "24f41ed750fc24003e31cd1e551256162119016ab299a94dda5e4c4fe030ff69"
|
|
12071
12081
|
},
|
|
12072
12082
|
{
|
|
12073
12083
|
"path": "tests/core_upgrade_owner_door.mjs",
|
|
@@ -12077,7 +12087,7 @@
|
|
|
12077
12087
|
{
|
|
12078
12088
|
"path": "tests/core_upgrade_runner.mjs",
|
|
12079
12089
|
"mode": "0000644",
|
|
12080
|
-
"sha256": "
|
|
12090
|
+
"sha256": "525e38bbafc2dd1419f3e41131dd89c6adc4f063f8585b193e2be72801bede5a"
|
|
12081
12091
|
},
|
|
12082
12092
|
{
|
|
12083
12093
|
"path": "tests/cosmos_token_contrast.mjs",
|
|
@@ -14692,7 +14702,7 @@
|
|
|
14692
14702
|
{
|
|
14693
14703
|
"path": "tests/provision.mjs",
|
|
14694
14704
|
"mode": "0000644",
|
|
14695
|
-
"sha256": "
|
|
14705
|
+
"sha256": "6b4405a5a4e0263f321a49e04d25cad87a042d5d9a296ba65ff3d14a1941deb0"
|
|
14696
14706
|
},
|
|
14697
14707
|
{
|
|
14698
14708
|
"path": "tests/provision_account_identity.mjs",
|
|
@@ -16094,6 +16104,11 @@
|
|
|
16094
16104
|
"mode": "0000644",
|
|
16095
16105
|
"sha256": "d3f5b941b89ede78ebb3cd7af43467363e5d6c6d35c7ad26a2977683ecf869e7"
|
|
16096
16106
|
},
|
|
16107
|
+
{
|
|
16108
|
+
"path": "tests/upgrade_outcome.mjs",
|
|
16109
|
+
"mode": "0000644",
|
|
16110
|
+
"sha256": "cec960e84573efe77003beb324c0679900f321249148418e7a04daf7a7a40215"
|
|
16111
|
+
},
|
|
16097
16112
|
{
|
|
16098
16113
|
"path": "tests/upgrade_persist_pin.mjs",
|
|
16099
16114
|
"mode": "0000644",
|
|
@@ -2715,5 +2715,7 @@ is load-bearing: the script throws rather than guess if it is missing, and
|
|
|
2715
2715
|
landed since 1.20.27 with no explicit bump. run 36772266063. (task 1002620)
|
|
2716
2716
|
1.20.29 — CI auto-patch (publish-on-merge, ADR 0161): carrier for merges
|
|
2717
2717
|
landed since 1.20.28 with no explicit bump. run 36775277990. (task 1002620)
|
|
2718
|
+
1.20.30 — CI auto-patch (publish-on-merge, ADR 0161): carrier for merges
|
|
2719
|
+
landed since 1.20.29 with no explicit bump. run 36777226204. (task 1002620)
|
|
2718
2720
|
---------------------------------------------------------------------------
|
|
2719
2721
|
```
|
|
@@ -361,6 +361,11 @@ function previewFromSnapshot(inst, snap, openIntent, lastMove) {
|
|
|
361
361
|
// `git status` (the dirty-tree one lists every file). Never set on a DONE move:
|
|
362
362
|
// a done row's last_error is the note below, and a move that worked is not an error.
|
|
363
363
|
error: lastMove.state !== 'done' && lastMove.last_error ? String(lastMove.last_error).slice(0, 600) : null,
|
|
364
|
+
// WHAT KIND of failure it was (task 1004446, migration provisioning_030): the class the
|
|
365
|
+
// page and the self-heal steps key on, and its fixed reason code. null on a move that
|
|
366
|
+
// worked, and on one resolved before the columns existed.
|
|
367
|
+
failure: lastMove.state !== 'done' && lastMove.failure_class
|
|
368
|
+
? { class: lastMove.failure_class, reason: lastMove.failure_reason || null } : null,
|
|
364
369
|
// It LANDED, but GitHub did not get the new pin (task 1004065) — parseRepoNote.
|
|
365
370
|
warning: lastMove.state === 'done' ? parseRepoNote(lastMove.last_error) : null,
|
|
366
371
|
} : null,
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
-- provisioning_030_intent_failure_class.sql — WHAT KIND of failure stopped an intent
|
|
2
|
+
-- (task 1004446, goal 1000090 deploy self-heal).
|
|
3
|
+
--
|
|
4
|
+
-- WHY. A failed core move was one sentence in last_error, and every non-zero exit read
|
|
5
|
+
-- "failed and was rolled back" — a refusal that changed nothing, a real rollback, and a
|
|
6
|
+
-- rollback that itself failed were indistinguishable to the runner and to the page. These
|
|
7
|
+
-- two columns carry scripts/gds/upgrade-outcome.js's classification beside the sentence:
|
|
8
|
+
--
|
|
9
|
+
-- failure_class transient | known_wedge | needs_decision | unknown — what the runner
|
|
10
|
+
-- may do about it (retry later, fix it itself, ask the owner, ask a person)
|
|
11
|
+
-- failure_reason a fixed reason code (dirty_tree, outside_channel, rollback_failed, …) —
|
|
12
|
+
-- vocabulary only, never free text, so it is safe to index and to show
|
|
13
|
+
--
|
|
14
|
+
-- NULL on both means "not a classified failure": a done intent, a pending one, or a row
|
|
15
|
+
-- written before this migration. Written by the drain loop (scripts/gds/provision.js).
|
|
16
|
+
--
|
|
17
|
+
-- Additive and namespaced (ADR 0083): no down-migration, idempotent — safe to re-run.
|
|
18
|
+
|
|
19
|
+
BEGIN;
|
|
20
|
+
|
|
21
|
+
ALTER TABLE provisioning_intents ADD COLUMN IF NOT EXISTS failure_class text;
|
|
22
|
+
ALTER TABLE provisioning_intents ADD COLUMN IF NOT EXISTS failure_reason text;
|
|
23
|
+
|
|
24
|
+
ALTER TABLE provisioning_intents DROP CONSTRAINT IF EXISTS provisioning_intents_failure_class_check;
|
|
25
|
+
ALTER TABLE provisioning_intents ADD CONSTRAINT provisioning_intents_failure_class_check
|
|
26
|
+
CHECK (failure_class IS NULL OR failure_class IN ('transient', 'known_wedge', 'needs_decision', 'unknown'));
|
|
27
|
+
|
|
28
|
+
COMMIT;
|
package/package-lock.json
CHANGED
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@bongos/core",
|
|
3
|
-
"version": "1.20.
|
|
3
|
+
"version": "1.20.30",
|
|
4
4
|
"lockfileVersion": 3,
|
|
5
5
|
"requires": true,
|
|
6
6
|
"packages": {
|
|
7
7
|
"": {
|
|
8
8
|
"name": "@bongos/core",
|
|
9
|
-
"version": "1.20.
|
|
9
|
+
"version": "1.20.30",
|
|
10
10
|
"license": "AGPL-3.0-or-later",
|
|
11
11
|
"dependencies": {
|
|
12
12
|
"express": "^4.21.2",
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@bongos/core",
|
|
3
|
-
"version": "1.20.
|
|
3
|
+
"version": "1.20.30",
|
|
4
4
|
"description": "Cloud Bongos — the AI-first build platform core (GDS + platform surfaces + module system), installed as a versioned dependency (ADR 0108).",
|
|
5
5
|
"license": "AGPL-3.0-or-later",
|
|
6
6
|
"main": "src/platform-server.js",
|
package/release-notes.json
CHANGED
|
@@ -8246,5 +8246,11 @@
|
|
|
8246
8246
|
"id": "1004445",
|
|
8247
8247
|
"text": "Each project now has its own update rule. A project can be allowed to take bigger step-up updates (like 1.19 to 1.20) instead of being stuck on small fixes only, and its owner can change the rule."
|
|
8248
8248
|
}
|
|
8249
|
+
],
|
|
8250
|
+
"1.20.30": [
|
|
8251
|
+
{
|
|
8252
|
+
"id": "1004446",
|
|
8253
|
+
"text": "When a software update fails, the system now records what kind of failure it was — a temporary glitch, a known snag, a rule only the owner can change, or something unknown — and tells the truth about it instead of always sayin"
|
|
8254
|
+
}
|
|
8249
8255
|
]
|
|
8250
8256
|
}
|
|
@@ -38,6 +38,7 @@ const { appUser, dbName, instanceUser, parseTargetRef, resolveOwnerGithubToken,
|
|
|
38
38
|
const { ensurePinWriteKey, hasOwnAccount, pinKeyPath, pinSshCommand } = require('./provision-pin-key.js');
|
|
39
39
|
const { isValidSlug } = require('../../modules/provisioning/provisioning.js');
|
|
40
40
|
const { RELEASE_NOTES_FILE, RELEASE_NOTES_MAX_BYTES, parseChangeRecords, notesBetween } = require('./release-notes.js');
|
|
41
|
+
const outcome = require('./upgrade-outcome.js'); // how far a failed move got, as a class (task 1004446)
|
|
41
42
|
|
|
42
43
|
// The shapes whose box this runner can actually reach. Mirrors UPGRADE_SHAPES in the
|
|
43
44
|
// web-tier half, and deliberately EXCLUDES 'dedicated': a dedicated droplet is stood up
|
|
@@ -150,6 +151,12 @@ function instanceChannel(inst, channelMod) {
|
|
|
150
151
|
return channelMod.normalizeChannel(inst && inst.update_channel != null ? inst.update_channel : channelMod.DEFAULT_CHANNEL);
|
|
151
152
|
}
|
|
152
153
|
|
|
154
|
+
// A classified refusal (task 1004446): the drain loop stores `failure` beside last_error,
|
|
155
|
+
// so the rest of the self-heal chain acts on a class, not on a sentence. PURE.
|
|
156
|
+
function failed(reason, error, extra = {}) {
|
|
157
|
+
return { ok: false, error, failure: outcome.classifyFailure(reason), ...extra };
|
|
158
|
+
}
|
|
159
|
+
|
|
153
160
|
// The builder id a queued move is attributed to, from the actor the web tier recorded.
|
|
154
161
|
// PURE. Two doors reach the move (ADR 0339): an Archon (`api:archon:<id>`) and, since
|
|
155
162
|
// task 1004174, the project's own owner (`api:owner:<id>`). Both carry the id, because
|
|
@@ -189,7 +196,7 @@ async function coreUpgradeInstance(inst, deps, intent) {
|
|
|
189
196
|
// anything. The guard's real job is the OTHER actions on the self row, enforced at
|
|
190
197
|
// enqueueIntent where every intent passes.
|
|
191
198
|
if (!UPGRADE_SHAPES.has(inst.hosting_shape)) {
|
|
192
|
-
return
|
|
199
|
+
return failed('shape_not_automated', `core-upgrade is not automated for a '${inst.hosting_shape}' instance — whoever runs that machine has to upgrade it there`);
|
|
193
200
|
}
|
|
194
201
|
if (inst.status !== 'active') {
|
|
195
202
|
// Not an error: a teardown or a stand-up that raced the request must not burn the
|
|
@@ -198,7 +205,7 @@ async function coreUpgradeInstance(inst, deps, intent) {
|
|
|
198
205
|
return { ok: true, noop: true };
|
|
199
206
|
}
|
|
200
207
|
if (!to) {
|
|
201
|
-
return
|
|
208
|
+
return failed('no_target', 'core-upgrade intent carries no target_version — refusing to guess which version was meant');
|
|
202
209
|
}
|
|
203
210
|
// THE SLUG IS INTERPOLATED INTO A SHELL COMMAND BELOW, so it is re-checked HERE
|
|
204
211
|
// rather than trusted. `isValidSlug` is enforced on the create route, but the
|
|
@@ -209,7 +216,7 @@ async function coreUpgradeInstance(inst, deps, intent) {
|
|
|
209
216
|
// two interpolated values are already constrained — the version by the column's own
|
|
210
217
|
// CHECK and by VERSION_RE, the actor by the digits-only extraction below.
|
|
211
218
|
if (!isValidSlug(inst.slug)) {
|
|
212
|
-
return
|
|
219
|
+
return failed('bad_slug', `instance slug '${inst.slug}' is not a valid slug — refusing to build a command from it`);
|
|
213
220
|
}
|
|
214
221
|
|
|
215
222
|
// ---- 2. is the target real, and permitted -----------------------------------------
|
|
@@ -227,7 +234,7 @@ async function coreUpgradeInstance(inst, deps, intent) {
|
|
|
227
234
|
const listed = channelMod.listAvailableVersions({ cwd: instanceDir });
|
|
228
235
|
const available = (listed && listed.versions) || [];
|
|
229
236
|
const fault = targetFault(to, available, installed, channel, channelMod.channelAllows);
|
|
230
|
-
if (fault) return
|
|
237
|
+
if (fault) return failed(outcome.runnerReason(fault), fault);
|
|
231
238
|
if (installed && installed === to) {
|
|
232
239
|
// `bongos upgrade` would no-op on this too, but saying so here keeps a re-drained
|
|
233
240
|
// intent from restarting a healthy service to land exactly where it already is.
|
|
@@ -265,10 +272,17 @@ async function coreUpgradeInstance(inst, deps, intent) {
|
|
|
265
272
|
log(` [upgrade] (in ${runDir}) ${cmd}`);
|
|
266
273
|
boxExec(cmd, { cwd: runDir });
|
|
267
274
|
} catch (e) {
|
|
268
|
-
//
|
|
269
|
-
//
|
|
270
|
-
//
|
|
271
|
-
|
|
275
|
+
// HOW FAR IT GOT, from upgrade.js's own outcome line (task 1004446) — never assumed.
|
|
276
|
+
// It used to say "failed and was rolled back" for every exit, including refusals that
|
|
277
|
+
// fire before the pin moves (nothing changed) and rollbacks that themselves failed
|
|
278
|
+
// (the project may be down). ok:false, never a throw: the drain loop's catch marks the
|
|
279
|
+
// INSTANCE broken, which is only ever right for the rollback_failed case, and that one
|
|
280
|
+
// is reported (and retired, never retried unattended) rather than guessed at here.
|
|
281
|
+
const text = `${(e && e.stderr) || ''}\n${(e && e.message) || e}`;
|
|
282
|
+
const o = outcome.parseOutcomeLine(text);
|
|
283
|
+
const said = (/✖ ([^\n]+)/.exec(text) || [])[1] || String((e && e.message) || e).split('\n')[0];
|
|
284
|
+
const reason = o ? o.reason : 'unknown';
|
|
285
|
+
return failed(reason, outcome.describeUpgradeFailure(to, o, said), { stage: o ? o.stage : null, terminal: reason === 'rollback_failed' });
|
|
272
286
|
}
|
|
273
287
|
|
|
274
288
|
// ---- 4. read back what the BOX serves (ADR 0293 D6) --------------------------------
|
|
@@ -287,7 +301,7 @@ async function coreUpgradeInstance(inst, deps, intent) {
|
|
|
287
301
|
if (served && served !== to) {
|
|
288
302
|
// upgrade.js's own --version-url gate should have caught and reverted this, so
|
|
289
303
|
// reaching here means the two disagree. Report what was SEEN rather than a verdict.
|
|
290
|
-
return
|
|
304
|
+
return failed('served_mismatch', `the box is serving core ${served}, not the requested ${to} — the move did not take`);
|
|
291
305
|
}
|
|
292
306
|
|
|
293
307
|
// ---- 5. the pin reaches the project's own repo, pushed from HERE (task 1004065) ----
|
package/scripts/gds/provision.js
CHANGED
|
@@ -1200,8 +1200,10 @@ async function cmdRunIntents(deps) {
|
|
|
1200
1200
|
// instance is never marked broken over a domain hiccup, but the card
|
|
1201
1201
|
// can say why the work stopped.
|
|
1202
1202
|
await provisioning.setInstanceStatus(db, intent.instance_id, inst.status, { error_note: result.error || 'op failed', error_note_action: intent.action }).catch(() => {});
|
|
1203
|
+
if (result.failure) await recordFailureClass(db, intent.id, result.failure); // WHAT KIND of failure, beside the sentence (task 1004446)
|
|
1203
1204
|
} else {
|
|
1204
1205
|
await db.query(`UPDATE provisioning_intents SET state='pending', last_error=$2, updated_at=now() WHERE id=$1`, [intent.id, result.error || 'op failed']);
|
|
1206
|
+
if (result.failure) await recordFailureClass(db, intent.id, result.failure);
|
|
1205
1207
|
}
|
|
1206
1208
|
errors++;
|
|
1207
1209
|
} else { await provisioning.resolveIntent(db, intent.id, 'done', (result && result.doneNote) || null); ok++; } // done WITH a note: it landed, and this is what it could not finish (task 1004065)
|
|
@@ -1293,6 +1295,15 @@ box (task 2074). Set PROVISION_CONTROL_PLANE_IP to this box's own public IP so a
|
|
|
1293
1295
|
standalone instance's A record points HERE, not at the co-hosting box's PROVISION_PUBLIC_IP.`);
|
|
1294
1296
|
}
|
|
1295
1297
|
|
|
1298
|
+
// Store an intent's failure class + reason code (migration provisioning_030, task 1004446),
|
|
1299
|
+
// only for a result that carries one. Best-effort: a runner on a schema without the columns
|
|
1300
|
+
// must still drain, so a failed write is swallowed — last_error still says it either way. A
|
|
1301
|
+
// class left on a row that later lands 'done' is harmless: readers ignore it on a done row.
|
|
1302
|
+
async function recordFailureClass(db, intentId, failure) {
|
|
1303
|
+
await db.query('UPDATE provisioning_intents SET failure_class = $2, failure_reason = $3 WHERE id = $1',
|
|
1304
|
+
[intentId, failure ? failure.class : null, failure ? failure.reason : null]).catch(() => {});
|
|
1305
|
+
}
|
|
1306
|
+
|
|
1296
1307
|
async function main() {
|
|
1297
1308
|
const args = process.argv.slice(2);
|
|
1298
1309
|
const cmd = args[0];
|
|
@@ -0,0 +1,155 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
//
|
|
3
|
+
// scripts/gds/upgrade-outcome.js — what a failed core move actually WAS, as a class and a
|
|
4
|
+
// reason code the deploy runner can act on (task 1004446, goal 1000090 deploy self-heal).
|
|
5
|
+
//
|
|
6
|
+
// WHY. Every non-zero exit from `bongos upgrade` reached the owner as "failed and was rolled
|
|
7
|
+
// back", including the refusals that fire BEFORE the pin moves (a dirty tree, the artist
|
|
8
|
+
// gate, a downgrade, the database identity check) — nothing was changed, so nothing was
|
|
9
|
+
// rolled back. And a rollback that itself failed ("manual intervention needed") read exactly
|
|
10
|
+
// like one that worked. The runner could not tell a wait-and-retry from a decision only the
|
|
11
|
+
// owner can make from an outage, so it treated all of them the same: retry, then give up.
|
|
12
|
+
//
|
|
13
|
+
// TWO HALVES, ONE FILE, so the words cannot drift apart:
|
|
14
|
+
// • upgrade.js ends a failed run with ONE machine-readable stderr line (outcomeLine) saying
|
|
15
|
+
// how far it got: refused (nothing moved) · rolled_back (moved, restored) ·
|
|
16
|
+
// rollback_failed (moved, NOT restored). The runner invokes upgrade.js from its own
|
|
17
|
+
// checkout, so the two halves are always the same version.
|
|
18
|
+
// • the runner reads it back (parseOutcomeLine) and classifies (classifyFailure).
|
|
19
|
+
//
|
|
20
|
+
// THE FOUR CLASSES the rest of the self-heal chain keys on:
|
|
21
|
+
// transient — try again later and it may simply work (registry blip, not published yet)
|
|
22
|
+
// known_wedge — a known snag with a known fix the runner may apply itself (dirty tree, full disk)
|
|
23
|
+
// needs_decision — a rule the owner set is refusing it; only the owner can change that
|
|
24
|
+
// unknown — anything else, and every rollback that failed; a person should look
|
|
25
|
+
//
|
|
26
|
+
// PURE: no I/O, no requires. Reason codes are a fixed vocabulary — never builder text — so
|
|
27
|
+
// they are safe to store, index and show.
|
|
28
|
+
|
|
29
|
+
const OUTCOME_TAG = 'BONGOS_UPGRADE_OUTCOME';
|
|
30
|
+
const STAGES = Object.freeze(['refused', 'rolled_back', 'rollback_failed']);
|
|
31
|
+
const FAILURE_CLASSES = Object.freeze(['transient', 'known_wedge', 'needs_decision', 'unknown']);
|
|
32
|
+
|
|
33
|
+
// reason → class. Anything not listed is `unknown`, which is the safe direction: an unknown
|
|
34
|
+
// failure reaches a person; a mislabelled one could be retried or "healed" wrongly.
|
|
35
|
+
const REASON_CLASS = Object.freeze({
|
|
36
|
+
// the runner's own refusals (provision-core-upgrade.js)
|
|
37
|
+
registry_unreadable: 'transient',
|
|
38
|
+
not_published: 'transient',
|
|
39
|
+
outside_channel: 'needs_decision',
|
|
40
|
+
shape_not_automated: 'needs_decision',
|
|
41
|
+
no_target: 'unknown',
|
|
42
|
+
bad_slug: 'unknown',
|
|
43
|
+
installed_unreadable: 'unknown',
|
|
44
|
+
served_mismatch: 'unknown',
|
|
45
|
+
// upgrade.js refusals, before the pin moves
|
|
46
|
+
dirty_tree: 'known_wedge',
|
|
47
|
+
downgrade: 'needs_decision',
|
|
48
|
+
artist_gate: 'needs_decision',
|
|
49
|
+
module_incompatible: 'needs_decision',
|
|
50
|
+
already_on: 'unknown',
|
|
51
|
+
db_unresolvable: 'unknown',
|
|
52
|
+
db_identity: 'unknown',
|
|
53
|
+
health_url_missing: 'unknown',
|
|
54
|
+
core_source: 'unknown',
|
|
55
|
+
// after the pin moved (rolled back)
|
|
56
|
+
disk_full: 'known_wedge',
|
|
57
|
+
npm_install: 'transient',
|
|
58
|
+
integrity_pin: 'unknown',
|
|
59
|
+
migrate: 'unknown',
|
|
60
|
+
restart: 'unknown',
|
|
61
|
+
health: 'unknown',
|
|
62
|
+
// the one that must never be retried unattended
|
|
63
|
+
rollback_failed: 'unknown',
|
|
64
|
+
});
|
|
65
|
+
|
|
66
|
+
// upgrade.js error text → reason. Ordered; first match wins. Each pattern is the PREFIX of
|
|
67
|
+
// a message upgrade.js builds itself (runUpgrade), so these track that file, not the world.
|
|
68
|
+
const UPGRADE_REASONS = [
|
|
69
|
+
[/no space left on device|ENOSPC/i, 'disk_full'],
|
|
70
|
+
[/^refusing to DOWNGRADE/, 'downgrade'],
|
|
71
|
+
[/^already on /, 'already_on'],
|
|
72
|
+
[/^working tree not clean/, 'dirty_tree'],
|
|
73
|
+
[/^module pre-check failed/, 'module_incompatible'],
|
|
74
|
+
[/^cannot determine which database/, 'db_unresolvable'],
|
|
75
|
+
[/^post-restart health check is mandatory/, 'health_url_missing'],
|
|
76
|
+
[/^(--registry and --from|--from tarball not found|no core tarball at)/, 'core_source'],
|
|
77
|
+
[/^npm install failed/, 'npm_install'],
|
|
78
|
+
[/^integrity pin/, 'integrity_pin'],
|
|
79
|
+
[/^migrate failed/, 'migrate'],
|
|
80
|
+
[/^restart failed/, 'restart'],
|
|
81
|
+
[/^(health check failed after restart|served core version did not change)/, 'health'],
|
|
82
|
+
];
|
|
83
|
+
|
|
84
|
+
/** The reason code for an upgrade.js failure result (runUpgrade's `{ ok:false, … }`). PURE. */
|
|
85
|
+
function upgradeReason(res) {
|
|
86
|
+
if (!res) return 'unknown';
|
|
87
|
+
if (res.downgrade) return 'downgrade';
|
|
88
|
+
if (res.artistGate) return 'artist_gate';
|
|
89
|
+
const text = String(res.error || '');
|
|
90
|
+
for (const [re, reason] of UPGRADE_REASONS) if (re.test(text)) return reason;
|
|
91
|
+
// A database-identity refusal carries preflightDbIdentity's own reason, which has no
|
|
92
|
+
// fixed prefix; it is the only pre-pin refusal left that names the database.
|
|
93
|
+
if (!res.rolledBack && /database|PGDATABASE/i.test(text)) return 'db_identity';
|
|
94
|
+
return 'unknown';
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
/** How far a failed run got. A rollback counts as failed if ANY of its steps did. PURE. */
|
|
98
|
+
function upgradeStage(res) {
|
|
99
|
+
if (!res || !res.rolledBack) return 'refused';
|
|
100
|
+
const rb = res.rollback || {};
|
|
101
|
+
const broken = rb.restoreOk === false || rb.reinstallOk === false || rb.restartOk === false || rb.healthOk === false;
|
|
102
|
+
return broken ? 'rollback_failed' : 'rolled_back';
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
/** The single stderr line upgrade.js ends a failed run with. PURE. */
|
|
106
|
+
function outcomeLine(res) {
|
|
107
|
+
const stage = upgradeStage(res);
|
|
108
|
+
const reason = stage === 'rollback_failed' ? 'rollback_failed' : upgradeReason(res);
|
|
109
|
+
return `${OUTCOME_TAG} ${JSON.stringify({ stage, reason })}\n`;
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
/** Read the outcome line back out of captured output, or null if there is none. PURE. */
|
|
113
|
+
function parseOutcomeLine(text) {
|
|
114
|
+
const lines = String(text || '').split(/\r?\n/).filter((l) => l.startsWith(`${OUTCOME_TAG} `));
|
|
115
|
+
if (!lines.length) return null;
|
|
116
|
+
try {
|
|
117
|
+
const o = JSON.parse(lines[lines.length - 1].slice(OUTCOME_TAG.length + 1));
|
|
118
|
+
if (!STAGES.includes(o.stage) || typeof o.reason !== 'string' || !/^[a-z_]{1,40}$/.test(o.reason)) return null;
|
|
119
|
+
return { stage: o.stage, reason: o.reason };
|
|
120
|
+
} catch { return null; }
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
/** `{ class, reason }` for a reason code; an unlisted code is `unknown`. PURE. */
|
|
124
|
+
function classifyFailure(reason) {
|
|
125
|
+
const r = typeof reason === 'string' && Object.prototype.hasOwnProperty.call(REASON_CLASS, reason) ? reason : 'unknown';
|
|
126
|
+
return { class: REASON_CLASS[r] || 'unknown', reason: r };
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
/** The runner's own refusals (targetFault's sentences) → reason code. PURE. */
|
|
130
|
+
function runnerReason(text) {
|
|
131
|
+
const s = String(text || '');
|
|
132
|
+
if (/^could not read the published core versions/.test(s)) return 'registry_unreadable';
|
|
133
|
+
if (/is not published yet/.test(s)) return 'not_published';
|
|
134
|
+
if (/could not read the core version installed/.test(s)) return 'installed_unreadable';
|
|
135
|
+
if (/update channel/.test(s)) return 'outside_channel';
|
|
136
|
+
return 'unknown';
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
/**
|
|
140
|
+
* The sentence the owner reads for a failed upgrade.js run. Says what actually happened,
|
|
141
|
+
* never more: "rolled back" only when a rollback ran and finished. PURE.
|
|
142
|
+
*/
|
|
143
|
+
function describeUpgradeFailure(to, outcome, detail) {
|
|
144
|
+
const d = String(detail || '').trim();
|
|
145
|
+
const tail = d ? `: ${d}` : '';
|
|
146
|
+
if (!outcome) return `core upgrade to ${to} failed, and the upgrade did not report how far it got — check the project before retrying${tail}`;
|
|
147
|
+
if (outcome.stage === 'refused') return `core upgrade to ${to} was refused before anything changed — the project is still on its previous core${tail}`;
|
|
148
|
+
if (outcome.stage === 'rolled_back') return `core upgrade to ${to} failed and was rolled back — the project is back on its previous core${tail}`;
|
|
149
|
+
return `core upgrade to ${to} failed AND the rollback did not finish — the project may be down or on a half-installed core, and needs a person${tail}`;
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
module.exports = {
|
|
153
|
+
OUTCOME_TAG, STAGES, FAILURE_CLASSES, REASON_CLASS,
|
|
154
|
+
upgradeReason, upgradeStage, outcomeLine, parseOutcomeLine, classifyFailure, runnerReason, describeUpgradeFailure,
|
|
155
|
+
};
|
package/scripts/gds/upgrade.js
CHANGED
|
@@ -1480,7 +1480,7 @@ async function main(argv = process.argv.slice(2)) {
|
|
|
1480
1480
|
return require('./module').cmdUpgrade(argv);
|
|
1481
1481
|
}
|
|
1482
1482
|
const res = await runUpgrade(parseArgs(argv));
|
|
1483
|
-
if (!res.ok) { process.stderr.write(`\n✖ ${res.error || 'upgrade failed'}\n`); return 1; }
|
|
1483
|
+
if (!res.ok) { process.stderr.write(`\n✖ ${res.error || 'upgrade failed'}\n${require('./upgrade-outcome').outcomeLine(res)}`); return 1; } // the outcome line: how far it got, for the deploy runner (task 1004446)
|
|
1484
1484
|
return 0;
|
|
1485
1485
|
}
|
|
1486
1486
|
|
package/src/module-api.js
CHANGED
|
@@ -75,7 +75,7 @@ const { responsibilityFor, ROLE_RESPONSIBILITIES } = require('./role-responsibil
|
|
|
75
75
|
// MAJOR (see allowBoxScope below): passes the request through untouched.
|
|
76
76
|
function deprecatedNoopMiddleware(_req, _res, next) { next(); }
|
|
77
77
|
|
|
78
|
-
const CORE_VERSION = '1.20.
|
|
78
|
+
const CORE_VERSION = '1.20.30'; // CI auto-patch carrier (ADR 0161); changelog: docs/module-api-changelog.md
|
|
79
79
|
|
|
80
80
|
// A namespaced logger so a module's log lines are attributable + consistent.
|
|
81
81
|
// Usage: const log = api.logger('discord'); log.info('mounted');
|
|
@@ -964,5 +964,14 @@ await t('the route is gated like the move: the owner door OR core.pin.move', ()
|
|
|
964
964
|
assert.match(block, /requireOwnProjectOrPin\(\{ pinGate: api\.requirePermission\('core\.pin\.move'\)/);
|
|
965
965
|
});
|
|
966
966
|
|
|
967
|
+
await t('a failed move carries its class to the page; a move that worked carries none (task 1004446)', () => {
|
|
968
|
+
const failedMove = { target_version: '1.20.1', state: 'error', last_error: 'x', failure_class: 'needs_decision', failure_reason: 'outside_channel' };
|
|
969
|
+
assert.deepEqual(CU.previewFromSnapshot(ACTIVE, null, null, failedMove).last_move.failure, { class: 'needs_decision', reason: 'outside_channel' });
|
|
970
|
+
const done = { ...failedMove, state: 'done' };
|
|
971
|
+
assert.equal(CU.previewFromSnapshot(ACTIVE, null, null, done).last_move.failure, null);
|
|
972
|
+
const old = { target_version: '1.20.1', state: 'error', last_error: 'x' };
|
|
973
|
+
assert.equal(CU.previewFromSnapshot(ACTIVE, null, null, old).last_move.failure, null, 'a row from before the columns');
|
|
974
|
+
});
|
|
975
|
+
|
|
967
976
|
console.log(`\ncore_upgrade_door: ${passed} passed, ${failed} failed`);
|
|
968
977
|
if (failed) process.exit(1);
|
|
@@ -1449,5 +1449,71 @@ await t('the snapshot OFFERS what the project\'s rule admits — the button and
|
|
|
1449
1449
|
assert.equal(await offered(undefined), '1.19.1082', 'a pre-column row behaves exactly as before');
|
|
1450
1450
|
});
|
|
1451
1451
|
|
|
1452
|
+
console.log('\nwhat a failed upgrade.js run is reported as (task 1004446):');
|
|
1453
|
+
|
|
1454
|
+
// execSync's error: message "Command failed: <cmd>\n<stderr>", stderr a Buffer.
|
|
1455
|
+
const upgradeThrows = (stderr) => (d) => {
|
|
1456
|
+
const base = d.exec;
|
|
1457
|
+
d.exec = (cmd, opts) => {
|
|
1458
|
+
if (/upgrade\.js/.test(cmd)) { const e = new Error(`Command failed: ${cmd}\n${stderr}`); e.stderr = Buffer.from(stderr); e.status = 1; throw e; }
|
|
1459
|
+
return base(cmd, opts);
|
|
1460
|
+
};
|
|
1461
|
+
return d;
|
|
1462
|
+
};
|
|
1463
|
+
|
|
1464
|
+
await t('a refusal before the pin moved is classified, and never called "rolled back"', async () => {
|
|
1465
|
+
const d = upgradeThrows('\n✖ working tree not clean — commit/stash first\nBONGOS_UPGRADE_OUTCOME {"stage":"refused","reason":"dirty_tree"}\n')(makeDeps());
|
|
1466
|
+
const r = await R.coreUpgradeInstance(inst(), d, intent());
|
|
1467
|
+
assert.equal(r.ok, false);
|
|
1468
|
+
assert.deepEqual(r.failure, { class: 'known_wedge', reason: 'dirty_tree' });
|
|
1469
|
+
assert.equal(r.stage, 'refused');
|
|
1470
|
+
assert.doesNotMatch(r.error, /rolled back/);
|
|
1471
|
+
assert.match(r.error, /working tree not clean/, 'the upgrade\'s own sentence is kept');
|
|
1472
|
+
assert.ok(!r.terminal);
|
|
1473
|
+
});
|
|
1474
|
+
|
|
1475
|
+
await t('a real rollback says rolled back', async () => {
|
|
1476
|
+
const d = upgradeThrows('\n✖ migrate failed (exit 3)\nBONGOS_UPGRADE_OUTCOME {"stage":"rolled_back","reason":"migrate"}\n')(makeDeps());
|
|
1477
|
+
const r = await R.coreUpgradeInstance(inst(), d, intent());
|
|
1478
|
+
assert.deepEqual(r.failure, { class: 'unknown', reason: 'migrate' });
|
|
1479
|
+
assert.match(r.error, /failed and was rolled back/);
|
|
1480
|
+
});
|
|
1481
|
+
|
|
1482
|
+
await t('a FAILED rollback is its own outcome, and terminal — never retried unattended', async () => {
|
|
1483
|
+
const d = upgradeThrows('\n✖ health check failed after restart\nBONGOS_UPGRADE_OUTCOME {"stage":"rollback_failed","reason":"rollback_failed"}\n')(makeDeps());
|
|
1484
|
+
const r = await R.coreUpgradeInstance(inst(), d, intent());
|
|
1485
|
+
assert.deepEqual(r.failure, { class: 'unknown', reason: 'rollback_failed' });
|
|
1486
|
+
assert.equal(r.stage, 'rollback_failed');
|
|
1487
|
+
assert.equal(r.terminal, true);
|
|
1488
|
+
assert.match(r.error, /rollback did not finish/);
|
|
1489
|
+
});
|
|
1490
|
+
|
|
1491
|
+
await t('an exit with NO outcome line (sudo refused before node ran) is unknown and claims no rollback', async () => {
|
|
1492
|
+
const d = upgradeThrows('sudo: unknown user bongos-test4\n')(makeDeps());
|
|
1493
|
+
const r = await R.coreUpgradeInstance(inst(), d, intent());
|
|
1494
|
+
assert.deepEqual(r.failure, { class: 'unknown', reason: 'unknown' });
|
|
1495
|
+
assert.doesNotMatch(r.error, /rolled back/);
|
|
1496
|
+
});
|
|
1497
|
+
|
|
1498
|
+
await t('the runner\'s own refusals are classified too', async () => {
|
|
1499
|
+
const d = withRealChannel({ installed: '1.19.1081', served: '1.19.1081', available: ['1.19.1081', '1.20.1'] });
|
|
1500
|
+
const r = await R.coreUpgradeInstance(inst({ update_channel: 'patch' }), d, intent({ target_version: '1.20.1' }));
|
|
1501
|
+
assert.deepEqual(r.failure, { class: 'needs_decision', reason: 'outside_channel' });
|
|
1502
|
+
const n = await R.coreUpgradeInstance(inst(), makeDeps({ available: ['1.19.760'] }), intent({ target_version: '1.19.999' }));
|
|
1503
|
+
assert.deepEqual(n.failure, { class: 'transient', reason: 'not_published' });
|
|
1504
|
+
const s = await R.coreUpgradeInstance(inst({ hosting_shape: 'dedicated' }), makeDeps(), intent());
|
|
1505
|
+
assert.deepEqual(s.failure, { class: 'needs_decision', reason: 'shape_not_automated' });
|
|
1506
|
+
});
|
|
1507
|
+
|
|
1508
|
+
await t('upgrade.js ends a failed run with the outcome line', () => {
|
|
1509
|
+
const src = fs.readFileSync(new URL('../scripts/gds/upgrade.js', import.meta.url), 'utf8');
|
|
1510
|
+
assert.match(src, /if \(!res\.ok\) \{ process\.stderr\.write\([^\n]*outcomeLine\(res\)/);
|
|
1511
|
+
});
|
|
1512
|
+
|
|
1513
|
+
await t('the drain loop writes the class beside the sentence (behaviour: tests/provision.mjs)', () => {
|
|
1514
|
+
const src = fs.readFileSync(new URL('../scripts/gds/provision.js', import.meta.url), 'utf8');
|
|
1515
|
+
assert.match(src, /SET failure_class = \$2, failure_reason = \$3/);
|
|
1516
|
+
});
|
|
1517
|
+
|
|
1452
1518
|
console.log(`\ncore_upgrade_runner: ${passed} passed, ${failed} failed`);
|
|
1453
1519
|
if (failed) process.exit(1);
|
package/tests/provision.mjs
CHANGED
|
@@ -2564,6 +2564,40 @@ await ta('F19: a NON-final failure re-pends the intent and writes nothing on the
|
|
|
2564
2564
|
assert.equal(statusWrites.length, 0, 'no instance write until the intent actually retires');
|
|
2565
2565
|
});
|
|
2566
2566
|
|
|
2567
|
+
// A failed core move stores WHAT KIND of failure beside its sentence (task 1004446).
|
|
2568
|
+
const classifiedRun = async (attempts) => {
|
|
2569
|
+
let claims = 0; const resolved = []; const classWrites = [];
|
|
2570
|
+
const deps = {
|
|
2571
|
+
apply: true, log: () => {},
|
|
2572
|
+
db: { query: async (sql, params) => { if (/SET failure_class/.test(sql)) classWrites.push(params); return { rows: [] }; } },
|
|
2573
|
+
provisioning: {
|
|
2574
|
+
claimNextIntent: async () => (claims++ === 0
|
|
2575
|
+
? { id: 11, instance_id: 18, action: 'core-upgrade', target_version: '1.20.1', attempts } : null),
|
|
2576
|
+
// 'dedicated' is a shape the runner cannot move — a classified refusal, no exec needed.
|
|
2577
|
+
getInstanceById: async () => ({ ...activeStandalone, hosting_shape: 'dedicated' }),
|
|
2578
|
+
resolveIntent: async (_db, id, state) => { resolved.push({ id, state }); },
|
|
2579
|
+
setInstanceStatus: async () => {}, recordEvent: async () => {},
|
|
2580
|
+
claimNextOAuthManifest: async () => null, resolveOAuthManifest: async () => {},
|
|
2581
|
+
},
|
|
2582
|
+
exec: () => ({ ok: true }), writeFile: () => ({ ok: true }),
|
|
2583
|
+
checkDnsTokenScope: async () => ({ reachable: true, zone: 'z' }), checkCaddyWiring: () => null,
|
|
2584
|
+
};
|
|
2585
|
+
await P.cmdRunIntents(deps);
|
|
2586
|
+
return { resolved, classWrites };
|
|
2587
|
+
};
|
|
2588
|
+
|
|
2589
|
+
await ta('a re-pended core move records its failure class and reason', async () => {
|
|
2590
|
+
const { resolved, classWrites } = await classifiedRun(1);
|
|
2591
|
+
assert.equal(resolved.length, 0);
|
|
2592
|
+
assert.deepEqual(classWrites, [[11, 'needs_decision', 'shape_not_automated']]);
|
|
2593
|
+
});
|
|
2594
|
+
|
|
2595
|
+
await ta('a RETIRED core move records its failure class and reason too', async () => {
|
|
2596
|
+
const { resolved, classWrites } = await classifiedRun(3); // = MAX_INTENT_ATTEMPTS: the last real attempt, retired after it runs
|
|
2597
|
+
assert.equal(resolved[0].state, 'error');
|
|
2598
|
+
assert.deepEqual(classWrites, [[11, 'needs_decision', 'shape_not_automated']]);
|
|
2599
|
+
});
|
|
2600
|
+
|
|
2567
2601
|
console.log('\nreview-panel hardening (task 1002720 — the loopback/latch/birth-note seams):');
|
|
2568
2602
|
|
|
2569
2603
|
await ta('a PASSING loopback probe on a DOMAIN-bearing instance records NO verdict — the address claim belongs to the sweep', async () => {
|
|
@@ -0,0 +1,129 @@
|
|
|
1
|
+
// tests/upgrade_outcome.mjs — every failed core move gets a class and honest wording
|
|
2
|
+
// (task 1004446, goal 1000090 deploy self-heal).
|
|
3
|
+
//
|
|
4
|
+
// WHAT IS AT RISK. The deploy page said "failed and was rolled back" for every exit of
|
|
5
|
+
// `bongos upgrade`, including refusals that changed nothing and rollbacks that themselves
|
|
6
|
+
// failed. The rest of the self-heal chain (retry later, fix it, escalate) keys on the class
|
|
7
|
+
// this file assigns, so a mislabel there is a retry of something that must not be retried,
|
|
8
|
+
// or an automatic "fix" for something that needed a person.
|
|
9
|
+
//
|
|
10
|
+
// Run: node tests/upgrade_outcome.mjs
|
|
11
|
+
|
|
12
|
+
import { strict as assert } from 'node:assert';
|
|
13
|
+
import { createRequire } from 'node:module';
|
|
14
|
+
import {
|
|
15
|
+
outcomeLine, parseOutcomeLine, upgradeReason, upgradeStage, classifyFailure, runnerReason,
|
|
16
|
+
describeUpgradeFailure, FAILURE_CLASSES, REASON_CLASS, STAGES, OUTCOME_TAG,
|
|
17
|
+
} from '../scripts/gds/upgrade-outcome.js';
|
|
18
|
+
|
|
19
|
+
const require = createRequire(import.meta.url);
|
|
20
|
+
const R = require('../scripts/gds/provision-core-upgrade.js');
|
|
21
|
+
|
|
22
|
+
let passed = 0;
|
|
23
|
+
let failed = 0;
|
|
24
|
+
const t = async (name, fn) => {
|
|
25
|
+
try { await fn(); passed += 1; console.log(` PASS ${name}`); }
|
|
26
|
+
catch (e) { failed += 1; console.log(` FAIL ${name}\n ${e.message}`); }
|
|
27
|
+
};
|
|
28
|
+
|
|
29
|
+
// Each row: an upgrade.js failure result as runUpgrade returns it → the stage, reason and
|
|
30
|
+
// class it must get. These are the branches of runUpgrade's failure table (upgrade.js).
|
|
31
|
+
const OK_RB = { restoreOk: true, reinstallOk: true, restartOk: true, healthOk: true };
|
|
32
|
+
const TABLE = [
|
|
33
|
+
['downgrade refusal', { ok: false, downgrade: true, error: 'refusing to DOWNGRADE: --to 1.19.1 is older' }, 'refused', 'downgrade', 'needs_decision'],
|
|
34
|
+
['already on the target', { ok: false, error: 'already on 1.20.1 (pass --force to re-run the steps anyway)' }, 'refused', 'already_on', 'unknown'],
|
|
35
|
+
['dirty working tree', { ok: false, error: 'working tree not clean — commit/stash first, or pass --force.\n M package.json' }, 'refused', 'dirty_tree', 'known_wedge'],
|
|
36
|
+
['module pre-check', { ok: false, error: 'module pre-check failed: game incompatible with core 1.20.1' }, 'refused', 'module_incompatible', 'needs_decision'],
|
|
37
|
+
['database unresolvable', { ok: false, error: 'cannot determine which database to migrate.\n Set PGDATABASE' }, 'refused', 'db_unresolvable', 'unknown'],
|
|
38
|
+
['database identity', { ok: false, error: 'database "cb" is not reachable as the current user: role "root" does not exist' }, 'refused', 'db_identity', 'unknown'],
|
|
39
|
+
['artist gate hold', { ok: false, artistGate: true, error: 'a visual review is holding this deploy' }, 'refused', 'artist_gate', 'needs_decision'],
|
|
40
|
+
['health url missing', { ok: false, error: 'post-restart health check is mandatory for a bump that restarts the service' }, 'refused', 'health_url_missing', 'unknown'],
|
|
41
|
+
['npm install failed, rolled back', { ok: false, error: 'npm install failed (exit 1)', rolledBack: true, rollback: OK_RB }, 'rolled_back', 'npm_install', 'transient'],
|
|
42
|
+
['disk full, rolled back', { ok: false, error: 'npm install failed (ENOSPC: no space left on device)', rolledBack: true, rollback: OK_RB }, 'rolled_back', 'disk_full', 'known_wedge'],
|
|
43
|
+
['integrity pin mismatch', { ok: false, error: 'integrity pin MISMATCH (side-car) — installed core does not match its pin: x', rolledBack: true, rollback: OK_RB }, 'rolled_back', 'integrity_pin', 'unknown'],
|
|
44
|
+
['migrate failed', { ok: false, error: 'migrate failed (exit 3)', rolledBack: true, rollback: OK_RB }, 'rolled_back', 'migrate', 'unknown'],
|
|
45
|
+
['restart failed', { ok: false, error: 'restart failed (exit 1) — the instance is still running the previous core', rolledBack: true, rollback: OK_RB }, 'rolled_back', 'restart', 'unknown'],
|
|
46
|
+
['health failed', { ok: false, error: 'health check failed after restart — rolled back to 1.19.1081', rolledBack: true, rollback: OK_RB }, 'rolled_back', 'health', 'unknown'],
|
|
47
|
+
['served version unchanged', { ok: false, error: 'served core version did not change (still 1.19.1081, expected 1.20.1) — rolled back', rolledBack: true, rollback: OK_RB }, 'rolled_back', 'health', 'unknown'],
|
|
48
|
+
['ROLLBACK FAILED (health still bad)', { ok: false, error: 'health check failed after restart — rolled back', rolledBack: true, rollback: { ...OK_RB, healthOk: false } }, 'rollback_failed', 'rollback_failed', 'unknown'],
|
|
49
|
+
['ROLLBACK FAILED (reinstall)', { ok: false, error: 'migrate failed (exit 3)', rolledBack: true, rollback: { ...OK_RB, reinstallOk: false } }, 'rollback_failed', 'rollback_failed', 'unknown'],
|
|
50
|
+
];
|
|
51
|
+
|
|
52
|
+
console.log('\nevery branch of the upgrade failure table:');
|
|
53
|
+
|
|
54
|
+
for (const [name, res, stage, reason, cls] of TABLE) {
|
|
55
|
+
await t(`${name} → ${stage} / ${reason} / ${cls}`, () => {
|
|
56
|
+
const o = parseOutcomeLine(outcomeLine(res));
|
|
57
|
+
assert.deepEqual(o, { stage, reason });
|
|
58
|
+
assert.equal(classifyFailure(o.reason).class, cls);
|
|
59
|
+
});
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
console.log('\nthe vocabulary:');
|
|
63
|
+
|
|
64
|
+
await t('every reason maps to one of the four classes', () => {
|
|
65
|
+
for (const [r, c] of Object.entries(REASON_CLASS)) assert.ok(FAILURE_CLASSES.includes(c), `${r} → ${c}`);
|
|
66
|
+
});
|
|
67
|
+
|
|
68
|
+
await t('an unlisted reason is unknown — the safe direction', () => {
|
|
69
|
+
assert.deepEqual(classifyFailure('something_new'), { class: 'unknown', reason: 'unknown' });
|
|
70
|
+
assert.deepEqual(classifyFailure(undefined), { class: 'unknown', reason: 'unknown' });
|
|
71
|
+
assert.deepEqual(classifyFailure('__proto__'), { class: 'unknown', reason: 'unknown' });
|
|
72
|
+
});
|
|
73
|
+
|
|
74
|
+
await t('the migration CHECK carries exactly the four classes', async () => {
|
|
75
|
+
const { readFileSync } = await import('node:fs');
|
|
76
|
+
const sql = readFileSync(new URL('../modules/provisioning/migrations/provisioning_030_intent_failure_class.sql', import.meta.url), 'utf8');
|
|
77
|
+
for (const c of FAILURE_CLASSES) assert.match(sql, new RegExp(`'${c}'`));
|
|
78
|
+
});
|
|
79
|
+
|
|
80
|
+
await t('a stage is only what upgrade.js reports: no rollback flag means nothing moved', () => {
|
|
81
|
+
assert.equal(upgradeStage({ ok: false, error: 'x' }), 'refused');
|
|
82
|
+
assert.equal(upgradeStage({ ok: false, rolledBack: true, rollback: OK_RB }), 'rolled_back');
|
|
83
|
+
assert.equal(upgradeStage({ ok: false, rolledBack: true, rollback: { restartOk: false } }), 'rollback_failed');
|
|
84
|
+
assert.equal(upgradeReason(null), 'unknown');
|
|
85
|
+
});
|
|
86
|
+
|
|
87
|
+
await t('parseOutcomeLine takes the LAST line, and refuses anything off-vocabulary', () => {
|
|
88
|
+
const two = `${OUTCOME_TAG} {"stage":"refused","reason":"dirty_tree"}\nnoise\n${OUTCOME_TAG} {"stage":"rolled_back","reason":"migrate"}\n`;
|
|
89
|
+
assert.deepEqual(parseOutcomeLine(two), { stage: 'rolled_back', reason: 'migrate' });
|
|
90
|
+
assert.equal(parseOutcomeLine('no tag here'), null);
|
|
91
|
+
assert.equal(parseOutcomeLine(`${OUTCOME_TAG} {"stage":"exploded","reason":"x"}`), null);
|
|
92
|
+
assert.equal(parseOutcomeLine(`${OUTCOME_TAG} {"stage":"refused","reason":"<script>"}`), null);
|
|
93
|
+
assert.equal(parseOutcomeLine(`${OUTCOME_TAG} not json`), null);
|
|
94
|
+
assert.deepEqual([...STAGES], ['refused', 'rolled_back', 'rollback_failed']);
|
|
95
|
+
});
|
|
96
|
+
|
|
97
|
+
await t('the runner\'s own refusals get their reason from targetFault\'s sentences', () => {
|
|
98
|
+
assert.equal(runnerReason(R.targetFault('1.20.1', [], '1.19.1', 'patch', () => true)), 'registry_unreadable');
|
|
99
|
+
assert.equal(runnerReason(R.targetFault('1.20.1', ['1.19.1'], '1.19.1', 'patch', () => true)), 'not_published');
|
|
100
|
+
assert.equal(runnerReason(R.targetFault('1.20.1', ['1.20.1'], null, 'patch', () => true)), 'installed_unreadable');
|
|
101
|
+
assert.equal(runnerReason(R.targetFault('1.20.1', ['1.20.1'], '1.19.1', 'patch', () => false)), 'outside_channel');
|
|
102
|
+
});
|
|
103
|
+
|
|
104
|
+
console.log('\nthe wording says what happened, and no more:');
|
|
105
|
+
|
|
106
|
+
await t('a refusal never says "rolled back"', () => {
|
|
107
|
+
const s = describeUpgradeFailure('1.20.1', { stage: 'refused', reason: 'dirty_tree' }, 'working tree not clean');
|
|
108
|
+
assert.doesNotMatch(s, /rolled back/);
|
|
109
|
+
assert.match(s, /refused before anything changed/);
|
|
110
|
+
});
|
|
111
|
+
|
|
112
|
+
await t('a real rollback says so', () => {
|
|
113
|
+
assert.match(describeUpgradeFailure('1.20.1', { stage: 'rolled_back', reason: 'migrate' }, 'migrate failed'), /failed and was rolled back/);
|
|
114
|
+
});
|
|
115
|
+
|
|
116
|
+
await t('a failed rollback says the project may be down and needs a person', () => {
|
|
117
|
+
const s = describeUpgradeFailure('1.20.1', { stage: 'rollback_failed', reason: 'rollback_failed' }, '');
|
|
118
|
+
assert.match(s, /rollback did not finish/);
|
|
119
|
+
assert.match(s, /needs a person/);
|
|
120
|
+
});
|
|
121
|
+
|
|
122
|
+
await t('no outcome line claims nothing about a rollback', () => {
|
|
123
|
+
const s = describeUpgradeFailure('1.20.1', null, 'sudo: unknown user bongos-test4');
|
|
124
|
+
assert.doesNotMatch(s, /rolled back/);
|
|
125
|
+
assert.match(s, /did not report how far it got/);
|
|
126
|
+
});
|
|
127
|
+
|
|
128
|
+
console.log(`\nupgrade_outcome: ${passed} passed, ${failed} failed`);
|
|
129
|
+
if (failed) process.exit(1);
|