@bongos/core 1.20.29 → 1.20.31
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.bongos-core.json +50 -20
- package/docs/module-api-changelog.md +4 -0
- package/modules/provisioning/core-upgrade.js +18 -1
- package/modules/provisioning/migrations/provisioning_030_intent_failure_class.sql +28 -0
- package/modules/provisioning/migrations/provisioning_031_intent_not_before.sql +19 -0
- package/modules/provisioning/provisioning.js +1 -1
- package/package-lock.json +2 -2
- package/package.json +1 -1
- package/release-notes.json +12 -0
- package/scripts/gds/intent-retry.js +52 -0
- package/scripts/gds/provision-core-upgrade.js +23 -9
- package/scripts/gds/provision.js +20 -4
- package/scripts/gds/upgrade-outcome.js +155 -0
- package/scripts/gds/upgrade.js +1 -1
- package/src/module-api.js +1 -1
- package/tests/core_upgrade_door.mjs +44 -1
- package/tests/core_upgrade_runner.mjs +66 -0
- package/tests/intent_retry.mjs +57 -0
- package/tests/provision.mjs +59 -0
- package/tests/upgrade_outcome.mjs +129 -0
package/.bongos-core.json
CHANGED
|
@@ -2,22 +2,22 @@
|
|
|
2
2
|
"artifact": "bongos-core",
|
|
3
3
|
"manifest_schema": 1,
|
|
4
4
|
"generator": "scripts/gds/package-core.js",
|
|
5
|
-
"core_version": "1.20.
|
|
6
|
-
"core_contract": "1.20.
|
|
7
|
-
"source_commit": "
|
|
5
|
+
"core_version": "1.20.31",
|
|
6
|
+
"core_contract": "1.20.31",
|
|
7
|
+
"source_commit": "7d9880eac50e580874638a8cca10da6606c18228",
|
|
8
8
|
"source_ref": "HEAD",
|
|
9
|
-
"built_at": "2026-09-
|
|
9
|
+
"built_at": "2026-09-30T21:24:24.438Z",
|
|
10
10
|
"redaction": {
|
|
11
11
|
"model": "docs-redacted+functional-verbatim",
|
|
12
12
|
"docs_redacted": 562,
|
|
13
13
|
"agent_docs_stubbed": 25,
|
|
14
|
-
"functional_verbatim":
|
|
14
|
+
"functional_verbatim": 2666,
|
|
15
15
|
"rules": 3,
|
|
16
16
|
"gate_literals": 3,
|
|
17
17
|
"gate": "passed"
|
|
18
18
|
},
|
|
19
|
-
"file_count":
|
|
20
|
-
"tree_sha256": "
|
|
19
|
+
"file_count": 3254,
|
|
20
|
+
"tree_sha256": "88a5431f2eb9f9248c57014423f4881035e2d1b9a6cee88740a6800ab933ba38",
|
|
21
21
|
"files": [
|
|
22
22
|
{
|
|
23
23
|
"path": ".claude/skills/ask-for-help/SKILL.md",
|
|
@@ -2802,7 +2802,7 @@
|
|
|
2802
2802
|
{
|
|
2803
2803
|
"path": "docs/module-api-changelog.md",
|
|
2804
2804
|
"mode": "0000644",
|
|
2805
|
-
"sha256": "
|
|
2805
|
+
"sha256": "fd78a2ab7d52f77fbd7181a64af68b8cf4c45c25693bf7dbf31e1b9bba69ada7"
|
|
2806
2806
|
},
|
|
2807
2807
|
{
|
|
2808
2808
|
"path": "docs/modules-contract.md",
|
|
@@ -7282,7 +7282,7 @@
|
|
|
7282
7282
|
{
|
|
7283
7283
|
"path": "modules/provisioning/core-upgrade.js",
|
|
7284
7284
|
"mode": "0000644",
|
|
7285
|
-
"sha256": "
|
|
7285
|
+
"sha256": "5467ab673dfad5969dec0029188d0be18ea82732a26f99bca720de6680e1d618"
|
|
7286
7286
|
},
|
|
7287
7287
|
{
|
|
7288
7288
|
"path": "modules/provisioning/cost-ledger.js",
|
|
@@ -7459,6 +7459,16 @@
|
|
|
7459
7459
|
"mode": "0000644",
|
|
7460
7460
|
"sha256": "b667ec5165e611bd6f0d86b09465c5d9e632de853becce139bac49f5a88e5c1e"
|
|
7461
7461
|
},
|
|
7462
|
+
{
|
|
7463
|
+
"path": "modules/provisioning/migrations/provisioning_030_intent_failure_class.sql",
|
|
7464
|
+
"mode": "0000644",
|
|
7465
|
+
"sha256": "7287181b4b42d3e391cd97e19ffa5fbc76026e81ad6f54648eb12a69dfea55a1"
|
|
7466
|
+
},
|
|
7467
|
+
{
|
|
7468
|
+
"path": "modules/provisioning/migrations/provisioning_031_intent_not_before.sql",
|
|
7469
|
+
"mode": "0000644",
|
|
7470
|
+
"sha256": "a93d8946cbe4631b2af64b2e3638dca188981032a08f5d29d63e4d0adb22d235"
|
|
7471
|
+
},
|
|
7462
7472
|
{
|
|
7463
7473
|
"path": "modules/provisioning/module.json",
|
|
7464
7474
|
"mode": "0000644",
|
|
@@ -7507,7 +7517,7 @@
|
|
|
7507
7517
|
{
|
|
7508
7518
|
"path": "modules/provisioning/provisioning.js",
|
|
7509
7519
|
"mode": "0000644",
|
|
7510
|
-
"sha256": "
|
|
7520
|
+
"sha256": "fb4f09b621654eebe595dcbca305773ea11ed063506a6e4116391e43566a540d"
|
|
7511
7521
|
},
|
|
7512
7522
|
{
|
|
7513
7523
|
"path": "modules/provisioning/public-refusal.js",
|
|
@@ -8977,12 +8987,12 @@
|
|
|
8977
8987
|
{
|
|
8978
8988
|
"path": "package-lock.json",
|
|
8979
8989
|
"mode": "0000644",
|
|
8980
|
-
"sha256": "
|
|
8990
|
+
"sha256": "892ad66ccaa864ee07860cd8578f134a72f8cc48625f728f186eaa0f994ac8ad"
|
|
8981
8991
|
},
|
|
8982
8992
|
{
|
|
8983
8993
|
"path": "package.json",
|
|
8984
8994
|
"mode": "0000644",
|
|
8985
|
-
"sha256": "
|
|
8995
|
+
"sha256": "fdebee1582c5673a44b97471f05814ab6bc8e850958ca3131e4a356f3a663d90"
|
|
8986
8996
|
},
|
|
8987
8997
|
{
|
|
8988
8998
|
"path": "public-docs/index.html",
|
|
@@ -9002,7 +9012,7 @@
|
|
|
9002
9012
|
{
|
|
9003
9013
|
"path": "release-notes.json",
|
|
9004
9014
|
"mode": "0000644",
|
|
9005
|
-
"sha256": "
|
|
9015
|
+
"sha256": "4398d89d420cf704f3030db8752e1f9e67321e43e49b0765e23329c26160385e"
|
|
9006
9016
|
},
|
|
9007
9017
|
{
|
|
9008
9018
|
"path": "scripts/bongos-mcp.js",
|
|
@@ -9734,6 +9744,11 @@
|
|
|
9734
9744
|
"mode": "0000644",
|
|
9735
9745
|
"sha256": "7b2402543c3b1eadbafc14907e6dc6689b434ac65fb9c709115dd81a92eb13eb"
|
|
9736
9746
|
},
|
|
9747
|
+
{
|
|
9748
|
+
"path": "scripts/gds/intent-retry.js",
|
|
9749
|
+
"mode": "0000644",
|
|
9750
|
+
"sha256": "056622ecd8327527f4fb98b37ce8af90778d42c816455431ed3dd2f76800a75b"
|
|
9751
|
+
},
|
|
9737
9752
|
{
|
|
9738
9753
|
"path": "scripts/gds/key-card.js",
|
|
9739
9754
|
"mode": "0000644",
|
|
@@ -9962,7 +9977,7 @@
|
|
|
9962
9977
|
{
|
|
9963
9978
|
"path": "scripts/gds/provision-core-upgrade.js",
|
|
9964
9979
|
"mode": "0000644",
|
|
9965
|
-
"sha256": "
|
|
9980
|
+
"sha256": "58599968201f98723a531c19f77985c0b26b9afa6d6448ae4803074497a6f2ad"
|
|
9966
9981
|
},
|
|
9967
9982
|
{
|
|
9968
9983
|
"path": "scripts/gds/provision-disconnect.js",
|
|
@@ -10017,7 +10032,7 @@
|
|
|
10017
10032
|
{
|
|
10018
10033
|
"path": "scripts/gds/provision.js",
|
|
10019
10034
|
"mode": "0000644",
|
|
10020
|
-
"sha256": "
|
|
10035
|
+
"sha256": "4da6314b41905bcffbc51abe127911512743ad14f8130b4c6a25c23600a3eb7f"
|
|
10021
10036
|
},
|
|
10022
10037
|
{
|
|
10023
10038
|
"path": "scripts/gds/publish-credential-check.js",
|
|
@@ -10629,10 +10644,15 @@
|
|
|
10629
10644
|
"mode": "0000644",
|
|
10630
10645
|
"sha256": "1f0a8d52622328e29bd3bdbe0402f680f425480e37e29ee42ceee907368710af"
|
|
10631
10646
|
},
|
|
10647
|
+
{
|
|
10648
|
+
"path": "scripts/gds/upgrade-outcome.js",
|
|
10649
|
+
"mode": "0000644",
|
|
10650
|
+
"sha256": "a4f05b5e1042cf3d8c7096ef413b47fd039a02b78f2e4dbefb5bd002973784cb"
|
|
10651
|
+
},
|
|
10632
10652
|
{
|
|
10633
10653
|
"path": "scripts/gds/upgrade.js",
|
|
10634
10654
|
"mode": "0000644",
|
|
10635
|
-
"sha256": "
|
|
10655
|
+
"sha256": "399c2225543fc0d37505bca9cd96e33cf5598aa1f364a458d5b5efb3a588ddb3"
|
|
10636
10656
|
},
|
|
10637
10657
|
{
|
|
10638
10658
|
"path": "scripts/gds/validate-design.js",
|
|
@@ -11122,7 +11142,7 @@
|
|
|
11122
11142
|
{
|
|
11123
11143
|
"path": "src/module-api.js",
|
|
11124
11144
|
"mode": "0000644",
|
|
11125
|
-
"sha256": "
|
|
11145
|
+
"sha256": "8acd38044812dee3022427fad27e12cc7f9d81a90c1a0d921e860135a30f9e43"
|
|
11126
11146
|
},
|
|
11127
11147
|
{
|
|
11128
11148
|
"path": "src/module-loader/catalog.js",
|
|
@@ -12067,7 +12087,7 @@
|
|
|
12067
12087
|
{
|
|
12068
12088
|
"path": "tests/core_upgrade_door.mjs",
|
|
12069
12089
|
"mode": "0000644",
|
|
12070
|
-
"sha256": "
|
|
12090
|
+
"sha256": "8c8b03b8a3bb83b0c733726f3aa1256b6af5ac61c76e6af465fe2bc4f792ea6b"
|
|
12071
12091
|
},
|
|
12072
12092
|
{
|
|
12073
12093
|
"path": "tests/core_upgrade_owner_door.mjs",
|
|
@@ -12077,7 +12097,7 @@
|
|
|
12077
12097
|
{
|
|
12078
12098
|
"path": "tests/core_upgrade_runner.mjs",
|
|
12079
12099
|
"mode": "0000644",
|
|
12080
|
-
"sha256": "
|
|
12100
|
+
"sha256": "525e38bbafc2dd1419f3e41131dd89c6adc4f063f8585b193e2be72801bede5a"
|
|
12081
12101
|
},
|
|
12082
12102
|
{
|
|
12083
12103
|
"path": "tests/cosmos_token_contrast.mjs",
|
|
@@ -13734,6 +13754,11 @@
|
|
|
13734
13754
|
"mode": "0000644",
|
|
13735
13755
|
"sha256": "9e69603c05fae6c39e2db7336d6569327f17365ff46ef05b1074943dc34a49e7"
|
|
13736
13756
|
},
|
|
13757
|
+
{
|
|
13758
|
+
"path": "tests/intent_retry.mjs",
|
|
13759
|
+
"mode": "0000644",
|
|
13760
|
+
"sha256": "750745cf866dcf1012f2677881738aa64ae2b8ceaed078ab688612a8b0d2208c"
|
|
13761
|
+
},
|
|
13737
13762
|
{
|
|
13738
13763
|
"path": "tests/interaction_posture.mjs",
|
|
13739
13764
|
"mode": "0000644",
|
|
@@ -14692,7 +14717,7 @@
|
|
|
14692
14717
|
{
|
|
14693
14718
|
"path": "tests/provision.mjs",
|
|
14694
14719
|
"mode": "0000644",
|
|
14695
|
-
"sha256": "
|
|
14720
|
+
"sha256": "46b2235ee186ba304c18f5cf3002a9e5311f7bda11fa230bbada8a7a43bec3cd"
|
|
14696
14721
|
},
|
|
14697
14722
|
{
|
|
14698
14723
|
"path": "tests/provision_account_identity.mjs",
|
|
@@ -16094,6 +16119,11 @@
|
|
|
16094
16119
|
"mode": "0000644",
|
|
16095
16120
|
"sha256": "d3f5b941b89ede78ebb3cd7af43467363e5d6c6d35c7ad26a2977683ecf869e7"
|
|
16096
16121
|
},
|
|
16122
|
+
{
|
|
16123
|
+
"path": "tests/upgrade_outcome.mjs",
|
|
16124
|
+
"mode": "0000644",
|
|
16125
|
+
"sha256": "cec960e84573efe77003beb324c0679900f321249148418e7a04daf7a7a40215"
|
|
16126
|
+
},
|
|
16097
16127
|
{
|
|
16098
16128
|
"path": "tests/upgrade_persist_pin.mjs",
|
|
16099
16129
|
"mode": "0000644",
|
|
@@ -2715,5 +2715,9 @@ is load-bearing: the script throws rather than guess if it is missing, and
|
|
|
2715
2715
|
landed since 1.20.27 with no explicit bump. run 36772266063. (task 1002620)
|
|
2716
2716
|
1.20.29 — CI auto-patch (publish-on-merge, ADR 0161): carrier for merges
|
|
2717
2717
|
landed since 1.20.28 with no explicit bump. run 36775277990. (task 1002620)
|
|
2718
|
+
1.20.30 — CI auto-patch (publish-on-merge, ADR 0161): carrier for merges
|
|
2719
|
+
landed since 1.20.29 with no explicit bump. run 36777226204. (task 1002620)
|
|
2720
|
+
1.20.31 — CI auto-patch (publish-on-merge, ADR 0161): carrier for merges
|
|
2721
|
+
landed since 1.20.30 with no explicit bump. run 36779142919. (task 1002620)
|
|
2718
2722
|
---------------------------------------------------------------------------
|
|
2719
2723
|
```
|
|
@@ -128,6 +128,16 @@ function coreUpgradeFault(inst, to, live = null) {
|
|
|
128
128
|
async function enqueueCoreUpgrade(provisioning, db, instanceId, to, requestedBy, action = 'core-upgrade') {
|
|
129
129
|
const { intent, created } = await provisioning.enqueueIntent(db, instanceId, action, requestedBy, to);
|
|
130
130
|
if (created) return { intent, created: true };
|
|
131
|
+
// THE SAME MOVE, WAITING OUT A RETRY (task 1004447). A transient failure re-pends the
|
|
132
|
+
// intent with a not_before up to six hours away, and it keeps the single open slot all
|
|
133
|
+
// that time — so without this, the owner pressing Move again was told "a run is pending,
|
|
134
|
+
// try again later" about the very move they were asking for. Same action, same target,
|
|
135
|
+
// still pending: bring its next attempt forward instead. Anything else stays a conflict.
|
|
136
|
+
if (intent && intent.state === 'pending' && intent.action === action && String(intent.target_version || '') === String(to)
|
|
137
|
+
&& intent.not_before && new Date(intent.not_before).getTime() > Date.now()) {
|
|
138
|
+
await db.query("UPDATE provisioning_intents SET not_before = NULL, updated_at = now() WHERE id = $1 AND state = 'pending'", [intent.id]);
|
|
139
|
+
return { intent: { ...intent, not_before: null }, created: false, retryNow: true };
|
|
140
|
+
}
|
|
131
141
|
return { intent, created: false, conflict: 'open_intent' };
|
|
132
142
|
}
|
|
133
143
|
|
|
@@ -336,7 +346,9 @@ function previewFromSnapshot(inst, snap, openIntent, lastMove) {
|
|
|
336
346
|
// and a stale pass is worse than no answer because it reads as a guarantee.
|
|
337
347
|
preflight: preflightForOffer(pf, served, recommended),
|
|
338
348
|
preflight_at: pf && recommended && pf.target === recommended ? (snap.preflight_at || null) : null,
|
|
339
|
-
|
|
349
|
+
// not_before: a failed move waiting out its retry backoff (task 1004447) — the page can
|
|
350
|
+
// say "trying again at …" instead of an unexplained "pending" for up to six hours.
|
|
351
|
+
open_intent: openIntent ? { action: openIntent.action, state: openIntent.state, target_version: openIntent.target_version || null, not_before: openIntent.not_before || null, last_error: openIntent.not_before && openIntent.last_error ? String(openIntent.last_error).slice(0, 600) : null } : null,
|
|
340
352
|
// THE LAST MOVE, WHATEVER BECAME OF IT (task 1004145).
|
|
341
353
|
//
|
|
342
354
|
// `open_intent` is `state IN ('pending','running')` — by the index's own definition.
|
|
@@ -361,6 +373,11 @@ function previewFromSnapshot(inst, snap, openIntent, lastMove) {
|
|
|
361
373
|
// `git status` (the dirty-tree one lists every file). Never set on a DONE move:
|
|
362
374
|
// a done row's last_error is the note below, and a move that worked is not an error.
|
|
363
375
|
error: lastMove.state !== 'done' && lastMove.last_error ? String(lastMove.last_error).slice(0, 600) : null,
|
|
376
|
+
// WHAT KIND of failure it was (task 1004446, migration provisioning_030): the class the
|
|
377
|
+
// page and the self-heal steps key on, and its fixed reason code. null on a move that
|
|
378
|
+
// worked, and on one resolved before the columns existed.
|
|
379
|
+
failure: lastMove.state !== 'done' && lastMove.failure_class
|
|
380
|
+
? { class: lastMove.failure_class, reason: lastMove.failure_reason || null } : null,
|
|
364
381
|
// It LANDED, but GitHub did not get the new pin (task 1004065) — parseRepoNote.
|
|
365
382
|
warning: lastMove.state === 'done' ? parseRepoNote(lastMove.last_error) : null,
|
|
366
383
|
} : null,
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
-- provisioning_030_intent_failure_class.sql — WHAT KIND of failure stopped an intent
|
|
2
|
+
-- (task 1004446, goal 1000090 deploy self-heal).
|
|
3
|
+
--
|
|
4
|
+
-- WHY. A failed core move was one sentence in last_error, and every non-zero exit read
|
|
5
|
+
-- "failed and was rolled back" — a refusal that changed nothing, a real rollback, and a
|
|
6
|
+
-- rollback that itself failed were indistinguishable to the runner and to the page. These
|
|
7
|
+
-- two columns carry scripts/gds/upgrade-outcome.js's classification beside the sentence:
|
|
8
|
+
--
|
|
9
|
+
-- failure_class transient | known_wedge | needs_decision | unknown — what the runner
|
|
10
|
+
-- may do about it (retry later, fix it itself, ask the owner, ask a person)
|
|
11
|
+
-- failure_reason a fixed reason code (dirty_tree, outside_channel, rollback_failed, …) —
|
|
12
|
+
-- vocabulary only, never free text, so it is safe to index and to show
|
|
13
|
+
--
|
|
14
|
+
-- NULL on both means "not a classified failure": a done intent, a pending one, or a row
|
|
15
|
+
-- written before this migration. Written by the drain loop (scripts/gds/provision.js).
|
|
16
|
+
--
|
|
17
|
+
-- Additive and namespaced (ADR 0083): no down-migration, idempotent — safe to re-run.
|
|
18
|
+
|
|
19
|
+
BEGIN;
|
|
20
|
+
|
|
21
|
+
ALTER TABLE provisioning_intents ADD COLUMN IF NOT EXISTS failure_class text;
|
|
22
|
+
ALTER TABLE provisioning_intents ADD COLUMN IF NOT EXISTS failure_reason text;
|
|
23
|
+
|
|
24
|
+
ALTER TABLE provisioning_intents DROP CONSTRAINT IF EXISTS provisioning_intents_failure_class_check;
|
|
25
|
+
ALTER TABLE provisioning_intents ADD CONSTRAINT provisioning_intents_failure_class_check
|
|
26
|
+
CHECK (failure_class IS NULL OR failure_class IN ('transient', 'known_wedge', 'needs_decision', 'unknown'));
|
|
27
|
+
|
|
28
|
+
COMMIT;
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
-- provisioning_031_intent_not_before.sql — a failed intent waits before its next attempt
|
|
2
|
+
-- (task 1004447, goal 1000090 deploy self-heal).
|
|
3
|
+
--
|
|
4
|
+
-- WHY. A failed intent was re-pended with no delay, and the drain loop claims the oldest
|
|
5
|
+
-- pending row — so one runner tick ran the same failing move up to MAX_INTENT_ATTEMPTS
|
|
6
|
+
-- times back to back, restarting the service each time a pin had moved, and then gave up.
|
|
7
|
+
-- A registry blip or a release still publishing needs minutes, not milliseconds.
|
|
8
|
+
--
|
|
9
|
+
-- not_before: the earliest time the runner may claim this row again. NULL = now (every row
|
|
10
|
+
-- written before this column, and every fresh request). claimNextIntent skips a row whose
|
|
11
|
+
-- not_before is still in the future; scripts/gds/provision.js sets it on each re-pend.
|
|
12
|
+
--
|
|
13
|
+
-- Additive and namespaced (ADR 0083): no down-migration, idempotent — safe to re-run.
|
|
14
|
+
|
|
15
|
+
BEGIN;
|
|
16
|
+
|
|
17
|
+
ALTER TABLE provisioning_intents ADD COLUMN IF NOT EXISTS not_before timestamptz;
|
|
18
|
+
|
|
19
|
+
COMMIT;
|
|
@@ -1254,7 +1254,7 @@ async function claimNextIntent(db) {
|
|
|
1254
1254
|
SET state = 'running', attempts = attempts + 1, updated_at = now()
|
|
1255
1255
|
WHERE id = (
|
|
1256
1256
|
SELECT id FROM provisioning_intents
|
|
1257
|
-
WHERE state = 'pending'
|
|
1257
|
+
WHERE state = 'pending' AND (not_before IS NULL OR not_before <= now()) -- a re-pended failure waits out its backoff (task 1004447)
|
|
1258
1258
|
ORDER BY created_at
|
|
1259
1259
|
FOR UPDATE SKIP LOCKED
|
|
1260
1260
|
LIMIT 1
|
package/package-lock.json
CHANGED
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@bongos/core",
|
|
3
|
-
"version": "1.20.
|
|
3
|
+
"version": "1.20.31",
|
|
4
4
|
"lockfileVersion": 3,
|
|
5
5
|
"requires": true,
|
|
6
6
|
"packages": {
|
|
7
7
|
"": {
|
|
8
8
|
"name": "@bongos/core",
|
|
9
|
-
"version": "1.20.
|
|
9
|
+
"version": "1.20.31",
|
|
10
10
|
"license": "AGPL-3.0-or-later",
|
|
11
11
|
"dependencies": {
|
|
12
12
|
"express": "^4.21.2",
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@bongos/core",
|
|
3
|
-
"version": "1.20.
|
|
3
|
+
"version": "1.20.31",
|
|
4
4
|
"description": "Cloud Bongos — the AI-first build platform core (GDS + platform surfaces + module system), installed as a versioned dependency (ADR 0108).",
|
|
5
5
|
"license": "AGPL-3.0-or-later",
|
|
6
6
|
"main": "src/platform-server.js",
|
package/release-notes.json
CHANGED
|
@@ -8246,5 +8246,17 @@
|
|
|
8246
8246
|
"id": "1004445",
|
|
8247
8247
|
"text": "Each project now has its own update rule. A project can be allowed to take bigger step-up updates (like 1.19 to 1.20) instead of being stuck on small fixes only, and its owner can change the rule."
|
|
8248
8248
|
}
|
|
8249
|
+
],
|
|
8250
|
+
"1.20.30": [
|
|
8251
|
+
{
|
|
8252
|
+
"id": "1004446",
|
|
8253
|
+
"text": "When a software update fails, the system now records what kind of failure it was — a temporary glitch, a known snag, a rule only the owner can change, or something unknown — and tells the truth about it instead of always sayin"
|
|
8254
|
+
}
|
|
8255
|
+
],
|
|
8256
|
+
"1.20.31": [
|
|
8257
|
+
{
|
|
8258
|
+
"id": "1004447",
|
|
8259
|
+
"text": "A software update that fails for a temporary reason now waits and tries again later — after 15 minutes, then an hour, then six hours — instead of hammering the same failing step three times in a row. Failures that retrying can"
|
|
8260
|
+
}
|
|
8249
8261
|
]
|
|
8250
8262
|
}
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
//
|
|
3
|
+
// scripts/gds/intent-retry.js — when a failed provisioning intent is tried again, and when it
|
|
4
|
+
// is retired instead (task 1004447, goal 1000090 deploy self-heal).
|
|
5
|
+
//
|
|
6
|
+
// WHY. The drain loop re-pended a failure with no delay and claims the oldest pending row, so
|
|
7
|
+
// one tick ran the same failing move up to MAX_INTENT_ATTEMPTS times back to back — seconds
|
|
8
|
+
// apart, restarting the service after every pin move — and then gave up. That is the wrong
|
|
9
|
+
// answer for each kind of failure upgrade-outcome.js now names:
|
|
10
|
+
// transient — the registry blipped, the release is still publishing: WAIT, then retry
|
|
11
|
+
// (about 15 minutes, 1 hour, 6 hours), which is what a person would do.
|
|
12
|
+
// known_wedge, needs_decision, unknown — trying the same thing again cannot change the
|
|
13
|
+
// answer, so the intent is RETIRED at once and the failure left standing
|
|
14
|
+
// for the next step (a fix, the owner, a person) to act on.
|
|
15
|
+
// A failure with no class (every action other than a core move) keeps its old attempt budget,
|
|
16
|
+
// but each retry now waits DEFAULT_RETRY_DELAY_MS — at least the next tick, never the same one.
|
|
17
|
+
//
|
|
18
|
+
// PURE: no I/O. The drain loop (scripts/gds/provision.js cmdRunIntents) asks and obeys.
|
|
19
|
+
|
|
20
|
+
const MINUTE = 60 * 1000;
|
|
21
|
+
const TRANSIENT_BACKOFF_MS = Object.freeze([15 * MINUTE, 60 * MINUTE, 6 * 60 * MINUTE]);
|
|
22
|
+
const DEFAULT_RETRY_DELAY_MS = MINUTE;
|
|
23
|
+
|
|
24
|
+
/**
|
|
25
|
+
* What to do with an intent whose run just failed. PURE.
|
|
26
|
+
* @param {object} p
|
|
27
|
+
* @param {number} p.attempts the attempt that just ran (claimNextIntent increments first)
|
|
28
|
+
* @param {object} [p.failure] `{ class, reason }` from the result, when classified
|
|
29
|
+
* @param {boolean} [p.terminal] the leg said it must not be retried unattended
|
|
30
|
+
* @param {number} p.maxAttempts the runner's budget for unclassified failures
|
|
31
|
+
* @returns {{ action: 'retire' } | { action: 'retry', delayMs: number }}
|
|
32
|
+
*/
|
|
33
|
+
function retryDecision({ attempts, failure, terminal, maxAttempts }) {
|
|
34
|
+
if (terminal) return { action: 'retire' };
|
|
35
|
+
if (failure && failure.class) {
|
|
36
|
+
if (failure.class !== 'transient') return { action: 'retire' };
|
|
37
|
+
const i = Math.max(0, (attempts || 1) - 1);
|
|
38
|
+
return i < TRANSIENT_BACKOFF_MS.length ? { action: 'retry', delayMs: TRANSIENT_BACKOFF_MS[i] } : { action: 'retire' };
|
|
39
|
+
}
|
|
40
|
+
return (attempts || 0) >= maxAttempts ? { action: 'retire' } : { action: 'retry', delayMs: DEFAULT_RETRY_DELAY_MS };
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
/**
|
|
44
|
+
* The attempt ceiling checked BEFORE an intent runs. A row waiting out a transient backoff
|
|
45
|
+
* has one attempt per backoff step plus the first, so the pre-run guard must not retire it
|
|
46
|
+
* on the default budget before its last scheduled try. PURE.
|
|
47
|
+
*/
|
|
48
|
+
function maxAttemptsFor(intent, maxAttempts) {
|
|
49
|
+
return intent && intent.failure_class === 'transient' ? Math.max(maxAttempts, TRANSIENT_BACKOFF_MS.length + 1) : maxAttempts;
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
module.exports = { TRANSIENT_BACKOFF_MS, DEFAULT_RETRY_DELAY_MS, retryDecision, maxAttemptsFor };
|
|
@@ -38,6 +38,7 @@ const { appUser, dbName, instanceUser, parseTargetRef, resolveOwnerGithubToken,
|
|
|
38
38
|
const { ensurePinWriteKey, hasOwnAccount, pinKeyPath, pinSshCommand } = require('./provision-pin-key.js');
|
|
39
39
|
const { isValidSlug } = require('../../modules/provisioning/provisioning.js');
|
|
40
40
|
const { RELEASE_NOTES_FILE, RELEASE_NOTES_MAX_BYTES, parseChangeRecords, notesBetween } = require('./release-notes.js');
|
|
41
|
+
const outcome = require('./upgrade-outcome.js'); // how far a failed move got, as a class (task 1004446)
|
|
41
42
|
|
|
42
43
|
// The shapes whose box this runner can actually reach. Mirrors UPGRADE_SHAPES in the
|
|
43
44
|
// web-tier half, and deliberately EXCLUDES 'dedicated': a dedicated droplet is stood up
|
|
@@ -150,6 +151,12 @@ function instanceChannel(inst, channelMod) {
|
|
|
150
151
|
return channelMod.normalizeChannel(inst && inst.update_channel != null ? inst.update_channel : channelMod.DEFAULT_CHANNEL);
|
|
151
152
|
}
|
|
152
153
|
|
|
154
|
+
// A classified refusal (task 1004446): the drain loop stores `failure` beside last_error,
|
|
155
|
+
// so the rest of the self-heal chain acts on a class, not on a sentence. PURE.
|
|
156
|
+
function failed(reason, error, extra = {}) {
|
|
157
|
+
return { ok: false, error, failure: outcome.classifyFailure(reason), ...extra };
|
|
158
|
+
}
|
|
159
|
+
|
|
153
160
|
// The builder id a queued move is attributed to, from the actor the web tier recorded.
|
|
154
161
|
// PURE. Two doors reach the move (ADR 0339): an Archon (`api:archon:<id>`) and, since
|
|
155
162
|
// task 1004174, the project's own owner (`api:owner:<id>`). Both carry the id, because
|
|
@@ -189,7 +196,7 @@ async function coreUpgradeInstance(inst, deps, intent) {
|
|
|
189
196
|
// anything. The guard's real job is the OTHER actions on the self row, enforced at
|
|
190
197
|
// enqueueIntent where every intent passes.
|
|
191
198
|
if (!UPGRADE_SHAPES.has(inst.hosting_shape)) {
|
|
192
|
-
return
|
|
199
|
+
return failed('shape_not_automated', `core-upgrade is not automated for a '${inst.hosting_shape}' instance — whoever runs that machine has to upgrade it there`);
|
|
193
200
|
}
|
|
194
201
|
if (inst.status !== 'active') {
|
|
195
202
|
// Not an error: a teardown or a stand-up that raced the request must not burn the
|
|
@@ -198,7 +205,7 @@ async function coreUpgradeInstance(inst, deps, intent) {
|
|
|
198
205
|
return { ok: true, noop: true };
|
|
199
206
|
}
|
|
200
207
|
if (!to) {
|
|
201
|
-
return
|
|
208
|
+
return failed('no_target', 'core-upgrade intent carries no target_version — refusing to guess which version was meant');
|
|
202
209
|
}
|
|
203
210
|
// THE SLUG IS INTERPOLATED INTO A SHELL COMMAND BELOW, so it is re-checked HERE
|
|
204
211
|
// rather than trusted. `isValidSlug` is enforced on the create route, but the
|
|
@@ -209,7 +216,7 @@ async function coreUpgradeInstance(inst, deps, intent) {
|
|
|
209
216
|
// two interpolated values are already constrained — the version by the column's own
|
|
210
217
|
// CHECK and by VERSION_RE, the actor by the digits-only extraction below.
|
|
211
218
|
if (!isValidSlug(inst.slug)) {
|
|
212
|
-
return
|
|
219
|
+
return failed('bad_slug', `instance slug '${inst.slug}' is not a valid slug — refusing to build a command from it`);
|
|
213
220
|
}
|
|
214
221
|
|
|
215
222
|
// ---- 2. is the target real, and permitted -----------------------------------------
|
|
@@ -227,7 +234,7 @@ async function coreUpgradeInstance(inst, deps, intent) {
|
|
|
227
234
|
const listed = channelMod.listAvailableVersions({ cwd: instanceDir });
|
|
228
235
|
const available = (listed && listed.versions) || [];
|
|
229
236
|
const fault = targetFault(to, available, installed, channel, channelMod.channelAllows);
|
|
230
|
-
if (fault) return
|
|
237
|
+
if (fault) return failed(outcome.runnerReason(fault), fault);
|
|
231
238
|
if (installed && installed === to) {
|
|
232
239
|
// `bongos upgrade` would no-op on this too, but saying so here keeps a re-drained
|
|
233
240
|
// intent from restarting a healthy service to land exactly where it already is.
|
|
@@ -265,10 +272,17 @@ async function coreUpgradeInstance(inst, deps, intent) {
|
|
|
265
272
|
log(` [upgrade] (in ${runDir}) ${cmd}`);
|
|
266
273
|
boxExec(cmd, { cwd: runDir });
|
|
267
274
|
} catch (e) {
|
|
268
|
-
//
|
|
269
|
-
//
|
|
270
|
-
//
|
|
271
|
-
|
|
275
|
+
// HOW FAR IT GOT, from upgrade.js's own outcome line (task 1004446) — never assumed.
|
|
276
|
+
// It used to say "failed and was rolled back" for every exit, including refusals that
|
|
277
|
+
// fire before the pin moves (nothing changed) and rollbacks that themselves failed
|
|
278
|
+
// (the project may be down). ok:false, never a throw: the drain loop's catch marks the
|
|
279
|
+
// INSTANCE broken, which is only ever right for the rollback_failed case, and that one
|
|
280
|
+
// is reported (and retired, never retried unattended) rather than guessed at here.
|
|
281
|
+
const text = `${(e && e.stderr) || ''}\n${(e && e.message) || e}`;
|
|
282
|
+
const o = outcome.parseOutcomeLine(text);
|
|
283
|
+
const said = (/✖ ([^\n]+)/.exec(text) || [])[1] || String((e && e.message) || e).split('\n')[0];
|
|
284
|
+
const reason = o ? o.reason : 'unknown';
|
|
285
|
+
return failed(reason, outcome.describeUpgradeFailure(to, o, said), { stage: o ? o.stage : null, terminal: reason === 'rollback_failed' });
|
|
272
286
|
}
|
|
273
287
|
|
|
274
288
|
// ---- 4. read back what the BOX serves (ADR 0293 D6) --------------------------------
|
|
@@ -287,7 +301,7 @@ async function coreUpgradeInstance(inst, deps, intent) {
|
|
|
287
301
|
if (served && served !== to) {
|
|
288
302
|
// upgrade.js's own --version-url gate should have caught and reverted this, so
|
|
289
303
|
// reaching here means the two disagree. Report what was SEEN rather than a verdict.
|
|
290
|
-
return
|
|
304
|
+
return failed('served_mismatch', `the box is serving core ${served}, not the requested ${to} — the move did not take`);
|
|
291
305
|
}
|
|
292
306
|
|
|
293
307
|
// ---- 5. the pin reaches the project's own repo, pushed from HERE (task 1004065) ----
|
package/scripts/gds/provision.js
CHANGED
|
@@ -46,6 +46,7 @@ const path = require('node:path');
|
|
|
46
46
|
const cp = require('node:child_process');
|
|
47
47
|
const crypto = require('node:crypto');
|
|
48
48
|
|
|
49
|
+
const retry = require('./intent-retry.js'); // when a failed intent is tried again (task 1004447)
|
|
49
50
|
const { CONFIG, MANIFEST_UA, softFailResult, MANIFEST_VERIFY_INTERVAL_MS, MANIFEST_VERIFY_TRIES, MAX_INTENT_ATTEMPTS, REPO_ROOT, coreVersionSafe, hasFlag, loadDeps, oauthSecret, provisionerBotEmail } = require('./provision-config.js');
|
|
50
51
|
const { accountPreflight, appUser, alreadyScaffolded, buildCorePinRefreshCommit, coreCheckoutHasGit, dbCreateCmd, dbName, dbRoleCmd, dbRoleCmdShown, dbRoleGrantsCmd, ensureInstanceUserCmd, generateDbPassword, grantInstanceRepoReadCmd, instanceDbRole, instanceStateDir, instanceUser, deployKeyPath, deployKeyTitle, ensurePrivateRepoAccess, installedCorePackDir, installedCoreTarball, instanceInitSpec, instanceRepoRemote, migrateCmd, onboardMode, ownerLoginOf, parseTargetRef, planCorePinRefresh, refreshStandaloneCorePin, resolveOwnerGithubToken, resolveVendorableCoreTarball, safeVersionLabel, scaffoldStandaloneRepo, seedFirstVersionCmd, standaloneInstallCmd, standaloneMigrateCmd, standalonePullCmd, standaloneRegenDocsCmd, standaloneRoot } = require('./provision-repo.js');
|
|
51
52
|
const { backupScriptPath, backupService, backupServicePath, backupTimer, backupTimerPath, backupUnitName, instanceManifestCmd, originEnvVarsFor, serviceUnit, serviceUnitPath, settingsConsumed, settingsEnvVarsFor, renamedEnvCarry, upsertEnvVars, webEnvBody, webEnvPath } = require('./provision-units.js');
|
|
@@ -1171,8 +1172,8 @@ async function cmdRunIntents(deps) {
|
|
|
1171
1172
|
const intent = await provisioning.claimNextIntent(db);
|
|
1172
1173
|
if (!intent) break;
|
|
1173
1174
|
ran++;
|
|
1174
|
-
if (intent.attempts > MAX_INTENT_ATTEMPTS) {
|
|
1175
|
-
await provisioning.resolveIntent(db, intent.id, 'error', `exceeded ${MAX_INTENT_ATTEMPTS} attempts`);
|
|
1175
|
+
if (intent.attempts > retry.maxAttemptsFor(intent, MAX_INTENT_ATTEMPTS)) { // a transient core move gets one try per backoff step (task 1004447)
|
|
1176
|
+
await provisioning.resolveIntent(db, intent.id, 'error', `exceeded ${retry.maxAttemptsFor(intent, MAX_INTENT_ATTEMPTS)} attempts`);
|
|
1176
1177
|
errors++; log(` ✖ intent #${intent.id} retired (too many attempts)`); continue;
|
|
1177
1178
|
}
|
|
1178
1179
|
try {
|
|
@@ -1192,7 +1193,11 @@ async function cmdRunIntents(deps) {
|
|
|
1192
1193
|
else if (intent.action === 'repo-private' || intent.action === 'render-standup') result = intent.action === 'repo-private' ? await require('./provision-repo-private.js').repoPrivateInstance(inst, deps) : await require('./provision-render.js').renderStandupInstance(inst, deps, intent); // wire the deploy key, THEN flip (task 1004193) · spend the borrowed Render key once (task 1004183); required here — this file is at the size ratchet
|
|
1193
1194
|
else result = { ok: false, error: `unknown_action '${intent.action}' — runner core ${coreVersionSafe() || '?'} does not know it (runner too old?); refusing to guess` };
|
|
1194
1195
|
if (result && result.ok === false) {
|
|
1195
|
-
|
|
1196
|
+
// WAIT OR RETIRE (task 1004447, intent-retry.js): a transient failure waits ~15m/1h/6h; a
|
|
1197
|
+
// classified non-transient one is retired at once (retrying cannot change its answer);
|
|
1198
|
+
// an unclassified one keeps its budget but never re-runs in the same tick.
|
|
1199
|
+
const next = retry.retryDecision({ attempts: intent.attempts, failure: result.failure, terminal: result.terminal, maxAttempts: MAX_INTENT_ATTEMPTS });
|
|
1200
|
+
if (next.action === 'retire') { // terminal: a leg that must not be retried unattended (the Render leg's key is already gone — task 1004183)
|
|
1196
1201
|
await provisioning.resolveIntent(db, intent.id, 'error', result.error || 'op failed');
|
|
1197
1202
|
// A retired intent stops silently otherwise (F19): record the reason
|
|
1198
1203
|
// on the instance WITHOUT changing its lifecycle status — passing the
|
|
@@ -1200,8 +1205,10 @@ async function cmdRunIntents(deps) {
|
|
|
1200
1205
|
// instance is never marked broken over a domain hiccup, but the card
|
|
1201
1206
|
// can say why the work stopped.
|
|
1202
1207
|
await provisioning.setInstanceStatus(db, intent.instance_id, inst.status, { error_note: result.error || 'op failed', error_note_action: intent.action }).catch(() => {});
|
|
1208
|
+
if (result.failure) await recordFailureClass(db, intent.id, result.failure); // WHAT KIND of failure, beside the sentence (task 1004446)
|
|
1203
1209
|
} else {
|
|
1204
|
-
await db.query(`UPDATE provisioning_intents SET state='pending', last_error=$2, updated_at=now() WHERE id=$1`, [intent.id, result.error || 'op failed']);
|
|
1210
|
+
await db.query(`UPDATE provisioning_intents SET state='pending', last_error=$2, not_before = now() + ($3 * interval '1 millisecond'), updated_at=now() WHERE id=$1`, [intent.id, result.error || 'op failed', next.delayMs]);
|
|
1211
|
+
if (result.failure) await recordFailureClass(db, intent.id, result.failure);
|
|
1205
1212
|
}
|
|
1206
1213
|
errors++;
|
|
1207
1214
|
} else { await provisioning.resolveIntent(db, intent.id, 'done', (result && result.doneNote) || null); ok++; } // done WITH a note: it landed, and this is what it could not finish (task 1004065)
|
|
@@ -1293,6 +1300,15 @@ box (task 2074). Set PROVISION_CONTROL_PLANE_IP to this box's own public IP so a
|
|
|
1293
1300
|
standalone instance's A record points HERE, not at the co-hosting box's PROVISION_PUBLIC_IP.`);
|
|
1294
1301
|
}
|
|
1295
1302
|
|
|
1303
|
+
// Store an intent's failure class + reason code (migration provisioning_030, task 1004446),
|
|
1304
|
+
// only for a result that carries one. Best-effort: a runner on a schema without the columns
|
|
1305
|
+
// must still drain, so a failed write is swallowed — last_error still says it either way. A
|
|
1306
|
+
// class left on a row that later lands 'done' is harmless: readers ignore it on a done row.
|
|
1307
|
+
async function recordFailureClass(db, intentId, failure) {
|
|
1308
|
+
await db.query('UPDATE provisioning_intents SET failure_class = $2, failure_reason = $3 WHERE id = $1',
|
|
1309
|
+
[intentId, failure ? failure.class : null, failure ? failure.reason : null]).catch(() => {});
|
|
1310
|
+
}
|
|
1311
|
+
|
|
1296
1312
|
async function main() {
|
|
1297
1313
|
const args = process.argv.slice(2);
|
|
1298
1314
|
const cmd = args[0];
|
|
@@ -0,0 +1,155 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
//
|
|
3
|
+
// scripts/gds/upgrade-outcome.js — what a failed core move actually WAS, as a class and a
|
|
4
|
+
// reason code the deploy runner can act on (task 1004446, goal 1000090 deploy self-heal).
|
|
5
|
+
//
|
|
6
|
+
// WHY. Every non-zero exit from `bongos upgrade` reached the owner as "failed and was rolled
|
|
7
|
+
// back", including the refusals that fire BEFORE the pin moves (a dirty tree, the artist
|
|
8
|
+
// gate, a downgrade, the database identity check) — nothing was changed, so nothing was
|
|
9
|
+
// rolled back. And a rollback that itself failed ("manual intervention needed") read exactly
|
|
10
|
+
// like one that worked. The runner could not tell a wait-and-retry from a decision only the
|
|
11
|
+
// owner can make from an outage, so it treated all of them the same: retry, then give up.
|
|
12
|
+
//
|
|
13
|
+
// TWO HALVES, ONE FILE, so the words cannot drift apart:
|
|
14
|
+
// • upgrade.js ends a failed run with ONE machine-readable stderr line (outcomeLine) saying
|
|
15
|
+
// how far it got: refused (nothing moved) · rolled_back (moved, restored) ·
|
|
16
|
+
// rollback_failed (moved, NOT restored). The runner invokes upgrade.js from its own
|
|
17
|
+
// checkout, so the two halves are always the same version.
|
|
18
|
+
// • the runner reads it back (parseOutcomeLine) and classifies (classifyFailure).
|
|
19
|
+
//
|
|
20
|
+
// THE FOUR CLASSES the rest of the self-heal chain keys on:
|
|
21
|
+
// transient — try again later and it may simply work (registry blip, not published yet)
|
|
22
|
+
// known_wedge — a known snag with a known fix the runner may apply itself (dirty tree, full disk)
|
|
23
|
+
// needs_decision — a rule the owner set is refusing it; only the owner can change that
|
|
24
|
+
// unknown — anything else, and every rollback that failed; a person should look
|
|
25
|
+
//
|
|
26
|
+
// PURE: no I/O, no requires. Reason codes are a fixed vocabulary — never builder text — so
|
|
27
|
+
// they are safe to store, index and show.
|
|
28
|
+
|
|
29
|
+
const OUTCOME_TAG = 'BONGOS_UPGRADE_OUTCOME';
|
|
30
|
+
const STAGES = Object.freeze(['refused', 'rolled_back', 'rollback_failed']);
|
|
31
|
+
const FAILURE_CLASSES = Object.freeze(['transient', 'known_wedge', 'needs_decision', 'unknown']);
|
|
32
|
+
|
|
33
|
+
// reason → class. Anything not listed is `unknown`, which is the safe direction: an unknown
|
|
34
|
+
// failure reaches a person; a mislabelled one could be retried or "healed" wrongly.
|
|
35
|
+
const REASON_CLASS = Object.freeze({
|
|
36
|
+
// the runner's own refusals (provision-core-upgrade.js)
|
|
37
|
+
registry_unreadable: 'transient',
|
|
38
|
+
not_published: 'transient',
|
|
39
|
+
outside_channel: 'needs_decision',
|
|
40
|
+
shape_not_automated: 'needs_decision',
|
|
41
|
+
no_target: 'unknown',
|
|
42
|
+
bad_slug: 'unknown',
|
|
43
|
+
installed_unreadable: 'unknown',
|
|
44
|
+
served_mismatch: 'unknown',
|
|
45
|
+
// upgrade.js refusals, before the pin moves
|
|
46
|
+
dirty_tree: 'known_wedge',
|
|
47
|
+
downgrade: 'needs_decision',
|
|
48
|
+
artist_gate: 'needs_decision',
|
|
49
|
+
module_incompatible: 'needs_decision',
|
|
50
|
+
already_on: 'unknown',
|
|
51
|
+
db_unresolvable: 'unknown',
|
|
52
|
+
db_identity: 'unknown',
|
|
53
|
+
health_url_missing: 'unknown',
|
|
54
|
+
core_source: 'unknown',
|
|
55
|
+
// after the pin moved (rolled back)
|
|
56
|
+
disk_full: 'known_wedge',
|
|
57
|
+
npm_install: 'transient',
|
|
58
|
+
integrity_pin: 'unknown',
|
|
59
|
+
migrate: 'unknown',
|
|
60
|
+
restart: 'unknown',
|
|
61
|
+
health: 'unknown',
|
|
62
|
+
// the one that must never be retried unattended
|
|
63
|
+
rollback_failed: 'unknown',
|
|
64
|
+
});
|
|
65
|
+
|
|
66
|
+
// upgrade.js error text → reason. Ordered; first match wins. Each pattern is the PREFIX of
|
|
67
|
+
// a message upgrade.js builds itself (runUpgrade), so these track that file, not the world.
|
|
68
|
+
const UPGRADE_REASONS = [
|
|
69
|
+
[/no space left on device|ENOSPC/i, 'disk_full'],
|
|
70
|
+
[/^refusing to DOWNGRADE/, 'downgrade'],
|
|
71
|
+
[/^already on /, 'already_on'],
|
|
72
|
+
[/^working tree not clean/, 'dirty_tree'],
|
|
73
|
+
[/^module pre-check failed/, 'module_incompatible'],
|
|
74
|
+
[/^cannot determine which database/, 'db_unresolvable'],
|
|
75
|
+
[/^post-restart health check is mandatory/, 'health_url_missing'],
|
|
76
|
+
[/^(--registry and --from|--from tarball not found|no core tarball at)/, 'core_source'],
|
|
77
|
+
[/^npm install failed/, 'npm_install'],
|
|
78
|
+
[/^integrity pin/, 'integrity_pin'],
|
|
79
|
+
[/^migrate failed/, 'migrate'],
|
|
80
|
+
[/^restart failed/, 'restart'],
|
|
81
|
+
[/^(health check failed after restart|served core version did not change)/, 'health'],
|
|
82
|
+
];
|
|
83
|
+
|
|
84
|
+
/** The reason code for an upgrade.js failure result (runUpgrade's `{ ok:false, … }`). PURE. */
|
|
85
|
+
function upgradeReason(res) {
|
|
86
|
+
if (!res) return 'unknown';
|
|
87
|
+
if (res.downgrade) return 'downgrade';
|
|
88
|
+
if (res.artistGate) return 'artist_gate';
|
|
89
|
+
const text = String(res.error || '');
|
|
90
|
+
for (const [re, reason] of UPGRADE_REASONS) if (re.test(text)) return reason;
|
|
91
|
+
// A database-identity refusal carries preflightDbIdentity's own reason, which has no
|
|
92
|
+
// fixed prefix; it is the only pre-pin refusal left that names the database.
|
|
93
|
+
if (!res.rolledBack && /database|PGDATABASE/i.test(text)) return 'db_identity';
|
|
94
|
+
return 'unknown';
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
/** How far a failed run got. A rollback counts as failed if ANY of its steps did. PURE. */
|
|
98
|
+
function upgradeStage(res) {
|
|
99
|
+
if (!res || !res.rolledBack) return 'refused';
|
|
100
|
+
const rb = res.rollback || {};
|
|
101
|
+
const broken = rb.restoreOk === false || rb.reinstallOk === false || rb.restartOk === false || rb.healthOk === false;
|
|
102
|
+
return broken ? 'rollback_failed' : 'rolled_back';
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
/** The single stderr line upgrade.js ends a failed run with. PURE. */
|
|
106
|
+
function outcomeLine(res) {
|
|
107
|
+
const stage = upgradeStage(res);
|
|
108
|
+
const reason = stage === 'rollback_failed' ? 'rollback_failed' : upgradeReason(res);
|
|
109
|
+
return `${OUTCOME_TAG} ${JSON.stringify({ stage, reason })}\n`;
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
/** Read the outcome line back out of captured output, or null if there is none. PURE. */
|
|
113
|
+
function parseOutcomeLine(text) {
|
|
114
|
+
const lines = String(text || '').split(/\r?\n/).filter((l) => l.startsWith(`${OUTCOME_TAG} `));
|
|
115
|
+
if (!lines.length) return null;
|
|
116
|
+
try {
|
|
117
|
+
const o = JSON.parse(lines[lines.length - 1].slice(OUTCOME_TAG.length + 1));
|
|
118
|
+
if (!STAGES.includes(o.stage) || typeof o.reason !== 'string' || !/^[a-z_]{1,40}$/.test(o.reason)) return null;
|
|
119
|
+
return { stage: o.stage, reason: o.reason };
|
|
120
|
+
} catch { return null; }
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
/** `{ class, reason }` for a reason code; an unlisted code is `unknown`. PURE. */
|
|
124
|
+
function classifyFailure(reason) {
|
|
125
|
+
const r = typeof reason === 'string' && Object.prototype.hasOwnProperty.call(REASON_CLASS, reason) ? reason : 'unknown';
|
|
126
|
+
return { class: REASON_CLASS[r] || 'unknown', reason: r };
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
/** The runner's own refusals (targetFault's sentences) → reason code. PURE. */
|
|
130
|
+
function runnerReason(text) {
|
|
131
|
+
const s = String(text || '');
|
|
132
|
+
if (/^could not read the published core versions/.test(s)) return 'registry_unreadable';
|
|
133
|
+
if (/is not published yet/.test(s)) return 'not_published';
|
|
134
|
+
if (/could not read the core version installed/.test(s)) return 'installed_unreadable';
|
|
135
|
+
if (/update channel/.test(s)) return 'outside_channel';
|
|
136
|
+
return 'unknown';
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
/**
|
|
140
|
+
* The sentence the owner reads for a failed upgrade.js run. Says what actually happened,
|
|
141
|
+
* never more: "rolled back" only when a rollback ran and finished. PURE.
|
|
142
|
+
*/
|
|
143
|
+
function describeUpgradeFailure(to, outcome, detail) {
|
|
144
|
+
const d = String(detail || '').trim();
|
|
145
|
+
const tail = d ? `: ${d}` : '';
|
|
146
|
+
if (!outcome) return `core upgrade to ${to} failed, and the upgrade did not report how far it got — check the project before retrying${tail}`;
|
|
147
|
+
if (outcome.stage === 'refused') return `core upgrade to ${to} was refused before anything changed — the project is still on its previous core${tail}`;
|
|
148
|
+
if (outcome.stage === 'rolled_back') return `core upgrade to ${to} failed and was rolled back — the project is back on its previous core${tail}`;
|
|
149
|
+
return `core upgrade to ${to} failed AND the rollback did not finish — the project may be down or on a half-installed core, and needs a person${tail}`;
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
module.exports = {
|
|
153
|
+
OUTCOME_TAG, STAGES, FAILURE_CLASSES, REASON_CLASS,
|
|
154
|
+
upgradeReason, upgradeStage, outcomeLine, parseOutcomeLine, classifyFailure, runnerReason, describeUpgradeFailure,
|
|
155
|
+
};
|
package/scripts/gds/upgrade.js
CHANGED
|
@@ -1480,7 +1480,7 @@ async function main(argv = process.argv.slice(2)) {
|
|
|
1480
1480
|
return require('./module').cmdUpgrade(argv);
|
|
1481
1481
|
}
|
|
1482
1482
|
const res = await runUpgrade(parseArgs(argv));
|
|
1483
|
-
if (!res.ok) { process.stderr.write(`\n✖ ${res.error || 'upgrade failed'}\n`); return 1; }
|
|
1483
|
+
if (!res.ok) { process.stderr.write(`\n✖ ${res.error || 'upgrade failed'}\n${require('./upgrade-outcome').outcomeLine(res)}`); return 1; } // the outcome line: how far it got, for the deploy runner (task 1004446)
|
|
1484
1484
|
return 0;
|
|
1485
1485
|
}
|
|
1486
1486
|
|
package/src/module-api.js
CHANGED
|
@@ -75,7 +75,7 @@ const { responsibilityFor, ROLE_RESPONSIBILITIES } = require('./role-responsibil
|
|
|
75
75
|
// MAJOR (see allowBoxScope below): passes the request through untouched.
|
|
76
76
|
function deprecatedNoopMiddleware(_req, _res, next) { next(); }
|
|
77
77
|
|
|
78
|
-
const CORE_VERSION = '1.20.
|
|
78
|
+
const CORE_VERSION = '1.20.31'; // CI auto-patch carrier (ADR 0161); changelog: docs/module-api-changelog.md
|
|
79
79
|
|
|
80
80
|
// A namespaced logger so a module's log lines are attributable + consistent.
|
|
81
81
|
// Usage: const log = api.logger('discord'); log.info('mounted');
|
|
@@ -431,7 +431,7 @@ await t('schema_pending passes through as a count, and UNKNOWN stays null rather
|
|
|
431
431
|
await t('an open intent is surfaced with its target, so the page can say what is already running', () => {
|
|
432
432
|
const p = CU.previewFromSnapshot(previewInst, { served_version: '1.19.758', available: [], recommended: null },
|
|
433
433
|
{ action: 'core-upgrade', state: 'running', target_version: '1.19.762' });
|
|
434
|
-
assert.deepEqual(p.open_intent, { action: 'core-upgrade', state: 'running', target_version: '1.19.762' });
|
|
434
|
+
assert.deepEqual(p.open_intent, { action: 'core-upgrade', state: 'running', target_version: '1.19.762', not_before: null, last_error: null });
|
|
435
435
|
});
|
|
436
436
|
|
|
437
437
|
console.log('\nthe deploy page is gated on the same atom as its routes:');
|
|
@@ -964,5 +964,48 @@ await t('the route is gated like the move: the owner door OR core.pin.move', ()
|
|
|
964
964
|
assert.match(block, /requireOwnProjectOrPin\(\{ pinGate: api\.requirePermission\('core\.pin\.move'\)/);
|
|
965
965
|
});
|
|
966
966
|
|
|
967
|
+
await t('a failed move carries its class to the page; a move that worked carries none (task 1004446)', () => {
|
|
968
|
+
const failedMove = { target_version: '1.20.1', state: 'error', last_error: 'x', failure_class: 'needs_decision', failure_reason: 'outside_channel' };
|
|
969
|
+
assert.deepEqual(CU.previewFromSnapshot(ACTIVE, null, null, failedMove).last_move.failure, { class: 'needs_decision', reason: 'outside_channel' });
|
|
970
|
+
const done = { ...failedMove, state: 'done' };
|
|
971
|
+
assert.equal(CU.previewFromSnapshot(ACTIVE, null, null, done).last_move.failure, null);
|
|
972
|
+
const old = { target_version: '1.20.1', state: 'error', last_error: 'x' };
|
|
973
|
+
assert.equal(CU.previewFromSnapshot(ACTIVE, null, null, old).last_move.failure, null, 'a row from before the columns');
|
|
974
|
+
});
|
|
975
|
+
|
|
976
|
+
console.log('\na move waiting out its retry (task 1004447):');
|
|
977
|
+
|
|
978
|
+
const waiting = (over = {}) => ({ id: 5, action: 'core-upgrade', state: 'pending', target_version: '1.20.1', not_before: new Date(Date.now() + 3600e3).toISOString(), ...over });
|
|
979
|
+
const enqueueAgainst = async (open, to = '1.20.1') => {
|
|
980
|
+
const writes = [];
|
|
981
|
+
const provisioning = { enqueueIntent: async () => ({ intent: open, created: false }) };
|
|
982
|
+
const db = { query: async (sql, params) => { writes.push({ sql, params }); return { rows: [] }; } };
|
|
983
|
+
return { r: await CU.enqueueCoreUpgrade(provisioning, db, 7, to, 'api:archon:42'), writes };
|
|
984
|
+
};
|
|
985
|
+
|
|
986
|
+
await t('pressing Move for the SAME waiting move brings its retry forward instead of refusing', async () => {
|
|
987
|
+
const { r, writes } = await enqueueAgainst(waiting());
|
|
988
|
+
assert.equal(r.retryNow, true);
|
|
989
|
+
assert.ok(!r.conflict);
|
|
990
|
+
assert.match(writes[0].sql, /SET not_before = NULL/);
|
|
991
|
+
assert.deepEqual(writes[0].params, [5]);
|
|
992
|
+
});
|
|
993
|
+
|
|
994
|
+
await t('a DIFFERENT target, a running move, or one not waiting is still a conflict', async () => {
|
|
995
|
+
for (const open of [waiting({ target_version: '1.20.2' }), waiting({ state: 'running' }), waiting({ not_before: null }), waiting({ action: 'restart' })]) {
|
|
996
|
+
const { r, writes } = await enqueueAgainst(open);
|
|
997
|
+
assert.equal(r.conflict, 'open_intent');
|
|
998
|
+
assert.equal(writes.length, 0);
|
|
999
|
+
}
|
|
1000
|
+
});
|
|
1001
|
+
|
|
1002
|
+
await t('the preview says when a waiting move will try again, and why it is waiting', () => {
|
|
1003
|
+
const p = CU.previewFromSnapshot(ACTIVE, null, waiting({ last_error: 'core 1.20.1 is not published yet' }), null);
|
|
1004
|
+
assert.ok(p.open_intent.not_before);
|
|
1005
|
+
assert.match(p.open_intent.last_error, /not published yet/);
|
|
1006
|
+
assert.equal(CU.previewFromSnapshot(ACTIVE, null, waiting({ not_before: null, last_error: 'x' }), null).open_intent.last_error, null,
|
|
1007
|
+
'an ordinary pending intent shows no error');
|
|
1008
|
+
});
|
|
1009
|
+
|
|
967
1010
|
console.log(`\ncore_upgrade_door: ${passed} passed, ${failed} failed`);
|
|
968
1011
|
if (failed) process.exit(1);
|
|
@@ -1449,5 +1449,71 @@ await t('the snapshot OFFERS what the project\'s rule admits — the button and
|
|
|
1449
1449
|
assert.equal(await offered(undefined), '1.19.1082', 'a pre-column row behaves exactly as before');
|
|
1450
1450
|
});
|
|
1451
1451
|
|
|
1452
|
+
console.log('\nwhat a failed upgrade.js run is reported as (task 1004446):');
|
|
1453
|
+
|
|
1454
|
+
// execSync's error: message "Command failed: <cmd>\n<stderr>", stderr a Buffer.
|
|
1455
|
+
const upgradeThrows = (stderr) => (d) => {
|
|
1456
|
+
const base = d.exec;
|
|
1457
|
+
d.exec = (cmd, opts) => {
|
|
1458
|
+
if (/upgrade\.js/.test(cmd)) { const e = new Error(`Command failed: ${cmd}\n${stderr}`); e.stderr = Buffer.from(stderr); e.status = 1; throw e; }
|
|
1459
|
+
return base(cmd, opts);
|
|
1460
|
+
};
|
|
1461
|
+
return d;
|
|
1462
|
+
};
|
|
1463
|
+
|
|
1464
|
+
await t('a refusal before the pin moved is classified, and never called "rolled back"', async () => {
|
|
1465
|
+
const d = upgradeThrows('\n✖ working tree not clean — commit/stash first\nBONGOS_UPGRADE_OUTCOME {"stage":"refused","reason":"dirty_tree"}\n')(makeDeps());
|
|
1466
|
+
const r = await R.coreUpgradeInstance(inst(), d, intent());
|
|
1467
|
+
assert.equal(r.ok, false);
|
|
1468
|
+
assert.deepEqual(r.failure, { class: 'known_wedge', reason: 'dirty_tree' });
|
|
1469
|
+
assert.equal(r.stage, 'refused');
|
|
1470
|
+
assert.doesNotMatch(r.error, /rolled back/);
|
|
1471
|
+
assert.match(r.error, /working tree not clean/, 'the upgrade\'s own sentence is kept');
|
|
1472
|
+
assert.ok(!r.terminal);
|
|
1473
|
+
});
|
|
1474
|
+
|
|
1475
|
+
await t('a real rollback says rolled back', async () => {
|
|
1476
|
+
const d = upgradeThrows('\n✖ migrate failed (exit 3)\nBONGOS_UPGRADE_OUTCOME {"stage":"rolled_back","reason":"migrate"}\n')(makeDeps());
|
|
1477
|
+
const r = await R.coreUpgradeInstance(inst(), d, intent());
|
|
1478
|
+
assert.deepEqual(r.failure, { class: 'unknown', reason: 'migrate' });
|
|
1479
|
+
assert.match(r.error, /failed and was rolled back/);
|
|
1480
|
+
});
|
|
1481
|
+
|
|
1482
|
+
await t('a FAILED rollback is its own outcome, and terminal — never retried unattended', async () => {
|
|
1483
|
+
const d = upgradeThrows('\n✖ health check failed after restart\nBONGOS_UPGRADE_OUTCOME {"stage":"rollback_failed","reason":"rollback_failed"}\n')(makeDeps());
|
|
1484
|
+
const r = await R.coreUpgradeInstance(inst(), d, intent());
|
|
1485
|
+
assert.deepEqual(r.failure, { class: 'unknown', reason: 'rollback_failed' });
|
|
1486
|
+
assert.equal(r.stage, 'rollback_failed');
|
|
1487
|
+
assert.equal(r.terminal, true);
|
|
1488
|
+
assert.match(r.error, /rollback did not finish/);
|
|
1489
|
+
});
|
|
1490
|
+
|
|
1491
|
+
await t('an exit with NO outcome line (sudo refused before node ran) is unknown and claims no rollback', async () => {
|
|
1492
|
+
const d = upgradeThrows('sudo: unknown user bongos-test4\n')(makeDeps());
|
|
1493
|
+
const r = await R.coreUpgradeInstance(inst(), d, intent());
|
|
1494
|
+
assert.deepEqual(r.failure, { class: 'unknown', reason: 'unknown' });
|
|
1495
|
+
assert.doesNotMatch(r.error, /rolled back/);
|
|
1496
|
+
});
|
|
1497
|
+
|
|
1498
|
+
await t('the runner\'s own refusals are classified too', async () => {
|
|
1499
|
+
const d = withRealChannel({ installed: '1.19.1081', served: '1.19.1081', available: ['1.19.1081', '1.20.1'] });
|
|
1500
|
+
const r = await R.coreUpgradeInstance(inst({ update_channel: 'patch' }), d, intent({ target_version: '1.20.1' }));
|
|
1501
|
+
assert.deepEqual(r.failure, { class: 'needs_decision', reason: 'outside_channel' });
|
|
1502
|
+
const n = await R.coreUpgradeInstance(inst(), makeDeps({ available: ['1.19.760'] }), intent({ target_version: '1.19.999' }));
|
|
1503
|
+
assert.deepEqual(n.failure, { class: 'transient', reason: 'not_published' });
|
|
1504
|
+
const s = await R.coreUpgradeInstance(inst({ hosting_shape: 'dedicated' }), makeDeps(), intent());
|
|
1505
|
+
assert.deepEqual(s.failure, { class: 'needs_decision', reason: 'shape_not_automated' });
|
|
1506
|
+
});
|
|
1507
|
+
|
|
1508
|
+
await t('upgrade.js ends a failed run with the outcome line', () => {
|
|
1509
|
+
const src = fs.readFileSync(new URL('../scripts/gds/upgrade.js', import.meta.url), 'utf8');
|
|
1510
|
+
assert.match(src, /if \(!res\.ok\) \{ process\.stderr\.write\([^\n]*outcomeLine\(res\)/);
|
|
1511
|
+
});
|
|
1512
|
+
|
|
1513
|
+
await t('the drain loop writes the class beside the sentence (behaviour: tests/provision.mjs)', () => {
|
|
1514
|
+
const src = fs.readFileSync(new URL('../scripts/gds/provision.js', import.meta.url), 'utf8');
|
|
1515
|
+
assert.match(src, /SET failure_class = \$2, failure_reason = \$3/);
|
|
1516
|
+
});
|
|
1517
|
+
|
|
1452
1518
|
console.log(`\ncore_upgrade_runner: ${passed} passed, ${failed} failed`);
|
|
1453
1519
|
if (failed) process.exit(1);
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
// tests/intent_retry.mjs — a failed intent waits, or is retired, by what kind of failure it
|
|
2
|
+
// was (task 1004447, goal 1000090 deploy self-heal).
|
|
3
|
+
//
|
|
4
|
+
// WHAT IS AT RISK. One runner tick used to run the same failing core move up to three times
|
|
5
|
+
// back to back, restarting the service each time, then give up. These cases pin the policy
|
|
6
|
+
// that replaced it; tests/provision.mjs drives the drain loop with it.
|
|
7
|
+
//
|
|
8
|
+
// Run: node tests/intent_retry.mjs
|
|
9
|
+
|
|
10
|
+
import { strict as assert } from 'node:assert';
|
|
11
|
+
import { retryDecision, maxAttemptsFor, TRANSIENT_BACKOFF_MS, DEFAULT_RETRY_DELAY_MS } from '../scripts/gds/intent-retry.js';
|
|
12
|
+
|
|
13
|
+
let passed = 0;
|
|
14
|
+
let failed = 0;
|
|
15
|
+
const t = async (name, fn) => {
|
|
16
|
+
try { await fn(); passed += 1; console.log(` PASS ${name}`); }
|
|
17
|
+
catch (e) { failed += 1; console.log(` FAIL ${name}\n ${e.message}`); }
|
|
18
|
+
};
|
|
19
|
+
const MIN = 60 * 1000;
|
|
20
|
+
const transient = { class: 'transient', reason: 'not_published' };
|
|
21
|
+
|
|
22
|
+
await t('a transient failure waits about 15 minutes, then 1 hour, then 6 hours', () => {
|
|
23
|
+
assert.deepEqual([...TRANSIENT_BACKOFF_MS], [15 * MIN, 60 * MIN, 360 * MIN]);
|
|
24
|
+
for (const [attempts, ms] of [[1, 15 * MIN], [2, 60 * MIN], [3, 360 * MIN]]) {
|
|
25
|
+
assert.deepEqual(retryDecision({ attempts, failure: transient, maxAttempts: 3 }), { action: 'retry', delayMs: ms });
|
|
26
|
+
}
|
|
27
|
+
});
|
|
28
|
+
|
|
29
|
+
await t('and is retired after its last scheduled try', () => {
|
|
30
|
+
assert.deepEqual(retryDecision({ attempts: 4, failure: transient, maxAttempts: 3 }), { action: 'retire' });
|
|
31
|
+
});
|
|
32
|
+
|
|
33
|
+
await t('every NON-transient class is retired at once — retrying cannot change the answer', () => {
|
|
34
|
+
for (const c of ['known_wedge', 'needs_decision', 'unknown']) {
|
|
35
|
+
assert.deepEqual(retryDecision({ attempts: 1, failure: { class: c, reason: 'x' }, maxAttempts: 3 }), { action: 'retire' }, c);
|
|
36
|
+
}
|
|
37
|
+
});
|
|
38
|
+
|
|
39
|
+
await t('terminal always retires, even a transient class', () => {
|
|
40
|
+
assert.deepEqual(retryDecision({ attempts: 1, failure: transient, terminal: true, maxAttempts: 3 }), { action: 'retire' });
|
|
41
|
+
});
|
|
42
|
+
|
|
43
|
+
await t('an UNCLASSIFIED failure keeps its budget, but never retries in the same tick', () => {
|
|
44
|
+
assert.deepEqual(retryDecision({ attempts: 1, maxAttempts: 3 }), { action: 'retry', delayMs: DEFAULT_RETRY_DELAY_MS });
|
|
45
|
+
assert.ok(DEFAULT_RETRY_DELAY_MS >= MIN, 'at least a minute: the runner ticks about every two');
|
|
46
|
+
assert.deepEqual(retryDecision({ attempts: 3, maxAttempts: 3 }), { action: 'retire' });
|
|
47
|
+
});
|
|
48
|
+
|
|
49
|
+
await t('the pre-run guard gives a transient row room for its last scheduled try', () => {
|
|
50
|
+
assert.equal(maxAttemptsFor({ failure_class: 'transient' }, 3), 4);
|
|
51
|
+
assert.equal(maxAttemptsFor({ failure_class: 'unknown' }, 3), 3);
|
|
52
|
+
assert.equal(maxAttemptsFor({}, 3), 3);
|
|
53
|
+
assert.equal(maxAttemptsFor({ failure_class: 'transient' }, 9), 9, 'never lowers a configured budget');
|
|
54
|
+
});
|
|
55
|
+
|
|
56
|
+
console.log(`\nintent_retry: ${passed} passed, ${failed} failed`);
|
|
57
|
+
if (failed) process.exit(1);
|
package/tests/provision.mjs
CHANGED
|
@@ -2564,6 +2564,65 @@ await ta('F19: a NON-final failure re-pends the intent and writes nothing on the
|
|
|
2564
2564
|
assert.equal(statusWrites.length, 0, 'no instance write until the intent actually retires');
|
|
2565
2565
|
});
|
|
2566
2566
|
|
|
2567
|
+
// A failed core move stores WHAT KIND of failure beside its sentence (task 1004446), and
|
|
2568
|
+
// waits or retires by that kind (task 1004447). The queue here is STATEFUL — a fake
|
|
2569
|
+
// claimNextIntent that honours not_before the way the real query does — so "one tick runs a
|
|
2570
|
+
// failing move at most once" is measured, not assumed.
|
|
2571
|
+
const queueRun = async ({ attempts = 0, transient = false } = {}) => {
|
|
2572
|
+
const row = { id: 11, instance_id: 18, action: 'core-upgrade', target_version: '1.20.1', attempts, state: 'pending', not_before: null, failure_class: attempts && transient ? 'transient' : null }; // a re-tried row carries the class its last attempt wrote
|
|
2573
|
+
let runs = 0; const resolved = []; const classWrites = []; const repends = [];
|
|
2574
|
+
const deps = {
|
|
2575
|
+
apply: true, log: () => {},
|
|
2576
|
+
db: { query: async (sql, params) => {
|
|
2577
|
+
if (/SET failure_class/.test(sql)) { classWrites.push(params); row.failure_class = params[1]; }
|
|
2578
|
+
if (/SET state='pending'/.test(sql)) { repends.push(params); row.state = 'pending'; row.not_before = new Date(Date.now() + params[2]); }
|
|
2579
|
+
return { rows: [] };
|
|
2580
|
+
} },
|
|
2581
|
+
provisioning: {
|
|
2582
|
+
claimNextIntent: async () => {
|
|
2583
|
+
if (row.state !== 'pending' || (row.not_before && row.not_before > new Date())) return null;
|
|
2584
|
+
row.state = 'running'; row.attempts += 1; runs += 1; return { ...row };
|
|
2585
|
+
},
|
|
2586
|
+
// 'dedicated' is a shape the runner cannot move (needs_decision); the control plane with
|
|
2587
|
+
// an unreadable registry is the transient case. Neither needs an exec.
|
|
2588
|
+
getInstanceById: async () => (transient ? { ...activeStandalone, hosting_shape: 'control-plane' } : { ...activeStandalone, hosting_shape: 'dedicated' }),
|
|
2589
|
+
resolveIntent: async (_db, id, state) => { resolved.push({ id, state }); row.state = state; },
|
|
2590
|
+
setInstanceStatus: async () => {}, recordEvent: async () => {},
|
|
2591
|
+
claimNextOAuthManifest: async () => null, resolveOAuthManifest: async () => {},
|
|
2592
|
+
},
|
|
2593
|
+
exec: () => ({ ok: true }), writeFile: () => ({ ok: true }),
|
|
2594
|
+
readInstalledCoreVersion: () => '1.19.1081', selfInstanceRoot: () => '/srv/cb',
|
|
2595
|
+
updateChannel: { DEFAULT_CHANNEL: 'patch', normalizeChannel: (c) => c || 'patch', channelAllows: () => true, listAvailableVersions: () => ({ ok: false, versions: [] }) },
|
|
2596
|
+
checkDnsTokenScope: async () => ({ reachable: true, zone: 'z' }), checkCaddyWiring: () => null,
|
|
2597
|
+
};
|
|
2598
|
+
await P.cmdRunIntents(deps);
|
|
2599
|
+
return { runs, resolved, classWrites, repends, row };
|
|
2600
|
+
};
|
|
2601
|
+
|
|
2602
|
+
await ta('a needs-decision core move is RETIRED at once with its class — retrying cannot change it', async () => {
|
|
2603
|
+
const { runs, resolved, classWrites } = await queueRun();
|
|
2604
|
+
assert.equal(runs, 1);
|
|
2605
|
+
assert.equal(resolved[0].state, 'error');
|
|
2606
|
+
assert.deepEqual(classWrites, [[11, 'needs_decision', 'shape_not_automated']]);
|
|
2607
|
+
});
|
|
2608
|
+
|
|
2609
|
+
await ta('a TRANSIENT core move runs ONCE per tick and waits ~15 minutes before its next try', async () => {
|
|
2610
|
+
const { runs, resolved, repends, classWrites } = await queueRun({ transient: true });
|
|
2611
|
+
assert.equal(runs, 1, 'the same failing move is not re-run inside one tick');
|
|
2612
|
+
assert.equal(resolved.length, 0, 'not retired — it waits');
|
|
2613
|
+
assert.equal(repends.length, 1);
|
|
2614
|
+
assert.equal(repends[0][2], 15 * 60 * 1000, 'not_before = now + 15 minutes');
|
|
2615
|
+
assert.deepEqual(classWrites, [[11, 'transient', 'registry_unreadable']]);
|
|
2616
|
+
});
|
|
2617
|
+
|
|
2618
|
+
await ta('a transient move on its LAST scheduled try (the 6-hour one) is retired after it', async () => {
|
|
2619
|
+
const { runs, resolved } = await queueRun({ transient: true, attempts: 3 });
|
|
2620
|
+
// attempt 4 RUNS (the pre-run guard allows it for a transient row) and is then retired.
|
|
2621
|
+
assert.equal(runs, 1, 'the 6-hour try actually ran — not retired unrun by the pre-run guard');
|
|
2622
|
+
assert.equal(resolved.length, 1);
|
|
2623
|
+
assert.equal(resolved[0].state, 'error');
|
|
2624
|
+
});
|
|
2625
|
+
|
|
2567
2626
|
console.log('\nreview-panel hardening (task 1002720 — the loopback/latch/birth-note seams):');
|
|
2568
2627
|
|
|
2569
2628
|
await ta('a PASSING loopback probe on a DOMAIN-bearing instance records NO verdict — the address claim belongs to the sweep', async () => {
|
|
@@ -0,0 +1,129 @@
|
|
|
1
|
+
// tests/upgrade_outcome.mjs — every failed core move gets a class and honest wording
|
|
2
|
+
// (task 1004446, goal 1000090 deploy self-heal).
|
|
3
|
+
//
|
|
4
|
+
// WHAT IS AT RISK. The deploy page said "failed and was rolled back" for every exit of
|
|
5
|
+
// `bongos upgrade`, including refusals that changed nothing and rollbacks that themselves
|
|
6
|
+
// failed. The rest of the self-heal chain (retry later, fix it, escalate) keys on the class
|
|
7
|
+
// this file assigns, so a mislabel there is a retry of something that must not be retried,
|
|
8
|
+
// or an automatic "fix" for something that needed a person.
|
|
9
|
+
//
|
|
10
|
+
// Run: node tests/upgrade_outcome.mjs
|
|
11
|
+
|
|
12
|
+
import { strict as assert } from 'node:assert';
|
|
13
|
+
import { createRequire } from 'node:module';
|
|
14
|
+
import {
|
|
15
|
+
outcomeLine, parseOutcomeLine, upgradeReason, upgradeStage, classifyFailure, runnerReason,
|
|
16
|
+
describeUpgradeFailure, FAILURE_CLASSES, REASON_CLASS, STAGES, OUTCOME_TAG,
|
|
17
|
+
} from '../scripts/gds/upgrade-outcome.js';
|
|
18
|
+
|
|
19
|
+
const require = createRequire(import.meta.url);
|
|
20
|
+
const R = require('../scripts/gds/provision-core-upgrade.js');
|
|
21
|
+
|
|
22
|
+
let passed = 0;
|
|
23
|
+
let failed = 0;
|
|
24
|
+
const t = async (name, fn) => {
|
|
25
|
+
try { await fn(); passed += 1; console.log(` PASS ${name}`); }
|
|
26
|
+
catch (e) { failed += 1; console.log(` FAIL ${name}\n ${e.message}`); }
|
|
27
|
+
};
|
|
28
|
+
|
|
29
|
+
// Each row: an upgrade.js failure result as runUpgrade returns it → the stage, reason and
|
|
30
|
+
// class it must get. These are the branches of runUpgrade's failure table (upgrade.js).
|
|
31
|
+
const OK_RB = { restoreOk: true, reinstallOk: true, restartOk: true, healthOk: true };
|
|
32
|
+
const TABLE = [
|
|
33
|
+
['downgrade refusal', { ok: false, downgrade: true, error: 'refusing to DOWNGRADE: --to 1.19.1 is older' }, 'refused', 'downgrade', 'needs_decision'],
|
|
34
|
+
['already on the target', { ok: false, error: 'already on 1.20.1 (pass --force to re-run the steps anyway)' }, 'refused', 'already_on', 'unknown'],
|
|
35
|
+
['dirty working tree', { ok: false, error: 'working tree not clean — commit/stash first, or pass --force.\n M package.json' }, 'refused', 'dirty_tree', 'known_wedge'],
|
|
36
|
+
['module pre-check', { ok: false, error: 'module pre-check failed: game incompatible with core 1.20.1' }, 'refused', 'module_incompatible', 'needs_decision'],
|
|
37
|
+
['database unresolvable', { ok: false, error: 'cannot determine which database to migrate.\n Set PGDATABASE' }, 'refused', 'db_unresolvable', 'unknown'],
|
|
38
|
+
['database identity', { ok: false, error: 'database "cb" is not reachable as the current user: role "root" does not exist' }, 'refused', 'db_identity', 'unknown'],
|
|
39
|
+
['artist gate hold', { ok: false, artistGate: true, error: 'a visual review is holding this deploy' }, 'refused', 'artist_gate', 'needs_decision'],
|
|
40
|
+
['health url missing', { ok: false, error: 'post-restart health check is mandatory for a bump that restarts the service' }, 'refused', 'health_url_missing', 'unknown'],
|
|
41
|
+
['npm install failed, rolled back', { ok: false, error: 'npm install failed (exit 1)', rolledBack: true, rollback: OK_RB }, 'rolled_back', 'npm_install', 'transient'],
|
|
42
|
+
['disk full, rolled back', { ok: false, error: 'npm install failed (ENOSPC: no space left on device)', rolledBack: true, rollback: OK_RB }, 'rolled_back', 'disk_full', 'known_wedge'],
|
|
43
|
+
['integrity pin mismatch', { ok: false, error: 'integrity pin MISMATCH (side-car) — installed core does not match its pin: x', rolledBack: true, rollback: OK_RB }, 'rolled_back', 'integrity_pin', 'unknown'],
|
|
44
|
+
['migrate failed', { ok: false, error: 'migrate failed (exit 3)', rolledBack: true, rollback: OK_RB }, 'rolled_back', 'migrate', 'unknown'],
|
|
45
|
+
['restart failed', { ok: false, error: 'restart failed (exit 1) — the instance is still running the previous core', rolledBack: true, rollback: OK_RB }, 'rolled_back', 'restart', 'unknown'],
|
|
46
|
+
['health failed', { ok: false, error: 'health check failed after restart — rolled back to 1.19.1081', rolledBack: true, rollback: OK_RB }, 'rolled_back', 'health', 'unknown'],
|
|
47
|
+
['served version unchanged', { ok: false, error: 'served core version did not change (still 1.19.1081, expected 1.20.1) — rolled back', rolledBack: true, rollback: OK_RB }, 'rolled_back', 'health', 'unknown'],
|
|
48
|
+
['ROLLBACK FAILED (health still bad)', { ok: false, error: 'health check failed after restart — rolled back', rolledBack: true, rollback: { ...OK_RB, healthOk: false } }, 'rollback_failed', 'rollback_failed', 'unknown'],
|
|
49
|
+
['ROLLBACK FAILED (reinstall)', { ok: false, error: 'migrate failed (exit 3)', rolledBack: true, rollback: { ...OK_RB, reinstallOk: false } }, 'rollback_failed', 'rollback_failed', 'unknown'],
|
|
50
|
+
];
|
|
51
|
+
|
|
52
|
+
console.log('\nevery branch of the upgrade failure table:');
|
|
53
|
+
|
|
54
|
+
for (const [name, res, stage, reason, cls] of TABLE) {
|
|
55
|
+
await t(`${name} → ${stage} / ${reason} / ${cls}`, () => {
|
|
56
|
+
const o = parseOutcomeLine(outcomeLine(res));
|
|
57
|
+
assert.deepEqual(o, { stage, reason });
|
|
58
|
+
assert.equal(classifyFailure(o.reason).class, cls);
|
|
59
|
+
});
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
console.log('\nthe vocabulary:');
|
|
63
|
+
|
|
64
|
+
await t('every reason maps to one of the four classes', () => {
|
|
65
|
+
for (const [r, c] of Object.entries(REASON_CLASS)) assert.ok(FAILURE_CLASSES.includes(c), `${r} → ${c}`);
|
|
66
|
+
});
|
|
67
|
+
|
|
68
|
+
await t('an unlisted reason is unknown — the safe direction', () => {
|
|
69
|
+
assert.deepEqual(classifyFailure('something_new'), { class: 'unknown', reason: 'unknown' });
|
|
70
|
+
assert.deepEqual(classifyFailure(undefined), { class: 'unknown', reason: 'unknown' });
|
|
71
|
+
assert.deepEqual(classifyFailure('__proto__'), { class: 'unknown', reason: 'unknown' });
|
|
72
|
+
});
|
|
73
|
+
|
|
74
|
+
await t('the migration CHECK carries exactly the four classes', async () => {
|
|
75
|
+
const { readFileSync } = await import('node:fs');
|
|
76
|
+
const sql = readFileSync(new URL('../modules/provisioning/migrations/provisioning_030_intent_failure_class.sql', import.meta.url), 'utf8');
|
|
77
|
+
for (const c of FAILURE_CLASSES) assert.match(sql, new RegExp(`'${c}'`));
|
|
78
|
+
});
|
|
79
|
+
|
|
80
|
+
await t('a stage is only what upgrade.js reports: no rollback flag means nothing moved', () => {
|
|
81
|
+
assert.equal(upgradeStage({ ok: false, error: 'x' }), 'refused');
|
|
82
|
+
assert.equal(upgradeStage({ ok: false, rolledBack: true, rollback: OK_RB }), 'rolled_back');
|
|
83
|
+
assert.equal(upgradeStage({ ok: false, rolledBack: true, rollback: { restartOk: false } }), 'rollback_failed');
|
|
84
|
+
assert.equal(upgradeReason(null), 'unknown');
|
|
85
|
+
});
|
|
86
|
+
|
|
87
|
+
await t('parseOutcomeLine takes the LAST line, and refuses anything off-vocabulary', () => {
|
|
88
|
+
const two = `${OUTCOME_TAG} {"stage":"refused","reason":"dirty_tree"}\nnoise\n${OUTCOME_TAG} {"stage":"rolled_back","reason":"migrate"}\n`;
|
|
89
|
+
assert.deepEqual(parseOutcomeLine(two), { stage: 'rolled_back', reason: 'migrate' });
|
|
90
|
+
assert.equal(parseOutcomeLine('no tag here'), null);
|
|
91
|
+
assert.equal(parseOutcomeLine(`${OUTCOME_TAG} {"stage":"exploded","reason":"x"}`), null);
|
|
92
|
+
assert.equal(parseOutcomeLine(`${OUTCOME_TAG} {"stage":"refused","reason":"<script>"}`), null);
|
|
93
|
+
assert.equal(parseOutcomeLine(`${OUTCOME_TAG} not json`), null);
|
|
94
|
+
assert.deepEqual([...STAGES], ['refused', 'rolled_back', 'rollback_failed']);
|
|
95
|
+
});
|
|
96
|
+
|
|
97
|
+
await t('the runner\'s own refusals get their reason from targetFault\'s sentences', () => {
|
|
98
|
+
assert.equal(runnerReason(R.targetFault('1.20.1', [], '1.19.1', 'patch', () => true)), 'registry_unreadable');
|
|
99
|
+
assert.equal(runnerReason(R.targetFault('1.20.1', ['1.19.1'], '1.19.1', 'patch', () => true)), 'not_published');
|
|
100
|
+
assert.equal(runnerReason(R.targetFault('1.20.1', ['1.20.1'], null, 'patch', () => true)), 'installed_unreadable');
|
|
101
|
+
assert.equal(runnerReason(R.targetFault('1.20.1', ['1.20.1'], '1.19.1', 'patch', () => false)), 'outside_channel');
|
|
102
|
+
});
|
|
103
|
+
|
|
104
|
+
console.log('\nthe wording says what happened, and no more:');
|
|
105
|
+
|
|
106
|
+
await t('a refusal never says "rolled back"', () => {
|
|
107
|
+
const s = describeUpgradeFailure('1.20.1', { stage: 'refused', reason: 'dirty_tree' }, 'working tree not clean');
|
|
108
|
+
assert.doesNotMatch(s, /rolled back/);
|
|
109
|
+
assert.match(s, /refused before anything changed/);
|
|
110
|
+
});
|
|
111
|
+
|
|
112
|
+
await t('a real rollback says so', () => {
|
|
113
|
+
assert.match(describeUpgradeFailure('1.20.1', { stage: 'rolled_back', reason: 'migrate' }, 'migrate failed'), /failed and was rolled back/);
|
|
114
|
+
});
|
|
115
|
+
|
|
116
|
+
await t('a failed rollback says the project may be down and needs a person', () => {
|
|
117
|
+
const s = describeUpgradeFailure('1.20.1', { stage: 'rollback_failed', reason: 'rollback_failed' }, '');
|
|
118
|
+
assert.match(s, /rollback did not finish/);
|
|
119
|
+
assert.match(s, /needs a person/);
|
|
120
|
+
});
|
|
121
|
+
|
|
122
|
+
await t('no outcome line claims nothing about a rollback', () => {
|
|
123
|
+
const s = describeUpgradeFailure('1.20.1', null, 'sudo: unknown user bongos-test4');
|
|
124
|
+
assert.doesNotMatch(s, /rolled back/);
|
|
125
|
+
assert.match(s, /did not report how far it got/);
|
|
126
|
+
});
|
|
127
|
+
|
|
128
|
+
console.log(`\nupgrade_outcome: ${passed} passed, ${failed} failed`);
|
|
129
|
+
if (failed) process.exit(1);
|