@bongos/core 1.20.30 → 1.20.31
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.bongos-core.json +32 -17
- package/docs/module-api-changelog.md +2 -0
- package/modules/provisioning/core-upgrade.js +13 -1
- package/modules/provisioning/migrations/provisioning_031_intent_not_before.sql +19 -0
- package/modules/provisioning/provisioning.js +1 -1
- package/package-lock.json +2 -2
- package/package.json +1 -1
- package/release-notes.json +6 -0
- package/scripts/gds/intent-retry.js +52 -0
- package/scripts/gds/provision.js +9 -4
- package/src/module-api.js +1 -1
- package/tests/core_upgrade_door.mjs +35 -1
- package/tests/intent_retry.mjs +57 -0
- package/tests/provision.mjs +41 -16
package/.bongos-core.json
CHANGED
|
@@ -2,22 +2,22 @@
|
|
|
2
2
|
"artifact": "bongos-core",
|
|
3
3
|
"manifest_schema": 1,
|
|
4
4
|
"generator": "scripts/gds/package-core.js",
|
|
5
|
-
"core_version": "1.20.
|
|
6
|
-
"core_contract": "1.20.
|
|
7
|
-
"source_commit": "
|
|
5
|
+
"core_version": "1.20.31",
|
|
6
|
+
"core_contract": "1.20.31",
|
|
7
|
+
"source_commit": "7d9880eac50e580874638a8cca10da6606c18228",
|
|
8
8
|
"source_ref": "HEAD",
|
|
9
|
-
"built_at": "2026-09-30T21:
|
|
9
|
+
"built_at": "2026-09-30T21:24:24.438Z",
|
|
10
10
|
"redaction": {
|
|
11
11
|
"model": "docs-redacted+functional-verbatim",
|
|
12
12
|
"docs_redacted": 562,
|
|
13
13
|
"agent_docs_stubbed": 25,
|
|
14
|
-
"functional_verbatim":
|
|
14
|
+
"functional_verbatim": 2666,
|
|
15
15
|
"rules": 3,
|
|
16
16
|
"gate_literals": 3,
|
|
17
17
|
"gate": "passed"
|
|
18
18
|
},
|
|
19
|
-
"file_count":
|
|
20
|
-
"tree_sha256": "
|
|
19
|
+
"file_count": 3254,
|
|
20
|
+
"tree_sha256": "88a5431f2eb9f9248c57014423f4881035e2d1b9a6cee88740a6800ab933ba38",
|
|
21
21
|
"files": [
|
|
22
22
|
{
|
|
23
23
|
"path": ".claude/skills/ask-for-help/SKILL.md",
|
|
@@ -2802,7 +2802,7 @@
|
|
|
2802
2802
|
{
|
|
2803
2803
|
"path": "docs/module-api-changelog.md",
|
|
2804
2804
|
"mode": "0000644",
|
|
2805
|
-
"sha256": "
|
|
2805
|
+
"sha256": "fd78a2ab7d52f77fbd7181a64af68b8cf4c45c25693bf7dbf31e1b9bba69ada7"
|
|
2806
2806
|
},
|
|
2807
2807
|
{
|
|
2808
2808
|
"path": "docs/modules-contract.md",
|
|
@@ -7282,7 +7282,7 @@
|
|
|
7282
7282
|
{
|
|
7283
7283
|
"path": "modules/provisioning/core-upgrade.js",
|
|
7284
7284
|
"mode": "0000644",
|
|
7285
|
-
"sha256": "
|
|
7285
|
+
"sha256": "5467ab673dfad5969dec0029188d0be18ea82732a26f99bca720de6680e1d618"
|
|
7286
7286
|
},
|
|
7287
7287
|
{
|
|
7288
7288
|
"path": "modules/provisioning/cost-ledger.js",
|
|
@@ -7464,6 +7464,11 @@
|
|
|
7464
7464
|
"mode": "0000644",
|
|
7465
7465
|
"sha256": "7287181b4b42d3e391cd97e19ffa5fbc76026e81ad6f54648eb12a69dfea55a1"
|
|
7466
7466
|
},
|
|
7467
|
+
{
|
|
7468
|
+
"path": "modules/provisioning/migrations/provisioning_031_intent_not_before.sql",
|
|
7469
|
+
"mode": "0000644",
|
|
7470
|
+
"sha256": "a93d8946cbe4631b2af64b2e3638dca188981032a08f5d29d63e4d0adb22d235"
|
|
7471
|
+
},
|
|
7467
7472
|
{
|
|
7468
7473
|
"path": "modules/provisioning/module.json",
|
|
7469
7474
|
"mode": "0000644",
|
|
@@ -7512,7 +7517,7 @@
|
|
|
7512
7517
|
{
|
|
7513
7518
|
"path": "modules/provisioning/provisioning.js",
|
|
7514
7519
|
"mode": "0000644",
|
|
7515
|
-
"sha256": "
|
|
7520
|
+
"sha256": "fb4f09b621654eebe595dcbca305773ea11ed063506a6e4116391e43566a540d"
|
|
7516
7521
|
},
|
|
7517
7522
|
{
|
|
7518
7523
|
"path": "modules/provisioning/public-refusal.js",
|
|
@@ -8982,12 +8987,12 @@
|
|
|
8982
8987
|
{
|
|
8983
8988
|
"path": "package-lock.json",
|
|
8984
8989
|
"mode": "0000644",
|
|
8985
|
-
"sha256": "
|
|
8990
|
+
"sha256": "892ad66ccaa864ee07860cd8578f134a72f8cc48625f728f186eaa0f994ac8ad"
|
|
8986
8991
|
},
|
|
8987
8992
|
{
|
|
8988
8993
|
"path": "package.json",
|
|
8989
8994
|
"mode": "0000644",
|
|
8990
|
-
"sha256": "
|
|
8995
|
+
"sha256": "fdebee1582c5673a44b97471f05814ab6bc8e850958ca3131e4a356f3a663d90"
|
|
8991
8996
|
},
|
|
8992
8997
|
{
|
|
8993
8998
|
"path": "public-docs/index.html",
|
|
@@ -9007,7 +9012,7 @@
|
|
|
9007
9012
|
{
|
|
9008
9013
|
"path": "release-notes.json",
|
|
9009
9014
|
"mode": "0000644",
|
|
9010
|
-
"sha256": "
|
|
9015
|
+
"sha256": "4398d89d420cf704f3030db8752e1f9e67321e43e49b0765e23329c26160385e"
|
|
9011
9016
|
},
|
|
9012
9017
|
{
|
|
9013
9018
|
"path": "scripts/bongos-mcp.js",
|
|
@@ -9739,6 +9744,11 @@
|
|
|
9739
9744
|
"mode": "0000644",
|
|
9740
9745
|
"sha256": "7b2402543c3b1eadbafc14907e6dc6689b434ac65fb9c709115dd81a92eb13eb"
|
|
9741
9746
|
},
|
|
9747
|
+
{
|
|
9748
|
+
"path": "scripts/gds/intent-retry.js",
|
|
9749
|
+
"mode": "0000644",
|
|
9750
|
+
"sha256": "056622ecd8327527f4fb98b37ce8af90778d42c816455431ed3dd2f76800a75b"
|
|
9751
|
+
},
|
|
9742
9752
|
{
|
|
9743
9753
|
"path": "scripts/gds/key-card.js",
|
|
9744
9754
|
"mode": "0000644",
|
|
@@ -10022,7 +10032,7 @@
|
|
|
10022
10032
|
{
|
|
10023
10033
|
"path": "scripts/gds/provision.js",
|
|
10024
10034
|
"mode": "0000644",
|
|
10025
|
-
"sha256": "
|
|
10035
|
+
"sha256": "4da6314b41905bcffbc51abe127911512743ad14f8130b4c6a25c23600a3eb7f"
|
|
10026
10036
|
},
|
|
10027
10037
|
{
|
|
10028
10038
|
"path": "scripts/gds/publish-credential-check.js",
|
|
@@ -11132,7 +11142,7 @@
|
|
|
11132
11142
|
{
|
|
11133
11143
|
"path": "src/module-api.js",
|
|
11134
11144
|
"mode": "0000644",
|
|
11135
|
-
"sha256": "
|
|
11145
|
+
"sha256": "8acd38044812dee3022427fad27e12cc7f9d81a90c1a0d921e860135a30f9e43"
|
|
11136
11146
|
},
|
|
11137
11147
|
{
|
|
11138
11148
|
"path": "src/module-loader/catalog.js",
|
|
@@ -12077,7 +12087,7 @@
|
|
|
12077
12087
|
{
|
|
12078
12088
|
"path": "tests/core_upgrade_door.mjs",
|
|
12079
12089
|
"mode": "0000644",
|
|
12080
|
-
"sha256": "
|
|
12090
|
+
"sha256": "8c8b03b8a3bb83b0c733726f3aa1256b6af5ac61c76e6af465fe2bc4f792ea6b"
|
|
12081
12091
|
},
|
|
12082
12092
|
{
|
|
12083
12093
|
"path": "tests/core_upgrade_owner_door.mjs",
|
|
@@ -13744,6 +13754,11 @@
|
|
|
13744
13754
|
"mode": "0000644",
|
|
13745
13755
|
"sha256": "9e69603c05fae6c39e2db7336d6569327f17365ff46ef05b1074943dc34a49e7"
|
|
13746
13756
|
},
|
|
13757
|
+
{
|
|
13758
|
+
"path": "tests/intent_retry.mjs",
|
|
13759
|
+
"mode": "0000644",
|
|
13760
|
+
"sha256": "750745cf866dcf1012f2677881738aa64ae2b8ceaed078ab688612a8b0d2208c"
|
|
13761
|
+
},
|
|
13747
13762
|
{
|
|
13748
13763
|
"path": "tests/interaction_posture.mjs",
|
|
13749
13764
|
"mode": "0000644",
|
|
@@ -14702,7 +14717,7 @@
|
|
|
14702
14717
|
{
|
|
14703
14718
|
"path": "tests/provision.mjs",
|
|
14704
14719
|
"mode": "0000644",
|
|
14705
|
-
"sha256": "
|
|
14720
|
+
"sha256": "46b2235ee186ba304c18f5cf3002a9e5311f7bda11fa230bbada8a7a43bec3cd"
|
|
14706
14721
|
},
|
|
14707
14722
|
{
|
|
14708
14723
|
"path": "tests/provision_account_identity.mjs",
|
|
@@ -2717,5 +2717,7 @@ is load-bearing: the script throws rather than guess if it is missing, and
|
|
|
2717
2717
|
landed since 1.20.28 with no explicit bump. run 36775277990. (task 1002620)
|
|
2718
2718
|
1.20.30 — CI auto-patch (publish-on-merge, ADR 0161): carrier for merges
|
|
2719
2719
|
landed since 1.20.29 with no explicit bump. run 36777226204. (task 1002620)
|
|
2720
|
+
1.20.31 — CI auto-patch (publish-on-merge, ADR 0161): carrier for merges
|
|
2721
|
+
landed since 1.20.30 with no explicit bump. run 36779142919. (task 1002620)
|
|
2720
2722
|
---------------------------------------------------------------------------
|
|
2721
2723
|
```
|
|
@@ -128,6 +128,16 @@ function coreUpgradeFault(inst, to, live = null) {
|
|
|
128
128
|
async function enqueueCoreUpgrade(provisioning, db, instanceId, to, requestedBy, action = 'core-upgrade') {
|
|
129
129
|
const { intent, created } = await provisioning.enqueueIntent(db, instanceId, action, requestedBy, to);
|
|
130
130
|
if (created) return { intent, created: true };
|
|
131
|
+
// THE SAME MOVE, WAITING OUT A RETRY (task 1004447). A transient failure re-pends the
|
|
132
|
+
// intent with a not_before up to six hours away, and it keeps the single open slot all
|
|
133
|
+
// that time — so without this, the owner pressing Move again was told "a run is pending,
|
|
134
|
+
// try again later" about the very move they were asking for. Same action, same target,
|
|
135
|
+
// still pending: bring its next attempt forward instead. Anything else stays a conflict.
|
|
136
|
+
if (intent && intent.state === 'pending' && intent.action === action && String(intent.target_version || '') === String(to)
|
|
137
|
+
&& intent.not_before && new Date(intent.not_before).getTime() > Date.now()) {
|
|
138
|
+
await db.query("UPDATE provisioning_intents SET not_before = NULL, updated_at = now() WHERE id = $1 AND state = 'pending'", [intent.id]);
|
|
139
|
+
return { intent: { ...intent, not_before: null }, created: false, retryNow: true };
|
|
140
|
+
}
|
|
131
141
|
return { intent, created: false, conflict: 'open_intent' };
|
|
132
142
|
}
|
|
133
143
|
|
|
@@ -336,7 +346,9 @@ function previewFromSnapshot(inst, snap, openIntent, lastMove) {
|
|
|
336
346
|
// and a stale pass is worse than no answer because it reads as a guarantee.
|
|
337
347
|
preflight: preflightForOffer(pf, served, recommended),
|
|
338
348
|
preflight_at: pf && recommended && pf.target === recommended ? (snap.preflight_at || null) : null,
|
|
339
|
-
|
|
349
|
+
// not_before: a failed move waiting out its retry backoff (task 1004447) — the page can
|
|
350
|
+
// say "trying again at …" instead of an unexplained "pending" for up to six hours.
|
|
351
|
+
open_intent: openIntent ? { action: openIntent.action, state: openIntent.state, target_version: openIntent.target_version || null, not_before: openIntent.not_before || null, last_error: openIntent.not_before && openIntent.last_error ? String(openIntent.last_error).slice(0, 600) : null } : null,
|
|
340
352
|
// THE LAST MOVE, WHATEVER BECAME OF IT (task 1004145).
|
|
341
353
|
//
|
|
342
354
|
// `open_intent` is `state IN ('pending','running')` — by the index's own definition.
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
-- provisioning_031_intent_not_before.sql — a failed intent waits before its next attempt
|
|
2
|
+
-- (task 1004447, goal 1000090 deploy self-heal).
|
|
3
|
+
--
|
|
4
|
+
-- WHY. A failed intent was re-pended with no delay, and the drain loop claims the oldest
|
|
5
|
+
-- pending row — so one runner tick ran the same failing move up to MAX_INTENT_ATTEMPTS
|
|
6
|
+
-- times back to back, restarting the service each time a pin had moved, and then gave up.
|
|
7
|
+
-- A registry blip or a release still publishing needs minutes, not milliseconds.
|
|
8
|
+
--
|
|
9
|
+
-- not_before: the earliest time the runner may claim this row again. NULL = now (every row
|
|
10
|
+
-- written before this column, and every fresh request). claimNextIntent skips a row whose
|
|
11
|
+
-- not_before is still in the future; scripts/gds/provision.js sets it on each re-pend.
|
|
12
|
+
--
|
|
13
|
+
-- Additive and namespaced (ADR 0083): no down-migration, idempotent — safe to re-run.
|
|
14
|
+
|
|
15
|
+
BEGIN;
|
|
16
|
+
|
|
17
|
+
ALTER TABLE provisioning_intents ADD COLUMN IF NOT EXISTS not_before timestamptz;
|
|
18
|
+
|
|
19
|
+
COMMIT;
|
|
@@ -1254,7 +1254,7 @@ async function claimNextIntent(db) {
|
|
|
1254
1254
|
SET state = 'running', attempts = attempts + 1, updated_at = now()
|
|
1255
1255
|
WHERE id = (
|
|
1256
1256
|
SELECT id FROM provisioning_intents
|
|
1257
|
-
WHERE state = 'pending'
|
|
1257
|
+
WHERE state = 'pending' AND (not_before IS NULL OR not_before <= now()) -- a re-pended failure waits out its backoff (task 1004447)
|
|
1258
1258
|
ORDER BY created_at
|
|
1259
1259
|
FOR UPDATE SKIP LOCKED
|
|
1260
1260
|
LIMIT 1
|
package/package-lock.json
CHANGED
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@bongos/core",
|
|
3
|
-
"version": "1.20.
|
|
3
|
+
"version": "1.20.31",
|
|
4
4
|
"lockfileVersion": 3,
|
|
5
5
|
"requires": true,
|
|
6
6
|
"packages": {
|
|
7
7
|
"": {
|
|
8
8
|
"name": "@bongos/core",
|
|
9
|
-
"version": "1.20.
|
|
9
|
+
"version": "1.20.31",
|
|
10
10
|
"license": "AGPL-3.0-or-later",
|
|
11
11
|
"dependencies": {
|
|
12
12
|
"express": "^4.21.2",
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@bongos/core",
|
|
3
|
-
"version": "1.20.
|
|
3
|
+
"version": "1.20.31",
|
|
4
4
|
"description": "Cloud Bongos — the AI-first build platform core (GDS + platform surfaces + module system), installed as a versioned dependency (ADR 0108).",
|
|
5
5
|
"license": "AGPL-3.0-or-later",
|
|
6
6
|
"main": "src/platform-server.js",
|
package/release-notes.json
CHANGED
|
@@ -8252,5 +8252,11 @@
|
|
|
8252
8252
|
"id": "1004446",
|
|
8253
8253
|
"text": "When a software update fails, the system now records what kind of failure it was — a temporary glitch, a known snag, a rule only the owner can change, or something unknown — and tells the truth about it instead of always sayin"
|
|
8254
8254
|
}
|
|
8255
|
+
],
|
|
8256
|
+
"1.20.31": [
|
|
8257
|
+
{
|
|
8258
|
+
"id": "1004447",
|
|
8259
|
+
"text": "A software update that fails for a temporary reason now waits and tries again later — after 15 minutes, then an hour, then six hours — instead of hammering the same failing step three times in a row. Failures that retrying can"
|
|
8260
|
+
}
|
|
8255
8261
|
]
|
|
8256
8262
|
}
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
//
|
|
3
|
+
// scripts/gds/intent-retry.js — when a failed provisioning intent is tried again, and when it
|
|
4
|
+
// is retired instead (task 1004447, goal 1000090 deploy self-heal).
|
|
5
|
+
//
|
|
6
|
+
// WHY. The drain loop re-pended a failure with no delay and claims the oldest pending row, so
|
|
7
|
+
// one tick ran the same failing move up to MAX_INTENT_ATTEMPTS times back to back — seconds
|
|
8
|
+
// apart, restarting the service after every pin move — and then gave up. That is the wrong
|
|
9
|
+
// answer for each kind of failure upgrade-outcome.js now names:
|
|
10
|
+
// transient — the registry blipped, the release is still publishing: WAIT, then retry
|
|
11
|
+
// (about 15 minutes, 1 hour, 6 hours), which is what a person would do.
|
|
12
|
+
// known_wedge, needs_decision, unknown — trying the same thing again cannot change the
|
|
13
|
+
// answer, so the intent is RETIRED at once and the failure left standing
|
|
14
|
+
// for the next step (a fix, the owner, a person) to act on.
|
|
15
|
+
// A failure with no class (every action other than a core move) keeps its old attempt budget,
|
|
16
|
+
// but each retry now waits DEFAULT_RETRY_DELAY_MS — at least the next tick, never the same one.
|
|
17
|
+
//
|
|
18
|
+
// PURE: no I/O. The drain loop (scripts/gds/provision.js cmdRunIntents) asks and obeys.
|
|
19
|
+
|
|
20
|
+
const MINUTE = 60 * 1000;
|
|
21
|
+
const TRANSIENT_BACKOFF_MS = Object.freeze([15 * MINUTE, 60 * MINUTE, 6 * 60 * MINUTE]);
|
|
22
|
+
const DEFAULT_RETRY_DELAY_MS = MINUTE;
|
|
23
|
+
|
|
24
|
+
/**
|
|
25
|
+
* What to do with an intent whose run just failed. PURE.
|
|
26
|
+
* @param {object} p
|
|
27
|
+
* @param {number} p.attempts the attempt that just ran (claimNextIntent increments first)
|
|
28
|
+
* @param {object} [p.failure] `{ class, reason }` from the result, when classified
|
|
29
|
+
* @param {boolean} [p.terminal] the leg said it must not be retried unattended
|
|
30
|
+
* @param {number} p.maxAttempts the runner's budget for unclassified failures
|
|
31
|
+
* @returns {{ action: 'retire' } | { action: 'retry', delayMs: number }}
|
|
32
|
+
*/
|
|
33
|
+
function retryDecision({ attempts, failure, terminal, maxAttempts }) {
|
|
34
|
+
if (terminal) return { action: 'retire' };
|
|
35
|
+
if (failure && failure.class) {
|
|
36
|
+
if (failure.class !== 'transient') return { action: 'retire' };
|
|
37
|
+
const i = Math.max(0, (attempts || 1) - 1);
|
|
38
|
+
return i < TRANSIENT_BACKOFF_MS.length ? { action: 'retry', delayMs: TRANSIENT_BACKOFF_MS[i] } : { action: 'retire' };
|
|
39
|
+
}
|
|
40
|
+
return (attempts || 0) >= maxAttempts ? { action: 'retire' } : { action: 'retry', delayMs: DEFAULT_RETRY_DELAY_MS };
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
/**
|
|
44
|
+
* The attempt ceiling checked BEFORE an intent runs. A row waiting out a transient backoff
|
|
45
|
+
* has one attempt per backoff step plus the first, so the pre-run guard must not retire it
|
|
46
|
+
* on the default budget before its last scheduled try. PURE.
|
|
47
|
+
*/
|
|
48
|
+
function maxAttemptsFor(intent, maxAttempts) {
|
|
49
|
+
return intent && intent.failure_class === 'transient' ? Math.max(maxAttempts, TRANSIENT_BACKOFF_MS.length + 1) : maxAttempts;
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
module.exports = { TRANSIENT_BACKOFF_MS, DEFAULT_RETRY_DELAY_MS, retryDecision, maxAttemptsFor };
|
package/scripts/gds/provision.js
CHANGED
|
@@ -46,6 +46,7 @@ const path = require('node:path');
|
|
|
46
46
|
const cp = require('node:child_process');
|
|
47
47
|
const crypto = require('node:crypto');
|
|
48
48
|
|
|
49
|
+
const retry = require('./intent-retry.js'); // when a failed intent is tried again (task 1004447)
|
|
49
50
|
const { CONFIG, MANIFEST_UA, softFailResult, MANIFEST_VERIFY_INTERVAL_MS, MANIFEST_VERIFY_TRIES, MAX_INTENT_ATTEMPTS, REPO_ROOT, coreVersionSafe, hasFlag, loadDeps, oauthSecret, provisionerBotEmail } = require('./provision-config.js');
|
|
50
51
|
const { accountPreflight, appUser, alreadyScaffolded, buildCorePinRefreshCommit, coreCheckoutHasGit, dbCreateCmd, dbName, dbRoleCmd, dbRoleCmdShown, dbRoleGrantsCmd, ensureInstanceUserCmd, generateDbPassword, grantInstanceRepoReadCmd, instanceDbRole, instanceStateDir, instanceUser, deployKeyPath, deployKeyTitle, ensurePrivateRepoAccess, installedCorePackDir, installedCoreTarball, instanceInitSpec, instanceRepoRemote, migrateCmd, onboardMode, ownerLoginOf, parseTargetRef, planCorePinRefresh, refreshStandaloneCorePin, resolveOwnerGithubToken, resolveVendorableCoreTarball, safeVersionLabel, scaffoldStandaloneRepo, seedFirstVersionCmd, standaloneInstallCmd, standaloneMigrateCmd, standalonePullCmd, standaloneRegenDocsCmd, standaloneRoot } = require('./provision-repo.js');
|
|
51
52
|
const { backupScriptPath, backupService, backupServicePath, backupTimer, backupTimerPath, backupUnitName, instanceManifestCmd, originEnvVarsFor, serviceUnit, serviceUnitPath, settingsConsumed, settingsEnvVarsFor, renamedEnvCarry, upsertEnvVars, webEnvBody, webEnvPath } = require('./provision-units.js');
|
|
@@ -1171,8 +1172,8 @@ async function cmdRunIntents(deps) {
|
|
|
1171
1172
|
const intent = await provisioning.claimNextIntent(db);
|
|
1172
1173
|
if (!intent) break;
|
|
1173
1174
|
ran++;
|
|
1174
|
-
if (intent.attempts > MAX_INTENT_ATTEMPTS) {
|
|
1175
|
-
await provisioning.resolveIntent(db, intent.id, 'error', `exceeded ${MAX_INTENT_ATTEMPTS} attempts`);
|
|
1175
|
+
if (intent.attempts > retry.maxAttemptsFor(intent, MAX_INTENT_ATTEMPTS)) { // a transient core move gets one try per backoff step (task 1004447)
|
|
1176
|
+
await provisioning.resolveIntent(db, intent.id, 'error', `exceeded ${retry.maxAttemptsFor(intent, MAX_INTENT_ATTEMPTS)} attempts`);
|
|
1176
1177
|
errors++; log(` ✖ intent #${intent.id} retired (too many attempts)`); continue;
|
|
1177
1178
|
}
|
|
1178
1179
|
try {
|
|
@@ -1192,7 +1193,11 @@ async function cmdRunIntents(deps) {
|
|
|
1192
1193
|
else if (intent.action === 'repo-private' || intent.action === 'render-standup') result = intent.action === 'repo-private' ? await require('./provision-repo-private.js').repoPrivateInstance(inst, deps) : await require('./provision-render.js').renderStandupInstance(inst, deps, intent); // wire the deploy key, THEN flip (task 1004193) · spend the borrowed Render key once (task 1004183); required here — this file is at the size ratchet
|
|
1193
1194
|
else result = { ok: false, error: `unknown_action '${intent.action}' — runner core ${coreVersionSafe() || '?'} does not know it (runner too old?); refusing to guess` };
|
|
1194
1195
|
if (result && result.ok === false) {
|
|
1195
|
-
|
|
1196
|
+
// WAIT OR RETIRE (task 1004447, intent-retry.js): a transient failure waits ~15m/1h/6h; a
|
|
1197
|
+
// classified non-transient one is retired at once (retrying cannot change its answer);
|
|
1198
|
+
// an unclassified one keeps its budget but never re-runs in the same tick.
|
|
1199
|
+
const next = retry.retryDecision({ attempts: intent.attempts, failure: result.failure, terminal: result.terminal, maxAttempts: MAX_INTENT_ATTEMPTS });
|
|
1200
|
+
if (next.action === 'retire') { // terminal: a leg that must not be retried unattended (the Render leg's key is already gone — task 1004183)
|
|
1196
1201
|
await provisioning.resolveIntent(db, intent.id, 'error', result.error || 'op failed');
|
|
1197
1202
|
// A retired intent stops silently otherwise (F19): record the reason
|
|
1198
1203
|
// on the instance WITHOUT changing its lifecycle status — passing the
|
|
@@ -1202,7 +1207,7 @@ async function cmdRunIntents(deps) {
|
|
|
1202
1207
|
await provisioning.setInstanceStatus(db, intent.instance_id, inst.status, { error_note: result.error || 'op failed', error_note_action: intent.action }).catch(() => {});
|
|
1203
1208
|
if (result.failure) await recordFailureClass(db, intent.id, result.failure); // WHAT KIND of failure, beside the sentence (task 1004446)
|
|
1204
1209
|
} else {
|
|
1205
|
-
await db.query(`UPDATE provisioning_intents SET state='pending', last_error=$2, updated_at=now() WHERE id=$1`, [intent.id, result.error || 'op failed']);
|
|
1210
|
+
await db.query(`UPDATE provisioning_intents SET state='pending', last_error=$2, not_before = now() + ($3 * interval '1 millisecond'), updated_at=now() WHERE id=$1`, [intent.id, result.error || 'op failed', next.delayMs]);
|
|
1206
1211
|
if (result.failure) await recordFailureClass(db, intent.id, result.failure);
|
|
1207
1212
|
}
|
|
1208
1213
|
errors++;
|
package/src/module-api.js
CHANGED
|
@@ -75,7 +75,7 @@ const { responsibilityFor, ROLE_RESPONSIBILITIES } = require('./role-responsibil
|
|
|
75
75
|
// MAJOR (see allowBoxScope below): passes the request through untouched.
|
|
76
76
|
function deprecatedNoopMiddleware(_req, _res, next) { next(); }
|
|
77
77
|
|
|
78
|
-
const CORE_VERSION = '1.20.
|
|
78
|
+
const CORE_VERSION = '1.20.31'; // CI auto-patch carrier (ADR 0161); changelog: docs/module-api-changelog.md
|
|
79
79
|
|
|
80
80
|
// A namespaced logger so a module's log lines are attributable + consistent.
|
|
81
81
|
// Usage: const log = api.logger('discord'); log.info('mounted');
|
|
@@ -431,7 +431,7 @@ await t('schema_pending passes through as a count, and UNKNOWN stays null rather
|
|
|
431
431
|
await t('an open intent is surfaced with its target, so the page can say what is already running', () => {
|
|
432
432
|
const p = CU.previewFromSnapshot(previewInst, { served_version: '1.19.758', available: [], recommended: null },
|
|
433
433
|
{ action: 'core-upgrade', state: 'running', target_version: '1.19.762' });
|
|
434
|
-
assert.deepEqual(p.open_intent, { action: 'core-upgrade', state: 'running', target_version: '1.19.762' });
|
|
434
|
+
assert.deepEqual(p.open_intent, { action: 'core-upgrade', state: 'running', target_version: '1.19.762', not_before: null, last_error: null });
|
|
435
435
|
});
|
|
436
436
|
|
|
437
437
|
console.log('\nthe deploy page is gated on the same atom as its routes:');
|
|
@@ -973,5 +973,39 @@ await t('a failed move carries its class to the page; a move that worked carries
|
|
|
973
973
|
assert.equal(CU.previewFromSnapshot(ACTIVE, null, null, old).last_move.failure, null, 'a row from before the columns');
|
|
974
974
|
});
|
|
975
975
|
|
|
976
|
+
console.log('\na move waiting out its retry (task 1004447):');
|
|
977
|
+
|
|
978
|
+
const waiting = (over = {}) => ({ id: 5, action: 'core-upgrade', state: 'pending', target_version: '1.20.1', not_before: new Date(Date.now() + 3600e3).toISOString(), ...over });
|
|
979
|
+
const enqueueAgainst = async (open, to = '1.20.1') => {
|
|
980
|
+
const writes = [];
|
|
981
|
+
const provisioning = { enqueueIntent: async () => ({ intent: open, created: false }) };
|
|
982
|
+
const db = { query: async (sql, params) => { writes.push({ sql, params }); return { rows: [] }; } };
|
|
983
|
+
return { r: await CU.enqueueCoreUpgrade(provisioning, db, 7, to, 'api:archon:42'), writes };
|
|
984
|
+
};
|
|
985
|
+
|
|
986
|
+
await t('pressing Move for the SAME waiting move brings its retry forward instead of refusing', async () => {
|
|
987
|
+
const { r, writes } = await enqueueAgainst(waiting());
|
|
988
|
+
assert.equal(r.retryNow, true);
|
|
989
|
+
assert.ok(!r.conflict);
|
|
990
|
+
assert.match(writes[0].sql, /SET not_before = NULL/);
|
|
991
|
+
assert.deepEqual(writes[0].params, [5]);
|
|
992
|
+
});
|
|
993
|
+
|
|
994
|
+
await t('a DIFFERENT target, a running move, or one not waiting is still a conflict', async () => {
|
|
995
|
+
for (const open of [waiting({ target_version: '1.20.2' }), waiting({ state: 'running' }), waiting({ not_before: null }), waiting({ action: 'restart' })]) {
|
|
996
|
+
const { r, writes } = await enqueueAgainst(open);
|
|
997
|
+
assert.equal(r.conflict, 'open_intent');
|
|
998
|
+
assert.equal(writes.length, 0);
|
|
999
|
+
}
|
|
1000
|
+
});
|
|
1001
|
+
|
|
1002
|
+
await t('the preview says when a waiting move will try again, and why it is waiting', () => {
|
|
1003
|
+
const p = CU.previewFromSnapshot(ACTIVE, null, waiting({ last_error: 'core 1.20.1 is not published yet' }), null);
|
|
1004
|
+
assert.ok(p.open_intent.not_before);
|
|
1005
|
+
assert.match(p.open_intent.last_error, /not published yet/);
|
|
1006
|
+
assert.equal(CU.previewFromSnapshot(ACTIVE, null, waiting({ not_before: null, last_error: 'x' }), null).open_intent.last_error, null,
|
|
1007
|
+
'an ordinary pending intent shows no error');
|
|
1008
|
+
});
|
|
1009
|
+
|
|
976
1010
|
console.log(`\ncore_upgrade_door: ${passed} passed, ${failed} failed`);
|
|
977
1011
|
if (failed) process.exit(1);
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
// tests/intent_retry.mjs — a failed intent waits, or is retired, by what kind of failure it
|
|
2
|
+
// was (task 1004447, goal 1000090 deploy self-heal).
|
|
3
|
+
//
|
|
4
|
+
// WHAT IS AT RISK. One runner tick used to run the same failing core move up to three times
|
|
5
|
+
// back to back, restarting the service each time, then give up. These cases pin the policy
|
|
6
|
+
// that replaced it; tests/provision.mjs drives the drain loop with it.
|
|
7
|
+
//
|
|
8
|
+
// Run: node tests/intent_retry.mjs
|
|
9
|
+
|
|
10
|
+
import { strict as assert } from 'node:assert';
|
|
11
|
+
import { retryDecision, maxAttemptsFor, TRANSIENT_BACKOFF_MS, DEFAULT_RETRY_DELAY_MS } from '../scripts/gds/intent-retry.js';
|
|
12
|
+
|
|
13
|
+
let passed = 0;
|
|
14
|
+
let failed = 0;
|
|
15
|
+
const t = async (name, fn) => {
|
|
16
|
+
try { await fn(); passed += 1; console.log(` PASS ${name}`); }
|
|
17
|
+
catch (e) { failed += 1; console.log(` FAIL ${name}\n ${e.message}`); }
|
|
18
|
+
};
|
|
19
|
+
const MIN = 60 * 1000;
|
|
20
|
+
const transient = { class: 'transient', reason: 'not_published' };
|
|
21
|
+
|
|
22
|
+
await t('a transient failure waits about 15 minutes, then 1 hour, then 6 hours', () => {
|
|
23
|
+
assert.deepEqual([...TRANSIENT_BACKOFF_MS], [15 * MIN, 60 * MIN, 360 * MIN]);
|
|
24
|
+
for (const [attempts, ms] of [[1, 15 * MIN], [2, 60 * MIN], [3, 360 * MIN]]) {
|
|
25
|
+
assert.deepEqual(retryDecision({ attempts, failure: transient, maxAttempts: 3 }), { action: 'retry', delayMs: ms });
|
|
26
|
+
}
|
|
27
|
+
});
|
|
28
|
+
|
|
29
|
+
await t('and is retired after its last scheduled try', () => {
|
|
30
|
+
assert.deepEqual(retryDecision({ attempts: 4, failure: transient, maxAttempts: 3 }), { action: 'retire' });
|
|
31
|
+
});
|
|
32
|
+
|
|
33
|
+
await t('every NON-transient class is retired at once — retrying cannot change the answer', () => {
|
|
34
|
+
for (const c of ['known_wedge', 'needs_decision', 'unknown']) {
|
|
35
|
+
assert.deepEqual(retryDecision({ attempts: 1, failure: { class: c, reason: 'x' }, maxAttempts: 3 }), { action: 'retire' }, c);
|
|
36
|
+
}
|
|
37
|
+
});
|
|
38
|
+
|
|
39
|
+
await t('terminal always retires, even a transient class', () => {
|
|
40
|
+
assert.deepEqual(retryDecision({ attempts: 1, failure: transient, terminal: true, maxAttempts: 3 }), { action: 'retire' });
|
|
41
|
+
});
|
|
42
|
+
|
|
43
|
+
await t('an UNCLASSIFIED failure keeps its budget, but never retries in the same tick', () => {
|
|
44
|
+
assert.deepEqual(retryDecision({ attempts: 1, maxAttempts: 3 }), { action: 'retry', delayMs: DEFAULT_RETRY_DELAY_MS });
|
|
45
|
+
assert.ok(DEFAULT_RETRY_DELAY_MS >= MIN, 'at least a minute: the runner ticks about every two');
|
|
46
|
+
assert.deepEqual(retryDecision({ attempts: 3, maxAttempts: 3 }), { action: 'retire' });
|
|
47
|
+
});
|
|
48
|
+
|
|
49
|
+
await t('the pre-run guard gives a transient row room for its last scheduled try', () => {
|
|
50
|
+
assert.equal(maxAttemptsFor({ failure_class: 'transient' }, 3), 4);
|
|
51
|
+
assert.equal(maxAttemptsFor({ failure_class: 'unknown' }, 3), 3);
|
|
52
|
+
assert.equal(maxAttemptsFor({}, 3), 3);
|
|
53
|
+
assert.equal(maxAttemptsFor({ failure_class: 'transient' }, 9), 9, 'never lowers a configured budget');
|
|
54
|
+
});
|
|
55
|
+
|
|
56
|
+
console.log(`\nintent_retry: ${passed} passed, ${failed} failed`);
|
|
57
|
+
if (failed) process.exit(1);
|
package/tests/provision.mjs
CHANGED
|
@@ -2564,38 +2564,63 @@ await ta('F19: a NON-final failure re-pends the intent and writes nothing on the
|
|
|
2564
2564
|
assert.equal(statusWrites.length, 0, 'no instance write until the intent actually retires');
|
|
2565
2565
|
});
|
|
2566
2566
|
|
|
2567
|
-
// A failed core move stores WHAT KIND of failure beside its sentence (task 1004446)
|
|
2568
|
-
|
|
2569
|
-
|
|
2567
|
+
// A failed core move stores WHAT KIND of failure beside its sentence (task 1004446), and
|
|
2568
|
+
// waits or retires by that kind (task 1004447). The queue here is STATEFUL — a fake
|
|
2569
|
+
// claimNextIntent that honours not_before the way the real query does — so "one tick runs a
|
|
2570
|
+
// failing move at most once" is measured, not assumed.
|
|
2571
|
+
const queueRun = async ({ attempts = 0, transient = false } = {}) => {
|
|
2572
|
+
const row = { id: 11, instance_id: 18, action: 'core-upgrade', target_version: '1.20.1', attempts, state: 'pending', not_before: null, failure_class: attempts && transient ? 'transient' : null }; // a re-tried row carries the class its last attempt wrote
|
|
2573
|
+
let runs = 0; const resolved = []; const classWrites = []; const repends = [];
|
|
2570
2574
|
const deps = {
|
|
2571
2575
|
apply: true, log: () => {},
|
|
2572
|
-
db: { query: async (sql, params) => {
|
|
2576
|
+
db: { query: async (sql, params) => {
|
|
2577
|
+
if (/SET failure_class/.test(sql)) { classWrites.push(params); row.failure_class = params[1]; }
|
|
2578
|
+
if (/SET state='pending'/.test(sql)) { repends.push(params); row.state = 'pending'; row.not_before = new Date(Date.now() + params[2]); }
|
|
2579
|
+
return { rows: [] };
|
|
2580
|
+
} },
|
|
2573
2581
|
provisioning: {
|
|
2574
|
-
claimNextIntent: async () =>
|
|
2575
|
-
|
|
2576
|
-
|
|
2577
|
-
|
|
2578
|
-
|
|
2582
|
+
claimNextIntent: async () => {
|
|
2583
|
+
if (row.state !== 'pending' || (row.not_before && row.not_before > new Date())) return null;
|
|
2584
|
+
row.state = 'running'; row.attempts += 1; runs += 1; return { ...row };
|
|
2585
|
+
},
|
|
2586
|
+
// 'dedicated' is a shape the runner cannot move (needs_decision); the control plane with
|
|
2587
|
+
// an unreadable registry is the transient case. Neither needs an exec.
|
|
2588
|
+
getInstanceById: async () => (transient ? { ...activeStandalone, hosting_shape: 'control-plane' } : { ...activeStandalone, hosting_shape: 'dedicated' }),
|
|
2589
|
+
resolveIntent: async (_db, id, state) => { resolved.push({ id, state }); row.state = state; },
|
|
2579
2590
|
setInstanceStatus: async () => {}, recordEvent: async () => {},
|
|
2580
2591
|
claimNextOAuthManifest: async () => null, resolveOAuthManifest: async () => {},
|
|
2581
2592
|
},
|
|
2582
2593
|
exec: () => ({ ok: true }), writeFile: () => ({ ok: true }),
|
|
2594
|
+
readInstalledCoreVersion: () => '1.19.1081', selfInstanceRoot: () => '/srv/cb',
|
|
2595
|
+
updateChannel: { DEFAULT_CHANNEL: 'patch', normalizeChannel: (c) => c || 'patch', channelAllows: () => true, listAvailableVersions: () => ({ ok: false, versions: [] }) },
|
|
2583
2596
|
checkDnsTokenScope: async () => ({ reachable: true, zone: 'z' }), checkCaddyWiring: () => null,
|
|
2584
2597
|
};
|
|
2585
2598
|
await P.cmdRunIntents(deps);
|
|
2586
|
-
return { resolved, classWrites };
|
|
2599
|
+
return { runs, resolved, classWrites, repends, row };
|
|
2587
2600
|
};
|
|
2588
2601
|
|
|
2589
|
-
await ta('a
|
|
2590
|
-
const { resolved, classWrites } = await
|
|
2591
|
-
assert.equal(
|
|
2602
|
+
await ta('a needs-decision core move is RETIRED at once with its class — retrying cannot change it', async () => {
|
|
2603
|
+
const { runs, resolved, classWrites } = await queueRun();
|
|
2604
|
+
assert.equal(runs, 1);
|
|
2605
|
+
assert.equal(resolved[0].state, 'error');
|
|
2592
2606
|
assert.deepEqual(classWrites, [[11, 'needs_decision', 'shape_not_automated']]);
|
|
2593
2607
|
});
|
|
2594
2608
|
|
|
2595
|
-
await ta('a
|
|
2596
|
-
const { resolved, classWrites } = await
|
|
2609
|
+
await ta('a TRANSIENT core move runs ONCE per tick and waits ~15 minutes before its next try', async () => {
|
|
2610
|
+
const { runs, resolved, repends, classWrites } = await queueRun({ transient: true });
|
|
2611
|
+
assert.equal(runs, 1, 'the same failing move is not re-run inside one tick');
|
|
2612
|
+
assert.equal(resolved.length, 0, 'not retired — it waits');
|
|
2613
|
+
assert.equal(repends.length, 1);
|
|
2614
|
+
assert.equal(repends[0][2], 15 * 60 * 1000, 'not_before = now + 15 minutes');
|
|
2615
|
+
assert.deepEqual(classWrites, [[11, 'transient', 'registry_unreadable']]);
|
|
2616
|
+
});
|
|
2617
|
+
|
|
2618
|
+
await ta('a transient move on its LAST scheduled try (the 6-hour one) is retired after it', async () => {
|
|
2619
|
+
const { runs, resolved } = await queueRun({ transient: true, attempts: 3 });
|
|
2620
|
+
// attempt 4 RUNS (the pre-run guard allows it for a transient row) and is then retired.
|
|
2621
|
+
assert.equal(runs, 1, 'the 6-hour try actually ran — not retired unrun by the pre-run guard');
|
|
2622
|
+
assert.equal(resolved.length, 1);
|
|
2597
2623
|
assert.equal(resolved[0].state, 'error');
|
|
2598
|
-
assert.deepEqual(classWrites, [[11, 'needs_decision', 'shape_not_automated']]);
|
|
2599
2624
|
});
|
|
2600
2625
|
|
|
2601
2626
|
console.log('\nreview-panel hardening (task 1002720 — the loopback/latch/birth-note seams):');
|