@bongos/core 1.20.32 → 1.20.33
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.bongos-core.json +34 -24
- package/docs/module-api-changelog.md +2 -0
- package/docs/recipes/upgrading-the-core.md +1 -0
- package/modules/autonomy/cadence.js +9 -0
- package/modules/autonomy/digest.js +5 -2
- package/package-lock.json +2 -2
- package/package.json +1 -1
- package/release-notes.json +10 -0
- package/scripts/gds/autobongos-run.js +155 -44
- package/scripts/gds/claim.js +10 -2
- package/scripts/gds/move-escalation.js +141 -0
- package/scripts/gds/provision.js +2 -1
- package/src/module-api.js +1 -1
- package/tests/autobongos_cadence.mjs +9 -0
- package/tests/autobongos_loop.mjs +142 -1
- package/tests/autonomy_digest.mjs +10 -0
- package/tests/claim_error_surface.mjs +7 -0
- package/tests/cli_exit_no_abort.mjs +7 -2
- package/tests/move_escalation.mjs +191 -0
- package/tests/plain_cards.mjs +2 -2
package/.bongos-core.json
CHANGED
|
@@ -2,22 +2,22 @@
|
|
|
2
2
|
"artifact": "bongos-core",
|
|
3
3
|
"manifest_schema": 1,
|
|
4
4
|
"generator": "scripts/gds/package-core.js",
|
|
5
|
-
"core_version": "1.20.
|
|
6
|
-
"core_contract": "1.20.
|
|
7
|
-
"source_commit": "
|
|
5
|
+
"core_version": "1.20.33",
|
|
6
|
+
"core_contract": "1.20.33",
|
|
7
|
+
"source_commit": "9b3842cb58220f06a93271aaf4d5c78088cf4374",
|
|
8
8
|
"source_ref": "HEAD",
|
|
9
|
-
"built_at": "2026-09-
|
|
9
|
+
"built_at": "2026-09-30T22:01:27.383Z",
|
|
10
10
|
"redaction": {
|
|
11
11
|
"model": "docs-redacted+functional-verbatim",
|
|
12
12
|
"docs_redacted": 562,
|
|
13
13
|
"agent_docs_stubbed": 25,
|
|
14
|
-
"functional_verbatim":
|
|
14
|
+
"functional_verbatim": 2671,
|
|
15
15
|
"rules": 3,
|
|
16
16
|
"gate_literals": 3,
|
|
17
17
|
"gate": "passed"
|
|
18
18
|
},
|
|
19
|
-
"file_count":
|
|
20
|
-
"tree_sha256": "
|
|
19
|
+
"file_count": 3259,
|
|
20
|
+
"tree_sha256": "aed626c2d3ddceedaf350fb3e2b40ada409404d6f830499bb790de6dec0f03d6",
|
|
21
21
|
"files": [
|
|
22
22
|
{
|
|
23
23
|
"path": ".claude/skills/ask-for-help/SKILL.md",
|
|
@@ -2802,7 +2802,7 @@
|
|
|
2802
2802
|
{
|
|
2803
2803
|
"path": "docs/module-api-changelog.md",
|
|
2804
2804
|
"mode": "0000644",
|
|
2805
|
-
"sha256": "
|
|
2805
|
+
"sha256": "62c1b89b8efed5d249b25f70db5bfb238cb9240c748122406d74fdc718e8b903"
|
|
2806
2806
|
},
|
|
2807
2807
|
{
|
|
2808
2808
|
"path": "docs/modules-contract.md",
|
|
@@ -3057,7 +3057,7 @@
|
|
|
3057
3057
|
{
|
|
3058
3058
|
"path": "docs/recipes/upgrading-the-core.md",
|
|
3059
3059
|
"mode": "0000644",
|
|
3060
|
-
"sha256": "
|
|
3060
|
+
"sha256": "81df5ffa3f55b075b284c51790fd1fea78ae7cab6af46460fcbf648987f7eb11"
|
|
3061
3061
|
},
|
|
3062
3062
|
{
|
|
3063
3063
|
"path": "docs/recipes/windows-builders.md",
|
|
@@ -4077,7 +4077,7 @@
|
|
|
4077
4077
|
{
|
|
4078
4078
|
"path": "modules/autonomy/cadence.js",
|
|
4079
4079
|
"mode": "0000644",
|
|
4080
|
-
"sha256": "
|
|
4080
|
+
"sha256": "24e717247143b6fae013c344d36f8b74dc3e4a228396dcfff39c383c2a762e6c"
|
|
4081
4081
|
},
|
|
4082
4082
|
{
|
|
4083
4083
|
"path": "modules/autonomy/db.js",
|
|
@@ -4087,7 +4087,7 @@
|
|
|
4087
4087
|
{
|
|
4088
4088
|
"path": "modules/autonomy/digest.js",
|
|
4089
4089
|
"mode": "0000644",
|
|
4090
|
-
"sha256": "
|
|
4090
|
+
"sha256": "f40114fde91ae3453ed58cd4c37d0ce4b292ae1a84b3faa6af62c8f2e500a33c"
|
|
4091
4091
|
},
|
|
4092
4092
|
{
|
|
4093
4093
|
"path": "modules/autonomy/fence.js",
|
|
@@ -8992,12 +8992,12 @@
|
|
|
8992
8992
|
{
|
|
8993
8993
|
"path": "package-lock.json",
|
|
8994
8994
|
"mode": "0000644",
|
|
8995
|
-
"sha256": "
|
|
8995
|
+
"sha256": "faabe741a65a2f632b26900d4d5aeac7e489263d0140c9d8633a1edf9761b8a9"
|
|
8996
8996
|
},
|
|
8997
8997
|
{
|
|
8998
8998
|
"path": "package.json",
|
|
8999
8999
|
"mode": "0000644",
|
|
9000
|
-
"sha256": "
|
|
9000
|
+
"sha256": "de12fffa22b571c06d6757c17cc153ef9a3867bc44893d572141bf989b80c185"
|
|
9001
9001
|
},
|
|
9002
9002
|
{
|
|
9003
9003
|
"path": "public-docs/index.html",
|
|
@@ -9017,7 +9017,7 @@
|
|
|
9017
9017
|
{
|
|
9018
9018
|
"path": "release-notes.json",
|
|
9019
9019
|
"mode": "0000644",
|
|
9020
|
-
"sha256": "
|
|
9020
|
+
"sha256": "695627258a4d47ae703472c86e7fb573ed751e6e2d351e47441c9ff29be824e5"
|
|
9021
9021
|
},
|
|
9022
9022
|
{
|
|
9023
9023
|
"path": "scripts/bongos-mcp.js",
|
|
@@ -9177,7 +9177,7 @@
|
|
|
9177
9177
|
{
|
|
9178
9178
|
"path": "scripts/gds/autobongos-run.js",
|
|
9179
9179
|
"mode": "0000644",
|
|
9180
|
-
"sha256": "
|
|
9180
|
+
"sha256": "667d8acc1ca0b709a61aebabd0ab79171873d66c32b42dcff2e86829adc2cae0"
|
|
9181
9181
|
},
|
|
9182
9182
|
{
|
|
9183
9183
|
"path": "scripts/gds/autobongos-service.cmd",
|
|
@@ -9312,7 +9312,7 @@
|
|
|
9312
9312
|
{
|
|
9313
9313
|
"path": "scripts/gds/claim.js",
|
|
9314
9314
|
"mode": "0000644",
|
|
9315
|
-
"sha256": "
|
|
9315
|
+
"sha256": "77c5d63df2ba881c11e184f11e7972527bac310d856a82d0d7cd234de514665c"
|
|
9316
9316
|
},
|
|
9317
9317
|
{
|
|
9318
9318
|
"path": "scripts/gds/classify-discipline.js",
|
|
@@ -9864,6 +9864,11 @@
|
|
|
9864
9864
|
"mode": "0000644",
|
|
9865
9865
|
"sha256": "2bb8d87422e2b24db1ea9fa0c51556b09a7bea99da5af7ad3798c3b386809f4b"
|
|
9866
9866
|
},
|
|
9867
|
+
{
|
|
9868
|
+
"path": "scripts/gds/move-escalation.js",
|
|
9869
|
+
"mode": "0000644",
|
|
9870
|
+
"sha256": "de4251cf593b77a9dee3590de55319127b0e84d3bb9560ce7f07ee3f4628a23e"
|
|
9871
|
+
},
|
|
9867
9872
|
{
|
|
9868
9873
|
"path": "scripts/gds/newcomer-floor.config.json",
|
|
9869
9874
|
"mode": "0000644",
|
|
@@ -10037,7 +10042,7 @@
|
|
|
10037
10042
|
{
|
|
10038
10043
|
"path": "scripts/gds/provision.js",
|
|
10039
10044
|
"mode": "0000644",
|
|
10040
|
-
"sha256": "
|
|
10045
|
+
"sha256": "a014e09f61a19cf16380ae35d5ab93661b65e3fd037d7fddc0fe5ab046a6e699"
|
|
10041
10046
|
},
|
|
10042
10047
|
{
|
|
10043
10048
|
"path": "scripts/gds/publish-credential-check.js",
|
|
@@ -11152,7 +11157,7 @@
|
|
|
11152
11157
|
{
|
|
11153
11158
|
"path": "src/module-api.js",
|
|
11154
11159
|
"mode": "0000644",
|
|
11155
|
-
"sha256": "
|
|
11160
|
+
"sha256": "0969a3b47823d6b24c9eaaeb777a69c2e6d20688908bb1995dd5893cd5ae6cf2"
|
|
11156
11161
|
},
|
|
11157
11162
|
{
|
|
11158
11163
|
"path": "src/module-loader/catalog.js",
|
|
@@ -11597,7 +11602,7 @@
|
|
|
11597
11602
|
{
|
|
11598
11603
|
"path": "tests/autobongos_cadence.mjs",
|
|
11599
11604
|
"mode": "0000644",
|
|
11600
|
-
"sha256": "
|
|
11605
|
+
"sha256": "d6afd12b3d2da5e77f832e8206defb676840b340d6e7a5f064fc2bb96a7c2ffb"
|
|
11601
11606
|
},
|
|
11602
11607
|
{
|
|
11603
11608
|
"path": "tests/autobongos_fence.mjs",
|
|
@@ -11612,7 +11617,7 @@
|
|
|
11612
11617
|
{
|
|
11613
11618
|
"path": "tests/autobongos_loop.mjs",
|
|
11614
11619
|
"mode": "0000644",
|
|
11615
|
-
"sha256": "
|
|
11620
|
+
"sha256": "2d2017585d61ee15f217bd6eaae26025732ef099dbcfebd26f3729790b8c9078"
|
|
11616
11621
|
},
|
|
11617
11622
|
{
|
|
11618
11623
|
"path": "tests/autobongos_service_ps1.mjs",
|
|
@@ -11632,7 +11637,7 @@
|
|
|
11632
11637
|
{
|
|
11633
11638
|
"path": "tests/autonomy_digest.mjs",
|
|
11634
11639
|
"mode": "0000644",
|
|
11635
|
-
"sha256": "
|
|
11640
|
+
"sha256": "928a8a0461adab72037a03d184966e7c1e20838e72421b0ed93c6ce5b8c84520"
|
|
11636
11641
|
},
|
|
11637
11642
|
{
|
|
11638
11643
|
"path": "tests/autonomy_gate.mjs",
|
|
@@ -11837,7 +11842,7 @@
|
|
|
11837
11842
|
{
|
|
11838
11843
|
"path": "tests/claim_error_surface.mjs",
|
|
11839
11844
|
"mode": "0000644",
|
|
11840
|
-
"sha256": "
|
|
11845
|
+
"sha256": "7678a98254284a541684292b3615bd02729003dfddbdd9d7b24bd895f63fb908"
|
|
11841
11846
|
},
|
|
11842
11847
|
{
|
|
11843
11848
|
"path": "tests/claim_from_task_and_home.mjs",
|
|
@@ -11912,7 +11917,7 @@
|
|
|
11912
11917
|
{
|
|
11913
11918
|
"path": "tests/cli_exit_no_abort.mjs",
|
|
11914
11919
|
"mode": "0000644",
|
|
11915
|
-
"sha256": "
|
|
11920
|
+
"sha256": "bce64eee77c2fb5af0fdab2a1c4b7d59cd1465817724ca7065572ba466a16fe5"
|
|
11916
11921
|
},
|
|
11917
11922
|
{
|
|
11918
11923
|
"path": "tests/cli_lib_formatters.mjs",
|
|
@@ -14224,6 +14229,11 @@
|
|
|
14224
14229
|
"mode": "0000644",
|
|
14225
14230
|
"sha256": "ee165bd5e817b909be4299014646d3c8b5e34bf00854491ea42cc5afc663fdc3"
|
|
14226
14231
|
},
|
|
14232
|
+
{
|
|
14233
|
+
"path": "tests/move_escalation.mjs",
|
|
14234
|
+
"mode": "0000644",
|
|
14235
|
+
"sha256": "0406212bd27e4b878b27414625071b646e4e023af1694f8fc2449fdc61be243c"
|
|
14236
|
+
},
|
|
14227
14237
|
{
|
|
14228
14238
|
"path": "tests/my_projects_surface.mjs",
|
|
14229
14239
|
"mode": "0000644",
|
|
@@ -14467,7 +14477,7 @@
|
|
|
14467
14477
|
{
|
|
14468
14478
|
"path": "tests/plain_cards.mjs",
|
|
14469
14479
|
"mode": "0000644",
|
|
14470
|
-
"sha256": "
|
|
14480
|
+
"sha256": "731956f3836cd73878668d0d2707f18eaec64ed70495ce1beaf5410d3fb9af8a"
|
|
14471
14481
|
},
|
|
14472
14482
|
{
|
|
14473
14483
|
"path": "tests/planning_session_skill.mjs",
|
|
@@ -2721,5 +2721,7 @@ is load-bearing: the script throws rather than guess if it is missing, and
|
|
|
2721
2721
|
landed since 1.20.30 with no explicit bump. run 36779142919. (task 1002620)
|
|
2722
2722
|
1.20.32 — CI auto-patch (publish-on-merge, ADR 0161): carrier for merges
|
|
2723
2723
|
landed since 1.20.31 with no explicit bump. run 36781343271. (task 1002620)
|
|
2724
|
+
1.20.33 — CI auto-patch (publish-on-merge, ADR 0161): carrier for merges
|
|
2725
|
+
landed since 1.20.32 with no explicit bump. run 36783021915. (task 1002620)
|
|
2724
2726
|
---------------------------------------------------------------------------
|
|
2725
2727
|
```
|
|
@@ -180,6 +180,7 @@ running a go-live.
|
|
|
180
180
|
## Gotchas
|
|
181
181
|
|
|
182
182
|
- **The Deploy door fixes three known snags by itself, once** (task 1004448, `scripts/gds/wedge-remedy.js`). When a core move through `/deploy` fails on a **dirty tree** (it commits the tracked changes inside the project), a **full disk** (it prunes that project's own nightly dumps down to the newest 7) or a **database behind its code** (it migrates that database with its own `PGDATABASE` before the code moves), the runner applies the fix, records a `core-upgrade-remedy` event on the project, and tries the move once more on the next tick. If the retry fails for the same reason, or the fix itself fails, the move is retired as `unknown` (`remedy_did_not_hold` / `remedy_failed`) for a person — never a second fix. The manual fixes below still apply to the unattended nightly sweep, which does not go through the door.
|
|
183
|
+
- **A Deploy-door move nobody can fix files a blocker and posts to Discord — once** (task 1004449, `scripts/gds/move-escalation.js`). A move retired as `unknown` (including `remedy_did_not_hold`, `remedy_failed` and a failed rollback) files one blocker (`source=provisioning`, `source_ref=core-move:<instance id>:<intent id>`) with the reason and what the move reported, and posts one notice to `#ship-news` that names the project only if it is public. The next core move on that project that lands closes the blocker by itself and posts an all-clear. Refusals the owner can act on (`needs_decision`) are shown on the deploy page instead, not filed.
|
|
183
184
|
- **`--registry` needs no credential at all** (task 1003878). `@bongos/core` is **public** on npm (task 1003872), so `bongos upgrade --registry` resolves it from the default registry like any other dependency — no token, no login, and no instance `.npmrc`. An install failure on this path is therefore a *real* failure (network, registry outage, or a version that was never published), never a missing credential.
|
|
184
185
|
*History, so the symptom is recognisable on an old instance:* the core used to publish private, and the instance `.npmrc` read `_authToken=<redacted> npm gives that per-project key precedence, so with the env var unset it expanded EMPTY and **masked a perfectly valid literal token in `~/.npmrc`** — npm went anonymous, the private core 404'd, and the error blamed a credential the box actually held. Task 1002754 worked around it by resolving the token in-process; task 1003878 deleted the whole mechanism along with the `.npmrc` that caused it. An instance still carrying a committed `.npmrc` from a pre-1003878 scaffold can safely delete it.
|
|
185
186
|
- **A green `publish` run is not evidence of a publish.** The publish-on-merge lane succeeds on its no-op paths too (cascade terminator, armed-gate). Read the run's **job summary**, which now states the verdict explicitly: `PUBLISHED @bongos/core@<v>` vs `NO-OP — nothing was published this run`.
|
|
@@ -178,6 +178,15 @@ function classifyEvent(row = {}) {
|
|
|
178
178
|
|
|
179
179
|
if (event === 'verify_halt') return { class: 'transient', detail: 'the ledger could not be read back after a worker' };
|
|
180
180
|
|
|
181
|
+
if (event === 'queue_gated') {
|
|
182
|
+
// A stranded ship refuses EVERY claim (task 1004396). That is somebody's
|
|
183
|
+
// unlanded work, not this machine breaking, so it is quiet: escalating would
|
|
184
|
+
// slow the runner for a fault it cannot fix. The wait already wakes early when
|
|
185
|
+
// main moves, and a strand landing is a merge to main.
|
|
186
|
+
const on = Array.isArray(row.waiting_on) && row.waiting_on.length ? row.waiting_on.map((id) => `task ${id}`).join(', ') : 'a stranded task';
|
|
187
|
+
return { class: 'quiet', detail: `waiting for ${on} to land — every claim is refused until then` };
|
|
188
|
+
}
|
|
189
|
+
|
|
181
190
|
if (event === 'pick_failed' || event === 'worktree_failed' || event === 'claim_failed') {
|
|
182
191
|
return {
|
|
183
192
|
class: looksSystemic(reason) ? 'systemic' : 'transient',
|
|
@@ -173,7 +173,9 @@ function buildDigest(rows = [], { gauge = null, nowEpochS = null, sinceMs = null
|
|
|
173
173
|
}
|
|
174
174
|
}
|
|
175
175
|
|
|
176
|
-
|
|
176
|
+
// A queue gate (task 1004396) is a hold too: the runner chose not to claim, and
|
|
177
|
+
// the reason names the stranded task it is waiting on.
|
|
178
|
+
const held = all.filter((r) => r.event === 'hold' || r.event === 'queue_gated').map((r) => ({ reason: r.reason || null, sleepUntil: r.sleep_until ?? null }));
|
|
177
179
|
|
|
178
180
|
// 3. WHAT EACH WORKER DECIDED that its spec did not determine — grouped by task,
|
|
179
181
|
// because the SHAPE of this list is the signal. One task with six decisions is a
|
|
@@ -211,7 +213,8 @@ function buildDigest(rows = [], { gauge = null, nowEpochS = null, sinceMs = null
|
|
|
211
213
|
// How the runner spent the night, so "nothing shipped" can be told apart from
|
|
212
214
|
// "nothing ran".
|
|
213
215
|
const KNOWN = new Set(['worked', 'fenced', 'hold', 'nothing_claimable', 'dry_run',
|
|
214
|
-
'pick_failed', 'worktree_failed', 'claim_failed', 'verify_halt', 'cadence', 'boot_reconcile'
|
|
216
|
+
'pick_failed', 'worktree_failed', 'claim_failed', 'verify_halt', 'cadence', 'boot_reconcile',
|
|
217
|
+
'queue_gated', 'queue_gate_cleared']);
|
|
215
218
|
const byEvent = {};
|
|
216
219
|
let unrecognised = 0;
|
|
217
220
|
for (const r of all) {
|
package/package-lock.json
CHANGED
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@bongos/core",
|
|
3
|
-
"version": "1.20.
|
|
3
|
+
"version": "1.20.33",
|
|
4
4
|
"lockfileVersion": 3,
|
|
5
5
|
"requires": true,
|
|
6
6
|
"packages": {
|
|
7
7
|
"": {
|
|
8
8
|
"name": "@bongos/core",
|
|
9
|
-
"version": "1.20.
|
|
9
|
+
"version": "1.20.33",
|
|
10
10
|
"license": "AGPL-3.0-or-later",
|
|
11
11
|
"dependencies": {
|
|
12
12
|
"express": "^4.21.2",
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@bongos/core",
|
|
3
|
-
"version": "1.20.
|
|
3
|
+
"version": "1.20.33",
|
|
4
4
|
"description": "Cloud Bongos — the AI-first build platform core (GDS + platform surfaces + module system), installed as a versioned dependency (ADR 0108).",
|
|
5
5
|
"license": "AGPL-3.0-or-later",
|
|
6
6
|
"main": "src/platform-server.js",
|
package/release-notes.json
CHANGED
|
@@ -8264,5 +8264,15 @@
|
|
|
8264
8264
|
"id": "1004448",
|
|
8265
8265
|
"text": "When an update to a project gets stuck on one of three known problems (leftover file changes, a full disk, or a database that is behind), the system now fixes it by itself and tries again once. If that doesn't work, it stops a"
|
|
8266
8266
|
}
|
|
8267
|
+
],
|
|
8268
|
+
"1.20.33": [
|
|
8269
|
+
{
|
|
8270
|
+
"id": "1004396",
|
|
8271
|
+
"text": "The unattended runner no longer gets stuck retrying a task it isn't allowed to take: it moves on to the next one, and only waits when a stuck ship is blocking everyone, resuming by itself once that lands."
|
|
8272
|
+
},
|
|
8273
|
+
{
|
|
8274
|
+
"id": "1004449",
|
|
8275
|
+
"text": "When a project update fails in a way the system can't fix by itself, it now opens one blocker for a person and posts one message on Discord, instead of failing quietly. When a later update succeeds, the blocker closes by itsel"
|
|
8276
|
+
}
|
|
8267
8277
|
]
|
|
8268
8278
|
}
|
|
@@ -189,7 +189,12 @@ async function pickTask(goals, deps = {}) {
|
|
|
189
189
|
|
|
190
190
|
const candidates = Array.from(byId.values()).filter((t) => !seq.TERMINAL.has(t.status));
|
|
191
191
|
const { order } = seq.topoOrder(candidates);
|
|
192
|
-
|
|
192
|
+
// Tasks this process set aside after a refused claim (task 1004396) drop out
|
|
193
|
+
// AFTER ordering, so a dependent of one is still halted by its own unmet
|
|
194
|
+
// dependency rather than promoted by the task's absence.
|
|
195
|
+
const exclude = deps.exclude || null;
|
|
196
|
+
const open = exclude && exclude.size ? order.filter((t) => !exclude.has(String(t.id))) : order;
|
|
197
|
+
const { next, skipped } = seq.pickNext(open, { rank, claimableIds });
|
|
193
198
|
if (next) return { task: next, goalId, skipped };
|
|
194
199
|
for (const sk of skipped) allSkipped.push({ ...sk, goal_id: goalId });
|
|
195
200
|
}
|
|
@@ -369,6 +374,75 @@ async function readFence(deps = {}) {
|
|
|
369
374
|
return { raw, graderBypassed };
|
|
370
375
|
}
|
|
371
376
|
|
|
377
|
+
// ── what one process remembers between iterations (task 1004396) ────────────
|
|
378
|
+
//
|
|
379
|
+
// Owner ruling, 2026-09-30: a refused claim must not put the runner into waiting
|
|
380
|
+
// by default. It sets THAT task aside and tries the next one. It waits only when
|
|
381
|
+
// there is a block and nothing else it can claim.
|
|
382
|
+
//
|
|
383
|
+
// The two refusals are told apart by claim.js's exit code, because the message is
|
|
384
|
+
// prose and the first cut of this read prose. The run that found this burned
|
|
385
|
+
// ~30 attempts on task 1004254, whose refusal quoted "Your work changes the
|
|
386
|
+
// tools…" — which reads like a per-task protected-path verdict and was in fact
|
|
387
|
+
// the plain-words line of a QUEUE-WIDE gate about a different, stranded task.
|
|
388
|
+
// Setting 1004254 aside would have moved the runner on to the next task, which
|
|
389
|
+
// the same gate refuses too.
|
|
390
|
+
//
|
|
391
|
+
// QUEUE_GATED_EXIT is pinned against claim.js's own value in a test rather than
|
|
392
|
+
// required from it: requiring the CLI would load its whole dependency graph into
|
|
393
|
+
// a process that runs for weeks, to read one integer.
|
|
394
|
+
const QUEUE_GATED_EXIT = 4;
|
|
395
|
+
// How long a task refused ON ITS OWN is left alone. Not forever: most per-task
|
|
396
|
+
// refusals clear (a dependency ships, a claim is released, a blocker resolves),
|
|
397
|
+
// and a process that remembered them for its whole life would shelve work it can
|
|
398
|
+
// do. Half an hour is several wakes at full speed, so a refused head of the queue
|
|
399
|
+
// costs one attempt per half hour instead of one per wake.
|
|
400
|
+
const CLAIM_SET_ASIDE_MS = 30 * 60 * 1000;
|
|
401
|
+
// A bound on claim attempts in ONE pass, so a queue of refusals cannot turn one
|
|
402
|
+
// iteration into dozens of worktree creations.
|
|
403
|
+
const MAX_CLAIM_ATTEMPTS = 5;
|
|
404
|
+
|
|
405
|
+
function newRunnerState() {
|
|
406
|
+
return { setAside: new Map(), queueGate: null };
|
|
407
|
+
}
|
|
408
|
+
const RUNNER_STATE = newRunnerState();
|
|
409
|
+
|
|
410
|
+
// The stranded task ids a queue-gated refusal names (" - task 777 Title"). An
|
|
411
|
+
// empty list is a legal answer: the gate is still a gate, the runner simply has
|
|
412
|
+
// nothing to poll and tries one claim per wake instead.
|
|
413
|
+
function owedTaskIds(text) {
|
|
414
|
+
const ids = [];
|
|
415
|
+
const re = /^\s*-\s*task\s+(\d+)/gm;
|
|
416
|
+
let m;
|
|
417
|
+
while ((m = re.exec(String(text || ''))) !== null) ids.push(m[1]);
|
|
418
|
+
return ids;
|
|
419
|
+
}
|
|
420
|
+
|
|
421
|
+
// Does the gate still hold? Asked of the LEDGER, one read per stranded task, and
|
|
422
|
+
// never by attempting a claim: a claim attempt per wake is the loop this replaces.
|
|
423
|
+
// An unreadable task keeps the wait — guessing "cleared" would restart the
|
|
424
|
+
// attempts on a read failure, the opposite of backing off.
|
|
425
|
+
async function queueGateHolds(queueGate, deps = {}) {
|
|
426
|
+
if (!queueGate.owed.length) return false;
|
|
427
|
+
const api = deps.api && deps.api.tasks ? deps.api : await (require('./cli-lib').cliClient)();
|
|
428
|
+
for (const id of queueGate.owed) {
|
|
429
|
+
let r = null;
|
|
430
|
+
try { r = await api.tasks.getTasksId({ id: Number(id) }); } catch (_) { return true; }
|
|
431
|
+
if (!r || !r.ok || !r.data || !r.data.task) return true;
|
|
432
|
+
if (r.data.task.status === 'confirmed') return true;
|
|
433
|
+
}
|
|
434
|
+
return false;
|
|
435
|
+
}
|
|
436
|
+
|
|
437
|
+
function queueGatedRow(queueGate, now) {
|
|
438
|
+
const on = queueGate.owed.length ? queueGate.owed.map((id) => `task ${id}`).join(', ') : 'a stranded confirmed task';
|
|
439
|
+
return {
|
|
440
|
+
event: 'queue_gated', waiting_on: queueGate.owed,
|
|
441
|
+
waited_s: Math.max(0, Math.round((now - queueGate.since) / 1000)),
|
|
442
|
+
reason: `every claim is refused until ${on} lands — waiting, not claiming (${queueGate.reason})`,
|
|
443
|
+
};
|
|
444
|
+
}
|
|
445
|
+
|
|
372
446
|
async function iteration(opts, deps = {}) {
|
|
373
447
|
const runCmd = deps.run || run;
|
|
374
448
|
const spawnWork = deps.spawnWorker || spawnWorker;
|
|
@@ -430,53 +504,89 @@ async function iteration(opts, deps = {}) {
|
|
|
430
504
|
}
|
|
431
505
|
}
|
|
432
506
|
|
|
433
|
-
//
|
|
434
|
-
//
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
});
|
|
444
|
-
}
|
|
445
|
-
const task = picked.task;
|
|
446
|
-
|
|
447
|
-
// 2b. The per-TASK half of the fence. sequence.js's halt matrix already refuses
|
|
448
|
-
// a protected-path task, and this re-asks anyway — not from distrust of the
|
|
449
|
-
// matrix, but because the matrix is a shared component with its own callers and
|
|
450
|
-
// the fence must not be a property of whoever happens to be picking. A gate that
|
|
451
|
-
// only holds while a collaborator keeps calling it is not a gate.
|
|
452
|
-
//
|
|
453
|
-
// Both halves read the task's DECLARED touches[], which is a claim, not a fact —
|
|
454
|
-
// see the header of modules/autonomy/fence.js for why that is acceptable as the
|
|
455
|
-
// OUTER of three layers and would not be as the only one.
|
|
456
|
-
const taskGate = fence.decideTask({
|
|
457
|
-
task, goals: gate.goals, protectedHits: matchProtected(task.touches || []),
|
|
458
|
-
});
|
|
459
|
-
if (!taskGate.go) {
|
|
460
|
-
return log({ event: 'fenced', code: taskGate.code, reason: taskGate.reason, task_id: task.id, goal_id: picked.goalId, globs: taskGate.globs || [] });
|
|
507
|
+
// 1c. A QUEUE-WIDE gate from an earlier pass (task 1004396). While the
|
|
508
|
+
// stranded task it named is still `confirmed`, every claim would be refused, so
|
|
509
|
+
// none is attempted. The wait wakes early when main moves (waitOrJump), which is
|
|
510
|
+
// exactly what the strand landing does.
|
|
511
|
+
const state = deps.state || RUNNER_STATE;
|
|
512
|
+
const now = deps.now ? deps.now() : Date.now();
|
|
513
|
+
if (state.queueGate) {
|
|
514
|
+
if (await queueGateHolds(state.queueGate, deps)) return log(queueGatedRow(state.queueGate, now));
|
|
515
|
+
log({ event: 'queue_gate_cleared', waited_on: state.queueGate.owed, waited_s: Math.max(0, Math.round((now - state.queueGate.since) / 1000)) });
|
|
516
|
+
state.queueGate = null;
|
|
461
517
|
}
|
|
518
|
+
for (const [id, until] of state.setAside) if (until <= now) state.setAside.delete(id);
|
|
519
|
+
|
|
520
|
+
// 2–4, up to MAX_CLAIM_ATTEMPTS times: pick, fence, tree, claim. A task refused
|
|
521
|
+
// on its own is set aside and the NEXT one is tried in the same pass.
|
|
522
|
+
let task = null; let picked = null; let wtName = null; let wtPath = null;
|
|
523
|
+
let lastRefusal = null;
|
|
524
|
+
for (let attempt = 0; attempt < MAX_CLAIM_ATTEMPTS && !task; attempt++) {
|
|
525
|
+
const exclude = new Set(state.setAside.keys());
|
|
526
|
+
// 2. Pick — from the ALLOWLISTED goals, never from what the command line asked
|
|
527
|
+
// for. gate.goals is already the intersection.
|
|
528
|
+
const got = await pick(gate.goals, { ...(deps.pickDeps || {}), exclude });
|
|
529
|
+
if (got.error) return log({ event: 'pick_failed', reason: got.error });
|
|
530
|
+
// A picker that hands back a set-aside task is treated as having none left:
|
|
531
|
+
// the exclusion is this function's contract, not the picker's courtesy.
|
|
532
|
+
if (got.none || exclude.has(String(got.task.id))) {
|
|
533
|
+
// Everything left was refused in THIS pass: return the last refusal, not
|
|
534
|
+
// "nothing claimable". Every claim failing at once is also what an outage
|
|
535
|
+
// looks like, and it must still reach the failure ladder (task 1004179).
|
|
536
|
+
if (lastRefusal) return lastRefusal;
|
|
537
|
+
return log({
|
|
538
|
+
event: 'nothing_claimable', goals: gate.goals,
|
|
539
|
+
// Carried so the cadence can tell an EMPTY queue from one where every task
|
|
540
|
+
// halts — the reasons are the matrix's own codes, safe to print.
|
|
541
|
+
skipped: (got.skipped || []).map((sk) => ({ id: sk.id, goal_id: sk.goal_id, reasons: (sk.reasons || []).map((r) => r.code) })),
|
|
542
|
+
set_aside: Array.from(state.setAside, ([id, until]) => ({ id, until: new Date(until).toISOString() })),
|
|
543
|
+
});
|
|
544
|
+
}
|
|
545
|
+
const candidate = got.task;
|
|
546
|
+
|
|
547
|
+
// 2b. The per-TASK half of the fence. sequence.js's halt matrix already refuses
|
|
548
|
+
// a protected-path task, and this re-asks anyway — not from distrust of the
|
|
549
|
+
// matrix, but because the matrix is a shared component with its own callers and
|
|
550
|
+
// the fence must not be a property of whoever happens to be picking. A gate that
|
|
551
|
+
// only holds while a collaborator keeps calling it is not a gate.
|
|
552
|
+
//
|
|
553
|
+
// Both halves read the task's DECLARED touches[], which is a claim, not a fact —
|
|
554
|
+
// see the header of modules/autonomy/fence.js for why that is acceptable as the
|
|
555
|
+
// OUTER of three layers and would not be as the only one.
|
|
556
|
+
const taskGate = fence.decideTask({
|
|
557
|
+
task: candidate, goals: gate.goals, protectedHits: matchProtected(candidate.touches || []),
|
|
558
|
+
});
|
|
559
|
+
if (!taskGate.go) {
|
|
560
|
+
return log({ event: 'fenced', code: taskGate.code, reason: taskGate.reason, task_id: candidate.id, goal_id: got.goalId, globs: taskGate.globs || [] });
|
|
561
|
+
}
|
|
462
562
|
|
|
463
|
-
|
|
563
|
+
if (opts.dryRun) return log({ event: 'dry_run', task_id: candidate.id, title: candidate.title, goal_id: got.goalId });
|
|
464
564
|
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
565
|
+
// 3. The task's own working tree.
|
|
566
|
+
const name = deps.worktreeName ? deps.worktreeName(candidate.id) : worktreeName(candidate.id);
|
|
567
|
+
const made = await runCmd(process.execPath, [path.join('scripts', 'gds', 'worktree.js'), 'add', name, '--base', 'origin/main']);
|
|
568
|
+
if (!made.ok) {
|
|
569
|
+
return log({ event: 'worktree_failed', task_id: candidate.id, worktree: name, reason: failureReason(made, 'worktree.js') });
|
|
570
|
+
}
|
|
571
|
+
const treePath = path.join(REPO_ROOT, '.claude', 'worktrees', name);
|
|
572
|
+
|
|
573
|
+
// 4. Claim FROM that tree, so claim.js records it as the claim's tree and the
|
|
574
|
+
// ship lock later agrees with where the work actually is.
|
|
575
|
+
const claimed = await runCmd(process.execPath, [path.join(REPO_ROOT, 'scripts', 'gds', 'claim.js'), String(candidate.id)], { cwd: treePath });
|
|
576
|
+
if (!claimed.ok) {
|
|
577
|
+
await runCmd(process.execPath, [path.join('scripts', 'gds', 'worktree.js'), 'remove', name]);
|
|
578
|
+
const reason = failureReason(claimed, 'claim.js');
|
|
579
|
+
if (claimed.code === QUEUE_GATED_EXIT) {
|
|
580
|
+
state.queueGate = { owed: owedTaskIds(`${claimed.stderr || ''}\n${claimed.stdout || ''}`), since: now, reason };
|
|
581
|
+
return log({ ...queueGatedRow(state.queueGate, now), task_id: candidate.id, worktree: name });
|
|
582
|
+
}
|
|
583
|
+
state.setAside.set(String(candidate.id), now + CLAIM_SET_ASIDE_MS);
|
|
584
|
+
lastRefusal = log({ event: 'claim_failed', task_id: candidate.id, worktree: name, reason, set_aside_s: CLAIM_SET_ASIDE_MS / 1000 });
|
|
585
|
+
continue;
|
|
586
|
+
}
|
|
587
|
+
task = candidate; picked = got; wtName = name; wtPath = treePath;
|
|
479
588
|
}
|
|
589
|
+
if (!task) return lastRefusal;
|
|
480
590
|
|
|
481
591
|
// 5. Work, in that tree.
|
|
482
592
|
const started = Date.now();
|
|
@@ -975,4 +1085,5 @@ module.exports = {
|
|
|
975
1085
|
worktreeName, readFence, reconcileLeftoverClaims, pollSignals, waitOrJump, forever, CLAIM_PREFIX, countWorkedSince, codeDrifted, loadedHead, UPGRADE_EXIT_CODE, MIN_UPTIME_BEFORE_UPGRADE_MS,
|
|
976
1086
|
heartbeatPath, writeHeartbeat, readHeartbeat, pidAlive, anotherRunnerIsAlive, HEARTBEAT_STALE_MS,
|
|
977
1087
|
isRunnerTree, RUNNER_TREE_RE, postHeartbeat, RUNNER_STARTED_AT, failureReason,
|
|
1088
|
+
newRunnerState, QUEUE_GATED_EXIT, CLAIM_SET_ASIDE_MS, MAX_CLAIM_ATTEMPTS, owedTaskIds,
|
|
978
1089
|
};
|
package/scripts/gds/claim.js
CHANGED
|
@@ -282,6 +282,14 @@ function claimErrorInfo(r) {
|
|
|
282
282
|
// control characters that repaint a terminal, and a runaway one can bury the
|
|
283
283
|
// guidance it was meant to add. Clip to one line, strip C0/C1 controls, cap length.
|
|
284
284
|
const MAX_LISTED_TASKS = 20;
|
|
285
|
+
|
|
286
|
+
// QUEUE_GATED_EXIT — the one refusal that is about the QUEUE, not the task
|
|
287
|
+
// (task 1004396). REBASE_REQUIRED refuses every claim until a stranded confirmed
|
|
288
|
+
// task lands, so an unattended caller that reads it as "this task is refused"
|
|
289
|
+
// sets aside task after task for a fault none of them has. Every other refusal
|
|
290
|
+
// keeps exit 1. The autobongos runner is the consumer and pins the same value
|
|
291
|
+
// in a test.
|
|
292
|
+
const QUEUE_GATED_EXIT = 4;
|
|
285
293
|
function clip(s, max = 160) {
|
|
286
294
|
if (s == null) return '';
|
|
287
295
|
// eslint-disable-next-line no-control-regex
|
|
@@ -370,7 +378,7 @@ function claimFailureGuidance({ code, message, d }, r, taskId, ctx) {
|
|
|
370
378
|
if (t.reason) lines.push(` flagged: ${clip(t.reason)}`);
|
|
371
379
|
}
|
|
372
380
|
lines.push(` ${clip(d.hint, 600) || 'Clear it with /strand-fix N, or rebase the branch and re-ship — or /builder-release it to give up on one.'}`);
|
|
373
|
-
return { lines, exit:
|
|
381
|
+
return { lines, exit: QUEUE_GATED_EXIT };
|
|
374
382
|
}
|
|
375
383
|
if (code === 'ALREADY_CLAIMED') {
|
|
376
384
|
lines.push(`Task #${taskId} was just claimed by another session. Try a different task.`);
|
|
@@ -771,4 +779,4 @@ if (require.main === module) {
|
|
|
771
779
|
});
|
|
772
780
|
}
|
|
773
781
|
|
|
774
|
-
module.exports = { formatClaimFailure, claimErrorInfo, goalMembershipOfferPlan, materializeModeSkills, printRoleLine, worktreeDetectionNote };
|
|
782
|
+
module.exports = { formatClaimFailure, claimErrorInfo, goalMembershipOfferPlan, materializeModeSkills, printRoleLine, worktreeDetectionNote, QUEUE_GATED_EXIT };
|
|
@@ -0,0 +1,141 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
//
|
|
3
|
+
// scripts/gds/move-escalation.js — a core move nobody can heal reaches a person: one blocker
|
|
4
|
+
// and one Discord notice per failed move, closed by itself when a later move lands (task
|
|
5
|
+
// 1004449, goal 1000090 deploy self-heal).
|
|
6
|
+
//
|
|
7
|
+
// WHY. After task 1004448 a failed /deploy move ends one of three ways: it waits and retries
|
|
8
|
+
// (transient), it is fixed and retried (known_wedge), or it is retired. A retired
|
|
9
|
+
// needs_decision move is the owner's own rule refusing it, and the deploy page says so. But a
|
|
10
|
+
// move retired as `unknown` — an unlisted failure, a fix that did not hold, and every
|
|
11
|
+
// rollback that FAILED (the project may be down) — only wrote a line on the intent, which
|
|
12
|
+
// nobody reads until they open the page. That is exactly the failure that needs a person.
|
|
13
|
+
//
|
|
14
|
+
// EDGE-TRIGGERED, WITHOUT NEW STATE. The blocker's (source, source_ref) is unique, and
|
|
15
|
+
// source_ref names the INTENT: `core-move:<instance id>:<intent id>`. An intent retires once,
|
|
16
|
+
// so a failed move files one blocker however many retries led up to it, and a re-run of the
|
|
17
|
+
// same retire files nothing. The Discord notice is posted only when the insert actually
|
|
18
|
+
// created the row, so it is exactly as edge-triggered as the blocker.
|
|
19
|
+
//
|
|
20
|
+
// CLOSED BY SUCCESS. A later core move on the same project that lands resolves every open
|
|
21
|
+
// blocker this file filed for that project, with a note saying which move fixed it, and
|
|
22
|
+
// posts one all-clear. A person can still resolve one by hand; this never reopens it.
|
|
23
|
+
//
|
|
24
|
+
// The Discord copy names a project only when it is public on the platform's map (the rule
|
|
25
|
+
// liveness-notify.js keeps), and never carries upgrade output — that stays in the blocker,
|
|
26
|
+
// inside the trust boundary. Writes go through the runner's own db handle (the hub
|
|
27
|
+
// database, where the blockers table lives), not modules/ideas/blockers.js's shared pool,
|
|
28
|
+
// which this CLI process never opens; the INSERT is that file's createBlocker, verbatim.
|
|
29
|
+
|
|
30
|
+
const { execFile } = require('child_process');
|
|
31
|
+
const { REPO_ROOT } = require('./provision-config.js');
|
|
32
|
+
|
|
33
|
+
const BLOCKER_SOURCE = 'provisioning';
|
|
34
|
+
const REF_PREFIX = 'core-move';
|
|
35
|
+
const DETAIL_MAX = 1500;
|
|
36
|
+
|
|
37
|
+
/** Does this retired move need a person? PURE. */
|
|
38
|
+
function needsPerson({ intent, failure, terminal }) {
|
|
39
|
+
if (!intent || intent.action !== 'core-upgrade') return false;
|
|
40
|
+
return !!terminal || !!(failure && (failure.class === 'unknown' || failure.reason === 'rollback_failed'));
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
function sourceRef(inst, intent) { return `${REF_PREFIX}:${inst.id}:${intent.id}`; }
|
|
44
|
+
|
|
45
|
+
function projectName(inst, provisioning) {
|
|
46
|
+
try { return provisioning.effectiveSettings(inst).visibility === 'public' ? inst.slug : null; } catch { return null; }
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
/** The blocker a failed move files. PURE. */
|
|
50
|
+
function blockerFor({ inst, intent, failure, error }) {
|
|
51
|
+
const reason = (failure && failure.reason) || 'unknown';
|
|
52
|
+
const down = reason === 'rollback_failed';
|
|
53
|
+
const to = intent.target_version || '(no target)';
|
|
54
|
+
const detail = String(error || '').slice(0, DETAIL_MAX).replace(/```/g, "'''");
|
|
55
|
+
return {
|
|
56
|
+
title: down
|
|
57
|
+
? `${inst.slug}: the core move to ${to} failed AND its rollback did not finish — the project may be down`
|
|
58
|
+
: `${inst.slug}: the core move to ${to} failed and the runner cannot fix it`,
|
|
59
|
+
bodyMd: [
|
|
60
|
+
`**Project:** ${inst.slug} (instance ${inst.id}) · **target:** ${to} · **reason:** \`${reason}\` · **move:** intent ${intent.id}`,
|
|
61
|
+
'',
|
|
62
|
+
down
|
|
63
|
+
? 'The move changed the project, failed, and the automatic rollback also failed. It may be down or on a half-installed core. Check it before anything else.'
|
|
64
|
+
: 'The runner retired this move: it is not a known snag it can fix, or its one automatic fix did not work. Nothing is being retried.',
|
|
65
|
+
'',
|
|
66
|
+
'What the move reported:',
|
|
67
|
+
'```',
|
|
68
|
+
detail || '(nothing)',
|
|
69
|
+
'```',
|
|
70
|
+
'',
|
|
71
|
+
'This closes by itself when a later core move on this project succeeds.',
|
|
72
|
+
].join('\n'),
|
|
73
|
+
source: BLOCKER_SOURCE,
|
|
74
|
+
sourceRef: sourceRef(inst, intent),
|
|
75
|
+
};
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
/** The owner-facing Discord line. `name` null → described, never identified. PURE. */
|
|
79
|
+
function noticeText({ kind, name, to, down }) {
|
|
80
|
+
const who = name ? `**${name}**` : 'A private project';
|
|
81
|
+
if (kind === 'fixed') return `✅ ${who} took its core update${to ? ` to ${to}` : ''}; the earlier failed update is closed.`;
|
|
82
|
+
return down
|
|
83
|
+
? `🚨 ${who}: a core update failed and could not be undone. The project may be down. A blocker is open with the details.`
|
|
84
|
+
: `⚠️ ${who}: a core update${to ? ` to ${to}` : ''} failed in a way the platform cannot fix by itself. A blocker is open with the details. Nothing is being retried.`;
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
// Fire-and-forget, the liveness-notify.js postNotice shape: CHAT_DRY_RUN=1 short-circuits and
|
|
88
|
+
// a Discord hiccup never stalls the drain.
|
|
89
|
+
function postNotice(text, { log = console.log } = {}) {
|
|
90
|
+
if (!text) return;
|
|
91
|
+
if (process.env.CHAT_DRY_RUN === '1') { log(`[core-move] [broadcast dry-run] would post: ${text}`); return; }
|
|
92
|
+
try {
|
|
93
|
+
execFile('node', ['scripts/discord/post.js', text], { cwd: REPO_ROOT, timeout: 10_000 }, (err) => {
|
|
94
|
+
if (err) log(`[core-move] notice broadcast failed (non-blocking): ${err.message}`);
|
|
95
|
+
});
|
|
96
|
+
} catch (e) { log(`[core-move] notice broadcast failed (non-blocking): ${(e && e.message) || e}`); }
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
/** After a move is retired: file its blocker and notice, once. Never throws. */
|
|
100
|
+
async function afterRetire({ intent, inst, result, deps }) {
|
|
101
|
+
const failure = result && result.failure;
|
|
102
|
+
if (!deps.apply || !needsPerson({ intent, failure, terminal: result && result.terminal })) return { filed: false };
|
|
103
|
+
const b = blockerFor({ inst, intent, failure, error: result.error });
|
|
104
|
+
try {
|
|
105
|
+
const { rows } = await deps.db.query(
|
|
106
|
+
`INSERT INTO blockers (title, body_md, source, source_ref)
|
|
107
|
+
VALUES ($1, $2, $3, $4)
|
|
108
|
+
ON CONFLICT (source, source_ref) WHERE source IS NOT NULL AND source_ref IS NOT NULL
|
|
109
|
+
DO NOTHING
|
|
110
|
+
RETURNING id`, [b.title, b.bodyMd, b.source, b.sourceRef]);
|
|
111
|
+
if (!rows || !rows[0]) return { filed: false };
|
|
112
|
+
const down = (failure && failure.reason) === 'rollback_failed';
|
|
113
|
+
(deps.postNotice || postNotice)(noticeText({ kind: 'failed', name: projectName(inst, deps.provisioning), to: intent.target_version, down }), { log: deps.log });
|
|
114
|
+
(deps.log || (() => {}))(` ⚑ filed blocker ${rows[0].id} for ${inst.slug}'s failed move to ${intent.target_version}`);
|
|
115
|
+
return { filed: true, blockerId: rows[0].id };
|
|
116
|
+
} catch (e) {
|
|
117
|
+
(deps.log || (() => {}))(` ⚠ could not file a blocker for ${inst.slug}'s failed move: ${(e && e.message) || e}`);
|
|
118
|
+
return { filed: false };
|
|
119
|
+
}
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
/** After a move LANDS: close every open blocker this file filed for the project. Never throws. */
|
|
123
|
+
async function afterSuccess({ intent, inst, result, deps }) {
|
|
124
|
+
if (!deps.apply || !intent || intent.action !== 'core-upgrade' || !result || result.ok === false || result.noop) return { closed: 0 };
|
|
125
|
+
const to = result.served || intent.target_version;
|
|
126
|
+
try {
|
|
127
|
+
const { rows } = await deps.db.query(
|
|
128
|
+
`UPDATE blockers SET status = 'resolved', resolved_at = now(), resolution_note = $3
|
|
129
|
+
WHERE status = 'open' AND source = $1 AND source_ref LIKE $2
|
|
130
|
+
RETURNING id`,
|
|
131
|
+
[BLOCKER_SOURCE, `${REF_PREFIX}:${inst.id}:%`, `closed automatically: a later core move to ${to} succeeded (intent ${intent.id})`]);
|
|
132
|
+
const n = (rows || []).length;
|
|
133
|
+
if (n) (deps.postNotice || postNotice)(noticeText({ kind: 'fixed', name: projectName(inst, deps.provisioning), to }), { log: deps.log });
|
|
134
|
+
return { closed: n };
|
|
135
|
+
} catch (e) {
|
|
136
|
+
(deps.log || (() => {}))(` ⚠ could not close ${inst.slug}'s core-move blockers: ${(e && e.message) || e}`);
|
|
137
|
+
return { closed: 0 };
|
|
138
|
+
}
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
module.exports = { BLOCKER_SOURCE, REF_PREFIX, needsPerson, sourceRef, blockerFor, noticeText, postNotice, afterRetire, afterSuccess };
|
package/scripts/gds/provision.js
CHANGED
|
@@ -1209,12 +1209,13 @@ async function cmdRunIntents(deps) {
|
|
|
1209
1209
|
// can say why the work stopped.
|
|
1210
1210
|
await provisioning.setInstanceStatus(db, intent.instance_id, inst.status, { error_note: result.error || 'op failed', error_note_action: intent.action }).catch(() => {});
|
|
1211
1211
|
if (result.failure) await recordFailureClass(db, intent.id, result.failure); // WHAT KIND of failure, beside the sentence (task 1004446)
|
|
1212
|
+
await require('./move-escalation.js').afterRetire({ intent, inst, result, deps }); // nobody can heal it: one blocker + one notice per move (task 1004449)
|
|
1212
1213
|
} else {
|
|
1213
1214
|
await db.query(`UPDATE provisioning_intents SET state='pending', last_error=$2, not_before = now() + ($3 * interval '1 millisecond'), updated_at=now() WHERE id=$1`, [intent.id, result.error || 'op failed', next.delayMs]);
|
|
1214
1215
|
if (result.failure) await recordFailureClass(db, intent.id, result.failure);
|
|
1215
1216
|
}
|
|
1216
1217
|
errors++;
|
|
1217
|
-
} else { await provisioning.resolveIntent(db, intent.id, 'done', (result && result.doneNote) || null); ok++; } // done WITH a note: it landed, and this is what it could not finish (task 1004065)
|
|
1218
|
+
} else { await provisioning.resolveIntent(db, intent.id, 'done', (result && result.doneNote) || null); ok++; await require('./move-escalation.js').afterSuccess({ intent, inst, result, deps }); } // done WITH a note: it landed, and this is what it could not finish (task 1004065)
|
|
1218
1219
|
} catch (e) {
|
|
1219
1220
|
await provisioning.resolveIntent(db, intent.id, 'error', e.message).catch(() => {});
|
|
1220
1221
|
// Surface the failure on the instance too, so a stuck instance is visible.
|
package/src/module-api.js
CHANGED
|
@@ -75,7 +75,7 @@ const { responsibilityFor, ROLE_RESPONSIBILITIES } = require('./role-responsibil
|
|
|
75
75
|
// MAJOR (see allowBoxScope below): passes the request through untouched.
|
|
76
76
|
function deprecatedNoopMiddleware(_req, _res, next) { next(); }
|
|
77
77
|
|
|
78
|
-
const CORE_VERSION = '1.20.
|
|
78
|
+
const CORE_VERSION = '1.20.33'; // CI auto-patch carrier (ADR 0161); changelog: docs/module-api-changelog.md
|
|
79
79
|
|
|
80
80
|
// A namespaced logger so a module's log lines are attributable + consistent.
|
|
81
81
|
// Usage: const log = api.logger('discord'); log.info('mounted');
|
|
@@ -40,6 +40,15 @@ const walk = (classes, cfg) => {
|
|
|
40
40
|
|
|
41
41
|
// ── classification ──────────────────────────────────────────────────────────────
|
|
42
42
|
|
|
43
|
+
test('a queue-wide gate is a wait, not a failure (task 1004396)', () => {
|
|
44
|
+
// The strand behind it is somebody's unlanded ship, not this machine breaking.
|
|
45
|
+
// Escalating it would slow the runner down for a fault it cannot fix, and the
|
|
46
|
+
// wait already wakes early when main moves, which is what the strand landing does.
|
|
47
|
+
const seen = classifyEvent({ event: 'queue_gated', waiting_on: ['777'], reason: 'x' });
|
|
48
|
+
assert.equal(seen.class, 'quiet');
|
|
49
|
+
assert.match(seen.detail, /777/);
|
|
50
|
+
});
|
|
51
|
+
|
|
43
52
|
test('a worked task counts as progress only when the LEDGER agreed', () => {
|
|
44
53
|
assert.equal(classifyEvent({ event: 'worked', verified: true }).class, 'progress');
|
|
45
54
|
// A worker's claim the ledger never saw is not progress; counting it would hold
|
|
@@ -237,6 +237,12 @@ test('an unreadable claimable feed does not halt everything', async () => {
|
|
|
237
237
|
assert.equal(got.task.id, 9001, 'the not_claimable reason stays quiet when the feed cannot be read');
|
|
238
238
|
});
|
|
239
239
|
|
|
240
|
+
test('pickTask passes over a task the runner has set aside (task 1004396)', async () => {
|
|
241
|
+
const api = fakeApi({ tasks: [ready({ id: 9001 }), ready({ id: 9002 })], claimable: [{ id: 9001 }, { id: 9002 }] });
|
|
242
|
+
const got = await runner.pickTask([1000119], { api, rank: 'archon', exclude: new Set(['9001']) });
|
|
243
|
+
assert.equal(got.task.id, 9002, 'a refused task must not stay at the head of the order');
|
|
244
|
+
});
|
|
245
|
+
|
|
240
246
|
test('a goal whose tasks cannot be read is reported, not silently skipped', async () => {
|
|
241
247
|
const api = fakeApi({ tasks: [], claimable: [] });
|
|
242
248
|
api.tasks.getTasks = async () => { throw new Error('boom'); };
|
|
@@ -273,7 +279,11 @@ function harness(over = {}) {
|
|
|
273
279
|
// The LEDGER, faked. iteration() now reads the task back from Bongos after
|
|
274
280
|
// every worker (task 1003903) instead of believing the worker's own verdict,
|
|
275
281
|
// so a test that does not say what the ledger holds is not describing a run.
|
|
276
|
-
api: {},
|
|
282
|
+
api: over.api || {},
|
|
283
|
+
// Per-test runner memory (task 1004396): refused tasks and a queue gate
|
|
284
|
+
// outlive one iteration, so a shared module-level copy would leak between tests.
|
|
285
|
+
state: over.state || runner.newRunnerState(),
|
|
286
|
+
now: over.now,
|
|
277
287
|
verifyDeps: {
|
|
278
288
|
task: over.ledger !== undefined ? over.ledger : { id: 4242, status: 'shipped', updated_at: new Date().toISOString() },
|
|
279
289
|
probeArtifact: over.probeArtifact || (async () => ({ checked: true, onMain: true, sha: 'a'.repeat(40), subject: 'x (task 4242)' })),
|
|
@@ -422,6 +432,137 @@ test('failureReason: stdout is used when stderr is empty, whitespace is collapse
|
|
|
422
432
|
assert.equal(runner.failureReason({ code: 1, stderr: 'a'.repeat(400) }, 'x').length, 400);
|
|
423
433
|
});
|
|
424
434
|
|
|
435
|
+
// ── task 1004396: a refusal is not a reason to stop, unless it refuses everything ──
|
|
436
|
+
//
|
|
437
|
+
// Owner ruling (2026-09-30): a refused claim must not put the runner into waiting
|
|
438
|
+
// by default. It sets that task aside and tries the next one. It waits only when
|
|
439
|
+
// there is a block AND nothing else it can claim — which is exactly the shape of
|
|
440
|
+
// the queue-wide gate (a stranded confirmed task refuses EVERY claim).
|
|
441
|
+
|
|
442
|
+
const taskA = { ...aTask, id: 4242 };
|
|
443
|
+
const taskB = { ...aTask, id: 4343, title: 'The next thing' };
|
|
444
|
+
// A picker that honours the exclusion set, the way the real pickTask now does.
|
|
445
|
+
const pickAorB = async (_goals, pd = {}) => {
|
|
446
|
+
const ex = pd.exclude || new Set();
|
|
447
|
+
for (const t of [taskA, taskB]) if (!ex.has(String(t.id))) return { task: t, goalId: 1000119, skipped: [] };
|
|
448
|
+
return { none: true, skipped: [] };
|
|
449
|
+
};
|
|
450
|
+
const scriptOf = (args) => String(args[0]).split(/[\\/]/).pop();
|
|
451
|
+
const gatedCard = [
|
|
452
|
+
'Your queue is gated — 1 confirmed task(s) are waiting on your rebase:',
|
|
453
|
+
' - task 777 A stranded thing',
|
|
454
|
+
' flagged: strand:branch_modifies_executed_code',
|
|
455
|
+
].join(String.fromCharCode(10));
|
|
456
|
+
|
|
457
|
+
test('a task refused on its own is set aside and the NEXT task is worked in the same pass', async () => {
|
|
458
|
+
const claims = [];
|
|
459
|
+
const h = harness({
|
|
460
|
+
pickTask: pickAorB,
|
|
461
|
+
ledger: { id: 4343, status: 'shipped', updated_at: new Date().toISOString() },
|
|
462
|
+
run: async (bin, args) => {
|
|
463
|
+
if (scriptOf(args) === 'claim.js') {
|
|
464
|
+
claims.push(args[1]);
|
|
465
|
+
return args[1] === '4242' ? { ok: false, code: 1, stdout: '', stderr: 'DEPS_NOT_SHIPPED' } : { ok: true, code: 0, stdout: '', stderr: '' };
|
|
466
|
+
}
|
|
467
|
+
return { ok: true, code: 0, stdout: '', stderr: '' };
|
|
468
|
+
},
|
|
469
|
+
});
|
|
470
|
+
const row = await runner.iteration(opts, h.deps);
|
|
471
|
+
assert.deepEqual(claims, ['4242', '4343'], 'the refusal must not end the pass');
|
|
472
|
+
assert.equal(row.event, 'worked');
|
|
473
|
+
assert.equal(row.task_id, 4343);
|
|
474
|
+
const refused = h.rows.find((r) => r.event === 'claim_failed');
|
|
475
|
+
assert.equal(refused.task_id, 4242, 'the refusal is still logged, with its reason');
|
|
476
|
+
assert.ok(refused.set_aside_s > 0, 'and says the task is set aside');
|
|
477
|
+
});
|
|
478
|
+
|
|
479
|
+
test('a set-aside task is not re-tried on the next wake, and comes back once the set-aside expires', async () => {
|
|
480
|
+
let t = 1_000_000;
|
|
481
|
+
const state = runner.newRunnerState();
|
|
482
|
+
const claims = [];
|
|
483
|
+
const mk = () => harness({
|
|
484
|
+
state, now: () => t, pickTask: pickAorB,
|
|
485
|
+
run: async (bin, args) => {
|
|
486
|
+
if (scriptOf(args) === 'claim.js') { claims.push(args[1]); return { ok: false, code: 1, stdout: '', stderr: 'nope' }; }
|
|
487
|
+
return { ok: true, code: 0, stdout: '', stderr: '' };
|
|
488
|
+
},
|
|
489
|
+
});
|
|
490
|
+
await runner.iteration(opts, mk().deps);
|
|
491
|
+
assert.deepEqual(claims, ['4242', '4343']);
|
|
492
|
+
const second = await runner.iteration(opts, mk().deps);
|
|
493
|
+
assert.deepEqual(claims, ['4242', '4343'], 'no claim is re-attempted while both are set aside');
|
|
494
|
+
assert.equal(second.event, 'nothing_claimable');
|
|
495
|
+
assert.deepEqual(second.set_aside.map((s) => s.id).sort(), ['4242', '4343']);
|
|
496
|
+
t += runner.CLAIM_SET_ASIDE_MS + 1;
|
|
497
|
+
await runner.iteration(opts, mk().deps);
|
|
498
|
+
assert.deepEqual(claims.slice(2), ['4242', '4343'], 'after the set-aside they are tried again');
|
|
499
|
+
});
|
|
500
|
+
|
|
501
|
+
test('when EVERY candidate is refused, the last failure is returned so a real outage still escalates', async () => {
|
|
502
|
+
const h = harness({
|
|
503
|
+
pickTask: pickAorB,
|
|
504
|
+
run: async (bin, args) => (scriptOf(args) === 'claim.js'
|
|
505
|
+
? { ok: false, code: 1, stdout: '', stderr: 'fetch failed' }
|
|
506
|
+
: { ok: true, code: 0, stdout: '', stderr: '' }),
|
|
507
|
+
});
|
|
508
|
+
const row = await runner.iteration(opts, h.deps);
|
|
509
|
+
assert.equal(row.event, 'claim_failed', 'returning "nothing claimable" here would hide a network outage as idle');
|
|
510
|
+
assert.equal(h.rows.filter((r) => r.event === 'claim_failed').length, 2);
|
|
511
|
+
});
|
|
512
|
+
|
|
513
|
+
test('a queue-wide gate WAITS, names the stranded task, and makes no claim until it lands', async () => {
|
|
514
|
+
const state = runner.newRunnerState();
|
|
515
|
+
let status = 'confirmed';
|
|
516
|
+
let gated = true;
|
|
517
|
+
const claims = [];
|
|
518
|
+
const api = { tasks: { getTasksId: async ({ id }) => ({ ok: true, data: { task: { id, status } } }) } };
|
|
519
|
+
const mk = () => harness({
|
|
520
|
+
state, api, pickTask: pickAorB,
|
|
521
|
+
run: async (bin, args) => {
|
|
522
|
+
if (scriptOf(args) === 'claim.js') {
|
|
523
|
+
claims.push(args[1]);
|
|
524
|
+
return gated ? { ok: false, code: runner.QUEUE_GATED_EXIT, stdout: '', stderr: gatedCard } : { ok: true, code: 0, stdout: '', stderr: '' };
|
|
525
|
+
}
|
|
526
|
+
return { ok: true, code: 0, stdout: '', stderr: '' };
|
|
527
|
+
},
|
|
528
|
+
});
|
|
529
|
+
|
|
530
|
+
const first = await runner.iteration(opts, mk().deps);
|
|
531
|
+
assert.equal(first.event, 'queue_gated');
|
|
532
|
+
assert.deepEqual(first.waiting_on, ['777'], 'the wait names what it is waiting on');
|
|
533
|
+
assert.deepEqual(claims, ['4242'], 'a gate refuses every claim, so trying the next task is pointless');
|
|
534
|
+
|
|
535
|
+
const second = await runner.iteration(opts, mk().deps);
|
|
536
|
+
assert.equal(second.event, 'queue_gated');
|
|
537
|
+
assert.deepEqual(claims, ['4242'], 'no claim is attempted while the strand is still confirmed');
|
|
538
|
+
|
|
539
|
+
status = 'shipped'; gated = false;
|
|
540
|
+
const third = await runner.iteration(opts, mk().deps);
|
|
541
|
+
assert.equal(third.event, 'worked', 'once the strand lands the runner resumes by itself');
|
|
542
|
+
assert.deepEqual(claims, ['4242', '4242']);
|
|
543
|
+
assert.equal(state.queueGate, null, 'the gate is forgotten');
|
|
544
|
+
});
|
|
545
|
+
|
|
546
|
+
test('a gate whose stranded task cannot be read back keeps waiting rather than guessing', async () => {
|
|
547
|
+
const state = runner.newRunnerState();
|
|
548
|
+
state.queueGate = { owed: ['777'], since: Date.now(), reason: 'x' };
|
|
549
|
+
let claimed = false;
|
|
550
|
+
const h = harness({
|
|
551
|
+
state, pickTask: pickAorB,
|
|
552
|
+
api: { tasks: { getTasksId: async () => ({ ok: false, status: 502 }) } },
|
|
553
|
+
run: async (bin, args) => { if (scriptOf(args) === 'claim.js') claimed = true; return { ok: true, code: 0, stdout: '', stderr: '' }; },
|
|
554
|
+
});
|
|
555
|
+
const row = await runner.iteration(opts, h.deps);
|
|
556
|
+
assert.equal(row.event, 'queue_gated');
|
|
557
|
+
assert.equal(claimed, false);
|
|
558
|
+
});
|
|
559
|
+
|
|
560
|
+
test('the runner and claim.js agree on the queue-gated exit code', () => {
|
|
561
|
+
const claim = require('../scripts/gds/claim.js');
|
|
562
|
+
assert.equal(runner.QUEUE_GATED_EXIT, claim.QUEUE_GATED_EXIT);
|
|
563
|
+
assert.notEqual(runner.QUEUE_GATED_EXIT, 1, 'it must differ from the generic refusal, or the runner cannot tell them apart');
|
|
564
|
+
});
|
|
565
|
+
|
|
425
566
|
test('a worker that cannot finish frees the claim; one that really shipped does not', async () => {
|
|
426
567
|
// Which of these happens is now decided by the LEDGER, not by the worker's own
|
|
427
568
|
// verdict: a stuck worker leaves the task still `active`, a finished one leaves
|
|
@@ -97,6 +97,16 @@ test('shipped is the LEDGER’s answer, never the worker’s', () => {
|
|
|
97
97
|
assert.deepEqual(d.onHold.ranButDidNotShip.map((r) => r.shape), ['still_held', 'shipped_no_artifact']);
|
|
98
98
|
});
|
|
99
99
|
|
|
100
|
+
test('a queue-gate wait is a hold that names the stranded task, not an unrecognised event (task 1004396)', () => {
|
|
101
|
+
const d = buildDigest([
|
|
102
|
+
{ at: at(0), event: 'queue_gated', waiting_on: ['777'], reason: 'every claim is refused until task 777 lands — waiting, not claiming (x)' },
|
|
103
|
+
{ at: at(5), event: 'queue_gate_cleared', waited_on: ['777'], waited_s: 300 },
|
|
104
|
+
], { nowEpochS: NOW_S });
|
|
105
|
+
assert.equal(d.runner.unrecognised, 0);
|
|
106
|
+
assert.equal(d.onHold.held.length, 1);
|
|
107
|
+
assert.match(d.onHold.held[0].reason, /task 777/);
|
|
108
|
+
});
|
|
109
|
+
|
|
100
110
|
test('an unworkable queue is reported as a queue problem, not as breakage', () => {
|
|
101
111
|
const d = buildDigest([{ at: at(0), event: 'nothing_claimable', skipped: [{ id: 9, reasons: ['needs_migration'] }] }], { nowEpochS: NOW_S });
|
|
102
112
|
assert.equal(d.onHold.unworkable.length, 1);
|
|
@@ -231,6 +231,13 @@ t('REBASE_REQUIRED: names the owed tasks and relays the server hint', () => {
|
|
|
231
231
|
'must not fall through to the bare code echo');
|
|
232
232
|
});
|
|
233
233
|
|
|
234
|
+
t('REBASE_REQUIRED exits with its OWN code, so an unattended caller can tell a queue gate from a task refusal (task 1004396)', () => {
|
|
235
|
+
const { QUEUE_GATED_EXIT } = require('../scripts/gds/claim.js');
|
|
236
|
+
assert.equal(formatClaimFailure(envelope('REBASE_REQUIRED', {}), 5).exit, QUEUE_GATED_EXIT);
|
|
237
|
+
assert.notEqual(QUEUE_GATED_EXIT, 1);
|
|
238
|
+
assert.equal(formatClaimFailure(envelope('ALREADY_CLAIMED', {}), 5).exit, 1, 'a per-task refusal keeps exit 1');
|
|
239
|
+
});
|
|
240
|
+
|
|
234
241
|
t('REBASE_REQUIRED: still guides when the server sends no hint and no task list', () => {
|
|
235
242
|
const out = text(formatClaimFailure(envelope('REBASE_REQUIRED', {}), 5));
|
|
236
243
|
assert.match(out, /gated/i, 'must say the queue is gated');
|
|
@@ -348,7 +348,12 @@ test('land-watch.js exits 0 after a REAL request — the teardown that aborted 3
|
|
|
348
348
|
// throwaway session, so no real API is touched and the developer's own config is
|
|
349
349
|
// never read or written. `--here` because CI may check out the MAIN checkout,
|
|
350
350
|
// where the worktree guard would refuse at exit 2 before any request is made.
|
|
351
|
-
|
|
351
|
+
// The stub refuses with REBASE_REQUIRED, the queue-wide gate, which exits with its
|
|
352
|
+
// own code since task 1004396 (QUEUE_GATED_EXIT) so an unattended caller can tell a
|
|
353
|
+
// gate from a task refusal. The abort this guards against shows as 127 either way.
|
|
354
|
+
const { QUEUE_GATED_EXIT } = createRequire(import.meta.url)('../scripts/gds/claim.js');
|
|
355
|
+
|
|
356
|
+
test('claim.js exits with its refusal code without aborting on a REFUSED claim — measured 3/3 aborting before', async () => {
|
|
352
357
|
const { proc, port } = await startStub({ claimStatus: 409 });
|
|
353
358
|
try {
|
|
354
359
|
const home = makeHome({ apiBase: `http://127.0.0.1:${port}` });
|
|
@@ -361,7 +366,7 @@ test('claim.js exits 1 without aborting on a REFUSED claim — measured 3/3 abor
|
|
|
361
366
|
});
|
|
362
367
|
assert.notEqual(r.signal, 'SIGTERM', `run ${i + 1} timed out — draining left a handle armed`);
|
|
363
368
|
assert.doesNotMatch(r.stderr || '', ABORT_RE, `run ${i + 1} aborted natively:\n${r.stderr}`);
|
|
364
|
-
assert.equal(r.status,
|
|
369
|
+
assert.equal(r.status, QUEUE_GATED_EXIT, `a queue-gated claim must exit ${QUEUE_GATED_EXIT}, got ${r.status} (127 = the abort); stderr: ${r.stderr}`);
|
|
365
370
|
// End-to-end proof that the new REBASE_REQUIRED rendering reaches a terminal,
|
|
366
371
|
// not just the unit test: the stub refuses with that exact envelope.
|
|
367
372
|
//
|
|
@@ -0,0 +1,191 @@
|
|
|
1
|
+
// tests/move_escalation.mjs — a core move nobody can heal files ONE blocker and ONE notice,
|
|
2
|
+
// and a later move that lands closes it (task 1004449, goal 1000090 deploy self-heal).
|
|
3
|
+
//
|
|
4
|
+
// WHAT IS AT RISK. Two opposite failures: a person never hears about a move that left a
|
|
5
|
+
// project down, or hears about it on every runner tick until they mute the channel. The
|
|
6
|
+
// blockers table here is a fake that enforces the real (source, source_ref) uniqueness, and
|
|
7
|
+
// the drain loop is the real one, so "one per failed move" is measured across retries and
|
|
8
|
+
// re-runs rather than assumed.
|
|
9
|
+
//
|
|
10
|
+
// Run: node tests/move_escalation.mjs
|
|
11
|
+
|
|
12
|
+
import { strict as assert } from 'node:assert';
|
|
13
|
+
import { createRequire } from 'node:module';
|
|
14
|
+
|
|
15
|
+
process.env.PROVISION_APP_USER = process.env.PROVISION_APP_USER || 'lars'; // appUser() refuses a Windows account name
|
|
16
|
+
const require = createRequire(import.meta.url);
|
|
17
|
+
const E = require('../scripts/gds/move-escalation.js');
|
|
18
|
+
const P = require('../scripts/gds/provision.js');
|
|
19
|
+
|
|
20
|
+
let passed = 0;
|
|
21
|
+
let failed = 0;
|
|
22
|
+
const t = async (name, fn) => {
|
|
23
|
+
try { await fn(); passed += 1; console.log(` PASS ${name}`); }
|
|
24
|
+
catch (e) { failed += 1; console.log(` FAIL ${name}\n ${e.message}`); }
|
|
25
|
+
};
|
|
26
|
+
|
|
27
|
+
const inst = (over = {}) => ({ id: 7, slug: 'cloudbongos', db_name: 'cloudbongos', port: 3002, status: 'active', hosting_shape: 'control-plane', owner_builder_id: 1, ...over });
|
|
28
|
+
const unknownFail = { ok: false, error: 'core upgrade to 1.20.31 failed and was rolled back: health check failed after restart', failure: { class: 'unknown', reason: 'health' } };
|
|
29
|
+
|
|
30
|
+
// A blockers table with the real partial unique index on (source, source_ref).
|
|
31
|
+
function blockersTable() {
|
|
32
|
+
const rows = []; let next = 500;
|
|
33
|
+
const query = async (sql, params) => {
|
|
34
|
+
if (/INSERT INTO blockers/.test(sql)) {
|
|
35
|
+
const [title, body_md, source, source_ref] = params;
|
|
36
|
+
if (rows.some((r) => r.source === source && r.source_ref === source_ref)) return { rows: [] };
|
|
37
|
+
const r = { id: next++, title, body_md, source, source_ref, status: 'open' }; rows.push(r); return { rows: [{ id: r.id }] };
|
|
38
|
+
}
|
|
39
|
+
if (/UPDATE blockers/.test(sql)) {
|
|
40
|
+
const [source, like, note] = params; const prefix = like.replace(/%$/, '');
|
|
41
|
+
const hit = rows.filter((r) => r.status === 'open' && r.source === source && r.source_ref.startsWith(prefix));
|
|
42
|
+
for (const r of hit) { r.status = 'resolved'; r.resolution_note = note; }
|
|
43
|
+
return { rows: hit.map((r) => ({ id: r.id })) };
|
|
44
|
+
}
|
|
45
|
+
return { rows: [] };
|
|
46
|
+
};
|
|
47
|
+
return { rows, query };
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
console.log('\nwho needs a person (needsPerson):');
|
|
51
|
+
|
|
52
|
+
await t('unknown and rollback_failed escalate; every other class does not', () => {
|
|
53
|
+
const i = { action: 'core-upgrade' };
|
|
54
|
+
assert.equal(E.needsPerson({ intent: i, failure: { class: 'unknown', reason: 'health' } }), true);
|
|
55
|
+
assert.equal(E.needsPerson({ intent: i, failure: { class: 'unknown', reason: 'rollback_failed' }, terminal: true }), true);
|
|
56
|
+
assert.equal(E.needsPerson({ intent: i, failure: { class: 'unknown', reason: 'remedy_did_not_hold' } }), true, 'a fix that did not hold (task 1004448)');
|
|
57
|
+
for (const c of ['transient', 'known_wedge', 'needs_decision']) assert.equal(E.needsPerson({ intent: i, failure: { class: c, reason: 'x' } }), false, c);
|
|
58
|
+
assert.equal(E.needsPerson({ intent: { action: 'restart' }, failure: { class: 'unknown', reason: 'x' } }), false, 'core moves only');
|
|
59
|
+
});
|
|
60
|
+
|
|
61
|
+
console.log('\nwhat is said (blockerFor, noticeText):');
|
|
62
|
+
|
|
63
|
+
await t('the blocker names project, target, reason and what the move reported', () => {
|
|
64
|
+
const b = E.blockerFor({ inst: inst(), intent: { id: 11, target_version: '1.20.31' }, failure: unknownFail.failure, error: unknownFail.error });
|
|
65
|
+
assert.equal(b.source, 'provisioning');
|
|
66
|
+
assert.equal(b.sourceRef, 'core-move:7:11');
|
|
67
|
+
assert.match(b.title, /cloudbongos: the core move to 1\.20\.31 failed/);
|
|
68
|
+
assert.match(b.bodyMd, /`health`/);
|
|
69
|
+
assert.match(b.bodyMd, /health check failed after restart/);
|
|
70
|
+
assert.match(b.bodyMd, /closes by itself/);
|
|
71
|
+
});
|
|
72
|
+
|
|
73
|
+
await t('a failed rollback says the project may be down', () => {
|
|
74
|
+
const b = E.blockerFor({ inst: inst(), intent: { id: 11, target_version: '1.20.31' }, failure: { class: 'unknown', reason: 'rollback_failed' }, error: 'x' });
|
|
75
|
+
assert.match(b.title, /may be down/);
|
|
76
|
+
assert.match(E.noticeText({ kind: 'failed', name: 'x', down: true }), /may be down/);
|
|
77
|
+
});
|
|
78
|
+
|
|
79
|
+
await t('reported output cannot break out of its code block', () => {
|
|
80
|
+
const b = E.blockerFor({ inst: inst(), intent: { id: 1, target_version: '1' }, failure: unknownFail.failure, error: 'a```\n# injected' });
|
|
81
|
+
assert.equal((b.bodyMd.match(/```/g) || []).length, 2);
|
|
82
|
+
});
|
|
83
|
+
|
|
84
|
+
await t('Discord names only a PUBLIC project, and never carries the upgrade output', () => {
|
|
85
|
+
assert.match(E.noticeText({ kind: 'failed', name: null, to: '1.20.31' }), /^⚠️ A private project/);
|
|
86
|
+
assert.match(E.noticeText({ kind: 'failed', name: 'mercury', to: '1.20.31' }), /\*\*mercury\*\*/);
|
|
87
|
+
assert.doesNotMatch(E.noticeText({ kind: 'failed', name: 'm', to: '1' }), /health/);
|
|
88
|
+
});
|
|
89
|
+
|
|
90
|
+
console.log('\nonce per failed move, closed by success (afterRetire / afterSuccess):');
|
|
91
|
+
|
|
92
|
+
function deps(table, { visibility = 'private' } = {}) {
|
|
93
|
+
const posts = [];
|
|
94
|
+
return { posts, d: { apply: true, log: () => {}, db: { query: table.query }, postNotice: (text) => posts.push(text), provisioning: { effectiveSettings: () => ({ visibility }) } } };
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
await t('a retired unknown move files one blocker and one notice', async () => {
|
|
98
|
+
const table = blockersTable(); const { posts, d } = deps(table);
|
|
99
|
+
const r = await E.afterRetire({ intent: { id: 11, action: 'core-upgrade', target_version: '1.20.31' }, inst: inst(), result: unknownFail, deps: d });
|
|
100
|
+
assert.equal(r.filed, true);
|
|
101
|
+
assert.equal(table.rows.length, 1);
|
|
102
|
+
assert.equal(posts.length, 1);
|
|
103
|
+
});
|
|
104
|
+
|
|
105
|
+
await t('the same move retiring again files nothing and posts nothing', async () => {
|
|
106
|
+
const table = blockersTable(); const { posts, d } = deps(table);
|
|
107
|
+
const i = { id: 11, action: 'core-upgrade', target_version: '1.20.31' };
|
|
108
|
+
await E.afterRetire({ intent: i, inst: inst(), result: unknownFail, deps: d });
|
|
109
|
+
const again = await E.afterRetire({ intent: i, inst: inst(), result: unknownFail, deps: d });
|
|
110
|
+
assert.equal(again.filed, false);
|
|
111
|
+
assert.equal(table.rows.length, 1);
|
|
112
|
+
assert.equal(posts.length, 1, 'edge-triggered: the notice follows the insert');
|
|
113
|
+
});
|
|
114
|
+
|
|
115
|
+
await t('a later move that lands closes this project\'s blockers — and only this project\'s', async () => {
|
|
116
|
+
const table = blockersTable(); const { posts, d } = deps(table, { visibility: 'public' });
|
|
117
|
+
await E.afterRetire({ intent: { id: 11, action: 'core-upgrade', target_version: '1.20.31' }, inst: inst(), result: unknownFail, deps: d });
|
|
118
|
+
await E.afterRetire({ intent: { id: 12, action: 'core-upgrade', target_version: '1.20.31' }, inst: inst({ id: 70, slug: 'other' }), result: unknownFail, deps: d });
|
|
119
|
+
const r = await E.afterSuccess({ intent: { id: 13, action: 'core-upgrade', target_version: '1.20.32' }, inst: inst(), result: { ok: true, served: '1.20.32' }, deps: d });
|
|
120
|
+
assert.equal(r.closed, 1);
|
|
121
|
+
assert.deepEqual(table.rows.map((x) => x.status), ['resolved', 'open'], 'instance 70 is not instance 7 — a LIKE prefix must not over-match');
|
|
122
|
+
assert.match(table.rows[0].resolution_note, /later core move to 1\.20\.32 succeeded \(intent 13\)/);
|
|
123
|
+
assert.match(posts[posts.length - 1], /^✅ \*\*cloudbongos\*\* took its core update to 1\.20\.32/);
|
|
124
|
+
});
|
|
125
|
+
|
|
126
|
+
await t('a no-op or a failed result closes nothing; nothing open posts no all-clear', async () => {
|
|
127
|
+
const table = blockersTable(); const { posts, d } = deps(table);
|
|
128
|
+
await E.afterRetire({ intent: { id: 11, action: 'core-upgrade', target_version: '1' }, inst: inst(), result: unknownFail, deps: d });
|
|
129
|
+
assert.equal((await E.afterSuccess({ intent: { id: 13, action: 'core-upgrade' }, inst: inst(), result: { ok: true, noop: true }, deps: d })).closed, 0);
|
|
130
|
+
assert.equal((await E.afterSuccess({ intent: { id: 13, action: 'core-upgrade' }, inst: inst(), result: { ok: false }, deps: d })).closed, 0);
|
|
131
|
+
const empty = blockersTable(); const e2 = deps(empty);
|
|
132
|
+
await E.afterSuccess({ intent: { id: 13, action: 'core-upgrade' }, inst: inst(), result: { ok: true, served: '1' }, deps: e2.d });
|
|
133
|
+
assert.equal(e2.posts.length, 0);
|
|
134
|
+
});
|
|
135
|
+
|
|
136
|
+
await t('a database error never throws into the drain', async () => {
|
|
137
|
+
const { d } = deps({ query: async () => { throw new Error('boom'); } });
|
|
138
|
+
assert.deepEqual(await E.afterRetire({ intent: { id: 1, action: 'core-upgrade' }, inst: inst(), result: unknownFail, deps: d }), { filed: false });
|
|
139
|
+
assert.deepEqual(await E.afterSuccess({ intent: { id: 1, action: 'core-upgrade' }, inst: inst(), result: { ok: true, served: '1' }, deps: d }), { closed: 0 });
|
|
140
|
+
});
|
|
141
|
+
|
|
142
|
+
console.log('\nthe drain loop, end to end:');
|
|
143
|
+
|
|
144
|
+
// One move that fails `health` (rolled back, unknown), then — a later, separate move — lands.
|
|
145
|
+
async function drain({ table, row, upgradeFails }) {
|
|
146
|
+
const posts = []; const resolved = [];
|
|
147
|
+
const deps = {
|
|
148
|
+
apply: true, privileged: false, log: () => {}, sleep: async () => {},
|
|
149
|
+
postNotice: (text) => posts.push(text),
|
|
150
|
+
db: { query: async (sql, params) => {
|
|
151
|
+
if (/blockers/.test(sql)) return table.query(sql, params);
|
|
152
|
+
if (/SET state='pending'/.test(sql)) { row.state = 'pending'; row.not_before = new Date(Date.now() + params[params.length - 1]); }
|
|
153
|
+
return { rows: [] };
|
|
154
|
+
} },
|
|
155
|
+
provisioning: {
|
|
156
|
+
claimNextIntent: async () => { if (row.state !== 'pending' || (row.not_before && row.not_before > new Date())) return null; row.state = 'running'; row.attempts += 1; return { ...row }; },
|
|
157
|
+
getInstanceById: async () => inst(),
|
|
158
|
+
resolveIntent: async (_db, id, state) => { resolved.push(state); row.state = state; },
|
|
159
|
+
setInstanceStatus: async () => {}, recordEvent: async () => {}, clearErrorNote: async () => {},
|
|
160
|
+
effectiveSettings: () => ({ visibility: 'private' }),
|
|
161
|
+
claimNextOAuthManifest: async () => null, resolveOAuthManifest: async () => {},
|
|
162
|
+
},
|
|
163
|
+
exec: (cmd) => {
|
|
164
|
+
if (/upgrade\.js/.test(cmd) && upgradeFails) { const e = new Error('Command failed'); e.stderr = '✖ health check failed after restart\nBONGOS_UPGRADE_OUTCOME {"stage":"rolled_back","reason":"health"}\n'; throw e; }
|
|
165
|
+
return { ok: true, stdout: /\/version/.test(cmd) ? JSON.stringify({ coreVersion: upgradeFails ? '1.20.30' : row.target_version, schemaPending: 0 }) : '' };
|
|
166
|
+
},
|
|
167
|
+
writeFile: () => ({ ok: true }),
|
|
168
|
+
readInstalledCoreVersion: () => '1.20.30', selfInstanceRoot: () => '/srv/cb',
|
|
169
|
+
updateChannel: { DEFAULT_CHANNEL: 'patch', normalizeChannel: (c) => c || 'patch', channelAllows: () => true, listAvailableVersions: () => ({ ok: true, versions: ['1.20.30', '1.20.31', '1.20.32'] }) },
|
|
170
|
+
checkDnsTokenScope: async () => ({ reachable: true, zone: 'z' }), checkCaddyWiring: () => null,
|
|
171
|
+
};
|
|
172
|
+
await P.cmdRunIntents(deps);
|
|
173
|
+
return { posts, resolved };
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
await t('a move that fails unknown files ONE blocker; a later move that lands closes it', async () => {
|
|
177
|
+
const table = blockersTable();
|
|
178
|
+
const first = await drain({ table, upgradeFails: true, row: { id: 11, instance_id: 7, action: 'core-upgrade', target_version: '1.20.31', attempts: 0, state: 'pending' } });
|
|
179
|
+
assert.deepEqual(first.resolved, ['error'], 'retired at once — unknown is not retried');
|
|
180
|
+
assert.equal(table.rows.length, 1);
|
|
181
|
+
assert.equal(table.rows[0].source_ref, 'core-move:7:11');
|
|
182
|
+
assert.equal(first.posts.length, 1);
|
|
183
|
+
const later = await drain({ table, upgradeFails: false, row: { id: 12, instance_id: 7, action: 'core-upgrade', target_version: '1.20.32', attempts: 0, state: 'pending' } });
|
|
184
|
+
assert.deepEqual(later.resolved, ['done']);
|
|
185
|
+
assert.equal(table.rows[0].status, 'resolved');
|
|
186
|
+
assert.equal(later.posts.length, 1);
|
|
187
|
+
assert.match(later.posts[0], /^✅ A private project took its core update to 1\.20\.32/);
|
|
188
|
+
});
|
|
189
|
+
|
|
190
|
+
console.log(`\nmove_escalation: ${passed} passed, ${failed} failed`);
|
|
191
|
+
if (failed) process.exit(1);
|
package/tests/plain_cards.mjs
CHANGED
|
@@ -18,7 +18,7 @@ import * as shipLand from '../scripts/gds/ship-land.js';
|
|
|
18
18
|
|
|
19
19
|
const require = createRequire(import.meta.url);
|
|
20
20
|
const { JARGON_TERMS } = require('../modules/sessions/accessibility.js');
|
|
21
|
-
const { formatClaimFailure } = require('../scripts/gds/claim.js');
|
|
21
|
+
const { formatClaimFailure, QUEUE_GATED_EXIT } = require('../scripts/gds/claim.js');
|
|
22
22
|
const { formatGradeOutcome } = require('../scripts/gds/ship-grade-steps.js');
|
|
23
23
|
|
|
24
24
|
let passed = 0;
|
|
@@ -130,7 +130,7 @@ test('a card interpolates nothing — it is static text, so it adds no injection
|
|
|
130
130
|
test('a refused claim prints the engineer text first and the card after it', () => {
|
|
131
131
|
const r = { status: 409, data: { error: { code: 'REBASE_REQUIRED', message: 'x', details: { rebaseTasks: [{ id: 7, reason: 'strand:pr_not_landed' }] } } } };
|
|
132
132
|
const { lines, exit } = formatClaimFailure(r, 42);
|
|
133
|
-
assert.equal(exit,
|
|
133
|
+
assert.equal(exit, QUEUE_GATED_EXIT, 'the queue-wide gate keeps its own exit code (task 1004396)');
|
|
134
134
|
assert.match(lines[0], /queue is gated/);
|
|
135
135
|
const at = lines.findIndex((l) => l.includes(PLAIN_MARKER));
|
|
136
136
|
assert.ok(at > 0, 'card follows the existing guidance');
|