@bongos/core 1.19.955 → 1.19.957
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.bongos-core.json +24 -14
- package/docs/adr/0325-the-managed-fleet-control-plane-is-the-hub-we-already-run.md +2 -2
- package/docs/adr/README.md +1 -1
- package/docs/module-api-changelog.md +4 -0
- package/modules/provisioning/pollers/liveness-sweep.js +41 -1
- package/modules/provisioning/pollers/runner-drift-notify.js +203 -0
- package/package-lock.json +2 -2
- package/package.json +1 -1
- package/src/module-api.js +1 -1
- package/tests/provisioning_runner_drift_notice.mjs +292 -0
package/.bongos-core.json
CHANGED
|
@@ -2,22 +2,22 @@
|
|
|
2
2
|
"artifact": "bongos-core",
|
|
3
3
|
"manifest_schema": 1,
|
|
4
4
|
"generator": "scripts/gds/package-core.js",
|
|
5
|
-
"core_version": "1.19.
|
|
6
|
-
"core_contract": "1.19.
|
|
7
|
-
"source_commit": "
|
|
5
|
+
"core_version": "1.19.957",
|
|
6
|
+
"core_contract": "1.19.957",
|
|
7
|
+
"source_commit": "48e752cc136cf78939c24f100a08934f3516ed57",
|
|
8
8
|
"source_ref": "HEAD",
|
|
9
|
-
"built_at": "2026-09-22T05:
|
|
9
|
+
"built_at": "2026-09-22T05:59:48.593Z",
|
|
10
10
|
"redaction": {
|
|
11
11
|
"model": "docs-redacted+functional-verbatim",
|
|
12
12
|
"docs_redacted": 514,
|
|
13
13
|
"agent_docs_stubbed": 26,
|
|
14
|
-
"functional_verbatim":
|
|
14
|
+
"functional_verbatim": 2357,
|
|
15
15
|
"rules": 3,
|
|
16
16
|
"gate_literals": 3,
|
|
17
17
|
"gate": "passed"
|
|
18
18
|
},
|
|
19
|
-
"file_count":
|
|
20
|
-
"tree_sha256": "
|
|
19
|
+
"file_count": 2897,
|
|
20
|
+
"tree_sha256": "3689a1fe6b51b1bde5cd80dc5e42e2111685ccd0ebc8f39c411ac17f4e1ff040",
|
|
21
21
|
"files": [
|
|
22
22
|
{
|
|
23
23
|
"path": ".claude/skills/ask-for-help/SKILL.md",
|
|
@@ -2117,12 +2117,12 @@
|
|
|
2117
2117
|
{
|
|
2118
2118
|
"path": "docs/adr/0325-the-managed-fleet-control-plane-is-the-hub-we-already-run.md",
|
|
2119
2119
|
"mode": "0000644",
|
|
2120
|
-
"sha256": "
|
|
2120
|
+
"sha256": "6ae902e7626feb915300f4b21b6d20ae3369116a86bf624bd614ac9616c3ac11"
|
|
2121
2121
|
},
|
|
2122
2122
|
{
|
|
2123
2123
|
"path": "docs/adr/README.md",
|
|
2124
2124
|
"mode": "0000644",
|
|
2125
|
-
"sha256": "
|
|
2125
|
+
"sha256": "5db407e3565216666055365b11a5ff05b8901ff9c7fbaee011c93784c40762e6"
|
|
2126
2126
|
},
|
|
2127
2127
|
{
|
|
2128
2128
|
"path": "docs/api-reference.md",
|
|
@@ -2692,7 +2692,7 @@
|
|
|
2692
2692
|
{
|
|
2693
2693
|
"path": "docs/module-api-changelog.md",
|
|
2694
2694
|
"mode": "0000644",
|
|
2695
|
-
"sha256": "
|
|
2695
|
+
"sha256": "d24452f51a99862ee117d68ebed98474ef8599cbbad464ad4ea07f19e760f528"
|
|
2696
2696
|
},
|
|
2697
2697
|
{
|
|
2698
2698
|
"path": "docs/modules-contract.md",
|
|
@@ -6702,7 +6702,12 @@
|
|
|
6702
6702
|
{
|
|
6703
6703
|
"path": "modules/provisioning/pollers/liveness-sweep.js",
|
|
6704
6704
|
"mode": "0000644",
|
|
6705
|
-
"sha256": "
|
|
6705
|
+
"sha256": "b7db2ecae7e3df06646e04f5203a6ac8bbb61b301ccde833d2024d9d2ba64394"
|
|
6706
|
+
},
|
|
6707
|
+
{
|
|
6708
|
+
"path": "modules/provisioning/pollers/runner-drift-notify.js",
|
|
6709
|
+
"mode": "0000644",
|
|
6710
|
+
"sha256": "8d861e41d66c9c14995db3921ab0b225d3abdafceda5edc0029a4020d47fe3b4"
|
|
6706
6711
|
},
|
|
6707
6712
|
{
|
|
6708
6713
|
"path": "modules/provisioning/provisioning.js",
|
|
@@ -8072,12 +8077,12 @@
|
|
|
8072
8077
|
{
|
|
8073
8078
|
"path": "package-lock.json",
|
|
8074
8079
|
"mode": "0000644",
|
|
8075
|
-
"sha256": "
|
|
8080
|
+
"sha256": "e6926dbbee5927840a3de03f03dff14b05a04ecdc2dd974941f93cca760c745e"
|
|
8076
8081
|
},
|
|
8077
8082
|
{
|
|
8078
8083
|
"path": "package.json",
|
|
8079
8084
|
"mode": "0000644",
|
|
8080
|
-
"sha256": "
|
|
8085
|
+
"sha256": "4f9672ddf0e6238c689b8884f85fdb3a5c85174b934d0ab57c79d8bd49acab6a"
|
|
8081
8086
|
},
|
|
8082
8087
|
{
|
|
8083
8088
|
"path": "public-docs/index.html",
|
|
@@ -10082,7 +10087,7 @@
|
|
|
10082
10087
|
{
|
|
10083
10088
|
"path": "src/module-api.js",
|
|
10084
10089
|
"mode": "0000644",
|
|
10085
|
-
"sha256": "
|
|
10090
|
+
"sha256": "fdab385d81b456d0cf1197085a21cac0e3e57b6d89a8a7c890d10248a16f25bf"
|
|
10086
10091
|
},
|
|
10087
10092
|
{
|
|
10088
10093
|
"path": "src/module-loader/catalog.js",
|
|
@@ -13249,6 +13254,11 @@
|
|
|
13249
13254
|
"mode": "0000644",
|
|
13250
13255
|
"sha256": "f31a8fc12866f7d639b7472578422e29129db85bbd9ddecb663e753393d74e2a"
|
|
13251
13256
|
},
|
|
13257
|
+
{
|
|
13258
|
+
"path": "tests/provisioning_runner_drift_notice.mjs",
|
|
13259
|
+
"mode": "0000644",
|
|
13260
|
+
"sha256": "18662f6f446b33a677ffc71dd7115f8e59997d6f86adbadcd91e5e4bdac2fd76"
|
|
13261
|
+
},
|
|
13252
13262
|
{
|
|
13253
13263
|
"path": "tests/provisioning_settings.mjs",
|
|
13254
13264
|
"mode": "0000644",
|
|
@@ -7,7 +7,7 @@
|
|
|
7
7
|
- **Supersedes in part:** [ADR 0111](0111-instance-hosting-provisioning-module.md) §5/§7 (the "managed layer is greenfield, deferred, spec-first" sequencing)
|
|
8
8
|
- **Depends on:** [ADR 0323](0323-hosting-is-three-shapes-and-we-are-not-the-landlord.md) (the shapes and the platform this operates)
|
|
9
9
|
- **Gates:** [task 1004142](https://cloudbongos.com/builders#/task/1004142) (the platform leg) · [task 1001949](https://cloudbongos.com/builders#/task/1001949) (metering) · [task 1001621](https://cloudbongos.com/builders#/task/1001621) (the spend rollup)
|
|
10
|
-
- **Filed by this ADR:** [task
|
|
10
|
+
- **Filed by this ADR:** [task 1004148](https://cloudbongos.com/builders#/task/1004148) (the $0 cost ledger — decision 6) · [task 1004150](https://cloudbongos.com/builders#/task/1004150) (the subscription roster — decision 5)
|
|
11
11
|
|
|
12
12
|
## Context
|
|
13
13
|
|
|
@@ -63,7 +63,7 @@ This ADR records the direction and files it as [task 1004150](https://cloudbongo
|
|
|
63
63
|
|
|
64
64
|
`costEstimateUsd` returns **0** for `standalone` — correct when a project was a process on a box we already paid for, and wrong the moment it becomes a ~$14/month Render service. Until it is fixed the fleet cost ledger reports \$0 for every project the platform is about to be billed for, and it reports it with no error.
|
|
65
65
|
|
|
66
|
-
That is a defect, not a deferral, and it is filed as [task
|
|
66
|
+
That is a defect, not a deferral, and it is filed as [task 1004148](https://cloudbongos.com/builders#/task/1004148). The metering and invoicing track ([task 1001949](https://cloudbongos.com/builders#/task/1001949)) and the spend rollup ([task 1001621](https://cloudbongos.com/builders#/task/1001621)) stay separate work, as [ADR 0323](0323-hosting-is-three-shapes-and-we-are-not-the-landlord.md) already says.
|
|
67
67
|
|
|
68
68
|
## Consequences
|
|
69
69
|
|
package/docs/adr/README.md
CHANGED
|
@@ -416,4 +416,4 @@ This keeps the decision history honest and traceable.
|
|
|
416
416
|
| 0322 | [**The five states are derived once, and a remnant needs evidence** ([task 1004103](https://cloudbongos.com/builders#/task/1004103), goal 1000095 — BONGOS-V2). The ideator surface draws a thought as one of five objects — star, protostar, satellite, remnant, cold body — and three surfaces are about to draw them (the room, the descent, the sky). Every fact already existed: `idea_inbox.status` (`core_195`), [ADR 0321](0321-brightness-is-a-weighted-share-and-the-targets-are-a-row.md)’s `hasWork` / `share` / `attributable`, and `tasks.goal_id` + `tasks.status`. What was missing was ONE place that reads them: a state inferred in three renderers is three states, and the sky can say a thought went out while the room beside it still calls it alive, with nothing erroring. **Decision:** one pure resolver (`modules/ideas/idea-state.js`), brightness exposed on the `reward` port (ideas may not reach `credit_log` any other way — [ADR 0083](0083-modular-architecture-strangler-build-plan.md) rule 1 — and already declares `reward` in `consumes`), and the ORDER of the tests IS the decision. **MERGED wins over everything:** a folded thought keeps its own money (`trail-size.js` is emphatic) but is no longer its own point in the sky, so letting work win draws one idea twice, beside the thought that absorbed it. **WORK beats a DISCARD:** a cold body “never ignited, so it has no size at all” is a stated PROPERTY, and testing work first makes it true by construction — triage really can discard a duplicate whose twin already routed and paid, and `updateIdea` never touches the ledger. **A REMNANT needs EVIDENCE, never the absence of an answer:** `share: 0` is a confirmed removal, `share: null` is an unreadable one, and ADR 0321 pays for that distinction upstream — so an idea with work and no readable facts is a STAR carrying `attributable: false`, because reading a falsy share as death turns one unmounted module into a sky of hollow rings. **A PROTOSTAR is the absence of WORK, not of a status:** ignition is being PAID for something, so a promoted-but-unshipped idea is still unignited, which is exactly what the nod queue exists for. **Position is a second axis** (`well` / `open-space` / `awaitingNod`) because the design says the object alone cannot carry it, and the nod is asked of EVERY task, not only the one that named the well. **`suggested_goal_id` is deliberately NOT a well** — `projection.js` records it as strictly advisory, the routing hint on a filing that did not convert, so reading it would put a thought in a nod queue with no task to promote; the binding `tasks.goal_id` now rides on both of the trail’s task joins at no extra round trip. The page reads the ledger ONCE: the state attach hands economy back the `byTask` breakdown size just computed, all of them or none (a partial map reads a missing entry as “no work” and draws a paid star as a protostar). An absent `reward` port does not empty the sky — states still resolve, flagged unattributable. Nothing writes; ADR 0313 stands. Rejected: resolving in each renderer; deriving from `share` alone (it merges a protostar with a remnant, the exact BV2.R19 distinction); treating an unreadable share as zero; status-first ordering; reading the hint as a home; a sixth object for “unknown brightness” (the vocabulary is the owner’s and is five — the uncertainty is a flag on the answer, not a new thing in the sky).](0322-a-remnant-needs-evidence-and-merged-wins.md) | ideas / economy |
|
|
417
417
|
| 0323 | [**Hosting is three shapes, and we are not the landlord** ([task 1004140](https://cloudbongos.com/builders#/task/1004140), goal 1000106 — BONGOS-V2). Five projects ran as one Node process each on the control-plane droplet, about $0.80/month apiece — the cheapest arrangement available, bought with ~4,600 lines of `provision*.js` that write systemd units, create unix accounts, render Caddy snippets, `createdb`, install backup timers and count memory against a ceiling. That is the cost nobody priced: **we are a landlord.** The owner reframed the question — build on what others maintain, and let an owner keep the deployment platform they already use. **Seven decisions.** The shapes become `byo-host` (the owner runs it), `cloud-host` (we stand it up for them) and `dedicated-cloud-host` (the same path, larger plan); `co-tenant` is **retired, not renamed**, because it means *a process on our box from our checkout* — the opposite of bring-your-own-host, so reusing the name would make every existing comment lie. **DigitalOcean hosts hubs only**, which closes [ADR 0145](0145-free-hosted-project-tier-isolation-and-domain-separation.md) Finding 1 by removing its precondition rather than hardening the cookie: with no customer code on a `cloudbongos.com` subdomain there is no tenant to receive the hub session. **Render is the default** — its Postgres carries PITR and automated backups on every paid tier; Fly's managed Postgres starts ~$38/mo and Railway's is explicitly unmanaged, which is the exact responsibility being shed. **Cloudflare is the front door, not the compute**: Workers is a V8-isolate runtime that would require rewriting Express and `pg`, and Containers is built for bursty workloads that sleep, which a hall with background sweeps and a live event stream does not survive. **No deployment-platform adapter is built at all** — their application already deploys itself from the repository, so the platform we do not talk to is the platform we do not have to add. **Secrets: the platform is the system of record and Bongos stores names, never values** — a manifest of name, purpose and environment, no application secret, no platform API token, no "is it set?" probe, because each of those makes us custodian of other companies' credentials for convenience the manifest already delivers; `byo-host` marks the manifest unverifiable rather than opening an exception. **The free tier becomes a capped trial**, since ~$14/project cannot be subsidised indefinitely. Per-project cost rises ~17×, which is the price of not being a landlord and the reason metering stops being deferred work. Mostly a DELETION: [ADR 0306](0306-instance-supervision-is-a-seam-with-systemd-as-the-default.md)'s supervision seam is not extended (Render supervises its own processes), [ADR 0285](0285-a-shared-box-holds-about-twelve-projects-per-gb-and-memory-is-the-wall.md) stays true and stops being load-bearing with its queued resize held rather than spent, and the hub's offsite backup becomes the highest-consequence open item. Rejected: AWS/GCP (2–3× the cost for a bare instance, plus an IAM and billing surface to learn — maintenance added where the goal is to remove it); keeping the shared box and fixing only the cookie (cheaper, closes the finding, leaves the landlord runner and answers nothing else); a Vercel/Netlify/Render adapter layer (speculative pluggability); Bongos as the secrets vault pushing values through platform APIs.](0323-hosting-is-three-shapes-and-we-are-not-the-landlord.md) | hosting / provisioning / cost |
|
|
418
418
|
| 0324 | [**The public npm CLI channel is retired** ([task 1004144](https://cloudbongos.com/builders#/task/1004144), goal 1000091 — BONGOS-V2). The owner deleted `@cloudbongos/cli` from the registry on 2026-09-22 (`npm view` → E404; the `@cloudbongos` scope is empty). This supersedes [ADR 0258](0258-the-public-cli-is-a-generated-client-package-not-the-published-core.md) and [ADR 0292](0292-both-install-paths-stay-and-vendoring-is-the-production-one.md) §2 — the latter titled "`@cloudbongos/cli` STAYS, on a new justification" and written *because* task 1003872 proposed removing it once the public core flip dissolved its original reason. 0292's measured argument (~0.5 MB front door vs a 46 MB core pull to print a task list) is still true and is NOT what changed; the owner's decision about carrying a public channel at all is. `scripts/gds/build-cli-package.js` + `tests/cli_package.mjs` are deleted, no doc offers `npx @cloudbongos/cli`, and the way in is the hall at `<origin>/builders` or a checkout. The COMPILED binary channel (`build-bongos.js` → `/downloads/bongos/*` via `config/bongos-cli.json`) is a different artifact and is untouched — the same near-twin task 1003889 warned about. Three tests used the package's FILES manifest as a stand-in for "what a terminal user can run" and now read `bin/bongos.js`'s verb table, which is what that set always meant.](0324-the-public-npm-cli-channel-is-retired.md) | distribution / CLI |
|
|
419
|
-
| 0325 | [**The managed fleet control plane is the hub we already run** ([task 1001948](https://cloudbongos.com/builders#/task/1001948), goal 1000106 — BONGOS-V2). The spike was written against [ADR 0111](0111-instance-hosting-provisioning-module.md) §5/§7, which deferred the managed offering as **"genuinely greenfield (no fleet registry exists)"** behind two gates — Mercury and a real `bongos upgrade`. Both gates are shipped and **the premise is not**: measuring the call sites first changed the answer, as it did for [task 1004141](https://cloudbongos.com/builders#/task/1004141). The registry is `provisioning_instances` (owner, shape, domain, database, port, lifecycle, cost estimate, module selection, the liveness overlay and the served/available core-version snapshot), with `provisioning_intents` as the queue and `provisioning_events` as the audit trail; the operator is `scripts/gds/provision.js`, the single token-holder draining that queue behind ADR 0111 §2's trust boundary; oversight is `GET /provisioning/fleet` and `/fleet/cost-ledger`; and the update channel is not one mechanism but two shipped ones — the owner's per-instance deploy door ([ADR 0293](0293-the-owners-deploy-door-is-an-intent-the-control-plane-drains.md), health-gated with auto-rollback) and the opt-in patch-only sweep ([ADR 0136](0136-update-channel-subscription-policy.md)). **Decision: there is no greenfield fleet control plane to design, and none will be built — the hub IS it.** It never needed to sit *above* the per-repo databases [ADR 0100](0100-two-repo-instance-model.md) §3 locks: the roster is a hub table and the operator has never read one. The genuinely missing leg is a **platform leg** — `grep -rl RENDER_API scripts modules src` returns nothing, and after [ADR 0323](0323-hosting-is-three-shapes-and-we-are-not-the-landlord.md) a `cloud-host` is a Render service — and this ADR **deliberately specifies none of it**, because that is [task 1004142](https://cloudbongos.com/builders#/task/1004142)'s with those files open and a second spec for one implementation is how two drift. The fleet **driven** and the fleet **observed** are different populations: `byo-host` is in the registry and operated by nobody, its liveness the [ADR 0252](0252-the-self-hosted-visibility-report-is-a-pull-and-auto-appear-buys-a-retraction-lever.md) pull, so every operator surface must state which it counts — a roster that mixes them reports a fleet-wide upgrade as complete while an unknown number were never asked. The operator holds **exactly one** credential and it is **its own** (the platform account token, root-only on the control plane, never the web tier, never per-customer); ADR 0323 D6's "names, never values" rule is about the CUSTOMER's secrets and is unchanged. Two gaps are filed rather than fixed here: the subscription roster living in `config/update-subscriptions.json` is the one fleet fact the roster route cannot report and a restored backup can lose independently of the rows it describes ([task 1004150](https://cloudbongos.com/builders#/task/1004150), a recommendation and not a ruling), and **`costEstimateUsd` returns 0 for every shape but `dedicated`** — measured live, all 24 fleet rows report `$0`, confidently and with no error, for projects about to cost ~\$14/month each ([task
|
|
419
|
+
| 0325 | [**The managed fleet control plane is the hub we already run** ([task 1001948](https://cloudbongos.com/builders#/task/1001948), goal 1000106 — BONGOS-V2). The spike was written against [ADR 0111](0111-instance-hosting-provisioning-module.md) §5/§7, which deferred the managed offering as **"genuinely greenfield (no fleet registry exists)"** behind two gates — Mercury and a real `bongos upgrade`. Both gates are shipped and **the premise is not**: measuring the call sites first changed the answer, as it did for [task 1004141](https://cloudbongos.com/builders#/task/1004141). The registry is `provisioning_instances` (owner, shape, domain, database, port, lifecycle, cost estimate, module selection, the liveness overlay and the served/available core-version snapshot), with `provisioning_intents` as the queue and `provisioning_events` as the audit trail; the operator is `scripts/gds/provision.js`, the single token-holder draining that queue behind ADR 0111 §2's trust boundary; oversight is `GET /provisioning/fleet` and `/fleet/cost-ledger`; and the update channel is not one mechanism but two shipped ones — the owner's per-instance deploy door ([ADR 0293](0293-the-owners-deploy-door-is-an-intent-the-control-plane-drains.md), health-gated with auto-rollback) and the opt-in patch-only sweep ([ADR 0136](0136-update-channel-subscription-policy.md)). **Decision: there is no greenfield fleet control plane to design, and none will be built — the hub IS it.** It never needed to sit *above* the per-repo databases [ADR 0100](0100-two-repo-instance-model.md) §3 locks: the roster is a hub table and the operator has never read one. The genuinely missing leg is a **platform leg** — `grep -rl RENDER_API scripts modules src` returns nothing, and after [ADR 0323](0323-hosting-is-three-shapes-and-we-are-not-the-landlord.md) a `cloud-host` is a Render service — and this ADR **deliberately specifies none of it**, because that is [task 1004142](https://cloudbongos.com/builders#/task/1004142)'s with those files open and a second spec for one implementation is how two drift. The fleet **driven** and the fleet **observed** are different populations: `byo-host` is in the registry and operated by nobody, its liveness the [ADR 0252](0252-the-self-hosted-visibility-report-is-a-pull-and-auto-appear-buys-a-retraction-lever.md) pull, so every operator surface must state which it counts — a roster that mixes them reports a fleet-wide upgrade as complete while an unknown number were never asked. The operator holds **exactly one** credential and it is **its own** (the platform account token, root-only on the control plane, never the web tier, never per-customer); ADR 0323 D6's "names, never values" rule is about the CUSTOMER's secrets and is unchanged. Two gaps are filed rather than fixed here: the subscription roster living in `config/update-subscriptions.json` is the one fleet fact the roster route cannot report and a restored backup can lose independently of the rows it describes ([task 1004150](https://cloudbongos.com/builders#/task/1004150), a recommendation and not a ruling), and **`costEstimateUsd` returns 0 for every shape but `dedicated`** — measured live, all 24 fleet rows report `$0`, confidently and with no error, for projects about to cost ~\$14/month each ([task 1004148](https://cloudbongos.com/builders#/task/1004148)). **No fleet-wide "upgrade everything" button**: both channels are per-instance by construction, and one action that moves the whole fleet is the shape most likely to take every customer down at once. AGPL posture unchanged — ADR 0111 §5's one-sentence rule survives its own table. Rejected: building the separate control plane §5 described (a second registry and a second operator holding the same rows with nothing keeping them equal); specifying Render here; reading per-instance databases for fleet reporting; and continuing to defer the whole layer, which is now the reason the ledger reads \$0 for a fleet about to cost real money.](0325-the-managed-fleet-control-plane-is-the-hub-we-already-run.md) | hosting / fleet / provisioning |
|
|
@@ -2384,5 +2384,9 @@ is load-bearing: the script throws rather than guess if it is missing, and
|
|
|
2384
2384
|
landed since 1.19.953 with no explicit bump. run 35687914716. (task 1002620)
|
|
2385
2385
|
1.19.955 — CI auto-patch (publish-on-merge, ADR 0161): carrier for merges
|
|
2386
2386
|
landed since 1.19.954 with no explicit bump. run 35690089430. (task 1002620)
|
|
2387
|
+
1.19.956 — CI auto-patch (publish-on-merge, ADR 0161): carrier for merges
|
|
2388
|
+
landed since 1.19.955 with no explicit bump. run 35691908420. (task 1002620)
|
|
2389
|
+
1.19.957 — CI auto-patch (publish-on-merge, ADR 0161): carrier for merges
|
|
2390
|
+
landed since 1.19.956 with no explicit bump. run 35692861666. (task 1002620)
|
|
2387
2391
|
---------------------------------------------------------------------------
|
|
2388
2392
|
```
|
|
@@ -28,12 +28,24 @@ const { Agent: UndiciAgent } = require('undici');
|
|
|
28
28
|
const api = require('../../../src/module-api');
|
|
29
29
|
const provisioning = require('../provisioning');
|
|
30
30
|
const { planNotices, postNotice } = require('./liveness-notify');
|
|
31
|
+
// task 1004000: the runner-drift check rides this tick rather than installing a timer of
|
|
32
|
+
// its own — the sweep already runs every ~5 min on the one host that can see the systemd
|
|
33
|
+
// units. It reads the checker as a SUBPROCESS, not a require: `scripts/gds/runner-drift.js`
|
|
34
|
+
// is core-package territory (ADR 0083) and its systemctl reads are blocking, so requiring
|
|
35
|
+
// it would have put N×15s of event-loop work in the process serving live traffic. See
|
|
36
|
+
// ./runner-drift-notify.js's header.
|
|
37
|
+
const { planDriftNotices, formatDriftLine, readRunnerDrift } = require('./runner-drift-notify');
|
|
31
38
|
|
|
32
39
|
// Notice state between sweeps (task 1002894): id → { streak, notified }. In memory
|
|
33
40
|
// on purpose — it is seeded only by a fall this process observed, which is exactly
|
|
34
41
|
// what stops a restarted control plane re-announcing an outage it arrived to find.
|
|
35
42
|
const noticeState = new Map();
|
|
36
|
-
|
|
43
|
+
// Unit names already announced as drifting (task 1004000). In memory like its neighbour,
|
|
44
|
+
// but for the opposite reason: this one exists only to stop a stale runner posting every
|
|
45
|
+
// five minutes forever, NOT to suppress a first sighting — a drift that predates this
|
|
46
|
+
// process is exactly the thing nobody noticed, so a restart re-announcing it is correct.
|
|
47
|
+
const driftState = new Set();
|
|
48
|
+
function resetNoticeState() { noticeState.clear(); driftState.clear(); }
|
|
37
49
|
|
|
38
50
|
// Name a project in an owner-facing message only when it is public on the platform's
|
|
39
51
|
// map; a private or stealth one is described, never identified.
|
|
@@ -241,6 +253,34 @@ async function sweepOnce(deps = {}) {
|
|
|
241
253
|
for (const notice of plan.notices) {
|
|
242
254
|
try { notify(notice); } catch (e) { log(`[liveness] notice failed (non-blocking): ${e && e.message ? e.message : e}`); }
|
|
243
255
|
}
|
|
256
|
+
|
|
257
|
+
// ── runner drift (task 1004000) ───────────────────────────────────────────
|
|
258
|
+
// Placed at the TAIL, after the account is built and the liveness notices are out, so
|
|
259
|
+
// that a systemctl hiccup can never cost this tick its liveness verdict — the reason
|
|
260
|
+
// the whole block is wrapped. `collect` is injected so the composition stays hermetic
|
|
261
|
+
// in tests; production uses readRunnerDrift, which spawns the checker and answers null
|
|
262
|
+
// when there is no systemd to ask. AWAITED, and that is the point: the read is a
|
|
263
|
+
// subprocess rather than a blocking in-process require, so a hung systemctl costs this
|
|
264
|
+
// tick its drift check and never the event loop serving live requests. Not gated on
|
|
265
|
+
// `rows.length`: the runners are worth checking on a control plane with no tenant
|
|
266
|
+
// projects at all.
|
|
267
|
+
try {
|
|
268
|
+
const collect = deps.collectRunnerDrift || readRunnerDrift;
|
|
269
|
+
const instanceRoot = deps.instanceRoot || process.env.BONGOS_INSTANCE_ROOT || process.cwd();
|
|
270
|
+
const drift = planDriftNotices(await collect({ instanceRoot }), driftState);
|
|
271
|
+
const line = formatDriftLine(drift);
|
|
272
|
+
if (line) log(line);
|
|
273
|
+
for (const notice of drift.notices) {
|
|
274
|
+
try { notify(notice); } catch (e) { log(`[liveness] drift notice failed (non-blocking): ${e && e.message ? e.message : e}`); }
|
|
275
|
+
}
|
|
276
|
+
out.runnerDrift = drift.drifting;
|
|
277
|
+
out.runnerUnresolved = drift.unresolved;
|
|
278
|
+
} catch (e) {
|
|
279
|
+
// A drift check that throws must not take the sweep with it. Logged rather than
|
|
280
|
+
// swallowed: silence here would be indistinguishable from a healthy host, which is
|
|
281
|
+
// the exact failure this task was filed about.
|
|
282
|
+
log(`[drift] check failed (non-blocking): ${e && e.message ? e.message : e}`);
|
|
283
|
+
}
|
|
244
284
|
return out;
|
|
245
285
|
}
|
|
246
286
|
|
|
@@ -0,0 +1,203 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
// modules/provisioning/pollers/runner-drift-notify.js — say it out loud when an
|
|
4
|
+
// intent-runner unit is pointed at a different core than the instance installed
|
|
5
|
+
// (task 1004000, the follow-up the task 1002722 grade asked for).
|
|
6
|
+
//
|
|
7
|
+
// WHY. Three drift signals already existed and every one of them needed a human to go
|
|
8
|
+
// and look: `scripts/gds/runner-drift.js` exits 1, `bongos doctor` has an "Intent
|
|
9
|
+
// runners" section, and each run-intents tick writes a `⚠ DRIFT:` banner into the
|
|
10
|
+
// journal. A stale runner is exactly the condition nobody is looking for — it drains
|
|
11
|
+
// intents perfectly happily using last month's code — so a signal that waits to be read
|
|
12
|
+
// is a signal that is not doing its job.
|
|
13
|
+
//
|
|
14
|
+
// WHERE IT RUNS. Inside the liveness sweep, not on a timer of its own. The sweep already
|
|
15
|
+
// ticks every ~5 minutes on the control plane, which is the one host that can see the
|
|
16
|
+
// systemd units (CI cannot), and `collectRunnerDrift` is two `systemctl` reads. A second
|
|
17
|
+
// timer would be a second thing to install, a second thing to forget to install, and a
|
|
18
|
+
// second thing whose silence means nothing.
|
|
19
|
+
//
|
|
20
|
+
// HOW IT DIFFERS FROM ITS SIBLING, AND WHY. `./liveness-notify.js` is the pattern here,
|
|
21
|
+
// but two of its three rules are deliberately NOT copied:
|
|
22
|
+
//
|
|
23
|
+
// * IT ANNOUNCES ON FIRST SIGHT. Liveness stays silent about a project that was already
|
|
24
|
+
// down before this process started, so that a restarted control plane does not
|
|
25
|
+
// re-announce an outage it merely arrived to find. Drift is the opposite case: a
|
|
26
|
+
// runner that has been stale for a fortnight is precisely the thing nobody noticed,
|
|
27
|
+
// and this task exists because those three signals went unread. Waiting for an edge
|
|
28
|
+
// would mean the only drift ever announced is drift that begins while the process
|
|
29
|
+
// happens to be watching.
|
|
30
|
+
// * THERE IS NO CONFIRM STREAK. Liveness waits out a second sweep because one refused
|
|
31
|
+
// connection is a blip. Drift is a deterministic read of two files on local disk —
|
|
32
|
+
// there is no blip to wait out, and a delay would buy nothing but a later notice.
|
|
33
|
+
//
|
|
34
|
+
// The third rule IS copied, and matters more here than there:
|
|
35
|
+
//
|
|
36
|
+
// * COLLAPSE A FAN-OUT. Every unit drifting at once is one cause — an upgrade moved the
|
|
37
|
+
// installed core and the units still run the old checkout — so it is ONE message. A
|
|
38
|
+
// host with several runners would otherwise emit a burst that reads like several
|
|
39
|
+
// independent faults. Unconditional here rather than threshold-gated; see the note
|
|
40
|
+
// where liveness's FLEET_GUARD would have been.
|
|
41
|
+
//
|
|
42
|
+
// And one rule of its own:
|
|
43
|
+
//
|
|
44
|
+
// * `unresolved` IS NOT DRIFT. It means the checker could not tell (an ExecStart that
|
|
45
|
+
// runs no .js directly, no core version above the script, no installed core). That is
|
|
46
|
+
// the absence of a verdict, not a bad one, so announcing it would be a siren over
|
|
47
|
+
// nothing. It is counted into the sweep's log line instead, where it is visible
|
|
48
|
+
// without being urgent — and a host that is ALL unresolved says so in that line
|
|
49
|
+
// rather than reading as healthy.
|
|
50
|
+
//
|
|
51
|
+
// HOW IT READS SYSTEMD, AND WHY NOT BY require(). The checker itself is
|
|
52
|
+
// `scripts/gds/runner-drift.js`, which is CORE-package territory: a module may require
|
|
53
|
+
// only the published doorway `src/module-api.js` (ADR 0083, `docs/modules-contract.md`,
|
|
54
|
+
// and this module's own CLAUDE.md header). So this reads it the way its sibling reads
|
|
55
|
+
// `scripts/discord/post.js` — as a SUBPROCESS, `node scripts/gds/runner-drift.js --json`
|
|
56
|
+
// — which crosses no require boundary and needs no new doorway for a fact only the
|
|
57
|
+
// control plane's own host can answer.
|
|
58
|
+
//
|
|
59
|
+
// The subprocess is ALSO the reason this is safe inside the web process. `collectRunnerDrift`
|
|
60
|
+
// uses `spawnSync` with a 15-second timeout per `systemctl` call; requiring it would have
|
|
61
|
+
// put N×15s of blocking work on the event loop that serves live HTTP traffic, once every
|
|
62
|
+
// five minutes. `execFile` is async, so a hung systemctl costs this tick its drift check
|
|
63
|
+
// and nothing else.
|
|
64
|
+
//
|
|
65
|
+
// **A NON-ZERO EXIT IS NOT AN ERROR HERE.** `runner-drift.js` exits 1 precisely WHEN it
|
|
66
|
+
// finds drift — that is its contract as a CI-style gate — so treating a failed exit as a
|
|
67
|
+
// failed read would discard the only result worth reporting. The JSON on stdout is the
|
|
68
|
+
// answer; the exit code is ignored when it parses, and only an unparseable stdout counts
|
|
69
|
+
// as "could not look".
|
|
70
|
+
//
|
|
71
|
+
// PURE except `readRunnerDrift`, which spawns that subprocess, and `postNotice`, which
|
|
72
|
+
// ./liveness-sweep.js imports straight from ./liveness-notify.js — one Discord path, one
|
|
73
|
+
// dry-run switch, one place a webhook change lands.
|
|
74
|
+
|
|
75
|
+
const { execFile } = require('node:child_process');
|
|
76
|
+
const path = require('node:path');
|
|
77
|
+
|
|
78
|
+
// scripts/gds/runner-drift.js is invoked from the repo root regardless of cwd, the same
|
|
79
|
+
// way ./liveness-notify.js invokes scripts/discord/post.js.
|
|
80
|
+
const REPO_ROOT = path.resolve(__dirname, '..', '..', '..');
|
|
81
|
+
// Generous next to the checker's own per-call 15s, and it exists so a hung systemctl
|
|
82
|
+
// costs this tick its drift check rather than the tick.
|
|
83
|
+
const READ_TIMEOUT_MS = Number(process.env.PROVISIONING_DRIFT_TIMEOUT_MS) || 30_000;
|
|
84
|
+
|
|
85
|
+
// Ask the checker what the host's runner units look like. Resolves to the units array, or
|
|
86
|
+
// NULL for "could not look" — no systemd, an unparseable answer, a timeout, a missing
|
|
87
|
+
// script. Null is deliberately not an empty array: "no runner units" and "could not ask"
|
|
88
|
+
// must not read the same, and planDriftNotices treats them differently.
|
|
89
|
+
function readRunnerDrift({ instanceRoot, execFileImpl = execFile, timeoutMs = READ_TIMEOUT_MS } = {}) {
|
|
90
|
+
return new Promise((resolve) => {
|
|
91
|
+
let settled = false;
|
|
92
|
+
const done = (v) => { if (!settled) { settled = true; resolve(v); } };
|
|
93
|
+
try {
|
|
94
|
+
execFileImpl('node',
|
|
95
|
+
['scripts/gds/runner-drift.js', '--instance-root', String(instanceRoot || ''), '--json'],
|
|
96
|
+
{ cwd: REPO_ROOT, timeout: timeoutMs, maxBuffer: 4 * 1024 * 1024 },
|
|
97
|
+
(_err, stdout) => {
|
|
98
|
+
// _err is IGNORED on purpose: exit 1 IS the drift signal (see the header).
|
|
99
|
+
let parsed = null;
|
|
100
|
+
try { parsed = JSON.parse(String(stdout || '')); } catch (_) { parsed = null; }
|
|
101
|
+
if (!parsed || parsed.systemd !== true) return done(null);
|
|
102
|
+
done(Array.isArray(parsed.units) ? parsed.units : null);
|
|
103
|
+
});
|
|
104
|
+
} catch (_) {
|
|
105
|
+
done(null);
|
|
106
|
+
}
|
|
107
|
+
});
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
// NO FLEET-GUARD KNOB, and the absence is deliberate. ./liveness-notify.js needs one
|
|
111
|
+
// because it posts per PROJECT and a fleet can be large, so "how many at once means look
|
|
112
|
+
// at the platform" is a real threshold. Here the collapse is unconditional: a host runs
|
|
113
|
+
// two or three runner units, every drifting one has the same cause (an upgrade moved the
|
|
114
|
+
// installed core), and the notice is one message whose wording already branches on the
|
|
115
|
+
// count. A tunable that can only ever be 1 or 2 on a population of 2 is a knob that does
|
|
116
|
+
// nothing, and a knob that does nothing still has to be read and reasoned about.
|
|
117
|
+
|
|
118
|
+
// What the owner reads. Owner-audience and plain: the reader is being told their
|
|
119
|
+
// automation is quietly running old code, and what that means for them. A unit name is
|
|
120
|
+
// operator vocabulary, so it appears once, at the end, for whoever goes to fix it. PURE.
|
|
121
|
+
function driftNoticeText({ kind, units = [], installed = null } = {}) {
|
|
122
|
+
const n = units.length;
|
|
123
|
+
if (kind === 'clear') {
|
|
124
|
+
return n > 1
|
|
125
|
+
? `✅ All ${n} background runners are back on the current version.`
|
|
126
|
+
: `✅ The background runner is back on the current version.`;
|
|
127
|
+
}
|
|
128
|
+
const version = installed ? ` The project installed **${installed}**.` : '';
|
|
129
|
+
const which = units.length ? ` (${units.join(', ')})` : '';
|
|
130
|
+
if (n > 1) {
|
|
131
|
+
return `⚠️ ${n} background runners are running an older version of the platform than this project has installed.${version} When that many disagree at once it is almost always one upgrade that moved the code without restarting them, so they are still doing their work with last month's rules — worth pointing them at the new version${which}.`;
|
|
132
|
+
}
|
|
133
|
+
return `⚠️ A background runner is running an older version of the platform than this project has installed.${version} It will keep doing its work, but with the older rules, which is the kind of thing that goes unnoticed for weeks — worth pointing it at the new version${which}.`;
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
// The decision, given this tick's assessment (the rows `collectRunnerDrift` returns) and
|
|
137
|
+
// the notice state carried between ticks. MUTATES `state` (a Set of unit names already
|
|
138
|
+
// announced) and returns what to post; everything else is pure, so the rules above are
|
|
139
|
+
// unit-tested with no systemd, no sweep and no network.
|
|
140
|
+
//
|
|
141
|
+
// `units === null` means there is no systemd to ask — not a healthy host, and not a
|
|
142
|
+
// drifted one. It returns nothing and touches no state, so a checker that cannot run
|
|
143
|
+
// never clears an announcement it also cannot re-make.
|
|
144
|
+
function planDriftNotices(units, state) {
|
|
145
|
+
if (units === null || units === undefined) {
|
|
146
|
+
return { notices: [], drifting: 0, unresolved: 0, checked: 0, systemd: false };
|
|
147
|
+
}
|
|
148
|
+
const drifting = units.filter((u) => u && u.status === 'drift');
|
|
149
|
+
const unresolved = units.filter((u) => u && u.status === 'unresolved');
|
|
150
|
+
const names = drifting.map((u) => u.name);
|
|
151
|
+
|
|
152
|
+
// NEW drift only: a unit already announced stays silent until it recovers, so a stale
|
|
153
|
+
// runner does not post every five minutes for a fortnight.
|
|
154
|
+
const fresh = names.filter((name) => !state.has(name));
|
|
155
|
+
// Recovery: a unit we announced that is no longer drifting. Read from the units this
|
|
156
|
+
// tick actually assessed — a unit that has been REMOVED from the host is not a
|
|
157
|
+
// recovery and is dropped from the state without a notice, because "the unit is gone"
|
|
158
|
+
// and "the unit is fixed" are different facts and only one of them is good news.
|
|
159
|
+
const assessed = new Set(units.map((u) => u && u.name).filter(Boolean));
|
|
160
|
+
const stillDrifting = new Set(names);
|
|
161
|
+
const recovered = [...state].filter((name) => assessed.has(name) && !stillDrifting.has(name));
|
|
162
|
+
const vanished = [...state].filter((name) => !assessed.has(name));
|
|
163
|
+
|
|
164
|
+
const notices = [];
|
|
165
|
+
if (fresh.length) {
|
|
166
|
+
// The installed version is the same for every unit on the host, so the first one
|
|
167
|
+
// carries it for the message.
|
|
168
|
+
const installed = (drifting.find((u) => u.instanceVersion) || {}).instanceVersion || null;
|
|
169
|
+
notices.push({ kind: 'drift', units: fresh, text: driftNoticeText({ kind: 'drift', units: fresh, installed }) });
|
|
170
|
+
}
|
|
171
|
+
if (recovered.length) {
|
|
172
|
+
notices.push({ kind: 'clear', units: recovered, text: driftNoticeText({ kind: 'clear', units: recovered }) });
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
for (const name of fresh) state.add(name);
|
|
176
|
+
for (const name of recovered) state.delete(name);
|
|
177
|
+
for (const name of vanished) state.delete(name);
|
|
178
|
+
|
|
179
|
+
return {
|
|
180
|
+
notices,
|
|
181
|
+
drifting: drifting.length,
|
|
182
|
+
unresolved: unresolved.length,
|
|
183
|
+
checked: units.length,
|
|
184
|
+
systemd: true,
|
|
185
|
+
};
|
|
186
|
+
}
|
|
187
|
+
|
|
188
|
+
// The sweep's one-line account of what the drift check saw. Returns null when there is
|
|
189
|
+
// nothing worth a line — no systemd, or no runner units at all — so a vanilla control
|
|
190
|
+
// plane does not narrate an empty check every five minutes. A host whose every unit is
|
|
191
|
+
// `unresolved` DOES get a line: that is not health, and it must not read as health. PURE.
|
|
192
|
+
function formatDriftLine({ systemd, checked, drifting, unresolved }) {
|
|
193
|
+
if (!systemd || !checked) return null;
|
|
194
|
+
if (!drifting && !unresolved) return null;
|
|
195
|
+
const parts = [`runners=${checked}`];
|
|
196
|
+
if (drifting) parts.push(`drift=${drifting}`);
|
|
197
|
+
if (unresolved) parts.push(`unresolved=${unresolved} (cannot tell — not a verdict)`);
|
|
198
|
+
return `[drift] ${parts.join(' · ')}`;
|
|
199
|
+
}
|
|
200
|
+
|
|
201
|
+
// postNotice is NOT re-exported: ./liveness-sweep.js imports it straight from
|
|
202
|
+
// ./liveness-notify.js, so a re-export here would be API surface with no caller.
|
|
203
|
+
module.exports = { planDriftNotices, driftNoticeText, formatDriftLine, readRunnerDrift };
|
package/package-lock.json
CHANGED
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@bongos/core",
|
|
3
|
-
"version": "1.19.
|
|
3
|
+
"version": "1.19.957",
|
|
4
4
|
"lockfileVersion": 3,
|
|
5
5
|
"requires": true,
|
|
6
6
|
"packages": {
|
|
7
7
|
"": {
|
|
8
8
|
"name": "@bongos/core",
|
|
9
|
-
"version": "1.19.
|
|
9
|
+
"version": "1.19.957",
|
|
10
10
|
"license": "AGPL-3.0-or-later",
|
|
11
11
|
"dependencies": {
|
|
12
12
|
"express": "^4.21.2",
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@bongos/core",
|
|
3
|
-
"version": "1.19.
|
|
3
|
+
"version": "1.19.957",
|
|
4
4
|
"description": "Cloud Bongos — the AI-first build platform core (GDS + platform surfaces + module system), installed as a versioned dependency (ADR 0108).",
|
|
5
5
|
"license": "AGPL-3.0-or-later",
|
|
6
6
|
"main": "src/platform-server.js",
|
package/src/module-api.js
CHANGED
|
@@ -71,7 +71,7 @@ const { responsibilityFor, ROLE_RESPONSIBILITIES } = require('./role-responsibil
|
|
|
71
71
|
// there. scripts/gds/bump-version.js still rewrites the literal below; it appends
|
|
72
72
|
// the entry to that file. Look for a version's history there, not here.
|
|
73
73
|
// ---------------------------------------------------------------------------
|
|
74
|
-
const CORE_VERSION = '1.19.
|
|
74
|
+
const CORE_VERSION = '1.19.957'; // CI auto-patch carrier (ADR 0161); changelog: docs/module-api-changelog.md
|
|
75
75
|
|
|
76
76
|
// A namespaced logger so a module's log lines are attributable + consistent.
|
|
77
77
|
// Usage: const log = api.logger('dev-box'); log.info('mounted');
|
|
@@ -0,0 +1,292 @@
|
|
|
1
|
+
// tests/provisioning_runner_drift_notice.mjs
|
|
2
|
+
//
|
|
3
|
+
// An intent-runner unit pointed at a different core than the instance installed now says
|
|
4
|
+
// so, once, without anybody looking (task 1004000 — the follow-up the task 1002722 grade
|
|
5
|
+
// asked for).
|
|
6
|
+
//
|
|
7
|
+
// THE DONE-WHEN IS TWO SENTENCES AND BOTH ARE HERE: "a unit repointed at a different core
|
|
8
|
+
// than the installed one produces one notice within one tick, and a healthy host produces
|
|
9
|
+
// none." The first is asserted through the real sweep tick, with a fake `systemctl`
|
|
10
|
+
// reader, so what is proved is the wiring and not only the planner. The second is
|
|
11
|
+
// asserted the same way — silence is a result, and the only way to know it is not the
|
|
12
|
+
// silence of a check that never ran is to prove the tick DID run and chose to say
|
|
13
|
+
// nothing.
|
|
14
|
+
//
|
|
15
|
+
// Also pinned, because each is a way this kind of alert goes wrong:
|
|
16
|
+
// • ONE notice, not one every five minutes — the state suppresses a repeat
|
|
17
|
+
// • a FIRST SIGHT is announced, unlike liveness's deliberate silence about an outage
|
|
18
|
+
// that predates the process (the reason is in runner-drift-notify.js's header, and a
|
|
19
|
+
// test that got this backwards would look just as green)
|
|
20
|
+
// • `unresolved` is NOT drift — it is the absence of a verdict, and announcing it would
|
|
21
|
+
// be a siren over nothing; it reaches the log line instead
|
|
22
|
+
// • no systemd → nothing said and no state touched, so a checker that cannot run never
|
|
23
|
+
// clears an announcement it also cannot re-make
|
|
24
|
+
// • a unit that VANISHES is not a recovery — "it is gone" and "it is fixed" are
|
|
25
|
+
// different facts and only one is good news
|
|
26
|
+
// • a drift check that throws does not take the liveness sweep with it
|
|
27
|
+
//
|
|
28
|
+
// DB-free and network-free: the systemctl reader and the notifier are injected.
|
|
29
|
+
//
|
|
30
|
+
// Run: node tests/provisioning_runner_drift_notice.mjs
|
|
31
|
+
|
|
32
|
+
import { strict as assert } from 'node:assert';
|
|
33
|
+
import { createRequire } from 'node:module';
|
|
34
|
+
import { readFileSync } from 'node:fs';
|
|
35
|
+
|
|
36
|
+
const require = createRequire(import.meta.url);
|
|
37
|
+
const notify = require('../modules/provisioning/pollers/runner-drift-notify.js');
|
|
38
|
+
const sweep = require('../modules/provisioning/pollers/liveness-sweep.js');
|
|
39
|
+
|
|
40
|
+
let passed = 0, failed = 0;
|
|
41
|
+
function t(name, fn) {
|
|
42
|
+
try { fn(); passed++; console.log(` PASS ${name}`); }
|
|
43
|
+
catch (e) { failed++; console.log(` FAIL ${name}\n ${e.message}`); }
|
|
44
|
+
}
|
|
45
|
+
async function ta(name, fn) {
|
|
46
|
+
try { await fn(); passed++; console.log(` PASS ${name}`); }
|
|
47
|
+
catch (e) { failed++; console.log(` FAIL ${name}\n ${e.message}`); }
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
const DRIFTED = { name: 'bongos-intent-runner.service', status: 'drift', runnerVersion: '1.19.900', instanceVersion: '1.19.953', reason: 'runs core 1.19.900 but the instance installed 1.19.953' };
|
|
51
|
+
const HEALTHY = { name: 'bongos-intent-runner.service', status: 'ok', runnerVersion: '1.19.953', instanceVersion: '1.19.953', reason: 'runs core 1.19.953' };
|
|
52
|
+
const UNRESOLVED = { name: 'bongos-intent-runner.service', status: 'unresolved', runnerVersion: null, instanceVersion: '1.19.953', reason: 'ExecStart does not run a .js script directly' };
|
|
53
|
+
|
|
54
|
+
console.log('the planner:');
|
|
55
|
+
|
|
56
|
+
t('a drifted unit is announced on FIRST SIGHT, and the message names the installed version', () => {
|
|
57
|
+
const state = new Set();
|
|
58
|
+
const r = notify.planDriftNotices([DRIFTED], state);
|
|
59
|
+
assert.equal(r.notices.length, 1);
|
|
60
|
+
assert.equal(r.notices[0].kind, 'drift');
|
|
61
|
+
assert.match(r.notices[0].text, /older version/);
|
|
62
|
+
assert.match(r.notices[0].text, /1\.19\.953/, 'the installed version is in the message — it is what the runner should be on');
|
|
63
|
+
assert.match(r.notices[0].text, /bongos-intent-runner\.service/, 'and the unit name, once, for whoever fixes it');
|
|
64
|
+
assert.equal(r.drifting, 1);
|
|
65
|
+
// The opposite of liveness's rule, deliberately: drift that predates this process is
|
|
66
|
+
// exactly what nobody noticed, so a restart re-announcing it is correct.
|
|
67
|
+
assert.deepEqual([...state], ['bongos-intent-runner.service']);
|
|
68
|
+
});
|
|
69
|
+
|
|
70
|
+
t('the SECOND tick says nothing — one notice, not one every five minutes', () => {
|
|
71
|
+
const state = new Set();
|
|
72
|
+
notify.planDriftNotices([DRIFTED], state);
|
|
73
|
+
const again = notify.planDriftNotices([DRIFTED], state);
|
|
74
|
+
assert.deepEqual(again.notices, []);
|
|
75
|
+
assert.equal(again.drifting, 1, 'still counted in the account, just not re-announced');
|
|
76
|
+
});
|
|
77
|
+
|
|
78
|
+
t('a recovery is announced once, and then the unit is forgotten', () => {
|
|
79
|
+
const state = new Set();
|
|
80
|
+
notify.planDriftNotices([DRIFTED], state);
|
|
81
|
+
const fixed = notify.planDriftNotices([HEALTHY], state);
|
|
82
|
+
assert.equal(fixed.notices.length, 1);
|
|
83
|
+
assert.equal(fixed.notices[0].kind, 'clear');
|
|
84
|
+
assert.match(fixed.notices[0].text, /back on the current version/);
|
|
85
|
+
assert.deepEqual([...state], [], 'forgotten, so a later drift is a fresh first sight');
|
|
86
|
+
assert.deepEqual(notify.planDriftNotices([HEALTHY], state).notices, [], 'and the all-clear is not repeated either');
|
|
87
|
+
});
|
|
88
|
+
|
|
89
|
+
t('a unit that VANISHES is not a recovery', () => {
|
|
90
|
+
// "The unit is gone" and "the unit is fixed" are different facts and only one of them
|
|
91
|
+
// is good news. The state is still cleared, so a reinstated unit is a fresh first sight.
|
|
92
|
+
const state = new Set();
|
|
93
|
+
notify.planDriftNotices([DRIFTED], state);
|
|
94
|
+
const gone = notify.planDriftNotices([], state);
|
|
95
|
+
assert.deepEqual(gone.notices, [], 'no all-clear for a unit nobody fixed');
|
|
96
|
+
assert.deepEqual([...state], []);
|
|
97
|
+
});
|
|
98
|
+
|
|
99
|
+
t('several units drifting at once are ONE message, because they have one cause', () => {
|
|
100
|
+
const state = new Set();
|
|
101
|
+
const units = [DRIFTED, { ...DRIFTED, name: 'bongos-box-intent-runner.service' }];
|
|
102
|
+
const r = notify.planDriftNotices(units, state);
|
|
103
|
+
assert.equal(r.notices.length, 1, 'one message, not one per unit');
|
|
104
|
+
assert.match(r.notices[0].text, /2 background runners/);
|
|
105
|
+
assert.match(r.notices[0].text, /one upgrade that moved the code/, 'and it names the usual cause');
|
|
106
|
+
assert.equal(r.notices[0].units.length, 2);
|
|
107
|
+
});
|
|
108
|
+
|
|
109
|
+
t("'unresolved' is NOT drift — no notice, but it does reach the log line", () => {
|
|
110
|
+
const state = new Set();
|
|
111
|
+
const r = notify.planDriftNotices([UNRESOLVED], state);
|
|
112
|
+
assert.deepEqual(r.notices, [], 'the absence of a verdict is not a bad verdict');
|
|
113
|
+
assert.equal(r.unresolved, 1);
|
|
114
|
+
assert.equal(r.drifting, 0);
|
|
115
|
+
assert.deepEqual([...state], []);
|
|
116
|
+
// Visible without being urgent — and a host that is ALL unresolved must not read as
|
|
117
|
+
// healthy, which is why the line exists at all.
|
|
118
|
+
assert.match(notify.formatDriftLine(r), /unresolved=1 \(cannot tell — not a verdict\)/);
|
|
119
|
+
});
|
|
120
|
+
|
|
121
|
+
t('no systemd says nothing, touches no state, and reports systemd:false', () => {
|
|
122
|
+
const state = new Set(['bongos-intent-runner.service']);
|
|
123
|
+
for (const nothing of [null, undefined]) {
|
|
124
|
+
const r = notify.planDriftNotices(nothing, state);
|
|
125
|
+
assert.deepEqual(r.notices, []);
|
|
126
|
+
assert.equal(r.systemd, false);
|
|
127
|
+
assert.deepEqual([...state], ['bongos-intent-runner.service'],
|
|
128
|
+
'a checker that cannot run must not clear an announcement it also cannot re-make');
|
|
129
|
+
}
|
|
130
|
+
});
|
|
131
|
+
|
|
132
|
+
t('a healthy host produces no log line either — no narration every five minutes', () => {
|
|
133
|
+
const r = notify.planDriftNotices([HEALTHY], new Set());
|
|
134
|
+
assert.deepEqual(r.notices, []);
|
|
135
|
+
assert.equal(notify.formatDriftLine(r), null, 'a vanilla control plane stays quiet');
|
|
136
|
+
assert.equal(notify.formatDriftLine({ systemd: true, checked: 0, drifting: 0, unresolved: 0 }), null, 'and so does a host with no runner units');
|
|
137
|
+
assert.match(notify.formatDriftLine({ systemd: true, checked: 2, drifting: 1, unresolved: 0 }), /runners=2 · drift=1/);
|
|
138
|
+
});
|
|
139
|
+
|
|
140
|
+
console.log('\nthrough the real sweep tick:');
|
|
141
|
+
|
|
142
|
+
// The sweep's own dependency seam, with no projects to probe: this tick is about the
|
|
143
|
+
// runners. `collectRunnerDrift` is what the production path reads systemd through.
|
|
144
|
+
function tickDeps({ units, notifySpy, logs }) {
|
|
145
|
+
return {
|
|
146
|
+
// `pool`, not `db` — sweepOnce's own seam name. Passing the wrong one silently falls
|
|
147
|
+
// back to the real api.pool and the tick tries to reach Postgres, which is how these
|
|
148
|
+
// five cases first failed.
|
|
149
|
+
pool: { query: async () => ({ rows: [] }) },
|
|
150
|
+
log: (m) => logs.push(String(m)),
|
|
151
|
+
notify: (n) => notifySpy.push(n),
|
|
152
|
+
collectRunnerDrift: async () => units,
|
|
153
|
+
instanceRoot: '/srv/instance',
|
|
154
|
+
};
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
// No active instances to probe: this tick is about the runners, and the roster read is
|
|
158
|
+
// the only other thing sweepOnce does before reaching them.
|
|
159
|
+
const provisioning = require('../modules/provisioning/provisioning.js');
|
|
160
|
+
provisioning.listActiveInstancesForLiveness = async () => [];
|
|
161
|
+
|
|
162
|
+
await ta('DONE-WHEN 1: a repointed unit produces ONE notice within ONE tick', async () => {
|
|
163
|
+
sweep.resetNoticeState();
|
|
164
|
+
const notifySpy = [], logs = [];
|
|
165
|
+
const out = await sweep.sweepOnce(tickDeps({ units: [DRIFTED], notifySpy, logs }));
|
|
166
|
+
assert.equal(notifySpy.length, 1, `one notice from one tick, got ${notifySpy.length}`);
|
|
167
|
+
assert.equal(notifySpy[0].kind, 'drift');
|
|
168
|
+
assert.equal(out.runnerDrift, 1, 'and the tick reports it in its account');
|
|
169
|
+
assert.ok(logs.some((l) => /\[drift\].*drift=1/.test(l)), 'the journal carries the line too');
|
|
170
|
+
});
|
|
171
|
+
|
|
172
|
+
await ta('DONE-WHEN 2: a healthy host produces NONE — and the tick demonstrably ran', async () => {
|
|
173
|
+
sweep.resetNoticeState();
|
|
174
|
+
const notifySpy = [], logs = [];
|
|
175
|
+
let asked = 0;
|
|
176
|
+
const deps = tickDeps({ units: [HEALTHY], notifySpy, logs });
|
|
177
|
+
const out = await sweep.sweepOnce({ ...deps, collectRunnerDrift: async () => { asked++; return [HEALTHY]; } });
|
|
178
|
+
assert.equal(notifySpy.length, 0, 'nothing is said');
|
|
179
|
+
assert.equal(asked, 1, 'and this is NOT the silence of a check that never ran');
|
|
180
|
+
assert.equal(out.runnerDrift, 0);
|
|
181
|
+
assert.ok(!logs.some((l) => /\[drift\]/.test(l)), 'no line either — silence all the way down');
|
|
182
|
+
});
|
|
183
|
+
|
|
184
|
+
await ta('the same drift over two ticks is announced once', async () => {
|
|
185
|
+
sweep.resetNoticeState();
|
|
186
|
+
const notifySpy = [], logs = [];
|
|
187
|
+
const deps = tickDeps({ units: [DRIFTED], notifySpy, logs });
|
|
188
|
+
await sweep.sweepOnce(deps);
|
|
189
|
+
await sweep.sweepOnce(deps);
|
|
190
|
+
assert.equal(notifySpy.length, 1, `the second tick must not re-announce, got ${notifySpy.length}`);
|
|
191
|
+
});
|
|
192
|
+
|
|
193
|
+
await ta('a drift check that THROWS does not take the sweep with it', async () => {
|
|
194
|
+
sweep.resetNoticeState();
|
|
195
|
+
const notifySpy = [], logs = [];
|
|
196
|
+
const deps = tickDeps({ units: [], notifySpy, logs });
|
|
197
|
+
const out = await sweep.sweepOnce({ ...deps, collectRunnerDrift: async () => { throw new Error('systemctl exploded'); } });
|
|
198
|
+
assert.ok(out, 'the tick still returns its liveness account');
|
|
199
|
+
assert.equal(out.swept, 0);
|
|
200
|
+
// Logged, not swallowed: silence here is indistinguishable from a healthy host, which
|
|
201
|
+
// is the exact failure this task was filed about.
|
|
202
|
+
assert.ok(logs.some((l) => /\[drift\] check failed/.test(l) && /systemctl exploded/.test(l)),
|
|
203
|
+
`the failure is said out loud: ${logs.join(' | ')}`);
|
|
204
|
+
});
|
|
205
|
+
|
|
206
|
+
await ta('the check is not gated on there being tenant projects', async () => {
|
|
207
|
+
// A control plane with no tenants still has runners worth checking — and that is the
|
|
208
|
+
// host this runs on.
|
|
209
|
+
sweep.resetNoticeState();
|
|
210
|
+
const notifySpy = [], logs = [];
|
|
211
|
+
const out = await sweep.sweepOnce(tickDeps({ units: [DRIFTED], notifySpy, logs }));
|
|
212
|
+
assert.equal(out.swept, 0, 'no projects at all');
|
|
213
|
+
assert.equal(notifySpy.length, 1, 'and the drift was still found');
|
|
214
|
+
});
|
|
215
|
+
|
|
216
|
+
console.log('\nwiring:');
|
|
217
|
+
|
|
218
|
+
t('the module crosses NO require boundary into core — the checker is a subprocess', () => {
|
|
219
|
+
// A module may require only src/module-api.js (ADR 0083, docs/modules-contract.md, and
|
|
220
|
+
// this module's own CLAUDE.md header). scripts/ is core-package territory, so the
|
|
221
|
+
// checker is read the way the sibling reads scripts/discord/post.js: as a subprocess.
|
|
222
|
+
// fitness's checkModulesImportOnlyApi does not currently catch a scripts/ target, so
|
|
223
|
+
// this assertion is the only thing standing between the rule and a silent regression.
|
|
224
|
+
for (const f of ['runner-drift-notify.js', 'liveness-sweep.js']) {
|
|
225
|
+
const src = readFileSync(new URL(`../modules/provisioning/pollers/${f}`, import.meta.url), 'utf8');
|
|
226
|
+
const requires = [...src.matchAll(/require\((['"])([^'"]+)\1\)/g)].map((m) => m[2]);
|
|
227
|
+
for (const spec of requires) {
|
|
228
|
+
// src/module-api(.js) is THE doorway and is allowed; everything else under src/ or
|
|
229
|
+
// scripts/ is not.
|
|
230
|
+
if (/^(\.\.\/)+src\/module-api(\.js)?$/.test(spec)) continue;
|
|
231
|
+
assert.ok(!/(^|\/)scripts\//.test(spec) && !/(^|\/)src\//.test(spec),
|
|
232
|
+
`${f} requires "${spec}" — a module may require only src/module-api.js; spawn a script instead (ADR 0083)`);
|
|
233
|
+
}
|
|
234
|
+
}
|
|
235
|
+
const notifySrc = readFileSync(new URL('../modules/provisioning/pollers/runner-drift-notify.js', import.meta.url), 'utf8');
|
|
236
|
+
assert.match(notifySrc, /execFile/, 'the checker is spawned');
|
|
237
|
+
assert.match(notifySrc, /runner-drift\.js', '--instance-root'|scripts\/gds\/runner-drift\.js/, 'and it is the checker that is spawned');
|
|
238
|
+
});
|
|
239
|
+
|
|
240
|
+
t('postNotice is NOT re-exported — the sweep takes it straight from its owner', () => {
|
|
241
|
+
// A re-export with no production caller is API surface to keep true for nothing.
|
|
242
|
+
assert.equal(notify.postNotice, undefined);
|
|
243
|
+
const sweepSrc = readFileSync(new URL('../modules/provisioning/pollers/liveness-sweep.js', import.meta.url), 'utf8');
|
|
244
|
+
assert.match(sweepSrc, /const \{ planNotices, postNotice \} = require\('\.\/liveness-notify'\);/,
|
|
245
|
+
'one Discord path, imported from the file that owns it');
|
|
246
|
+
});
|
|
247
|
+
|
|
248
|
+
await ta('readRunnerDrift: a NON-ZERO exit is the drift signal, not a failed read', async () => {
|
|
249
|
+
// runner-drift.js exits 1 exactly WHEN it finds drift — that is its contract as a
|
|
250
|
+
// CI-style gate. Treating a failed exit as a failed read would discard the only result
|
|
251
|
+
// worth reporting, so the exit code is ignored whenever stdout parses.
|
|
252
|
+
const payload = JSON.stringify({ instanceRoot: '/srv/instance', systemd: true, units: [DRIFTED] });
|
|
253
|
+
const execFileImpl = (_cmd, _args, _opts, cb) => cb(Object.assign(new Error('Command failed'), { code: 1 }), payload, '');
|
|
254
|
+
const units = await notify.readRunnerDrift({ instanceRoot: '/srv/instance', execFileImpl });
|
|
255
|
+
assert.equal(units.length, 1);
|
|
256
|
+
assert.equal(units[0].status, 'drift');
|
|
257
|
+
});
|
|
258
|
+
|
|
259
|
+
await ta('readRunnerDrift: it spawns the checker with the instance root, from the repo root', async () => {
|
|
260
|
+
let seen = null;
|
|
261
|
+
const execFileImpl = (cmd, args, opts, cb) => { seen = { cmd, args, opts }; cb(null, JSON.stringify({ systemd: true, units: [] }), ''); };
|
|
262
|
+
const units = await notify.readRunnerDrift({ instanceRoot: '/srv/instance', execFileImpl });
|
|
263
|
+
assert.deepEqual(units, []);
|
|
264
|
+
assert.equal(seen.cmd, 'node');
|
|
265
|
+
assert.deepEqual(seen.args, ['scripts/gds/runner-drift.js', '--instance-root', '/srv/instance', '--json']);
|
|
266
|
+
assert.match(seen.opts.cwd, /cloud-bongos/, 'invoked from the repo root regardless of cwd');
|
|
267
|
+
assert.ok(seen.opts.timeout > 0, 'a hung systemctl costs the check, not the tick');
|
|
268
|
+
});
|
|
269
|
+
|
|
270
|
+
await ta('readRunnerDrift: "could not look" is NULL, never an empty array', async () => {
|
|
271
|
+
// "No runner units" and "could not ask" must not read the same — planDriftNotices
|
|
272
|
+
// treats them differently, and an empty array would silently clear announcements.
|
|
273
|
+
const cases = [
|
|
274
|
+
['unparseable stdout', (_c, _a, _o, cb) => cb(null, 'systemctl: command not found', '')],
|
|
275
|
+
['no systemd', (_c, _a, _o, cb) => cb(null, JSON.stringify({ systemd: false, units: [] }), '')],
|
|
276
|
+
['empty stdout', (_c, _a, _o, cb) => cb(new Error('timeout'), '', '')],
|
|
277
|
+
['a throwing spawn', () => { throw new Error('ENOENT'); }],
|
|
278
|
+
];
|
|
279
|
+
for (const [why, execFileImpl] of cases) {
|
|
280
|
+
assert.equal(await notify.readRunnerDrift({ instanceRoot: '/x', execFileImpl }), null, why);
|
|
281
|
+
}
|
|
282
|
+
// And the empty-array case really is distinct: a host with systemd and no units.
|
|
283
|
+
assert.deepEqual(await notify.readRunnerDrift({ instanceRoot: '/x', execFileImpl: (_c, _a, _o, cb) => cb(null, JSON.stringify({ systemd: true, units: [] }), '') }), []);
|
|
284
|
+
});
|
|
285
|
+
|
|
286
|
+
t('the sweep clears BOTH notice states on reset, or a test leaks into the next one', () => {
|
|
287
|
+
const src = readFileSync(new URL('../modules/provisioning/pollers/liveness-sweep.js', import.meta.url), 'utf8');
|
|
288
|
+
assert.match(src, /function resetNoticeState\(\) \{ noticeState\.clear\(\); driftState\.clear\(\); \}/);
|
|
289
|
+
});
|
|
290
|
+
|
|
291
|
+
console.log(`\nprovisioning_runner_drift_notice: ${passed} passed, ${failed} failed`);
|
|
292
|
+
if (failed) process.exit(1);
|