@bongos/core 1.19.839 → 1.19.841
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.bongos-core.json +73 -33
- package/clients/bongos-client/index.d.ts +1 -1
- package/config/scheduled-routines.json +29 -0
- package/docs/adr/0303-provisioning-verifies-the-base-caddyfile-and-never-edits-it.md +49 -0
- package/docs/adr/README.md +1 -0
- package/docs/api/openapi.json +2 -0
- package/docs/architecture.md +2 -2
- package/docs/module-api-changelog.md +4 -0
- package/modules/government/protected-surfaces.json +2 -1
- package/modules/lifecycle/github-push.js +2 -2
- package/modules/lifecycle/pr-assign.js +77 -0
- package/modules/lifecycle/routes/tasks.js +16 -1
- package/modules/provisioning/pollers/liveness-notify.js +110 -0
- package/modules/provisioning/pollers/liveness-sweep.js +27 -2
- package/modules/provisioning/provisioning.js +9 -9
- package/modules/provisioning/tests/liveness-notify.mjs +199 -0
- package/modules/provisioning/tests/liveness.mjs +4 -3
- package/package-lock.json +2 -2
- package/package.json +1 -1
- package/scripts/gds/paste-token.js +65 -14
- package/scripts/gds/provision-config.js +3 -0
- package/scripts/gds/provision-net.js +32 -0
- package/scripts/gds/provision.js +21 -4
- package/scripts/gds/publish-manifest.js +1 -0
- package/scripts/gds/ship-land.js +12 -0
- package/src/bongos/routes/auth.js +24 -3
- package/src/module-api.js +1 -1
- package/tests/bongos_cli.mjs +13 -4
- package/tests/cli_token_reissue.mjs +54 -7
- package/tests/government_protected_surfaces.mjs +181 -0
- package/tests/paste_token_api_base.mjs +87 -0
- package/tests/pr_assign.mjs +189 -0
- package/tests/provision.mjs +12 -0
- package/tests/provision_caddy_wiring.mjs +148 -0
|
@@ -46,6 +46,7 @@ const { makeAbandonHandler } = require('./abandon-handler');
|
|
|
46
46
|
// longer imports the grader — it resolves the `grade` kernel PORT at grade time
|
|
47
47
|
// (seams.resolveOptional('grade'), in POST /tasks/:id/grade below).
|
|
48
48
|
const githubPush = require('../github-push');
|
|
49
|
+
const prAssign = require('../pr-assign');
|
|
49
50
|
const shipAssurance = require('../ship-assurance');
|
|
50
51
|
const RECONCILE_PROOF = Object.freeze({ kind: 'proof', reason: 'publish-status reconcile: land proven' });
|
|
51
52
|
const seams = api;
|
|
@@ -1125,12 +1126,25 @@ module.exports = function buildTasksRouter() {
|
|
|
1125
1126
|
fs.writeFileSync(bundlePath, bundleBuf);
|
|
1126
1127
|
const { head_sha: pushedSha } = await githubPush.pushBundle({ bundlePath, branch, headSha });
|
|
1127
1128
|
const title = `task ${id}: ${deHashRefs(valueSummary) || branch}`.slice(0, 240);
|
|
1129
|
+
// task 1003991: the server's credential is the PR's AUTHOR, so the builder who
|
|
1130
|
+
// wrote the work is named in the body and made its assignee. The claim holder,
|
|
1131
|
+
// not the caller — an Archon re-shipping someone's strand did not write it. A
|
|
1132
|
+
// lookup that FAILS names nobody (a PR body is never rewritten, so a wrong name
|
|
1133
|
+
// is permanent); only a never-claimed task falls back to the caller.
|
|
1134
|
+
const builderLogin = await db.getLastClaimHolder(id)
|
|
1135
|
+
.then((h) => (h ? h.github_login : req.builder.github_login), () => null);
|
|
1136
|
+
const builderLine = prAssign.prBuilderLine(builderLogin);
|
|
1128
1137
|
const body =
|
|
1129
|
-
`Auto-opened by GDS server-mediated publish (ADR 0055) for task ${id}
|
|
1138
|
+
`Auto-opened by GDS server-mediated publish (ADR 0055) for task ${id}.` +
|
|
1139
|
+
`${builderLine ? ` ${builderLine}` : ''}\n\n` +
|
|
1130
1140
|
`${deHashRefs(valueSummary) || '(no value summary)'}\n\n` +
|
|
1131
1141
|
`Merging triggers .github/workflows/deploy-prod.yml -> ${branding().domains.publicOrigin}. ` +
|
|
1132
1142
|
`Auto-merge is enabled — GitHub lands it once the required checks pass.`;
|
|
1133
1143
|
const pr = await githubPush.openOrFindPullRequest({ branch, title, body });
|
|
1144
|
+
// Best-effort by contract (pr-assign.js never rejects); the catch is for a future
|
|
1145
|
+
// edit that forgets — an assignment must not be able to fail a pushed ship.
|
|
1146
|
+
const assignment = await prAssign.assignPullRequest({ number: pr.number, login: builderLogin })
|
|
1147
|
+
.catch(() => ({ assigned: false }));
|
|
1134
1148
|
// task 1001431: ask WHY when arming fails. `auto_merge: false` on its own told the
|
|
1135
1149
|
// builder nothing, and the commonest cause is a repo with allow_auto_merge OFF —
|
|
1136
1150
|
// a permanent condition that reads exactly like a transient blip.
|
|
@@ -1157,6 +1171,7 @@ module.exports = function buildTasksRouter() {
|
|
|
1157
1171
|
}
|
|
1158
1172
|
return res.status(201).json({
|
|
1159
1173
|
ok: true, pr_url: pr.url, pr_number: pr.number, head_sha: pushedSha, auto_merge: autoMerge,
|
|
1174
|
+
pr_assignee: assignment.assigned ? builderLogin : null,
|
|
1160
1175
|
// Only present when arming failed — the publish itself still succeeded (201).
|
|
1161
1176
|
...(autoMerge ? {} : { auto_merge_reason: autoMergeResult.reason }),
|
|
1162
1177
|
});
|
|
@@ -0,0 +1,110 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
// modules/provisioning/pollers/liveness-notify.js — the owner-facing half of the
|
|
4
|
+
// liveness sweep: when does an outage become something worth saying out loud, and
|
|
5
|
+
// in what words (task 1002894, from idea 1000703; the owner's YES of 2026-08-12,
|
|
6
|
+
// scoped out of task 1002696).
|
|
7
|
+
//
|
|
8
|
+
// WHY. The sweep has knocked on every active project's /healthz every five minutes
|
|
9
|
+
// since task 1002696 and told nobody — an owner learned their project had stopped
|
|
10
|
+
// answering by opening the projects page. This turns that record into one message
|
|
11
|
+
// on the DOWN edge, and one when it comes back.
|
|
12
|
+
//
|
|
13
|
+
// THE THREE RULES that keep a heartbeat from becoming a siren:
|
|
14
|
+
// * EDGE, not state. The fall is only visible in the row's PRIOR last_verified_up,
|
|
15
|
+
// which recordLivenessProbe now returns. A row that was already down when this
|
|
16
|
+
// process started has no edge to see, so a restarted control plane re-posts
|
|
17
|
+
// nothing — the notice state is deliberately in memory, seeded only by an edge
|
|
18
|
+
// this process observed.
|
|
19
|
+
// * CONFIRM before speaking. One failed probe is a blip; the notice waits for the
|
|
20
|
+
// outage to survive a second consecutive sweep, and a project that recovers in
|
|
21
|
+
// between was never announced, so nothing needs unsaying.
|
|
22
|
+
// * COLLAPSE a fan-out. When several projects fall in the same sweep, the cause is
|
|
23
|
+
// almost always on this side — a control-plane restart, a prober-side network
|
|
24
|
+
// blip — so they become ONE message that says so, never one per project.
|
|
25
|
+
//
|
|
26
|
+
// The copy is owner-audience and names a project only when the project is public on
|
|
27
|
+
// the platform's map; a private one is described, never identified.
|
|
28
|
+
|
|
29
|
+
const { execFile } = require('node:child_process');
|
|
30
|
+
const path = require('node:path');
|
|
31
|
+
|
|
32
|
+
// Consecutive down probes before an outage is announced. Two ≈ one sweep interval
|
|
33
|
+
// of patience, which is what makes a single refused connection cost nothing.
|
|
34
|
+
const DOWN_CONFIRM = Number(process.env.PROVISIONING_LIVENESS_DOWN_CONFIRM) || 2;
|
|
35
|
+
// Simultaneous falls that mean "look at the platform, not the projects".
|
|
36
|
+
const FLEET_GUARD = Number(process.env.PROVISIONING_LIVENESS_FLEET_GUARD) || 3;
|
|
37
|
+
// scripts/discord/post.js is invoked from the repo root regardless of cwd.
|
|
38
|
+
const REPO_ROOT = path.resolve(__dirname, '..', '..', '..');
|
|
39
|
+
|
|
40
|
+
// What the owner reads. `name` is null for a project that is not public — it is
|
|
41
|
+
// described rather than identified, so a private project's slug never rides a
|
|
42
|
+
// Discord channel. A `count` with no name is the collapsed fleet message. PURE.
|
|
43
|
+
function noticeText({ kind, name = null, count = 0 } = {}) {
|
|
44
|
+
if (!name && count > 1) {
|
|
45
|
+
return kind === 'up'
|
|
46
|
+
? `✅ All ${count} projects that went quiet are answering again.`
|
|
47
|
+
: `⚠️ ${count} projects stopped answering at the same moment. When that many go quiet together it is usually the platform's own check that is having trouble, not the projects themselves — worth a look here first.`;
|
|
48
|
+
}
|
|
49
|
+
const who = name ? `**${name}**` : 'A private project';
|
|
50
|
+
if (kind === 'up') return `✅ ${who} is answering again.`;
|
|
51
|
+
return `⚠️ ${who} has stopped answering. The platform knocked twice in a row and heard nothing back. It will say so here when the project answers again.`;
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
// One collapsed message, or one per project — never both. PURE.
|
|
55
|
+
function collapse(rows, kind, fleetGuard) {
|
|
56
|
+
if (!rows.length) return [];
|
|
57
|
+
if (rows.length >= fleetGuard) return [{ kind, ids: rows.map((r) => r.id), text: noticeText({ kind, count: rows.length }) }];
|
|
58
|
+
return rows.map((r) => ({ kind, ids: [r.id], text: noticeText({ kind, name: r.isPublic ? r.slug : null }) }));
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
// The decision, given this sweep's observations ({ id, slug, isPublic, up, prevUp })
|
|
62
|
+
// and the notice state carried between sweeps. MUTATES `state` (a Map, id → { streak,
|
|
63
|
+
// notified }) and returns what to post; everything else about it is pure, so the
|
|
64
|
+
// rules above are unit-tested without a sweep, a database or a network.
|
|
65
|
+
function planNotices(observations, state, { confirm = DOWN_CONFIRM, fleetGuard = FLEET_GUARD } = {}) {
|
|
66
|
+
const downs = [], ups = [];
|
|
67
|
+
for (const o of observations || []) {
|
|
68
|
+
const prev = state.get(o.id);
|
|
69
|
+
if (o.up) {
|
|
70
|
+
// Only a project we actually announced gets an all-clear; a blip that never
|
|
71
|
+
// reached a notice just drops its streak.
|
|
72
|
+
if (prev && prev.notified) ups.push(o);
|
|
73
|
+
state.delete(o.id);
|
|
74
|
+
continue;
|
|
75
|
+
}
|
|
76
|
+
let streak;
|
|
77
|
+
if (prev) streak = prev.streak + 1;
|
|
78
|
+
else if (o.prevUp === true) streak = 1; // the fall, observed in this process
|
|
79
|
+
else continue; // already down before we were watching
|
|
80
|
+
const notified = !!(prev && prev.notified);
|
|
81
|
+
if (!notified && streak >= confirm) downs.push(o);
|
|
82
|
+
state.set(o.id, { streak, notified: notified || streak >= confirm });
|
|
83
|
+
}
|
|
84
|
+
const notices = [...collapse(downs, 'down', fleetGuard), ...collapse(ups, 'up', fleetGuard)];
|
|
85
|
+
const collapsedCount = (list) => (list.length >= fleetGuard ? list.length : 0);
|
|
86
|
+
return { notices, suppressed: collapsedCount(downs) + collapsedCount(ups) };
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
// Fire-and-forget to Discord #ship-news — the owner-facing feed the build broadcast
|
|
90
|
+
// already uses (owner's call, 2026-09-17: no new channel, no new webhook to place).
|
|
91
|
+
// Mirrors fireDiscordBroadcast in src/bongos/routes/builders.js: non-blocking,
|
|
92
|
+
// CHAT_DRY_RUN=1 short-circuits for the smoke path, and a Discord hiccup can never
|
|
93
|
+
// stall a sweep tick.
|
|
94
|
+
function postNotice(notice, { log = console.log } = {}) {
|
|
95
|
+
const text = notice && notice.text;
|
|
96
|
+
if (!text) return;
|
|
97
|
+
if (process.env.CHAT_DRY_RUN === '1') {
|
|
98
|
+
log(`[liveness] [broadcast dry-run] would post: ${text}`);
|
|
99
|
+
return;
|
|
100
|
+
}
|
|
101
|
+
try {
|
|
102
|
+
execFile('node', ['scripts/discord/post.js', text], { cwd: REPO_ROOT, timeout: 10_000 }, (err) => {
|
|
103
|
+
if (err) log(`[liveness] notice broadcast failed (non-blocking): ${err.message}`);
|
|
104
|
+
});
|
|
105
|
+
} catch (e) {
|
|
106
|
+
log(`[liveness] notice broadcast failed (non-blocking): ${e && e.message ? e.message : e}`);
|
|
107
|
+
}
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
module.exports = { planNotices, noticeText, postNotice, DOWN_CONFIRM, FLEET_GUARD };
|
|
@@ -27,6 +27,19 @@ const dns = require('node:dns').promises;
|
|
|
27
27
|
const { Agent: UndiciAgent } = require('undici');
|
|
28
28
|
const api = require('../../../src/module-api');
|
|
29
29
|
const provisioning = require('../provisioning');
|
|
30
|
+
const { planNotices, postNotice } = require('./liveness-notify');
|
|
31
|
+
|
|
32
|
+
// Notice state between sweeps (task 1002894): id → { streak, notified }. In memory
|
|
33
|
+
// on purpose — it is seeded only by a fall this process observed, which is exactly
|
|
34
|
+
// what stops a restarted control plane re-announcing an outage it arrived to find.
|
|
35
|
+
const noticeState = new Map();
|
|
36
|
+
function resetNoticeState() { noticeState.clear(); }
|
|
37
|
+
|
|
38
|
+
// Name a project in an owner-facing message only when it is public on the platform's
|
|
39
|
+
// map; a private or stealth one is described, never identified.
|
|
40
|
+
function isPublicProject(row) {
|
|
41
|
+
try { return provisioning.effectiveSettings(row).visibility === 'public'; } catch (_) { return false; }
|
|
42
|
+
}
|
|
30
43
|
|
|
31
44
|
// ~5-minute cadence — matches the display contract's "fresh ≤ 2 cadences" window
|
|
32
45
|
// and is cheap: one HTTP GET per active instance. Override for tests/ops.
|
|
@@ -198,8 +211,8 @@ async function sweepOnce(deps = {}) {
|
|
|
198
211
|
pin = gate.pin || null;
|
|
199
212
|
}
|
|
200
213
|
const probe = await probeHealthz(url, { timeoutMs: deps.timeoutMs, fetchImpl: deps.fetchImpl, pin });
|
|
201
|
-
await provisioning.recordLivenessProbe(db, row.id, probe.up);
|
|
202
|
-
return { row, probed: true, ...probe };
|
|
214
|
+
const rec = await provisioning.recordLivenessProbe(db, row.id, probe.up);
|
|
215
|
+
return { row, probed: true, prevUp: rec ? rec.prev_up : null, ...probe };
|
|
203
216
|
}));
|
|
204
217
|
|
|
205
218
|
const probed = results.filter((r) => r.probed);
|
|
@@ -217,6 +230,17 @@ async function sweepOnce(deps = {}) {
|
|
|
217
230
|
// Nothing active ⇒ nothing to say. A vanilla control plane would otherwise emit
|
|
218
231
|
// an empty account every five minutes forever.
|
|
219
232
|
if (rows.length) log(formatSweepLine(out));
|
|
233
|
+
|
|
234
|
+
// Tell the owner what changed (task 1002894). Best-effort like everything else
|
|
235
|
+
// here: a broadcast that throws is logged and the tick still returns its account.
|
|
236
|
+
const notify = deps.notify || ((n) => postNotice(n, { log }));
|
|
237
|
+
const plan = planNotices(
|
|
238
|
+
probed.map((r) => ({ id: r.row.id, slug: r.row.slug, isPublic: isPublicProject(r.row), up: !!r.up, prevUp: r.prevUp })),
|
|
239
|
+
noticeState
|
|
240
|
+
);
|
|
241
|
+
for (const notice of plan.notices) {
|
|
242
|
+
try { notify(notice); } catch (e) { log(`[liveness] notice failed (non-blocking): ${e && e.message ? e.message : e}`); }
|
|
243
|
+
}
|
|
220
244
|
return out;
|
|
221
245
|
}
|
|
222
246
|
|
|
@@ -255,6 +279,7 @@ module.exports = {
|
|
|
255
279
|
stop,
|
|
256
280
|
// exported for tests
|
|
257
281
|
sweepOnce,
|
|
282
|
+
resetNoticeState,
|
|
258
283
|
probeHealthz,
|
|
259
284
|
healthzUrl,
|
|
260
285
|
formatSweepLine,
|
|
@@ -979,7 +979,7 @@ async function clearReleasedDomains(db, id) {
|
|
|
979
979
|
// against it would only ever record a false "down". `limit` is a runaway guard.
|
|
980
980
|
async function listActiveInstancesForLiveness(db, { limit = 500 } = {}) {
|
|
981
981
|
const { rows } = await db.query(
|
|
982
|
-
`SELECT id, slug, domain, port FROM provisioning_instances
|
|
982
|
+
`SELECT id, slug, domain, port, settings FROM provisioning_instances
|
|
983
983
|
WHERE status = 'active'
|
|
984
984
|
ORDER BY id
|
|
985
985
|
LIMIT $1`,
|
|
@@ -988,16 +988,16 @@ async function listActiveInstancesForLiveness(db, { limit = 500 } = {}) {
|
|
|
988
988
|
return rows;
|
|
989
989
|
}
|
|
990
990
|
|
|
991
|
-
// Record ONE probe observation. Deliberately narrow: it writes the two overlay
|
|
992
|
-
//
|
|
993
|
-
//
|
|
994
|
-
//
|
|
991
|
+
// Record ONE probe observation. Deliberately narrow: it writes the two overlay columns
|
|
992
|
+
// and nothing else — not `status`, and not `updated_at` either, so a 5-minute heartbeat
|
|
993
|
+
// can never reshuffle the owner's updated_at-ordered card list or fake activity on an
|
|
994
|
+
// instance nobody touched. `prev_up` rides the RETURNING (task 1002894): the FALL is visible
|
|
995
|
+
// only in the value this write replaces — one statement, so no separate SELECT races a sweep.
|
|
995
996
|
async function recordLivenessProbe(db, id, up) {
|
|
996
997
|
const { rows } = await db.query(
|
|
997
|
-
`UPDATE provisioning_instances
|
|
998
|
-
|
|
999
|
-
|
|
1000
|
-
RETURNING id, last_verified_at, last_verified_up`,
|
|
998
|
+
`UPDATE provisioning_instances p SET last_verified_at = now(), last_verified_up = $2
|
|
999
|
+
FROM (SELECT id, last_verified_up FROM provisioning_instances WHERE id = $1) prev WHERE p.id = prev.id
|
|
1000
|
+
RETURNING p.id, p.last_verified_at, p.last_verified_up, prev.last_verified_up AS prev_up`,
|
|
1001
1001
|
[id, !!up]
|
|
1002
1002
|
);
|
|
1003
1003
|
return rows[0] || null;
|
|
@@ -0,0 +1,199 @@
|
|
|
1
|
+
// modules/provisioning/tests/liveness-notify.mjs
|
|
2
|
+
//
|
|
3
|
+
// The owner-facing outage notice the liveness sweep posts (task 1002894, from
|
|
4
|
+
// idea 1000703). The sweep has probed every active instance since task 1002696 and
|
|
5
|
+
// told nobody. Covered here:
|
|
6
|
+
// • planNotices — the edge rules: one notice per outage, silence afterwards, a
|
|
7
|
+
// single-sweep blip never posts, a restarted poller never re-posts a fall it
|
|
8
|
+
// did not see, recovery only after a down notice actually went out
|
|
9
|
+
// • the fleet guard — many falls at once collapse into ONE message, because that
|
|
10
|
+
// shape is the platform's own checker, not N projects
|
|
11
|
+
// • noticeText — owner-audience copy that never names a non-public project
|
|
12
|
+
// • sweepOnce — the notices ride the sweep, and a broadcast that throws cannot
|
|
13
|
+
// stall a tick
|
|
14
|
+
//
|
|
15
|
+
// DB-free, network-free: `db` is a recording fake and `notify` is injected.
|
|
16
|
+
//
|
|
17
|
+
// Run standalone: node modules/provisioning/tests/liveness-notify.mjs
|
|
18
|
+
|
|
19
|
+
import { strict as assert } from 'node:assert';
|
|
20
|
+
import { createRequire } from 'node:module';
|
|
21
|
+
|
|
22
|
+
// ESM named imports, not createRequire: knip cannot trace a require()'d export from a
|
|
23
|
+
// test, so a test-only export would read as dead code in the CI ratchet.
|
|
24
|
+
import { planNotices, noticeText, DOWN_CONFIRM, FLEET_GUARD } from '../pollers/liveness-notify.js';
|
|
25
|
+
|
|
26
|
+
const require = createRequire(import.meta.url);
|
|
27
|
+
const liveness = require('../pollers/liveness-sweep.js');
|
|
28
|
+
|
|
29
|
+
let passed = 0, failed = 0;
|
|
30
|
+
function t(name, fn) {
|
|
31
|
+
try { fn(); passed++; console.log(` PASS ${name}`); }
|
|
32
|
+
catch (e) { failed++; console.log(` FAIL ${name}\n ${e.message}`); }
|
|
33
|
+
}
|
|
34
|
+
async function ta(name, fn) {
|
|
35
|
+
try { await fn(); passed++; console.log(` PASS ${name}`); }
|
|
36
|
+
catch (e) { failed++; console.log(` FAIL ${name}\n ${e.message}`); }
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
// One observation of one instance in one sweep. prevUp is what the row said
|
|
40
|
+
// BEFORE this probe was recorded — the transition is only visible there.
|
|
41
|
+
const obs = (over = {}) => ({ id: 1, slug: 'mercury', isPublic: true, up: false, prevUp: true, ...over });
|
|
42
|
+
// Drive N sweeps through one state Map and collect what each would post.
|
|
43
|
+
function sweeps(observations, { state = new Map(), opts = {} } = {}) {
|
|
44
|
+
return observations.map((row) => planNotices([].concat(row), state, opts).notices);
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
console.log('planNotices (the edge rules):');
|
|
48
|
+
|
|
49
|
+
t('a fall posts exactly ONE notice, after the outage is confirmed, then stays quiet', () => {
|
|
50
|
+
assert.equal(DOWN_CONFIRM >= 2, true, 'one probe is a blip, not an outage');
|
|
51
|
+
const rounds = sweeps([
|
|
52
|
+
obs(), // the fall: prior up, now down
|
|
53
|
+
obs({ prevUp: false }), // still down — confirms it
|
|
54
|
+
obs({ prevUp: false }), // and again
|
|
55
|
+
obs({ prevUp: false }),
|
|
56
|
+
]);
|
|
57
|
+
assert.deepEqual(rounds.map((n) => n.length), [0, 1, 0, 0]);
|
|
58
|
+
assert.equal(rounds[1][0].kind, 'down');
|
|
59
|
+
assert.match(rounds[1][0].text, /mercury/);
|
|
60
|
+
});
|
|
61
|
+
|
|
62
|
+
t('a one-sweep blip never posts — it recovers before the outage is confirmed', () => {
|
|
63
|
+
const rounds = sweeps([obs(), obs({ up: true, prevUp: false })]);
|
|
64
|
+
assert.deepEqual(rounds.map((n) => n.length), [0, 0], 'nothing was said, so nothing needs unsaying');
|
|
65
|
+
});
|
|
66
|
+
|
|
67
|
+
t('a restarted poller never re-posts an outage that fell before it started', () => {
|
|
68
|
+
// Fresh state (the restart) and a row that is ALREADY down: no edge to see.
|
|
69
|
+
const rounds = sweeps([obs({ prevUp: false }), obs({ prevUp: false }), obs({ prevUp: false })]);
|
|
70
|
+
assert.deepEqual(rounds.map((n) => n.length), [0, 0, 0]);
|
|
71
|
+
});
|
|
72
|
+
|
|
73
|
+
t('recovery posts once, and only when a down notice actually went out', () => {
|
|
74
|
+
const state = new Map();
|
|
75
|
+
const rounds = sweeps([obs(), obs({ prevUp: false }), obs({ up: true, prevUp: false }), obs({ up: true, prevUp: true })], { state });
|
|
76
|
+
assert.deepEqual(rounds.map((n) => n.length), [0, 1, 1, 0]);
|
|
77
|
+
assert.equal(rounds[2][0].kind, 'up');
|
|
78
|
+
assert.equal(state.size, 0, 'a healed instance leaves no state behind');
|
|
79
|
+
});
|
|
80
|
+
|
|
81
|
+
t('a second outage on the same instance posts again — the notice is per outage, not per lifetime', () => {
|
|
82
|
+
const state = new Map();
|
|
83
|
+
const rounds = sweeps([obs(), obs({ prevUp: false }), obs({ up: true, prevUp: false }), obs(), obs({ prevUp: false })], { state });
|
|
84
|
+
assert.deepEqual(rounds.map((n) => n.length), [0, 1, 1, 0, 1]);
|
|
85
|
+
assert.equal(rounds[4][0].kind, 'down');
|
|
86
|
+
});
|
|
87
|
+
|
|
88
|
+
t('the fleet guard collapses a mass fall into ONE message and never fans out', () => {
|
|
89
|
+
const state = new Map();
|
|
90
|
+
const fleet = Array.from({ length: FLEET_GUARD }, (_, i) => ({ id: i + 1, slug: `p${i + 1}`, isPublic: true, up: false, prevUp: true }));
|
|
91
|
+
assert.equal(planNotices(fleet, state).notices.length, 0, 'still unconfirmed on the first sweep');
|
|
92
|
+
const second = planNotices(fleet.map((r) => ({ ...r, prevUp: false })), state);
|
|
93
|
+
assert.equal(second.notices.length, 1);
|
|
94
|
+
assert.equal(second.suppressed, FLEET_GUARD);
|
|
95
|
+
assert.match(second.notices[0].text, new RegExp(`${FLEET_GUARD} projects`));
|
|
96
|
+
assert.doesNotMatch(second.notices[0].text, /p1|p2/, 'a fleet-wide message names no project');
|
|
97
|
+
// …and the recovery collapses the same way, rather than posting one per project.
|
|
98
|
+
const back = planNotices(fleet.map((r) => ({ ...r, up: true, prevUp: false })), state);
|
|
99
|
+
assert.equal(back.notices.length, 1);
|
|
100
|
+
assert.equal(back.notices[0].kind, 'up');
|
|
101
|
+
assert.equal(state.size, 0);
|
|
102
|
+
});
|
|
103
|
+
|
|
104
|
+
t('below the guard, each project still gets its own named notice', () => {
|
|
105
|
+
const state = new Map();
|
|
106
|
+
const few = Array.from({ length: FLEET_GUARD - 1 }, (_, i) => ({ id: i + 1, slug: `p${i + 1}`, isPublic: true, up: false, prevUp: true }));
|
|
107
|
+
planNotices(few, state);
|
|
108
|
+
const out = planNotices(few.map((r) => ({ ...r, prevUp: false })), state);
|
|
109
|
+
assert.equal(out.notices.length, few.length);
|
|
110
|
+
assert.equal(out.suppressed, 0);
|
|
111
|
+
assert.match(out.notices[0].text, /p1/);
|
|
112
|
+
});
|
|
113
|
+
|
|
114
|
+
console.log('\nnoticeText (owner-audience copy):');
|
|
115
|
+
|
|
116
|
+
t('a public project is named; a non-public one never is', () => {
|
|
117
|
+
const named = noticeText({ kind: 'down', name: 'mercury' });
|
|
118
|
+
assert.match(named, /mercury/);
|
|
119
|
+
const hidden = noticeText({ kind: 'down', name: null });
|
|
120
|
+
assert.doesNotMatch(hidden, /mercury|null|undefined/);
|
|
121
|
+
assert.match(hidden, /project/i);
|
|
122
|
+
});
|
|
123
|
+
|
|
124
|
+
t('the copy carries no jargon and no internals', () => {
|
|
125
|
+
for (const kind of ['down', 'up']) {
|
|
126
|
+
for (const name of ['mercury', null]) {
|
|
127
|
+
const text = noticeText({ kind, name, count: 4 });
|
|
128
|
+
assert.doesNotMatch(text, /healthz|probe|ECONNREFUSED|http \d|instance|sweep|poller|null|undefined/i, `${kind}/${name}: ${text}`);
|
|
129
|
+
assert.ok(text.length > 20 && text.length < 320, 'one short paragraph');
|
|
130
|
+
}
|
|
131
|
+
}
|
|
132
|
+
});
|
|
133
|
+
|
|
134
|
+
t('a non-public project is not named in the sweep observation either', () => {
|
|
135
|
+
const state = new Map();
|
|
136
|
+
const row = { id: 9, slug: 'secret-co', isPublic: false, up: false, prevUp: true };
|
|
137
|
+
planNotices([row], state);
|
|
138
|
+
const out = planNotices([{ ...row, prevUp: false }], state);
|
|
139
|
+
assert.equal(out.notices.length, 1);
|
|
140
|
+
assert.doesNotMatch(out.notices[0].text, /secret-co/);
|
|
141
|
+
});
|
|
142
|
+
|
|
143
|
+
console.log('\nsweepOnce (the notices ride the sweep):');
|
|
144
|
+
|
|
145
|
+
// A fake db that answers the sweep's two statements, with the prior value the
|
|
146
|
+
// edge rules read. `up` decides what the probe will find.
|
|
147
|
+
function fakeDb(rows, priorUp = new Map()) {
|
|
148
|
+
return {
|
|
149
|
+
query: async (sql, params) => {
|
|
150
|
+
if (/FROM provisioning_instances/.test(sql) && /WHERE status = 'active'/.test(sql)) return { rows };
|
|
151
|
+
if (/UPDATE provisioning_instances/.test(sql)) {
|
|
152
|
+
const prev = priorUp.has(params[0]) ? priorUp.get(params[0]) : true;
|
|
153
|
+
priorUp.set(params[0], params[1]);
|
|
154
|
+
return { rows: [{ id: params[0], last_verified_at: new Date(), last_verified_up: params[1], prev_up: prev }] };
|
|
155
|
+
}
|
|
156
|
+
throw new Error(`unexpected SQL: ${sql}`);
|
|
157
|
+
},
|
|
158
|
+
};
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
await ta('a down instance posts one notice on the confirming sweep, through the injected broadcaster', async () => {
|
|
162
|
+
liveness.resetNoticeState();
|
|
163
|
+
const rows = [{ id: 1, slug: 'mercury', domain: null, port: 1, settings: {} }];
|
|
164
|
+
const db = fakeDb(rows);
|
|
165
|
+
const posted = [];
|
|
166
|
+
const deps = { pool: db, log: () => {}, timeoutMs: 50, notify: (n) => posted.push(n), fetchImpl: async () => { throw new Error('ECONNREFUSED'); } };
|
|
167
|
+
await liveness.sweepOnce(deps);
|
|
168
|
+
assert.equal(posted.length, 0, 'the first down is not yet an outage');
|
|
169
|
+
await liveness.sweepOnce(deps);
|
|
170
|
+
assert.equal(posted.length, 1);
|
|
171
|
+
assert.equal(posted[0].kind, 'down');
|
|
172
|
+
await liveness.sweepOnce(deps);
|
|
173
|
+
assert.equal(posted.length, 1, 'and it stays quiet');
|
|
174
|
+
});
|
|
175
|
+
|
|
176
|
+
await ta('a broadcaster that throws cannot stall a sweep tick', async () => {
|
|
177
|
+
liveness.resetNoticeState();
|
|
178
|
+
const rows = [{ id: 1, slug: 'mercury', domain: null, port: 1, settings: {} }];
|
|
179
|
+
const db = fakeDb(rows);
|
|
180
|
+
const deps = { pool: db, log: () => {}, timeoutMs: 50, notify: () => { throw new Error('discord is down'); }, fetchImpl: async () => { throw new Error('ECONNREFUSED'); } };
|
|
181
|
+
await liveness.sweepOnce(deps);
|
|
182
|
+
const out = await liveness.sweepOnce(deps);
|
|
183
|
+
assert.equal(out.down, 1, 'the sweep still reports what it saw');
|
|
184
|
+
});
|
|
185
|
+
|
|
186
|
+
await ta('a gated project is never named in what the sweep posts', async () => {
|
|
187
|
+
liveness.resetNoticeState();
|
|
188
|
+
const rows = [{ id: 7, slug: 'secret-co', domain: null, port: 1, settings: { visibility: 'private' } }];
|
|
189
|
+
const db = fakeDb(rows);
|
|
190
|
+
const posted = [];
|
|
191
|
+
const deps = { pool: db, log: () => {}, timeoutMs: 50, notify: (n) => posted.push(n), fetchImpl: async () => { throw new Error('ECONNREFUSED'); } };
|
|
192
|
+
await liveness.sweepOnce(deps);
|
|
193
|
+
await liveness.sweepOnce(deps);
|
|
194
|
+
assert.equal(posted.length, 1);
|
|
195
|
+
assert.doesNotMatch(posted[0].text, /secret-co/);
|
|
196
|
+
});
|
|
197
|
+
|
|
198
|
+
console.log(`\nprovisioning liveness notice: ${passed} passed, ${failed} failed`);
|
|
199
|
+
process.exitCode = failed ? 1 : 0;
|
|
@@ -59,10 +59,11 @@ function fakeDb(rows) {
|
|
|
59
59
|
return {
|
|
60
60
|
updates,
|
|
61
61
|
query: async (sql, params) => {
|
|
62
|
-
if (/
|
|
63
|
-
if (
|
|
62
|
+
if (/FROM provisioning_instances/.test(sql) && /WHERE status = 'active'/.test(sql)) return { rows };
|
|
63
|
+
if (/^UPDATE provisioning_instances/.test(sql)) {
|
|
64
64
|
updates.push({ sql, id: params[0], up: params[1] });
|
|
65
|
-
|
|
65
|
+
// prev_up rides the write (task 1002894) — the notice reads the value replaced.
|
|
66
|
+
return { rows: [{ id: params[0], last_verified_at: new Date(), last_verified_up: params[1], prev_up: true }] };
|
|
66
67
|
}
|
|
67
68
|
throw new Error(`unexpected SQL: ${sql}`);
|
|
68
69
|
},
|
package/package-lock.json
CHANGED
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@bongos/core",
|
|
3
|
-
"version": "1.19.
|
|
3
|
+
"version": "1.19.841",
|
|
4
4
|
"lockfileVersion": 3,
|
|
5
5
|
"requires": true,
|
|
6
6
|
"packages": {
|
|
7
7
|
"": {
|
|
8
8
|
"name": "@bongos/core",
|
|
9
|
-
"version": "1.19.
|
|
9
|
+
"version": "1.19.841",
|
|
10
10
|
"license": "AGPL-3.0-or-later",
|
|
11
11
|
"dependencies": {
|
|
12
12
|
"express": "^4.21.2",
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@bongos/core",
|
|
3
|
-
"version": "1.19.
|
|
3
|
+
"version": "1.19.841",
|
|
4
4
|
"description": "Cloud Bongos — the AI-first build platform core (GDS + platform surfaces + module system), installed as a versioned dependency (ADR 0108).",
|
|
5
5
|
"license": "AGPL-3.0-or-later",
|
|
6
6
|
"main": "src/platform-server.js",
|
|
@@ -32,27 +32,71 @@ const { SESSION_FILENAMES, readSessionConfigSync } = require('../../src/instance
|
|
|
32
32
|
// WRITE the session to the configured instance dir (OTB → ~/.config/otb,
|
|
33
33
|
// unchanged; a fresh instance → its neutral dir). R61 / task 1200.
|
|
34
34
|
const SESSION_PATH = ic.configPath(SESSION_FILENAMES[0]);
|
|
35
|
-
|
|
36
|
-
//
|
|
37
|
-
//
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
const
|
|
35
|
+
|
|
36
|
+
// `--api-base <url>` / `--api-base=<url>`, plus the positional token. Hand-rolled
|
|
37
|
+
// because this script has no arg parser and one flag does not earn a dependency.
|
|
38
|
+
function parseArgs(argv) {
|
|
39
|
+
const out = { token: null, apiBase: null };
|
|
40
|
+
for (let i = 0; i < argv.length; i += 1) {
|
|
41
|
+
const a = String(argv[i]);
|
|
42
|
+
if (a === '--api-base') { out.apiBase = argv[i + 1] === undefined ? '' : String(argv[i + 1]); i += 1; continue; }
|
|
43
|
+
if (a.startsWith('--api-base=')) { out.apiBase = a.slice('--api-base='.length); continue; }
|
|
44
|
+
if (out.token === null && !a.startsWith('-')) out.token = a;
|
|
45
|
+
}
|
|
46
|
+
return out;
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
// WHICH INSTANCE IS THIS TOKEN FOR? (task 1003722)
|
|
50
|
+
//
|
|
51
|
+
// The old order was env → the EXISTING session's api_base → branding default, on the
|
|
52
|
+
// reasoning that a re-auth already knows which instance the builder is on. That holds
|
|
53
|
+
// right up until the builder uses a SECOND instance: the stale api_base then wins over
|
|
54
|
+
// the instance that actually minted the token, so the paste is verified against — and
|
|
55
|
+
// written as — the wrong host. The reported symptom was a token from this instance
|
|
56
|
+
// landing as a token for an unrelated one, after which every /builder-* command
|
|
57
|
+
// authenticated somewhere unintended.
|
|
58
|
+
//
|
|
59
|
+
// A token is minted BY an instance, so the issuer is a property of the token, not of
|
|
60
|
+
// the machine. The hall now stamps it into the paste command, and an explicit flag
|
|
61
|
+
// outranks every inferred source: it is per-invocation, it came from the minting
|
|
62
|
+
// instance, and it is the only source that is right by construction.
|
|
63
|
+
//
|
|
64
|
+
// Below the flag the old order is untouched, so an older instance whose paste command
|
|
65
|
+
// carries no flag behaves exactly as before — and so does the founder-host regression
|
|
66
|
+
// this file already guards (task 1002427: hardcoding a host here rejected a valid
|
|
67
|
+
// token with a 401).
|
|
68
|
+
function resolveBaseForPaste({ flag, resolve = () => ic.resolveApiBase() } = {}) {
|
|
69
|
+
const f = typeof flag === 'string' ? flag.trim() : '';
|
|
70
|
+
if (!f) return { base: resolve(), source: 'resolved' };
|
|
71
|
+
if (!/^https?:\/\/[^\s/]+/i.test(f)) return { base: null, source: 'invalid' };
|
|
72
|
+
return { base: f.replace(/\/+$/, ''), source: 'flag' };
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
const ARGS = parseArgs(process.argv.slice(2));
|
|
76
|
+
const RESOLVED = resolveBaseForPaste({ flag: ARGS.apiBase });
|
|
77
|
+
const API_BASE = RESOLVED.base;
|
|
42
78
|
|
|
43
79
|
function usage(extra) {
|
|
44
80
|
if (extra) console.error(extra);
|
|
45
81
|
console.error('');
|
|
46
|
-
console.error('Usage: bongos paste-token <token>');
|
|
82
|
+
console.error('Usage: bongos paste-token <token> [--api-base <url>]');
|
|
47
83
|
console.error(' (inside a core checkout: node scripts/gds/paste-token.js <token>)');
|
|
48
84
|
console.error('');
|
|
49
|
-
|
|
85
|
+
// Fall back to the inferred base for this hint when --api-base was the thing that
|
|
86
|
+
// was malformed — otherwise the error message that explains the bad flag would
|
|
87
|
+
// itself print "null/builders/settings".
|
|
88
|
+
const hintBase = API_BASE || ic.resolveApiBase();
|
|
89
|
+
console.error('Get a token on your Settings page: ' + hintBase + '/builders/settings');
|
|
50
90
|
console.error('— under "CLI Access" click "Re-issue CLI token" (you must be signed in).');
|
|
51
91
|
console.error('The page prints the exact paste-command — just run that line.');
|
|
52
92
|
}
|
|
53
93
|
|
|
54
94
|
async function main() {
|
|
55
|
-
|
|
95
|
+
if (RESOLVED.source === 'invalid') {
|
|
96
|
+
usage(`--api-base must be an http(s) URL, got: ${ARGS.apiBase}`);
|
|
97
|
+
process.exit(2);
|
|
98
|
+
}
|
|
99
|
+
const token = ARGS.token;
|
|
56
100
|
if (!token) {
|
|
57
101
|
usage('No token argument provided.');
|
|
58
102
|
process.exit(2);
|
|
@@ -138,7 +182,14 @@ async function main() {
|
|
|
138
182
|
}
|
|
139
183
|
}
|
|
140
184
|
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
185
|
+
// Run only as a script. Exporting the two pure helpers lets the decision that caused
|
|
186
|
+
// task 1003722 — WHICH instance a pasted token belongs to — be tested directly, rather
|
|
187
|
+
// than inferred from a text scan of this file.
|
|
188
|
+
if (require.main === module) {
|
|
189
|
+
main().catch((err) => {
|
|
190
|
+
console.error('paste-token failed:', err && err.message ? err.message : err);
|
|
191
|
+
process.exit(1);
|
|
192
|
+
});
|
|
193
|
+
}
|
|
194
|
+
|
|
195
|
+
module.exports = { parseArgs, resolveBaseForPaste };
|
|
@@ -55,6 +55,9 @@ const CONFIG = {
|
|
|
55
55
|
// <slug>.caddy file (idempotent overwrite, no fragile append to a shared file);
|
|
56
56
|
// the main Caddyfile carries a one-time `import <dir>/*.caddy` (operator setup).
|
|
57
57
|
caddySnippetDir: process.env.PROVISION_CADDY_SNIPPET_DIR || '/etc/caddy/sites',
|
|
58
|
+
// The base Caddyfile the running Caddy loads — read (never written) to prove it imports
|
|
59
|
+
// the snippets and carries the on-demand TLS ask (task 1002860, ADR 0303).
|
|
60
|
+
caddyfile: process.env.PROVISION_CADDYFILE || '/etc/caddy/Caddyfile',
|
|
58
61
|
// Where nightly pg_dump backups land (db-backup-nightly.sh's GDS_BACKUP_DIR
|
|
59
62
|
// default). Shared across co-tenant instances on one box — dumps are named
|
|
60
63
|
// `<db>-<ts>.sql.gz`, so each instance's files + retention prune stay scoped to
|
|
@@ -240,6 +240,36 @@ function caddyBlock(inst) {
|
|
|
240
240
|
function caddySnippetPath(inst, { snippetDir = CONFIG.caddySnippetDir } = {}) {
|
|
241
241
|
return path.posix.join(snippetDir, `${inst.slug}.caddy`);
|
|
242
242
|
}
|
|
243
|
+
// Does the base Caddyfile carry what a provisioned domain needs (task 1002860, ADR 0303)?
|
|
244
|
+
// Read from Caddy's RESOLVED config, not the file's text, so the answer holds whatever form
|
|
245
|
+
// the operator wrote it in: an `ask` (2.7) or a `permission` module (2.8+), an absolute or
|
|
246
|
+
// relative import, directives split across imported files. `requireHost` asks whether the
|
|
247
|
+
// domain is served, which only a written snippet can make true. Returns the operator
|
|
248
|
+
// instruction, or null when wired.
|
|
249
|
+
function caddyWiringGap(adaptedJson, { domain, requireHost = false, caddyfile = CONFIG.caddyfile, snippetDir = CONFIG.caddySnippetDir } = {}) {
|
|
250
|
+
let apps;
|
|
251
|
+
try { apps = JSON.parse(adaptedJson).apps || {}; } catch (_) {
|
|
252
|
+
return `could not read Caddy's resolved config ('caddy adapt --config ${caddyfile}' printed no JSON)`;
|
|
253
|
+
}
|
|
254
|
+
const onDemand = apps.tls && apps.tls.automation && apps.tls.automation.on_demand;
|
|
255
|
+
if (!JSON.stringify(onDemand || {}).includes('/provisioning/tls-check')) {
|
|
256
|
+
return `${caddyfile} has no on_demand_tls ask for the provisioning tls-check, so no provisioned domain can get a certificate — add 'on_demand_tls { ask <platform origin>/api/bongos/provisioning/tls-check }' to its global options block and reload Caddy`;
|
|
257
|
+
}
|
|
258
|
+
if (requireHost && !JSON.stringify(apps.http || {}).includes(JSON.stringify(domain))) {
|
|
259
|
+
return `${caddyfile} does not import ${snippetDir}/*.caddy, so Caddy will never serve ${domain} — add 'import ${snippetDir}/*.caddy' at the top level of ${caddyfile} and reload Caddy`;
|
|
260
|
+
}
|
|
261
|
+
return null;
|
|
262
|
+
}
|
|
263
|
+
// `caddy adapt` reads config files only: no privilege, no certificate keys, no running server.
|
|
264
|
+
function checkCaddyWiring(exec, opts = {}) {
|
|
265
|
+
const caddyfile = opts.caddyfile || CONFIG.caddyfile;
|
|
266
|
+
let res;
|
|
267
|
+
try { res = exec(`caddy adapt --config ${caddyfile} --adapter caddyfile`); } catch (e) {
|
|
268
|
+
const why = String((e && (e.stderr || e.message)) || e).trim().slice(0, 200);
|
|
269
|
+
return `'caddy adapt --config ${caddyfile}' failed (${why}) — the base Caddyfile must load before a site is added to it`;
|
|
270
|
+
}
|
|
271
|
+
return caddyWiringGap(res && res.stdout, { ...opts, caddyfile });
|
|
272
|
+
}
|
|
243
273
|
function healthzCmd(inst) {
|
|
244
274
|
return `curl -sS --max-time 5 http://127.0.0.1:${inst.port}/healthz`;
|
|
245
275
|
}
|
|
@@ -281,6 +311,8 @@ module.exports = {
|
|
|
281
311
|
DNS_BLACKHOLE_IP,
|
|
282
312
|
caddyBlock,
|
|
283
313
|
caddySnippetPath,
|
|
314
|
+
caddyWiringGap,
|
|
315
|
+
checkCaddyWiring,
|
|
284
316
|
checkDnsTokenScope,
|
|
285
317
|
classifyZoneScope,
|
|
286
318
|
dnsReleaseEnv,
|