@bongos/core 1.19.319 → 1.19.321
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.bongos-core.json +125 -40
- package/.claude/skills/merge-mode/SKILL.md +3 -1
- package/.devcontainer/init-firewall.sh +41 -5
- package/clients/bongos-client/examples/hello-world.mjs +11 -2
- package/docs/copy-inventory.md +20 -20
- package/docs/copy-registry.json +22 -22
- package/modules/dev-box/box-access.js +20 -0
- package/modules/hall-ui/public/builders.js +0 -1
- package/package-lock.json +2 -2
- package/package.json +1 -1
- package/scripts/gds/audit-replay.sh +20 -3
- package/scripts/gds/box.js +79 -86
- package/scripts/gds/db-backup-nightly.sh +17 -4
- package/scripts/gds/db-backup-offsite.sh +23 -6
- package/scripts/gds/discipline-modes.json +1 -1
- package/scripts/gds/doc-cli-guard.js +34 -2
- package/scripts/gds/fitness-ratchets.js +54 -6
- package/scripts/gds/fitness.js +8 -4
- package/scripts/gds/gds-db-backup.service +15 -4
- package/scripts/gds/gds-db-backup.timer +12 -1
- package/scripts/gds/gds-db-offsite.service +16 -5
- package/scripts/gds/gds-db-offsite.timer +12 -1
- package/scripts/gds/instance-apex.js +28 -0
- package/scripts/gds/newcomer-floor.templates.json +2 -2
- package/scripts/gds/provision.js +7 -3
- package/scripts/gds/ship-deploy-target.js +388 -0
- package/scripts/gds/ship-finish.js +377 -0
- package/scripts/gds/ship-flow.js +814 -0
- package/scripts/gds/ship-git.js +100 -0
- package/scripts/gds/ship-grade-steps.js +376 -0
- package/scripts/gds/ship-grade.js +319 -0
- package/scripts/gds/ship-io.js +100 -0
- package/scripts/gds/ship-land.js +539 -0
- package/scripts/gds/ship-memory-sync.js +192 -0
- package/scripts/gds/ship-merge.js +297 -0
- package/scripts/gds/ship-preflight-steps.js +323 -0
- package/scripts/gds/ship-regen.js +286 -0
- package/scripts/gds/ship-resume.js +151 -0
- package/scripts/gds/ship-session-upload.js +216 -0
- package/scripts/gds/ship-state.js +73 -0
- package/scripts/gds/ship-surprise.js +84 -0
- package/scripts/gds/ship.js +105 -4195
- package/scripts/status-probe.mjs +30 -2
- package/src/module-api.js +5 -1
- package/tests/box_access.mjs +46 -0
- package/tests/fitness_ratchets.mjs +82 -0
- package/tests/helpers.mjs +13 -0
- package/tests/provision.mjs +7 -1
- package/tests/regrade_eligibility.mjs +43 -0
- package/tests/ship_assurance.mjs +4 -2
- package/tests/ship_publish_error.mjs +43 -0
package/scripts/gds/box.js
CHANGED
|
@@ -1,12 +1,11 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
// scripts/gds/box.js — per-builder cloud dev box lifecycle CLI (ADR 0031 §2/§5, #598).
|
|
3
3
|
//
|
|
4
|
-
// The control-plane entrypoint for the box lifecycle. Like provision-staging.sh
|
|
5
|
-
//
|
|
6
|
-
//
|
|
7
|
-
//
|
|
8
|
-
//
|
|
9
|
-
// LIVE DB rank (the same value requireRank reads), so it is a real ADR-0016
|
|
4
|
+
// The control-plane entrypoint for the box lifecycle. Like provision-staging.sh it
|
|
5
|
+
// must RUN ON A TRUSTED CONTROL PLANE (the prod droplet or an operator's machine),
|
|
6
|
+
// NOT inside a builder box — a box provisioning boxes is privilege escalation, and
|
|
7
|
+
// the builder firewall doesn't even allow-list the DO API. The rank gate reads the
|
|
8
|
+
// builder's LIVE DB rank (what requireRank reads), so it is a real ADR-0016
|
|
10
9
|
// server-side check, not local prose.
|
|
11
10
|
//
|
|
12
11
|
// Subcommands:
|
|
@@ -21,10 +20,9 @@
|
|
|
21
20
|
//
|
|
22
21
|
// <builder> is a GitHub login or a numeric builder id.
|
|
23
22
|
//
|
|
24
|
-
// SAFETY: dry-run is the DEFAULT
|
|
25
|
-
//
|
|
26
|
-
//
|
|
27
|
-
//
|
|
23
|
+
// SAFETY: dry-run is the DEFAULT — nothing touches DigitalOcean or writes the DB
|
|
24
|
+
// without --apply, so run any command bare first to see the plan (as with
|
|
25
|
+
// `pulse.js --release-stale --dry-run`):
|
|
28
26
|
// node scripts/gds/box.js sweep-idle # plan only
|
|
29
27
|
// node scripts/gds/box.js sweep-idle --apply # actually park idle boxes
|
|
30
28
|
//
|
|
@@ -96,6 +94,9 @@ function resolveBoxInfraDir(opts = {}) {
|
|
|
96
94
|
return null;
|
|
97
95
|
}
|
|
98
96
|
|
|
97
|
+
// task 1002757 — the instance's own apex; null when unconfigured (callers refuse).
|
|
98
|
+
const { instanceApex } = require('./instance-apex');
|
|
99
|
+
|
|
99
100
|
const CONFIG = {
|
|
100
101
|
floor: process.env.BOX_PROVISION_MIN_RANK || 'xenos',
|
|
101
102
|
region: process.env.BOX_REGION || 'nyc3',
|
|
@@ -108,34 +109,35 @@ const CONFIG = {
|
|
|
108
109
|
sshKeyIds: (process.env.BOX_SSH_KEY_IDS || '').split(',').map((s) => s.trim()).filter(Boolean),
|
|
109
110
|
dnsHook: process.env.BOX_DNS_HOOK || null,
|
|
110
111
|
tag: process.env.BOX_TAG || 'otb-builder-box',
|
|
111
|
-
|
|
112
|
+
// task 1002757 settled this file's old carve-out: an operator-plane DNS zone is
|
|
113
|
+
// still a FUNCTIONAL DEFAULT, and a founder literal aimed another operator's
|
|
114
|
+
// fleet at the founder's zone. Derived from the instance apex, null when
|
|
115
|
+
// unconfigured — hostnameFor() then refuses instead of minting a dead FQDN.
|
|
116
|
+
dnsZone: process.env.BOX_DNS_ZONE || (instanceApex() ? `dev.${instanceApex()}` : null),
|
|
112
117
|
};
|
|
113
118
|
|
|
114
119
|
// Named-tunnel automation for the browser terminal (#771, ADR 0038 Part B).
|
|
115
120
|
// OPT-IN, default OFF — an unset BOX_NAMED_TUNNEL leaves the box on the tokenless
|
|
116
|
-
// quick tunnel (trycloudflare.com), the zero-secret #760 default. When the
|
|
117
|
-
//
|
|
118
|
-
//
|
|
119
|
-
//
|
|
120
|
-
//
|
|
121
|
+
// quick tunnel (trycloudflare.com), the zero-secret #760 default. When the operator
|
|
122
|
+
// turns it on AND supplies a Cloudflare token with Tunnel + DNS scope (the same
|
|
123
|
+
// CLOUDFLARE_API_TOKEN box-dns uses) + the account id, every provision creates a
|
|
124
|
+
// stable term-<login>.<terminalZone> tunnel and bakes its connector token into the
|
|
125
|
+
// box; deprovision tears it down.
|
|
121
126
|
const CF = {
|
|
122
127
|
enabled: /^(1|true|yes)$/i.test(process.env.BOX_NAMED_TUNNEL || ''),
|
|
123
128
|
apiToken: process.env.CLOUDFLARE_API_TOKEN || null,
|
|
124
129
|
accountId: process.env.CLOUDFLARE_ACCOUNT_ID || null,
|
|
125
|
-
// The Cloudflare ZONE the
|
|
126
|
-
//
|
|
127
|
-
//
|
|
128
|
-
zone: process.env.CLOUDFLARE_ZONE ||
|
|
129
|
-
//
|
|
130
|
-
//
|
|
131
|
-
//
|
|
132
|
-
//
|
|
133
|
-
//
|
|
134
|
-
//
|
|
135
|
-
|
|
136
|
-
// via BOX_TERMINAL_ZONE. (The SSH A-record path, box-dns-cloudflare.sh, stays on
|
|
137
|
-
// dev.* — it is proxied=false / direct-IP, so it needs no edge cert.)
|
|
138
|
-
terminalZone: process.env.BOX_TERMINAL_ZONE || 'amazonprimea.com',
|
|
130
|
+
// The Cloudflare ZONE the record lives in (the registrable apex) — the terminal
|
|
131
|
+
// host is a record WITHIN it, matching box-dns-cloudflare.sh. Distinct from
|
|
132
|
+
// terminalZone (the host suffix). Both resolve from the instance (task 1002757).
|
|
133
|
+
zone: process.env.CLOUDFLARE_ZONE || instanceApex(),
|
|
134
|
+
// Host suffix for term-<login>.<terminalZone>. DEFAULTS TO THE APEX, NOT dev.*:
|
|
135
|
+
// the terminal rides a PROXIED tunnel, and free Universal SSL covers <apex> +
|
|
136
|
+
// *.<apex> (ONE label) but not a TWO-label host like term-x.dev.<apex>, which
|
|
137
|
+
// fails TLS at the edge (verified live, #771). An operator holding an Advanced
|
|
138
|
+
// Certificate for *.dev.<apex> can override via BOX_TERMINAL_ZONE. (The SSH
|
|
139
|
+
// A-record path stays on dev.* — proxied=false, so it needs no edge cert.)
|
|
140
|
+
terminalZone: process.env.BOX_TERMINAL_ZONE || instanceApex(),
|
|
139
141
|
};
|
|
140
142
|
|
|
141
143
|
// Named-tunnel automation is live only when the operator opted in AND the
|
|
@@ -158,10 +160,9 @@ function makeCf(apply) {
|
|
|
158
160
|
// SECURITY PREFLIGHT (task 1002511, ADR 0053 / SR#11). The edge cookie-strip
|
|
159
161
|
// Transform Rule is applied BEST-EFFORT deep inside ensureNamedTunnel, so a
|
|
160
162
|
// CLOUDFLARE_API_TOKEN missing the zone "Transform Rules" edit scope fails as one
|
|
161
|
-
// buried log line
|
|
162
|
-
//
|
|
163
|
-
//
|
|
164
|
-
// live droplet listing) and shouts the exact fix.
|
|
163
|
+
// buried log line while the apex session cookie keeps reaching box subdomains. This
|
|
164
|
+
// probe runs BEFORE any DigitalOcean spend (dry-run included — a read-only lookup in
|
|
165
|
+
// a plan is the drift-reconcile precedent) and shouts the exact fix.
|
|
165
166
|
//
|
|
166
167
|
// HONESTY: the probe is a GET, so it can only DISPROVE the scope. A readable phase
|
|
167
168
|
// does NOT mean the rule can be written — only the PUT inside ensureNamedTunnel
|
|
@@ -278,18 +279,21 @@ function shortStamp() {
|
|
|
278
279
|
}
|
|
279
280
|
|
|
280
281
|
function dropletName(login) { return `otb-box-${login}`; }
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
//
|
|
289
|
-
//
|
|
290
|
-
//
|
|
291
|
-
//
|
|
292
|
-
//
|
|
282
|
+
// Refuses on an unresolved zone (task 1002757) — `box-x.undefined` would mint a
|
|
283
|
+
// DNS record and a box that can never be reached.
|
|
284
|
+
function hostnameFor(login) {
|
|
285
|
+
if (!CONFIG.dnsZone) throw new Error('box: no DNS zone configured — set BOX_DNS_ZONE, or give this instance a domains.publicOrigin in config/branding.json (task 1002757).');
|
|
286
|
+
return `box-${login}.${CONFIG.dnsZone}`;
|
|
287
|
+
}
|
|
288
|
+
|
|
289
|
+
// Run the optional DNS hook so the per-builder hostname tracks the new IP; with no
|
|
290
|
+
// hook configured, print the reminder — the stable-hostname CUTOVER is #599's, #598
|
|
291
|
+
// only records IP + hostname and offers the hook so an operator can wire DNS without
|
|
292
|
+
// editing this file. → { ok, error?, skipped? }. task 1180: callers MUST act on
|
|
293
|
+
// ok:false (box_events row + flag the box), never continue silently — that swallow
|
|
294
|
+
// left @kingaonline's A record pinned at a dead IP while provision reported success.
|
|
295
|
+
// The hook retries transient Cloudflare blips, so ok:false is a real, exhausted
|
|
296
|
+
// failure. `hook` is injectable so the contract is unit-testable (#1034).
|
|
293
297
|
function runDnsHook(hostname, ip, hook = CONFIG.dnsHook) {
|
|
294
298
|
if (!ip) return { ok: true, skipped: true };
|
|
295
299
|
if (!hook) {
|
|
@@ -316,21 +320,16 @@ function runDnsHook(hostname, ip, hook = CONFIG.dnsHook) {
|
|
|
316
320
|
// dead end to park a record at.
|
|
317
321
|
const DNS_BLACKHOLE_IP = '192.0.2.1';
|
|
318
322
|
|
|
319
|
-
// Park the per-builder A record at the blackhole sentinel on deprovision (idea
|
|
320
|
-
//
|
|
321
|
-
//
|
|
322
|
-
//
|
|
323
|
-
//
|
|
324
|
-
//
|
|
325
|
-
//
|
|
326
|
-
//
|
|
327
|
-
//
|
|
328
|
-
//
|
|
329
|
-
// stranger's IP. The next provision/wake PUTs the record straight to the live IP
|
|
330
|
-
// — a clean in-place repoint at the 60s record TTL, no gap. Best-effort (like the
|
|
331
|
-
// prior delete): a failure just leaves the old IP, which the next provision
|
|
332
|
-
// repoints anyway. Reuses the UPSERT path of runDnsHook; `hook` is injectable for
|
|
333
|
-
// tests. Returns { ok, error?, skipped? }.
|
|
323
|
+
// Park the per-builder A record at the blackhole sentinel on deprovision (idea 309,
|
|
324
|
+
// refining task 1180). 1180 DELETED the record so it couldn't dangle at a recycled
|
|
325
|
+
// DO IP, but delete-then-recreate across a reap→rebuild left an NXDOMAIN window that
|
|
326
|
+
// resolvers negative-cache for the zone's SOA neg-TTL (~30 min, not builder-editable)
|
|
327
|
+
// — a client querying then kept getting ENOTFOUND long after the box was back.
|
|
328
|
+
// REPOINTING to a non-routable sentinel keeps the name always-resolving (nothing to
|
|
329
|
+
// negative-cache) while preserving 1180's guarantee that a torn-down box's name can
|
|
330
|
+
// never dial a stranger's recycled IP; the next provision/wake PUTs the live IP
|
|
331
|
+
// in-place at the 60s TTL, no gap. Best-effort like the prior delete. Reuses
|
|
332
|
+
// runDnsHook's UPSERT path; `hook` injectable for tests. → { ok, error?, skipped? }
|
|
334
333
|
function runDnsHookBlackhole(hostname, hook = CONFIG.dnsHook) {
|
|
335
334
|
if (!hook || !hostname) return { ok: true, skipped: true };
|
|
336
335
|
return runDnsHook(hostname, DNS_BLACKHOLE_IP, hook);
|
|
@@ -510,14 +509,13 @@ async function cmdProvision(pool, boxes, boxOnboard, ref, { apply }) {
|
|
|
510
509
|
// Create (or reuse) the named tunnel BEFORE the droplet so its connector token
|
|
511
510
|
// bakes into the cloud-init box.env. A failure here aborts provision — better
|
|
512
511
|
// no box than a box that boots without its promised stable terminal.
|
|
513
|
-
// #774: on the named-tunnel path the control plane knows
|
|
514
|
-
// on-ramp needs at provision time — the stable hostname (here) + a password
|
|
515
|
-
// generate
|
|
516
|
-
//
|
|
517
|
-
// (setTerminalAccess)
|
|
518
|
-
//
|
|
519
|
-
//
|
|
520
|
-
// builder/infra credential is funneled into the box.
|
|
512
|
+
// #774: on the named-tunnel path the control plane knows everything the browser
|
|
513
|
+
// on-ramp needs at provision time — the stable hostname (here) + a password we
|
|
514
|
+
// generate. So bake the password into box.env (the box serves ttyd with it) and,
|
|
515
|
+
// once the box is active, RECORD {url, password} into the GDS ourselves
|
|
516
|
+
// (setTerminalAccess): the builder reads them from the hall, with no token placed
|
|
517
|
+
// on the box and no box-side reporter (the quick-tunnel fallback that caused the
|
|
518
|
+
// #771/#774 chicken-and-egg). ADR 0016 holds — no credential funneled into a box.
|
|
521
519
|
let boxEnv = '';
|
|
522
520
|
let terminalHostname = null;
|
|
523
521
|
let terminalCredential = null;
|
|
@@ -553,24 +551,19 @@ async function cmdProvision(pool, boxes, boxOnboard, ref, { apply }) {
|
|
|
553
551
|
}
|
|
554
552
|
}
|
|
555
553
|
|
|
556
|
-
// Bootstrap the builder's GDS session onto the box (#818). box-source-fetch
|
|
557
|
-
//
|
|
558
|
-
//
|
|
559
|
-
//
|
|
560
|
-
//
|
|
561
|
-
// un-cloned repo (the chicken-and-egg). So we mint a session here and bake it
|
|
562
|
-
// into cloud-init, exactly like the SSH keys / tunnel token / terminal credential.
|
|
554
|
+
// Bootstrap the builder's GDS session onto the box (#818). box-source-fetch (#600)
|
|
555
|
+
// authenticates as the BUILDER to pull the rank-scoped repo; with no session file
|
|
556
|
+
// it no-ops forever (repo never clones → no devcontainer, no /builder-* skills),
|
|
557
|
+
// and the session normally comes from /builder-setup, which lives in the un-cloned
|
|
558
|
+
// repo — the chicken-and-egg. So mint one here and bake it into cloud-init.
|
|
563
559
|
//
|
|
564
|
-
// SECURITY (#919, SR#2): this baked session is source='box' —
|
|
565
|
-
// refresh-on-use
|
|
566
|
-
//
|
|
567
|
-
//
|
|
568
|
-
//
|
|
569
|
-
//
|
|
570
|
-
//
|
|
571
|
-
// FROM the box (claim/ship) the builder runs /builder-reauth (device flow), minting
|
|
572
|
-
// a normal cli/web session — never auto-baked. user_agent='box-provision' is the
|
|
573
|
-
// marker migration 090 keys on to retroactively scope already-baked tokens.
|
|
560
|
+
// SECURITY (#919, SR#2): this baked session is source='box' — SCOPED, 24h
|
|
561
|
+
// refresh-on-use. The token sits ON the box (any on-box process + the metadata
|
|
562
|
+
// endpoint can read it), OUTSIDE the ADR 0016 trust boundary, so it must not carry
|
|
563
|
+
// the builder's full authority: requireBuilder denies a 'box' session everywhere
|
|
564
|
+
// except the 4 on-box cron routes (source-access, authorized-keys, heartbeat,
|
|
565
|
+
// terminal — via auth.allowBoxScope). Real work FROM the box needs /builder-reauth
|
|
566
|
+
// (device flow); user_agent='box-provision' is the marker migration 090 keys on.
|
|
574
567
|
const gdsDb = require(path.join(REPO_ROOT, 'src/bongos/db'));
|
|
575
568
|
const sessionToken = crypto.randomBytes(32).toString('hex');
|
|
576
569
|
await gdsDb.createSession({
|
|
@@ -6,8 +6,8 @@
|
|
|
6
6
|
# in /var/backups/gds/ as YYYY-MM-DD-HHMMSS.sql.gz (UTC timestamp), and
|
|
7
7
|
# files older than 30 days are pruned.
|
|
8
8
|
#
|
|
9
|
-
# Authentication: peer auth via the local Unix socket
|
|
10
|
-
# owns the
|
|
9
|
+
# Authentication: peer auth via the local Unix socket — the invoking unix
|
|
10
|
+
# user owns the instance's Postgres role; pg_dump finds the socket at
|
|
11
11
|
# /var/run/postgresql with no password needed. Matches the model in
|
|
12
12
|
# src/db.js.
|
|
13
13
|
#
|
|
@@ -36,8 +36,21 @@ set -euo pipefail
|
|
|
36
36
|
# Destination directory for dumps. Owned by lars:lars, mode 0750.
|
|
37
37
|
BACKUP_DIR="${GDS_BACKUP_DIR:-/var/backups/gds}"
|
|
38
38
|
|
|
39
|
-
# Database to dump
|
|
40
|
-
|
|
39
|
+
# Database to dump: explicit GDS_BACKUP_DB wins, else the instance's own branding
|
|
40
|
+
# pack names it — the SAME resolver as migrate.sh / pool.js, so the backup and the
|
|
41
|
+
# running server can never disagree (task 1002757). Deliberately NO literal
|
|
42
|
+
# fallback: this script ships to every instance, and a baked-in founder default
|
|
43
|
+
# aimed another instance's nightly pg_dump at a database that is not its own.
|
|
44
|
+
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
45
|
+
REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
|
|
46
|
+
DB_NAME="${GDS_BACKUP_DB:-}"
|
|
47
|
+
if [ -z "$DB_NAME" ]; then
|
|
48
|
+
DB_NAME="$(node -e "process.stdout.write(require(process.argv[1]).instanceDbName() || '')" "$REPO_ROOT/src/bongos/pool" 2>/dev/null || true)"
|
|
49
|
+
fi
|
|
50
|
+
if [ -z "$DB_NAME" ]; then
|
|
51
|
+
echo "db-backup: cannot determine which database to dump. Set GDS_BACKUP_DB, or give this instance a db.database in config/branding.json. Refusing to guess (task 1002757)." >&2
|
|
52
|
+
exit 2
|
|
53
|
+
fi
|
|
41
54
|
|
|
42
55
|
# Retention in days. Files older than this (by mtime) are deleted at the
|
|
43
56
|
# end of each successful run. 30 days = 30 daily dumps retained.
|
|
@@ -13,13 +13,13 @@
|
|
|
13
13
|
# scripts/gds/gds-db-offsite.service + .timer). Weekly cadence.
|
|
14
14
|
#
|
|
15
15
|
# Authentication: S3-style credentials for the Space live OUTSIDE the repo
|
|
16
|
-
# in /etc
|
|
17
|
-
# The systemd unit injects them via EnvironmentFile; a manual run
|
|
18
|
-
# the same file. Required vars:
|
|
16
|
+
# in /etc/<configDir>/spaces.env (chmod 600, owned by the service user), per
|
|
17
|
+
# ADR 0022. The systemd unit injects them via EnvironmentFile; a manual run
|
|
18
|
+
# sources the same file. Required vars:
|
|
19
19
|
# SPACES_KEY SPACES_SECRET SPACES_BUCKET SPACES_REGION SPACES_ENDPOINT
|
|
20
20
|
#
|
|
21
21
|
# Manual one-shot:
|
|
22
|
-
#
|
|
22
|
+
# <instance-root>/scripts/gds/db-backup-offsite.sh
|
|
23
23
|
#
|
|
24
24
|
# Systemd:
|
|
25
25
|
# sudo systemctl start gds-db-offsite.service # one-shot
|
|
@@ -32,7 +32,19 @@ set -euo pipefail
|
|
|
32
32
|
|
|
33
33
|
# Where the local nightly dumps land (must match db-backup-nightly.sh).
|
|
34
34
|
BACKUP_DIR="${GDS_BACKUP_DIR:-/var/backups/gds}"
|
|
35
|
-
|
|
35
|
+
# Database name: explicit GDS_BACKUP_DB wins, else the instance's branding pack —
|
|
36
|
+
# the same resolver + no-literal-fallback posture as db-backup-nightly.sh /
|
|
37
|
+
# migrate.sh (task 1002757).
|
|
38
|
+
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
39
|
+
REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
|
|
40
|
+
DB_NAME="${GDS_BACKUP_DB:-}"
|
|
41
|
+
if [ -z "$DB_NAME" ]; then
|
|
42
|
+
DB_NAME="$(node -e "process.stdout.write(require(process.argv[1]).instanceDbName() || '')" "$REPO_ROOT/src/bongos/pool" 2>/dev/null || true)"
|
|
43
|
+
fi
|
|
44
|
+
if [ -z "$DB_NAME" ]; then
|
|
45
|
+
echo "db-backup-offsite: cannot determine which database's dumps to ship. Set GDS_BACKUP_DB, or give this instance a db.database in config/branding.json. Refusing to guess (task 1002757)." >&2
|
|
46
|
+
exit 2
|
|
47
|
+
fi
|
|
36
48
|
|
|
37
49
|
# How many weekly offsite copies to retain in the Space before pruning the
|
|
38
50
|
# oldest. 4 = ~1 month of weekly snapshots (Archon decision, #415 claim 338).
|
|
@@ -45,7 +57,12 @@ PREFIX="${GDS_OFFSITE_PREFIX:-gds-offsite/}"
|
|
|
45
57
|
|
|
46
58
|
# Credentials file. Read only if the SPACES_* vars aren't already set
|
|
47
59
|
# (systemd injects them via EnvironmentFile; this covers manual runs).
|
|
48
|
-
|
|
60
|
+
# Default path derives from the instance's config dir (task 1002757 — the old
|
|
61
|
+
# founder-pinned /etc/<founder>/ default could never exist on another box);
|
|
62
|
+
# empty when the resolver is unavailable, in which case the SPACES_* vars (or
|
|
63
|
+
# an explicit GDS_SPACES_ENV) must carry the credentials.
|
|
64
|
+
CONFIG_DIR_NAME="$(node -e "process.stdout.write(require(process.argv[1]).configDirName() || '')" "$REPO_ROOT/src/instance-config" 2>/dev/null || true)"
|
|
65
|
+
ENV_FILE="${GDS_SPACES_ENV:-${CONFIG_DIR_NAME:+/etc/$CONFIG_DIR_NAME/spaces.env}}"
|
|
49
66
|
|
|
50
67
|
# ---- load credentials -------------------------------------------------------
|
|
51
68
|
|
|
@@ -14,7 +14,7 @@
|
|
|
14
14
|
"artist": {
|
|
15
15
|
"skill": "paint",
|
|
16
16
|
"label": "Art session",
|
|
17
|
-
"directive": "This is an ARTIST task. To work it, invoke the /paint skill — that skill is the operating playbook for art sessions, and it is the only place the experience is defined. It is show-first / low-text: Claude communicates in pictures (display the generated tile, the /art gallery at sandbox-<login
|
|
17
|
+
"directive": "This is an ARTIST task. To work it, invoke the /paint skill — that skill is the operating playbook for art sessions, and it is the only place the experience is defined. It is show-first / low-text: Claude communicates in pictures (display the generated tile, the /art gallery at sandbox-<login>.<instance-apex>/art, and the in-world sandbox), not paragraphs about what the pipeline did. It drives the existing pixel-art pipeline (otb-tile-generate / otb-design-review / otb-feedback-capture / art-stage.js) and keeps the locked palette + rubric sacred. If no human is present (autonomous / bypass-permissions), /paint generates, rubric-gates hard, stages passing assets to the gallery, and parks the visuals for human review. Open /paint now rather than working this task the default way; the gallery is at sandbox-<login>.<instance-apex>/art."
|
|
18
18
|
},
|
|
19
19
|
"ui": {
|
|
20
20
|
"skill": "design",
|
|
@@ -15,7 +15,7 @@
|
|
|
15
15
|
//
|
|
16
16
|
// WHAT IT CHECKS, deliberately narrow: for every `node scripts/gds/<x>.js` in a
|
|
17
17
|
// tracked .md, (a) the script exists, and (b) each `--flag` written next to it
|
|
18
|
-
// appears in that script's source. Nothing else — this is not a prose linter.
|
|
18
|
+
// appears in that script's source (or in a same-directory module it requires — a CLI may be carved into phase files). Nothing else — this is not a prose linter.
|
|
19
19
|
//
|
|
20
20
|
// STATIC ONLY, NEVER EXECUTED. The reason matters: running every documented CLI
|
|
21
21
|
// would be slow, would need a session, and a command that blocks on a prompt
|
|
@@ -203,6 +203,38 @@ const readFileOrNull = (rel) => {
|
|
|
203
203
|
|
|
204
204
|
const NAME = 'a doc that names a CLI matches the real CLI (G01)';
|
|
205
205
|
|
|
206
|
+
// readCliSource — a CLI's source PLUS the source of the same-directory modules
|
|
207
|
+
// it requires, concatenated.
|
|
208
|
+
//
|
|
209
|
+
// A big CLI is allowed to be more than one file: task 1003205 carved
|
|
210
|
+
// scripts/gds/ship.js into ship-*.js phase modules, and the flag parsing for
|
|
211
|
+
// e.g. `--no-surprise` moved with the phase that reads it. Scanning ship.js
|
|
212
|
+
// alone then reports a documented, genuinely-honoured flag as a violation —
|
|
213
|
+
// a false alarm that pushes the next agent toward silencing the rule via
|
|
214
|
+
// DOC_CLI_ALLOW, which is exactly the outcome the rule exists to prevent.
|
|
215
|
+
//
|
|
216
|
+
// One level of same-directory relative requires is enough (it is where a carve
|
|
217
|
+
// puts things) and keeps the read bounded. A missing sibling is skipped: the
|
|
218
|
+
// point is to find the flag, not to prove the require graph.
|
|
219
|
+
function readCliSource(rel) {
|
|
220
|
+
const src = readFileOrNull(rel);
|
|
221
|
+
if (src === null) return null;
|
|
222
|
+
const dir = path.dirname(rel);
|
|
223
|
+
const seen = new Set([rel]);
|
|
224
|
+
const parts = [src];
|
|
225
|
+
const RE = /require\(\s*['"](\.\/[^'"]+)['"]\s*\)/g;
|
|
226
|
+
let m;
|
|
227
|
+
while ((m = RE.exec(src))) {
|
|
228
|
+
let sib = path.join(dir, m[1]);
|
|
229
|
+
if (!sib.endsWith('.js')) sib += '.js';
|
|
230
|
+
if (seen.has(sib)) continue;
|
|
231
|
+
seen.add(sib);
|
|
232
|
+
const sibSrc = readFileOrNull(sib);
|
|
233
|
+
if (sibSrc !== null) parts.push(sibSrc);
|
|
234
|
+
}
|
|
235
|
+
return parts.join('\n');
|
|
236
|
+
}
|
|
237
|
+
|
|
206
238
|
function checkDocNamesRealCli({ files = null } = {}) {
|
|
207
239
|
const docs = files || trackedMarkdown();
|
|
208
240
|
// A lint that reports on nothing is worse than one that fails (the
|
|
@@ -217,7 +249,7 @@ function checkDocNamesRealCli({ files = null } = {}) {
|
|
|
217
249
|
}
|
|
218
250
|
|
|
219
251
|
const { violations, warnings, invocations } = scanDocs(docs, {
|
|
220
|
-
readDoc: readFileOrNull, readScript:
|
|
252
|
+
readDoc: readFileOrNull, readScript: readCliSource,
|
|
221
253
|
});
|
|
222
254
|
|
|
223
255
|
return {
|
|
@@ -215,12 +215,52 @@ function knipIssueCount() {
|
|
|
215
215
|
|
|
216
216
|
// ---------- baseline + ratchet ----------
|
|
217
217
|
|
|
218
|
+
// readBaselines — task 1003227. Returns { metrics, error }: `error` is a reason
|
|
219
|
+
// string when the file cannot be trusted, null when it read cleanly.
|
|
220
|
+
//
|
|
221
|
+
// This used to `catch { return {}; }`, which made the gate FAIL OPEN. An empty
|
|
222
|
+
// baseline map leaves every `floor === undefined`, and compareToBaselines classifies
|
|
223
|
+
// an absent floor as a WARNING — so a checkout with no config/ reported the ratchet
|
|
224
|
+
// as PASSED. Measured: with the baselines gone, a planted 5,000-line and 94-cycle
|
|
225
|
+
// regression produced 0 violations. That is not a hypothetical checkout — `config/`
|
|
226
|
+
// is absent from every dev box's sparse slice, so the builders who most needed the
|
|
227
|
+
// gate were the ones silently exempt from it, learning from red CI instead.
|
|
228
|
+
//
|
|
229
|
+
// Distinguish the two absences, because they are not the same fact:
|
|
230
|
+
// - the FILE is missing/malformed -> cannot verify anything -> hard fail (here);
|
|
231
|
+
// - a KEY is missing for one metric -> a genuinely new metric -> stays a warning
|
|
232
|
+
// in compareToBaselines, so adding a metric does not break the build.
|
|
233
|
+
//
|
|
234
|
+
// The file's own header says raising a baseline to pass CI is out of policy;
|
|
235
|
+
// deleting the file must not be the loophole that raising it is not.
|
|
218
236
|
function readBaselines() {
|
|
237
|
+
let raw;
|
|
219
238
|
try {
|
|
220
|
-
|
|
221
|
-
} catch {
|
|
222
|
-
|
|
239
|
+
raw = fs.readFileSync(BASELINE_PATH, 'utf8');
|
|
240
|
+
} catch (e) {
|
|
241
|
+
const missing = e && e.code === 'ENOENT';
|
|
242
|
+
return {
|
|
243
|
+
metrics: {},
|
|
244
|
+
error: missing
|
|
245
|
+
? `config/fitness-baselines.json is MISSING — the quality ratchet cannot verify anything, so it fails CLOSED rather than reporting a pass it did not check. `
|
|
246
|
+
+ `If this is a dev box, config/ was historically outside the sparse checkout (task 1003227); re-sync the box (\`node scripts/gds/box-sync.js\`). `
|
|
247
|
+
+ `To bootstrap a fresh baseline deliberately: node scripts/gds/fitness-ratchets.js --update-baselines`
|
|
248
|
+
: `config/fitness-baselines.json is unreadable (${e && e.message}) — failing CLOSED.`,
|
|
249
|
+
};
|
|
250
|
+
}
|
|
251
|
+
let parsed;
|
|
252
|
+
try {
|
|
253
|
+
parsed = JSON.parse(raw);
|
|
254
|
+
} catch (e) {
|
|
255
|
+
return { metrics: {}, error: `config/fitness-baselines.json is not valid JSON (${e && e.message}) — failing CLOSED.` };
|
|
256
|
+
}
|
|
257
|
+
const metrics = parsed && typeof parsed.metrics === 'object' && parsed.metrics !== null
|
|
258
|
+
? parsed.metrics
|
|
259
|
+
: null;
|
|
260
|
+
if (!metrics) {
|
|
261
|
+
return { metrics: {}, error: 'config/fitness-baselines.json has no "metrics" object — failing CLOSED.' };
|
|
223
262
|
}
|
|
263
|
+
return { metrics, error: null };
|
|
224
264
|
}
|
|
225
265
|
|
|
226
266
|
// Pure ratchet verdict — exported for the hermetic test.
|
|
@@ -256,7 +296,11 @@ function computeMetrics({ withDocs = true } = {}) {
|
|
|
256
296
|
// The fitness.js check (in-suite, hermetic — everything except knip).
|
|
257
297
|
function checkQualityRatchets() {
|
|
258
298
|
const metrics = computeMetrics();
|
|
259
|
-
const {
|
|
299
|
+
const { metrics: baselines, error: baselineError } = readBaselines();
|
|
300
|
+
const { violations, warnings } = compareToBaselines(metrics, baselines);
|
|
301
|
+
// task 1003227: an unreadable baseline file is a hard failure, not silence. Put it
|
|
302
|
+
// FIRST so it is the headline rather than buried under per-metric noise.
|
|
303
|
+
if (baselineError) violations.unshift(baselineError);
|
|
260
304
|
return {
|
|
261
305
|
name: 'quality ratchets — size, duplication, cycles, console, doc entropy only tighten (task 1003129)',
|
|
262
306
|
hardFail: violations.length > 0,
|
|
@@ -274,7 +318,9 @@ function main() {
|
|
|
274
318
|
const metrics = computeMetrics();
|
|
275
319
|
const knip = knipIssueCount();
|
|
276
320
|
if (knip !== null) metrics.knip_issue_count = knip;
|
|
277
|
-
|
|
321
|
+
// The bootstrap path deliberately tolerates an absent file — writing it is the
|
|
322
|
+
// whole point — so it reads the metrics and ignores any error (task 1003227).
|
|
323
|
+
const { metrics: existing } = readBaselines();
|
|
278
324
|
if (knip === null && existing.knip_issue_count !== undefined) metrics.knip_issue_count = existing.knip_issue_count;
|
|
279
325
|
fs.writeFileSync(
|
|
280
326
|
BASELINE_PATH,
|
|
@@ -290,7 +336,9 @@ function main() {
|
|
|
290
336
|
console.warn('fitness-ratchets: WARN — knip not runnable here (no node_modules / npx failure); failing OPEN.');
|
|
291
337
|
process.exit(0);
|
|
292
338
|
}
|
|
293
|
-
const {
|
|
339
|
+
const { metrics: knipBaselines, error: knipBaselineError } = readBaselines();
|
|
340
|
+
const { violations, warnings } = compareToBaselines({ knip_issue_count: value }, knipBaselines);
|
|
341
|
+
if (knipBaselineError) violations.unshift(knipBaselineError);
|
|
294
342
|
for (const w of warnings) console.warn(`fitness-ratchets: ${w}`);
|
|
295
343
|
if (violations.length) {
|
|
296
344
|
for (const v of violations) console.error(`fitness-ratchets: FAIL — ${v}`);
|
package/scripts/gds/fitness.js
CHANGED
|
@@ -1023,9 +1023,14 @@ const IDENTITY_SCAN_ALLOWED = {
|
|
|
1023
1023
|
'scripts/gds/fitness.js': 2,
|
|
1024
1024
|
'scripts/gds/package-core.js': 1,
|
|
1025
1025
|
'scripts/gds/seed-v3-tasks.js': 14,
|
|
1026
|
-
|
|
1027
|
-
|
|
1028
|
-
|
|
1026
|
+
// audit-replay.sh + db-backup-{nightly,offsite}.sh were carved out here as
|
|
1027
|
+
// "host-pinned ops"; task 1002757 RETIRED that justification for them — each
|
|
1028
|
+
// now resolves its database from the branding pack (the migrate.sh pattern) and
|
|
1029
|
+
// fails loudly rather than defaulting, so their counts ratcheted to zero and the
|
|
1030
|
+
// entries are gone. seed-staging.sh keeps its carve-out on the OPPOSITE
|
|
1031
|
+
// rationale: its literal is a fail-CLOSED safety interlock (the script wipes
|
|
1032
|
+
// STAGING_DB and refuses unless the name matches exactly), so on any other
|
|
1033
|
+
// instance it declines to run instead of destroying the wrong database.
|
|
1029
1034
|
'scripts/gds/seed-staging.sh': 4,
|
|
1030
1035
|
'scripts/gds/backfill-sessions.js': 2,
|
|
1031
1036
|
'scripts/gds/box.js': 3,
|
|
@@ -1649,7 +1654,6 @@ function checkWriteRoutesValidated() {
|
|
|
1649
1654
|
};
|
|
1650
1655
|
}
|
|
1651
1656
|
|
|
1652
|
-
|
|
1653
1657
|
// ── Check 19 — the ADR 0174 vocabulary wall (goal 1000068 R07, task 1003008) ──
|
|
1654
1658
|
//
|
|
1655
1659
|
// "Role" means ONE thing in this codebase: a craft — Artist, Ideator, or Builder.
|
|
@@ -1,13 +1,24 @@
|
|
|
1
|
+
# TEMPLATE — not installable as-is (task 1002757). The __TOKENS__ below MUST be
|
|
2
|
+
# substituted at install time; a provisioned instance gets a correct per-instance
|
|
3
|
+
# unit pair generated by scripts/gds/provision.js (task 2213) and never needs this
|
|
4
|
+
# file. Substitute:
|
|
5
|
+
# __SERVICE_USER__ the unix user that owns the instance + its Postgres role
|
|
6
|
+
# __INSTANCE_ROOT__ the instance checkout root (docs live under it)
|
|
7
|
+
# __CORE_SCRIPTS_DIR__ the deployed core's scripts/gds dir:
|
|
8
|
+
# standalone: <instance-root>/node_modules/@bongos/core/scripts/gds
|
|
9
|
+
# full clone: <instance-root>/scripts/gds
|
|
10
|
+
# __CONFIG_DIR__ the instance config dir name (offsite unit only)
|
|
11
|
+
# One-liner: sed -e 's|__SERVICE_USER__|<user>|g' -e … gds-db-backup.service | sudo tee /etc/systemd/system/…
|
|
1
12
|
[Unit]
|
|
2
13
|
Description=Nightly pg_dump of GDS Postgres DB (filed by #412)
|
|
3
|
-
Documentation=file
|
|
14
|
+
Documentation=file://__INSTANCE_ROOT__/docs/recipes/gds-db-backup.md
|
|
4
15
|
After=postgresql.service
|
|
5
16
|
Requires=postgresql.service
|
|
6
17
|
|
|
7
18
|
[Service]
|
|
8
19
|
Type=oneshot
|
|
9
|
-
User=
|
|
10
|
-
Group=
|
|
20
|
+
User=__SERVICE_USER__
|
|
21
|
+
Group=__SERVICE_USER__
|
|
11
22
|
# Run the canonical script from the deployed repo. Keep this path stable;
|
|
12
23
|
# the recipe doc points at it for restore + manual-one-shot use cases.
|
|
13
24
|
#
|
|
@@ -25,7 +36,7 @@ Group=lars
|
|
|
25
36
|
# vendored/standalone: /home/<user>/<instance>/node_modules/@bongos/core/scripts/gds/db-backup-nightly.sh
|
|
26
37
|
# full clone: /home/<user>/<repo>/scripts/gds/db-backup-nightly.sh
|
|
27
38
|
# See docs/recipes/gds-db-backup.md.
|
|
28
|
-
ExecStart=/usr/bin/env bash /
|
|
39
|
+
ExecStart=/usr/bin/env bash __CORE_SCRIPTS_DIR__/db-backup-nightly.sh
|
|
29
40
|
# Modest hardening — the script only needs read on the socket + write on
|
|
30
41
|
# /var/backups/gds. Anything else is gravy for forensics.
|
|
31
42
|
NoNewPrivileges=true
|
|
@@ -1,6 +1,17 @@
|
|
|
1
|
+
# TEMPLATE — not installable as-is (task 1002757). The __TOKENS__ below MUST be
|
|
2
|
+
# substituted at install time; a provisioned instance gets a correct per-instance
|
|
3
|
+
# unit pair generated by scripts/gds/provision.js (task 2213) and never needs this
|
|
4
|
+
# file. Substitute:
|
|
5
|
+
# __SERVICE_USER__ the unix user that owns the instance + its Postgres role
|
|
6
|
+
# __INSTANCE_ROOT__ the instance checkout root (docs live under it)
|
|
7
|
+
# __CORE_SCRIPTS_DIR__ the deployed core's scripts/gds dir:
|
|
8
|
+
# standalone: <instance-root>/node_modules/@bongos/core/scripts/gds
|
|
9
|
+
# full clone: <instance-root>/scripts/gds
|
|
10
|
+
# __CONFIG_DIR__ the instance config dir name (offsite unit only)
|
|
11
|
+
# One-liner: sed -e 's|__SERVICE_USER__|<user>|g' -e … gds-db-backup.service | sudo tee /etc/systemd/system/…
|
|
1
12
|
[Unit]
|
|
2
13
|
Description=Daily timer for GDS DB pg_dump (filed by #412)
|
|
3
|
-
Documentation=file
|
|
14
|
+
Documentation=file://__INSTANCE_ROOT__/docs/recipes/gds-db-backup.md
|
|
4
15
|
|
|
5
16
|
[Timer]
|
|
6
17
|
# Run every day at 04:00 UTC. That's a low-traffic window for a
|
|
@@ -1,13 +1,24 @@
|
|
|
1
|
+
# TEMPLATE — not installable as-is (task 1002757). The __TOKENS__ below MUST be
|
|
2
|
+
# substituted at install time; a provisioned instance gets a correct per-instance
|
|
3
|
+
# unit pair generated by scripts/gds/provision.js (task 2213) and never needs this
|
|
4
|
+
# file. Substitute:
|
|
5
|
+
# __SERVICE_USER__ the unix user that owns the instance + its Postgres role
|
|
6
|
+
# __INSTANCE_ROOT__ the instance checkout root (docs live under it)
|
|
7
|
+
# __CORE_SCRIPTS_DIR__ the deployed core's scripts/gds dir:
|
|
8
|
+
# standalone: <instance-root>/node_modules/@bongos/core/scripts/gds
|
|
9
|
+
# full clone: <instance-root>/scripts/gds
|
|
10
|
+
# __CONFIG_DIR__ the instance config dir name (offsite unit only)
|
|
11
|
+
# One-liner: sed -e 's|__SERVICE_USER__|<user>|g' -e … gds-db-backup.service | sudo tee /etc/systemd/system/…
|
|
1
12
|
[Unit]
|
|
2
13
|
Description=Weekly offsite archive of GDS Postgres dumps to DigitalOcean Spaces (filed by #415)
|
|
3
|
-
Documentation=file
|
|
14
|
+
Documentation=file://__INSTANCE_ROOT__/docs/recipes/gds-db-backup.md
|
|
4
15
|
After=network-online.target
|
|
5
16
|
Wants=network-online.target
|
|
6
17
|
|
|
7
18
|
[Service]
|
|
8
19
|
Type=oneshot
|
|
9
|
-
User=
|
|
10
|
-
Group=
|
|
20
|
+
User=__SERVICE_USER__
|
|
21
|
+
Group=__SERVICE_USER__
|
|
11
22
|
# Run the canonical script from the deployed repo. Keep this path stable;
|
|
12
23
|
# the recipe doc points at it for restore + manual-one-shot use cases.
|
|
13
24
|
#
|
|
@@ -18,12 +29,12 @@ Group=lars
|
|
|
18
29
|
# this unit would have hit it on its first run.
|
|
19
30
|
#
|
|
20
31
|
# EDIT THIS PATH when installing (layout differs — vendored vs full clone).
|
|
21
|
-
ExecStart=/usr/bin/env bash /
|
|
32
|
+
ExecStart=/usr/bin/env bash __CORE_SCRIPTS_DIR__/db-backup-offsite.sh
|
|
22
33
|
# S3 credentials for the Space live outside the repo (ADR 0022). systemd
|
|
23
34
|
# injects them; the file is chmod 600, owned by lars. Missing-file is
|
|
24
35
|
# tolerated here (the script fails loudly if the vars are unset) so a
|
|
25
36
|
# fresh install doesn't fail to load the unit before creds are dropped in.
|
|
26
|
-
EnvironmentFile=-/etc/
|
|
37
|
+
EnvironmentFile=-/etc/__CONFIG_DIR__/spaces.env
|
|
27
38
|
# Hardening — the script reads dumps from /var/backups/gds and talks to the
|
|
28
39
|
# network. It needs no write access to the local filesystem.
|
|
29
40
|
NoNewPrivileges=true
|