@bongos/core 1.21.174 → 1.21.176

package/scripts/bongos/db-backup-nightly.sh

@@ -0,0 +1,232 @@
+#!/bin/bash
+# Nightly pg_dump of the GDS Postgres DB.
+#
+# Runs as the unit's User= (the plane's app account) via the gds-db-backup.timer systemd unit (see
+# scripts/bongos/gds-db-backup.timer + .service in this repo). Output lands
+# in /var/backups/gds/ as <db>-YYYY-MM-DDTHHMMSSZ.sql.gz (UTC timestamp), and
+# files older than 30 days are pruned.
+#
+# Authentication: peer auth via the local Unix socket — the invoking unix
+# user owns the instance's Postgres role; pg_dump finds the socket at
+# /var/run/postgresql with no password needed. Matches the model in
+# src/db.js.
+#
+# Filed by #412 (V3.R62-precondition) — closes the "GDS DB has no
+# nightly backup" gap surfaced by ADR 0024. The OFFSITE copy of these
+# dumps is filed separately as #415 (V3.R62-offsite); this script only
+# handles the local-on-droplet half.
+#
+# Caveat: dumps land on the same physical disk as the live database
+# (/dev/vda1). That protects against corruption, accidental DROP, bad
+# migrations, and "oops I deleted a row" — but NOT whole-droplet loss.
+# #415 closes that remaining gap with a weekly offsite archive.
+#
+# Manual one-shot:
+#   /usr/local/sbin/gds-db-backup.sh
+#
+# Systemd:
+#   sudo systemctl start gds-db-backup.service   # one-shot
+#   sudo systemctl status gds-db-backup.timer    # check schedule
+#   journalctl -u gds-db-backup.service          # see history
+
+set -euo pipefail
+
+# ---- configuration ----------------------------------------------------------
+
+# Destination directory for dumps. Owned by that account and its primary group, mode 0750.
+BACKUP_DIR="${GDS_BACKUP_DIR:-/var/backups/gds}"
+
+# Database to dump: explicit GDS_BACKUP_DB wins, else the instance's own branding
+# pack names it — the SAME resolver as migrate.sh / pool.js, so the backup and the
+# running server can never disagree (task 1002757). Deliberately NO literal
+# fallback: this script ships to every instance, and a baked-in founder default
+# aimed another instance's nightly pg_dump at a database that is not its own.
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
+DB_NAME="${GDS_BACKUP_DB:-}"
+if [ -z "$DB_NAME" ]; then
+  DB_NAME="$(node -e "process.stdout.write(require(process.argv[1]).instanceDbName() || '')" "$REPO_ROOT/src/bongos/pool" 2>/dev/null || true)"
+fi
+if [ -z "$DB_NAME" ]; then
+  echo "db-backup: cannot determine which database to dump. Set GDS_BACKUP_DB, or give this instance a db.database in config/branding.json. Refusing to guess (task 1002757)." >&2
+  exit 2
+fi
+
+# Retention in days. Files older than this (by mtime) are deleted at the
+# end of each successful run. 30 days = 30 daily dumps retained.
+RETENTION_DAYS="${GDS_BACKUP_RETENTION_DAYS:-30}"
+
+# Timestamp format. UTC so dumps sort sanely regardless of TZ.
+TS=$(date -u +"%Y-%m-%dT%H%M%SZ")
+OUT="${BACKUP_DIR}/${DB_NAME}-${TS}.sql.gz"
+# Exactly the name above. Every project's unit shares BACKUP_DIR, so a looser
+# `${DB_NAME}-*` would let `test` prune and count `test-nk`'s dumps (task 1003948).
+D='[0-9]'
+DUMP_GLOB="${DB_NAME}-${D}${D}${D}${D}-${D}${D}-${D}${D}T${D}${D}${D}${D}${D}${D}Z.sql.gz"
+
+# ---- preconditions ----------------------------------------------------------
+
+if [ ! -d "$BACKUP_DIR" ]; then
+  echo "ERROR: backup dir $BACKUP_DIR does not exist." >&2
+  echo "       Create it with:" >&2
+  echo "         sudo install -d -o $(id -un) -g $(id -gn) -m 0750 $BACKUP_DIR" >&2
+  exit 2
+fi
+
+if [ ! -w "$BACKUP_DIR" ]; then
+  echo "ERROR: backup dir $BACKUP_DIR is not writable by $(whoami)." >&2
+  exit 2
+fi
+
+# Confirm pg_dump is reachable. Cheap fail-loudly so a missing pg or a
+# misconfigured PATH doesn't silently skip a night.
+if ! command -v pg_dump >/dev/null 2>&1; then
+  echo "ERROR: pg_dump not on PATH." >&2
+  exit 2
+fi
+
+# ---- the dump ---------------------------------------------------------------
+
+echo "[$(date -u +%FT%TZ)] gds-db-backup: dumping ${DB_NAME} → ${OUT}"
+
+# -Fc would give a custom-format dump (recommended for selective restore),
+# but plain SQL + gzip is easier for an Archon to eyeball and easier for the
+# restore-drill to verify.
+#
+# We dump everything with --clean so a restore into a fresh DB drops
+# existing objects first. --if-exists tolerates the no-existing-objects
+# case. --no-owner + --no-privileges keep the dump portable across role
+# names (useful if we ever restore into staging under a different user).
+#
+# --exclude-table-data=search_chunks (#973 review): search_chunks (migration
+# 103, the /recall lexical index) is REGENERABLE derived data — chunked copies
+# of DB prose (already dumped as tasks/learnings/… rows) and repo markdown
+# (already in git). Dumping its rows roughly duplicates the prose corpus and
+# was the dump's dominant growth vector as docs accumulated. Excluding only its
+# DATA keeps the CREATE TABLE + GIN indexes in the dump, so a restore recreates
+# the table empty and the index rebuilds from source: search-ingest-md.js on the
+# next deploy (doc chunks) + search-ingest-db.js nightly (DB prose) — or run both
+# by hand for immediate recall (see docs/recipes/gds-db-backup.md restore drill).
+# The tsv column is GENERATED (never dumped) and the GIN indexes are CREATE-INDEX
+# statements (not data), so the only thing dropped is the duplicated body text.
+#
+# stderr is deliberately NOT filtered (incident 2026-07-02, task 1711). A prior
+# `--verbose 2> >(grep -v "^pg_dump:" >&2)` threw away pg_dump's OWN error lines
+# too (they are `pg_dump: error:`-prefixed) — so a `permission denied` on one
+# orphaned table (credit_log_backup_task1317) failed EVERY dump silently for 12
+# days. Let pg_dump's stderr flow straight to journald; `set -o pipefail` still
+# aborts on a real failure, now visibly in `systemctl status` /
+# `journalctl -u gds-db-backup.service`.
+pg_dump \
+  --dbname="$DB_NAME" \
+  --format=plain \
+  --clean \
+  --if-exists \
+  --no-owner \
+  --no-privileges \
+  --exclude-table-data=search_chunks \
+  | gzip -9 > "$OUT.partial"
+
+# Atomic rename — only flip to the real filename once gzip finishes.
+# A SIGKILL mid-stream leaves a .partial that the next run will overwrite.
+mv "$OUT.partial" "$OUT"
+
+# Size in bytes via POSIX `wc -c`, not `stat -c '%s'`: that is GNU-only, and BSD/macOS
+# stat refuses it outright, so every run on a Mac died here (task 1004170). tr drops the
+# padding BSD wc prints in front of the number.
+BYTES=$(wc -c < "$OUT" | tr -d '[:space:]')
+echo "[$(date -u +%FT%TZ)] gds-db-backup: wrote ${OUT} (${BYTES} bytes)"
+
+# Completeness guard (incident 2026-07-02, task 1711; task 1003925). pipefail already
+# aborts on a pg_dump error; this is belt-and-suspenders against a dump cut short. A
+# plain-format dump always closes with pg_dump's own "dump complete" comment (patched
+# servers append a \unrestrict line after it), whatever its size. The fixed 1 MiB floor
+# this replaces was sized for the platform database and failed — and deleted — every
+# valid dump of a small project (~36 KB gzipped). Fail loudly and delete a broken dump
+# so it can never masquerade as a good one.
+DUMP_TAIL=$(gzip -dc "$OUT" 2>/dev/null | tail -n 10) || DUMP_TAIL=''
+FAIL_REASON=''
+case "$DUMP_TAIL" in
+  *'-- PostgreSQL database dump complete'*) ;;
+  *) FAIL_REASON="(${BYTES} bytes) is incomplete — no pg_dump completion line" ;;
+esac
+
+# Optional operator floor. Unset means none: no single size fits every instance.
+MIN_BYTES="${GDS_BACKUP_MIN_BYTES:-0}"
+if [ -z "$FAIL_REASON" ] && [ "$BYTES" -lt "$MIN_BYTES" ]; then
+  FAIL_REASON="is only ${BYTES} bytes (< ${MIN_BYTES} floor)"
+fi
+if [ -n "$FAIL_REASON" ]; then
+  echo "[$(date -u +%FT%TZ)] gds-db-backup: ERROR latest dump ${FAIL_REASON} — treating as FAILED." >&2
+  rm -f "$OUT"
+  exit 1
+fi
+
+# Size-threshold guard (#973 review). With search_chunks data excluded above the
+# dump stays sub-MB, so a gzipped dump past this threshold means something else
+# grew unexpectedly (a new large table, or the exclusion silently stopped
+# matching after a rename). WARN loudly to journald — never fail the backup, just
+# flag it for a review. Generous default (25 MiB gzipped vs the current sub-1 MB);
+# tune via GDS_BACKUP_WARN_BYTES once real post-exclusion sizes are observed.
+WARN_BYTES="${GDS_BACKUP_WARN_BYTES:-26214400}"
+if [ "$BYTES" -gt "$WARN_BYTES" ]; then
+  echo "[$(date -u +%FT%TZ)] gds-db-backup: WARN latest dump is ${BYTES} bytes (> ${WARN_BYTES} threshold) — review what grew; is it a regenerable table excludable like search_chunks?" >&2
+fi
+
+# ---- bug attachments (task 1003918) -----------------------------------------
+
+# The DB dump above is not the whole state. Re-hosted #bugs screenshots live on
+# disk under var/bug-attachments, outside Postgres and outside git, and they are
+# the one artefact here that CANNOT be regenerated: the Discord CDN URL they were
+# copied from is signed and expires in about a day, so our copy IS the image.
+#
+# Nothing backed them up. All five this instance ever stored are gone — the
+# 2026-07-05 droplet move carried the database and not var/ — and every task
+# description embedding one now renders a broken image in the hall.
+#
+# Tarred beside the dump, same timestamp, same retention, so a restore drill that
+# picks a date gets the database and the images that go with it. Best-effort by
+# design: a missing or empty attachments dir is the ordinary case on a fresh
+# instance and must never fail a night of database backup.
+ATTACH_DIR="${BUG_ATTACHMENTS_DIR:-$REPO_ROOT/var/bug-attachments}"
+ATTACH_OUT="${BACKUP_DIR}/${DB_NAME}-attachments-${TS}.tar.gz"
+ATTACH_GLOB="${DB_NAME}-attachments-${D}${D}${D}${D}-${D}${D}-${D}${D}T${D}${D}${D}${D}${D}${D}Z.tar.gz"
+
+if [ -d "$ATTACH_DIR" ]; then
+  # Every count here goes through tr for the same reason as BYTES: BSD wc pads it, which
+  # put "pruned        1 dump(s)" in a Mac's log where Linux writes "pruned 1 dump(s)".
+  ATTACH_COUNT=$(find "$ATTACH_DIR" -maxdepth 1 -type f | wc -l | tr -d '[:space:]')
+  if [ "$ATTACH_COUNT" -gt 0 ]; then
+    if tar -czf "$ATTACH_OUT.partial" -C "$ATTACH_DIR" . 2>/dev/null; then
+      mv "$ATTACH_OUT.partial" "$ATTACH_OUT"
+      echo "[$(date -u +%FT%TZ)] gds-db-backup: wrote ${ATTACH_OUT} (${ATTACH_COUNT} attachment(s))"
+    else
+      rm -f "$ATTACH_OUT.partial"
+      echo "[$(date -u +%FT%TZ)] gds-db-backup: WARN could not archive ${ATTACH_DIR} — database dump is unaffected" >&2
+    fi
+  else
+    echo "[$(date -u +%FT%TZ)] gds-db-backup: no bug attachments to archive"
+  fi
+else
+  echo "[$(date -u +%FT%TZ)] gds-db-backup: no bug-attachment dir at ${ATTACH_DIR} — nothing to archive"
+fi
+
+# ---- retention prune --------------------------------------------------------
+
+# Delete dumps older than RETENTION_DAYS. -mtime +N means strictly older
+# than N days (so RETENTION_DAYS=30 keeps the most recent ~30 dumps).
+# Done AFTER the new dump lands so a streak of failed runs can't erase
+# everything: we never delete on a day we didn't successfully dump.
+PRUNED=$(find "$BACKUP_DIR" -maxdepth 1 -type f \
+    \( -name "$DUMP_GLOB" -o -name "$ATTACH_GLOB" \) \
+    -mtime +"${RETENTION_DAYS}" \
+    -print -delete | wc -l | tr -d '[:space:]')
+
+if [ "$PRUNED" -gt 0 ]; then
+  echo "[$(date -u +%FT%TZ)] gds-db-backup: pruned ${PRUNED} dump(s) older than ${RETENTION_DAYS}d"
+fi
+
+# ---- emit a one-line summary for journald + monitoring ---------------------
+
+REMAINING=$(find "$BACKUP_DIR" -maxdepth 1 -type f -name "$DUMP_GLOB" | wc -l | tr -d '[:space:]')
+echo "[$(date -u +%FT%TZ)] gds-db-backup: ok · ${REMAINING} dump(s) retained · ${BYTES} bytes latest"

package/scripts/bongos/db-backup-offsite.sh

@@ -0,0 +1,267 @@
+#!/bin/bash
+# Weekly OFFSITE archive of the GDS Postgres dumps.
+#
+# Filed by #415 (V3.R62-offsite) — closes the whole-droplet-loss gap that
+# the local nightly backup (#412, scripts/bongos/db-backup-nightly.sh) leaves
+# open. This script does NOT take its own pg_dump: it copies the most
+# recent local dump produced by db-backup-nightly.sh up to a DigitalOcean
+# Space (S3-compatible object storage, off the droplet's disk).
+#
+# Vendor decision: DigitalOcean Spaces over Backblaze B2 — see ADR 0025.
+#
+# Runs as the unit's User= (the plane's app account) via the gds-db-offsite.timer systemd unit (see
+# scripts/bongos/gds-db-offsite.service + .timer). Weekly cadence.
+#
+# Authentication: S3-style credentials for the Space live OUTSIDE the repo
+# in /etc/<configDir>/spaces.env (chmod 600, owned by the service user), per
+# ADR 0022. The systemd unit injects them via EnvironmentFile; a manual run
+# sources the same file. Required vars:
+#   SPACES_KEY SPACES_SECRET SPACES_BUCKET SPACES_REGION SPACES_ENDPOINT
+#   GDS_OFFSITE_AGE_RECIPIENT
+#
+# ENCRYPTION IS MANDATORY (task 1003372, audit ref A1). Until that task this
+# script uploaded the raw pg_dump to the Space. A database dump is not "a backup"
+# from a security standpoint — it is every secret the platform holds, in one
+# file, on someone else's disk: OAuth credentials, the GitHub App private key,
+# and (before core_249) every builder's live session bearer in plaintext. A
+# breach of the Space, a mis-set bucket ACL, or a leaked read-only S3 key was a
+# total compromise of the instance with no trace in any log the platform keeps.
+#
+# The dump is encrypted to a PUBLIC key (`age`), so this box can produce backups
+# it cannot itself read. That asymmetry is the point: an attacker who owns the
+# droplet gets the live database anyway, but they do not also get the archive of
+# every previous state, and they cannot decrypt what is already in the Space.
+# Keep the private key OFF this machine.
+#
+# One-time setup, on a machine that is NOT the server:
+#   age-keygen -o bongos-backup.key      # store this somewhere safe and offline
+#   # it prints:  Public key: age1xxxxxxxx...
+# Then on the server, add to $ENV_FILE:
+#   GDS_OFFSITE_AGE_RECIPIENT=age1xxxxxxxx...
+#
+# To restore:
+#   age -d -i bongos-backup.key <dump>.sql.gz.age | gunzip | psql <db>
+#
+# WITHOUT A RECIPIENT THIS SCRIPT REFUSES TO RUN. It does not fall back to a
+# plaintext upload: that fallback is the vulnerability, and a silent one, since
+# a backup that "worked" looks identical either way. A failing timer is visible
+# in `systemctl status gds-db-offsite.timer`; an unencrypted dump in a bucket is
+# visible to nobody until it matters.
+#
+# Manual one-shot:
+#   <instance-root>/scripts/bongos/db-backup-offsite.sh
+#
+# Systemd:
+#   sudo systemctl start gds-db-offsite.service   # one-shot
+#   sudo systemctl status gds-db-offsite.timer    # check schedule
+#   journalctl -u gds-db-offsite.service          # see history
+
+set -euo pipefail
+
+# ---- configuration ----------------------------------------------------------
+
+# Where the local nightly dumps land (must match db-backup-nightly.sh).
+BACKUP_DIR="${GDS_BACKUP_DIR:-/var/backups/gds}"
+# Database name: explicit GDS_BACKUP_DB wins, else the instance's branding pack —
+# the same resolver + no-literal-fallback posture as db-backup-nightly.sh /
+# migrate.sh (task 1002757).
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
+DB_NAME="${GDS_BACKUP_DB:-}"
+if [ -z "$DB_NAME" ]; then
+  DB_NAME="$(node -e "process.stdout.write(require(process.argv[1]).instanceDbName() || '')" "$REPO_ROOT/src/bongos/pool" 2>/dev/null || true)"
+fi
+if [ -z "$DB_NAME" ]; then
+  echo "db-backup-offsite: cannot determine which database's dumps to ship. Set GDS_BACKUP_DB, or give this instance a db.database in config/branding.json. Refusing to guess (task 1002757)." >&2
+  exit 2
+fi
+
+# How many weekly offsite copies to retain in the Space before pruning the
+# oldest. 4 = ~1 month of weekly snapshots (Archon decision, #415 claim 338).
+KEEP="${GDS_OFFSITE_KEEP:-4}"
+
+# Key prefix inside the bucket. Keeps offsite dumps namespaced and lets the
+# prune logic scope its list to exactly our objects (and not, say, the dead
+# pg_dump_* objects the retired #7 cron would have written).
+PREFIX="${GDS_OFFSITE_PREFIX:-gds-offsite/}"
+
+# Credentials file. Read only if the SPACES_* vars aren't already set
+# (systemd injects them via EnvironmentFile; this covers manual runs).
+# Default path derives from the instance's config dir (task 1002757 — the old
+# founder-pinned /etc/<founder>/ default could never exist on another box);
+# empty when the resolver is unavailable, in which case the SPACES_* vars (or
+# an explicit GDS_SPACES_ENV) must carry the credentials.
+CONFIG_DIR_NAME="$(node -e "process.stdout.write(require(process.argv[1]).configDirName() || '')" "$REPO_ROOT/src/instance-config" 2>/dev/null || true)"
+ENV_FILE="${GDS_SPACES_ENV:-${CONFIG_DIR_NAME:+/etc/$CONFIG_DIR_NAME/spaces.env}}"
+
+# ---- load credentials -------------------------------------------------------
+
+# Load the env file WITHOUT executing it. We deliberately do NOT `source` it:
+# sourcing runs arbitrary shell if ENV_FILE ever points at hostile content
+# (e.g. a manual run with an attacker-controlled GDS_SPACES_ENV). Instead we
+# extract only the five expected KEY=VALUE lines. `read -r k v` with IFS='='
+# splits on the FIRST '=' only, so base64 secrets containing '/', '+' or '='
+# survive intact.
+# Read the file when EITHER the credentials or the encryption recipient is still
+# unset. Keying this on SPACES_KEY alone would skip the file on a box where
+# systemd injects the S3 vars but the recipient was added to the file later —
+# and the script would then refuse to run while the key it needs sat right there.
+if { [ -z "${SPACES_KEY:-}" ] || [ -z "${GDS_OFFSITE_AGE_RECIPIENT:-}" ]; } && [ -f "$ENV_FILE" ]; then
+  while IFS='=' read -r _k _v; do
+    case "$_k" in
+      SPACES_KEY|SPACES_SECRET|SPACES_BUCKET|SPACES_REGION|SPACES_ENDPOINT|GDS_OFFSITE_AGE_RECIPIENT)
+        export "$_k=$_v" ;;
+    esac
+  done < <(grep -E '^(SPACES_KEY|SPACES_SECRET|SPACES_BUCKET|SPACES_REGION|SPACES_ENDPOINT|GDS_OFFSITE_AGE_RECIPIENT)=' "$ENV_FILE" || true)
+fi
+
+missing=()
+for v in SPACES_KEY SPACES_SECRET SPACES_BUCKET SPACES_REGION SPACES_ENDPOINT; do
+  if [ -z "${!v:-}" ]; then missing+=("$v"); fi
+done
+if [ "${#missing[@]}" -gt 0 ]; then
+  echo "ERROR: missing Spaces credentials: ${missing[*]}" >&2
+  echo "       Set them in $ENV_FILE (chmod 600, owned by $(id -un)). See ADR 0022 + 0025." >&2
+  exit 2
+fi
+
+# The encryption recipient is as required as the credentials — a dump this box
+# can upload but nobody can decrypt is useless, and a dump it uploads in the
+# clear is the finding this refusal exists to close (task 1003372, audit A1).
+if [ -z "${GDS_OFFSITE_AGE_RECIPIENT:-}" ]; then
+  echo "ERROR: GDS_OFFSITE_AGE_RECIPIENT is not set — refusing to ship an unencrypted database dump offsite." >&2
+  echo "       A dump is every secret this platform holds, in one file, on storage the platform does not own." >&2
+  echo "" >&2
+  echo "       Fix, on a machine that is NOT this server:" >&2
+  echo "         age-keygen -o bongos-backup.key     # keep this file safe and OFFLINE — it is the only way back" >&2
+  echo "       then add the age1... public key it prints to $ENV_FILE:" >&2
+  echo "         GDS_OFFSITE_AGE_RECIPIENT=age1..." >&2
+  echo "" >&2
+  echo "       This is deliberately not optional: falling back to a plaintext upload would look identical in the logs." >&2
+  exit 2
+fi
+
+
+# ---- preconditions ----------------------------------------------------------
+
+if ! command -v age >/dev/null 2>&1; then
+  echo "ERROR: 'age' not on PATH — it is what encrypts the dump before it leaves this box." >&2
+  echo "       Install with: apt install age   (or fetch the static binary from https://github.com/FiloSottile/age/releases)" >&2
+  exit 2
+fi
+if ! command -v python3 >/dev/null 2>&1; then
+  echo "ERROR: python3 not on PATH (needed for the boto3 S3 client)." >&2
+  exit 2
+fi
+if ! python3 -c "import boto3" >/dev/null 2>&1; then
+  echo "ERROR: python3 'boto3' module not installed." >&2
+  echo "       Install with: pip3 install --user boto3   (or: apt install python3-boto3)" >&2
+  exit 2
+fi
+
+# Pick the most recent local dump. We rely on db-backup-nightly.sh having
+# run; if there are no dumps, fail loudly rather than archive nothing.
+LATEST=$(ls -1t "${BACKUP_DIR}/${DB_NAME}-"*.sql.gz 2>/dev/null | head -1 || true)
+if [ -z "$LATEST" ]; then
+  echo "ERROR: no local dumps found in ${BACKUP_DIR} (${DB_NAME}-*.sql.gz)." >&2
+  echo "       The local nightly backup (#412) must run before the offsite copy." >&2
+  exit 2
+fi
+
+BASENAME=$(basename "$LATEST")
+# Sizes via POSIX `wc -c`, never GNU-only `stat -c '%s'`, which BSD/macOS stat refuses
+# (task 1004170). tr drops the padding BSD wc prints in front of the number.
+BYTES=$(wc -c < "$LATEST" | tr -d '[:space:]')
+
+# ---- encrypt before it leaves the box ---------------------------------------
+#
+# To a temp file under the backup dir (same filesystem, so no cross-device copy
+# of plaintext through /tmp) with a 0600 umask, and removed on every exit path
+# including failure — the cleartext dump already lives beside it, so the
+# ciphertext is not sensitive in itself, but leaving files behind on a failing
+# weekly timer fills a disk nobody is watching.
+umask 077
+ENC="$(mktemp "${BACKUP_DIR}/.${BASENAME}.age.XXXXXX")"
+trap 'rm -f "$ENC"' EXIT
+
+if ! age -r "$GDS_OFFSITE_AGE_RECIPIENT" -o "$ENC" "$LATEST"; then
+  echo "ERROR: age refused to encrypt — is GDS_OFFSITE_AGE_RECIPIENT a valid age1... public key?" >&2
+  exit 2
+fi
+
+# A zero-byte or missing ciphertext would upload happily and restore to nothing.
+# Checked here rather than trusted, because the failure is silent for a month.
+ENC_BYTES=$(wc -c < "$ENC" | tr -d '[:space:]')
+if [ "$ENC_BYTES" -lt 1 ]; then
+  echo "ERROR: encryption produced an empty file — refusing to upload it as a backup." >&2
+  exit 2
+fi
+
+KEY="${PREFIX}${BASENAME}.age"
+echo "[$(date -u +%FT%TZ)] gds-db-offsite: archiving ${LATEST} (${BYTES} bytes → ${ENC_BYTES} encrypted) → s3://${SPACES_BUCKET}/${KEY}"
+
+# ---- upload + prune (single boto3 session) ----------------------------------
+
+GDS_OFFSITE_LOCAL="$ENC" \
+GDS_OFFSITE_KEY="$KEY" \
+GDS_OFFSITE_KEEP="$KEEP" \
+GDS_OFFSITE_PREFIX="$PREFIX" \
+python3 - <<'PYEOF'
+import os, sys
+import boto3
+
+key      = os.environ["SPACES_KEY"]
+secret   = os.environ["SPACES_SECRET"]
+bucket   = os.environ["SPACES_BUCKET"]
+region   = os.environ["SPACES_REGION"]
+endpoint = os.environ["SPACES_ENDPOINT"]
+
+local    = os.environ["GDS_OFFSITE_LOCAL"]
+objkey   = os.environ["GDS_OFFSITE_KEY"]
+prefix   = os.environ["GDS_OFFSITE_PREFIX"]
+keep     = int(os.environ["GDS_OFFSITE_KEEP"])
+
+s3 = boto3.client(
+    "s3",
+    region_name=region,
+    endpoint_url=endpoint,
+    aws_access_key_id=key,
+    aws_secret_access_key=secret,
+)
+
+# Upload. Idempotent: re-running on the same day overwrites the same key.
+s3.upload_file(local, bucket, objkey)
+print(f"[gds-db-offsite] uploaded {objkey}")
+
+resp = s3.list_objects_v2(Bucket=bucket, Prefix=prefix)
+contents = resp.get("Contents", [])
+
+# PURGE THE PLAINTEXT ALREADY UP THERE (task 1003372, audit A1). Every `.sql.gz`
+# under this prefix was uploaded unencrypted by an earlier version of this
+# script, and each one is a live copy of every secret the platform held on the
+# day it was taken — including, before core_249, directly replayable session
+# bearers. Encrypting from now on does nothing about them. They are deleted
+# OUTSIDE the retention count, and unconditionally: retaining four of them
+# "for safety" would retain the finding. The local nightly dumps on the droplet
+# are untouched, so this removes copies, never the last copy.
+legacy = [o for o in contents if o["Key"].endswith(".sql.gz")]
+for obj in legacy:
+    s3.delete_object(Bucket=bucket, Key=obj["Key"])
+    print(f"[gds-db-offsite] PURGED unencrypted legacy dump {obj['Key']}")
+
+# Prune: keep the newest `keep` ENCRYPTED objects under our prefix.
+objects = sorted(
+    [o for o in contents if o["Key"].endswith(".sql.gz.age")],
+    key=lambda o: o["LastModified"],
+)
+to_delete = objects[:-keep] if len(objects) > keep else []
+for obj in to_delete:
+    s3.delete_object(Bucket=bucket, Key=obj["Key"])
+    print(f"[gds-db-offsite] pruned {obj['Key']}")
+
+kept = min(len(objects), keep)
+print(f"[gds-db-offsite] ok · {kept} offsite dump(s) retained · pruned {len(to_delete)}"
+      + (f" · purged {len(legacy)} unencrypted legacy dump(s)" if legacy else ""))
+PYEOF
+
+echo "[$(date -u +%FT%TZ)] gds-db-offsite: done"

package/scripts/bongos/demote.js

@@ -0,0 +1,45 @@
+#!/usr/bin/env node
+// Builder demote — the inverse of promote: move a task ready -> backlog.
+//
+// Usage:
+//   node scripts/bongos/demote.js <task-id>
+//
+// Use case: a dependency-gated task that was wrongly promoted to `ready` needs to
+// go back to `backlog` (so it stops appearing as claimable) without hand-SQL on
+// prod. Strictly ready -> backlog — the server refuses any other status (a
+// claimed/active task, or a later lifecycle state). Metic+ only (mirrors promote).
+
+const { cliClient, errCode, errDetails, requireSession } = require('./cli-lib');
+
+(async () => {
+  await requireSession();
+  const args = process.argv.slice(2);
+  const taskId = Number(args[0]);
+  if (!Number.isFinite(taskId)) {
+    console.error('usage: node scripts/bongos/demote.js <task-id>');
+    process.exit(2);
+  }
+
+  const api = await cliClient();
+  const r = await api.tasks.postTasksIdDemote({ id: taskId });
+  if (!r.ok) {
+    // ADR 0116 nests BOTH the slug and its payload — see the note in cli-lib errCode
+    // (task 1003658). current_status moved to error.details with the code, so reading
+    // it off the top level printed 'undefined' even once the branch was reachable.
+    const code = errCode(r.data);
+    const details = errDetails(r.data);
+    if (code === 'cannot_demote') {
+      console.error(`Cannot demote task #${taskId}: it is '${details.current_status}', not 'ready'.`);
+      console.error("demote only moves a task ready -> backlog (the inverse of promote).");
+    } else if (code === 'task_not_found') {
+      console.error(`No task #${taskId}.`);
+    } else {
+      console.error(`demote failed (${r.status}):`, r.data);
+    }
+    process.exit(1);
+  }
+  console.log(`Demoted task #${taskId}: status returned to 'backlog'.`);
+})().catch((err) => {
+  console.error('fatal:', err);
+  process.exit(1);
+});