@clearance/management 0.1.4 → 0.2.1

package/dist/ops/deploy/compose/docker-compose.production.yml

@@ -0,0 +1,129 @@
+# Production overlay for Clearance.
+#
+# Usage (after validating secrets):
+#   scripts/validate-production-env.sh
+#   docker compose -f docker-compose.yml -f deploy/compose/docker-compose.production.yml config
+#   docker compose -f docker-compose.yml -f deploy/compose/docker-compose.production.yml up -d
+#
+# Fail-closed rules:
+# - NODE_ENV is forced to production
+# - DATABASE_URL must be a complete, operator-supplied connection string
+#   (no shell construction of password-bearing URLs — avoids @:/#? interpolation breakage)
+# - Every secret is required with no defaults
+# - Postgres is NOT published to the host by default (ports reset)
+# - Credential encryption key + key id are mandatory
+# - No VAR-with-default secret fallbacks (only fail-closed ${VAR:?...} forms)
+services:
+  postgres:
+    environment:
+      POSTGRES_USER: ${CLEARANCE_DB_USER:?set CLEARANCE_DB_USER}
+      POSTGRES_PASSWORD: ${CLEARANCE_DB_PASSWORD:?set CLEARANCE_DB_PASSWORD}
+      POSTGRES_DB: ${CLEARANCE_DB_NAME:?set CLEARANCE_DB_NAME}
+    # Production default: internal network only. Host publish requires a local override file.
+    ports: !reset []
+    volumes:
+      - clearance_pg:/var/lib/postgresql/data
+    healthcheck:
+      # $$ defers expansion to the container shell (avoids host-side compose interpolation)
+      test: ["CMD-SHELL", "pg_isready -U \"$$POSTGRES_USER\" -d \"$$POSTGRES_DB\""]
+      interval: 5s
+      timeout: 5s
+      retries: 12
+    restart: always
+    stop_grace_period: 30s
+
+  api:
+    build: !reset null
+    image: "${CLEARANCE_IMAGE_REPOSITORY:?set CLEARANCE_IMAGE_REPOSITORY}@${CLEARANCE_IMAGE_DIGEST:?set signed CLEARANCE_IMAGE_DIGEST}"
+    environment:
+      NODE_ENV: production
+      CLEARANCE_STRICT_SECRETS: "1"
+      CLEARANCE_API_PORT: "3200"
+      CLEARANCE_OPERATOR_TOKEN: ${CLEARANCE_OPERATOR_TOKEN:?set CLEARANCE_OPERATOR_TOKEN}
+      CLEARANCE_SECRET: ${CLEARANCE_SECRET:?set CLEARANCE_SECRET}
+      CLEARANCE_CREDENTIAL_KEY: ${CLEARANCE_CREDENTIAL_KEY:?set CLEARANCE_CREDENTIAL_KEY}
+      CLEARANCE_CREDENTIAL_KEY_ID: ${CLEARANCE_CREDENTIAL_KEY_ID:?set CLEARANCE_CREDENTIAL_KEY_ID}
+      CLEARANCE_BASE_URL: ${CLEARANCE_BASE_URL:?set CLEARANCE_BASE_URL}
+      CLEARANCE_CONSOLE_URL: ${CLEARANCE_CONSOLE_URL:?set CLEARANCE_CONSOLE_URL}
+      CLEARANCE_CORS_ORIGINS: ${CLEARANCE_CORS_ORIGINS:?set CLEARANCE_CORS_ORIGINS}
+      # Full connection string required — do not embed raw passwords via compose interpolation.
+      DATABASE_URL: ${DATABASE_URL:?set DATABASE_URL to a full postgres URL with percent-encoded password}
+      # Fail-closed override of the dev base file: this profile publishes the
+      # API port, so x-forwarded-for from arbitrary clients must NOT be
+      # trusted for rate-limit keying. Hardcoded (this overlay forbids
+      # ${VAR:-default} interpolation); edit this line to "1" only if your
+      # deployment makes the API reachable EXCLUSIVELY via trusted proxies
+      # (e.g. the console BFF or a fronting load balancer).
+      CLEARANCE_TRUSTED_PROXY: "0"
+    # Replace the base loopback mapping instead of merging a second binding.
+    ports: !override
+      - "127.0.0.1:${CLEARANCE_API_PORT:?set CLEARANCE_API_PORT}:3200"
+    restart: always
+    stop_grace_period: 30s
+
+  console:
+    build: !reset null
+    image: "${CLEARANCE_IMAGE_REPOSITORY:?set CLEARANCE_IMAGE_REPOSITORY}@${CLEARANCE_IMAGE_DIGEST:?set signed CLEARANCE_IMAGE_DIGEST}"
+    environment:
+      NODE_ENV: production
+      CLEARANCE_STRICT_SECRETS: "1"
+      CLEARANCE_CONSOLE_PORT: "3100"
+      CLEARANCE_API_URL: http://api:3200
+      CLEARANCE_OPERATOR_TOKEN: ${CLEARANCE_OPERATOR_TOKEN:?set CLEARANCE_OPERATOR_TOKEN}
+      CLEARANCE_CONSOLE_ADMIN_USER: ${CLEARANCE_CONSOLE_ADMIN_USER:?set CLEARANCE_CONSOLE_ADMIN_USER}
+      CLEARANCE_CONSOLE_ADMIN_PASSWORD: ${CLEARANCE_CONSOLE_ADMIN_PASSWORD:?set CLEARANCE_CONSOLE_ADMIN_PASSWORD}
+      CLEARANCE_CONSOLE_SESSION_SECRET: ${CLEARANCE_CONSOLE_SESSION_SECRET:?set CLEARANCE_CONSOLE_SESSION_SECRET}
+    # Replace the base loopback mapping instead of merging a second binding.
+    ports: !override
+      - "127.0.0.1:${CLEARANCE_CONSOLE_PORT:?set CLEARANCE_CONSOLE_PORT}:3100"
+    restart: always
+    stop_grace_period: 30s
+
+  sample-b2b:
+    build: !reset null
+    image: "${CLEARANCE_IMAGE_REPOSITORY:?set CLEARANCE_IMAGE_REPOSITORY}@${CLEARANCE_IMAGE_DIGEST:?set signed CLEARANCE_IMAGE_DIGEST}"
+    environment:
+      NODE_ENV: production
+      CLEARANCE_STRICT_SECRETS: "1"
+      SAMPLE_APP_PORT: "3000"
+      CLEARANCE_SECRET: ${CLEARANCE_SECRET:?set CLEARANCE_SECRET}
+      CLEARANCE_BASE_URL: ${CLEARANCE_BASE_URL:?set CLEARANCE_BASE_URL}
+      # Optional social providers. The application fails startup on incomplete pairs.
+      CLEARANCE_GITHUB_CLIENT_ID: ${CLEARANCE_GITHUB_CLIENT_ID-}
+      CLEARANCE_GITHUB_CLIENT_SECRET: ${CLEARANCE_GITHUB_CLIENT_SECRET-}
+      CLEARANCE_GOOGLE_CLIENT_ID: ${CLEARANCE_GOOGLE_CLIENT_ID-}
+      CLEARANCE_GOOGLE_CLIENT_SECRET: ${CLEARANCE_GOOGLE_CLIENT_SECRET-}
+      DATABASE_URL: ${DATABASE_URL:?set DATABASE_URL to a full postgres URL with percent-encoded password}
+    # Replace the base loopback mapping instead of merging a second binding.
+    ports: !override
+      - "127.0.0.1:${CLEARANCE_SAMPLE_PORT:?set CLEARANCE_SAMPLE_PORT}:3000"
+    restart: always
+
+  # One-shot scheduled job. Invoke from cron/systemd:
+  # docker compose ... --profile backup run --rm backup
+  # The script fails closed unless CLEARANCE_BACKUP_COPY_COMMAND succeeds.
+  backup:
+    profiles: ["backup"]
+    build: !reset null
+    image: "${CLEARANCE_BACKUP_IMAGE_REPOSITORY:?set CLEARANCE_BACKUP_IMAGE_REPOSITORY}@${CLEARANCE_BACKUP_IMAGE_DIGEST:?set signed CLEARANCE_BACKUP_IMAGE_DIGEST}"
+    command: ["bash", "scripts/backup-scheduled.sh", "--dir", "/backups"]
+    environment:
+      DATABASE_URL: ${DATABASE_URL:?set DATABASE_URL to a full postgres URL with percent-encoded password}
+      CLEARANCE_BACKUP_COPY_COMMAND: ${CLEARANCE_BACKUP_COPY_COMMAND-}
+      CLEARANCE_BACKUP_RETENTION_DAYS: ${CLEARANCE_BACKUP_RETENTION_DAYS-30}
+      CLEARANCE_BACKUP_RESTORE_VERIFY: "1"
+    volumes:
+      - clearance_backups:/backups
+    tmpfs:
+      - /tmp
+    read_only: true
+    depends_on:
+      postgres:
+        condition: service_healthy
+    restart: "no"
+
+volumes:
+  clearance_pg:
+    name: ${CLEARANCE_PG_VOLUME:?set CLEARANCE_PG_VOLUME to an explicit volume name}
+  clearance_backups:
+    name: ${CLEARANCE_BACKUP_VOLUME:?set CLEARANCE_BACKUP_VOLUME to an explicit volume name}

package/dist/ops/deploy/upgrades/README.md

@@ -0,0 +1,24 @@
+# Upgrade step hooks
+
+`scripts/upgrade-apply.sh` requires a hook:
+
+```text
+deploy/upgrades/steps/<targetVersion>/apply.sh
+```
+
+The currently supported transition is `0.1.4` to `0.2.1`. Its shipped hook is
+`deploy/upgrades/steps/0.2.1/apply.sh`; it verifies the release marker is at
+`0.1.4`, advances it to `0.2.1`, and verifies the result. The Clearance CLI
+packages this hook under `dist/ops/deploy/upgrades/steps/0.2.1/apply.sh`.
+
+## Contract
+
+- Invoked only after preflight and a **verified** Postgres backup (with isolated restore verification).
+- Args: `--plan PATH --from CURRENT --to TARGET`
+- Exit non-zero to fail closed (no silent partial apply).
+- Must not drop or overwrite the active database; destructive changes require explicit operator runbooks.
+- Rollback reference is always the verified backup id recorded in the plan state sidecar.
+
+Without a hook, apply fails after creating and verifying its rollback backup. It never records a no-op version transition as applied.
+
+`scripts/upgrade-rollback.sh` is a rollback **drill**: it verifies the deterministic backup reference through an isolated restore and leaves the active environment unchanged. An actual rollback remains an operator-runbook action until active restore automation is designed and tested separately.

package/dist/ops/deploy/upgrades/steps/0.2.0/apply.sh

@@ -0,0 +1,66 @@
+#!/usr/bin/env bash
+# Shipped 0.1.4 -> 0.2.0 database transition.
+set -Eeuo pipefail
+
+ROOT="$(cd "$(dirname "$0")/../../../.." && pwd)"
+# shellcheck source=scripts/lib/ops-common.sh
+source "$ROOT/scripts/lib/ops-common.sh"
+
+PLAN=""
+FROM=""
+TO=""
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --plan) PLAN="$2"; shift 2 ;;
+    --from) FROM="$2"; shift 2 ;;
+    --to) TO="$2"; shift 2 ;;
+    *) die "unknown upgrade hook argument: $1" ;;
+  esac
+done
+
+[[ -f "$PLAN" ]] || die "upgrade plan is missing"
+[[ "$FROM" == "0.1.4" && "$TO" == "0.2.0" ]] \
+  || die "this hook only supports 0.1.4 to 0.2.0"
+
+require_pg_client
+require_cmd node
+URL="$(resolve_database_url)"
+PLAN_ID="$(node -e 'const j=require(process.argv[1]); process.stdout.write(j.planId)' "$PLAN")"
+PLAN_SHA="$(sha256_file "$PLAN")"
+[[ "$PLAN_ID" =~ ^upg_[0-9TZ]+_[a-f0-9]+$ ]] || die "unsafe plan id"
+require_application_release "$URL" "$FROM"
+
+# This is the application-owned migration contract: the durable management
+# snapshot used by the running API advances atomically with an append-only
+# migration ledger. It cannot pass against the old test-only ops marker.
+psql_q "$URL" \
+  -v from_version="$FROM" -v to_version="$TO" \
+  -v plan_id="$PLAN_ID" -v plan_sha="$PLAN_SHA" <<'SQL' >/dev/null
+BEGIN;
+CREATE TABLE IF NOT EXISTS clearance_schema_migrations (
+  version text PRIMARY KEY,
+  applied_at timestamptz NOT NULL DEFAULT now(),
+  plan_id text NOT NULL,
+  plan_sha256 text NOT NULL,
+  from_version text
+);
+INSERT INTO clearance_schema_migrations(version, plan_id, plan_sha256, from_version)
+VALUES (:'from_version', 'application-baseline', :'plan_sha', NULL)
+ON CONFLICT (version) DO NOTHING;
+WITH advanced AS (
+  UPDATE clearance_management_snapshot
+  SET data = jsonb_set(data, '{releaseVersion}', to_jsonb(:'to_version'::text), false),
+      revision = revision + 1,
+      updated_at = now()
+  WHERE id = 1 AND data->>'releaseVersion' = :'from_version'
+  RETURNING 1
+)
+INSERT INTO clearance_schema_migrations(version, plan_id, plan_sha256, from_version)
+SELECT :'to_version', :'plan_id', :'plan_sha', :'from_version'
+FROM advanced;
+COMMIT;
+SQL
+
+require_application_release "$URL" "$TO"
+ledger_plan="$(psql_q "$URL" -At -c "SELECT plan_id FROM clearance_schema_migrations WHERE version = '${TO}'")"
+[[ "$ledger_plan" == "$PLAN_ID" ]] || die "migration ledger did not record plan $PLAN_ID"

package/dist/ops/deploy/upgrades/steps/0.2.1/apply.sh

@@ -0,0 +1,66 @@
+#!/usr/bin/env bash
+# Shipped 0.1.4 -> 0.2.1 database transition.
+set -Eeuo pipefail
+
+ROOT="$(cd "$(dirname "$0")/../../../.." && pwd)"
+# shellcheck source=scripts/lib/ops-common.sh
+source "$ROOT/scripts/lib/ops-common.sh"
+
+PLAN=""
+FROM=""
+TO=""
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --plan) PLAN="$2"; shift 2 ;;
+    --from) FROM="$2"; shift 2 ;;
+    --to) TO="$2"; shift 2 ;;
+    *) die "unknown upgrade hook argument: $1" ;;
+  esac
+done
+
+[[ -f "$PLAN" ]] || die "upgrade plan is missing"
+[[ "$FROM" == "0.1.4" && "$TO" == "0.2.1" ]] \
+  || die "this hook only supports 0.1.4 to 0.2.1"
+
+require_pg_client
+require_cmd node
+URL="$(resolve_database_url)"
+PLAN_ID="$(node -e 'const j=require(process.argv[1]); process.stdout.write(j.planId)' "$PLAN")"
+PLAN_SHA="$(sha256_file "$PLAN")"
+[[ "$PLAN_ID" =~ ^upg_[0-9TZ]+_[a-f0-9]+$ ]] || die "unsafe plan id"
+require_application_release "$URL" "$FROM"
+
+# This is the application-owned migration contract: the durable management
+# snapshot used by the running API advances atomically with an append-only
+# migration ledger. It cannot pass against the old test-only ops marker.
+psql_q "$URL" \
+  -v from_version="$FROM" -v to_version="$TO" \
+  -v plan_id="$PLAN_ID" -v plan_sha="$PLAN_SHA" <<'SQL' >/dev/null
+BEGIN;
+CREATE TABLE IF NOT EXISTS clearance_schema_migrations (
+  version text PRIMARY KEY,
+  applied_at timestamptz NOT NULL DEFAULT now(),
+  plan_id text NOT NULL,
+  plan_sha256 text NOT NULL,
+  from_version text
+);
+INSERT INTO clearance_schema_migrations(version, plan_id, plan_sha256, from_version)
+VALUES (:'from_version', 'application-baseline', :'plan_sha', NULL)
+ON CONFLICT (version) DO NOTHING;
+WITH advanced AS (
+  UPDATE clearance_management_snapshot
+  SET data = jsonb_set(data, '{releaseVersion}', to_jsonb(:'to_version'::text), false),
+      revision = revision + 1,
+      updated_at = now()
+  WHERE id = 1 AND data->>'releaseVersion' = :'from_version'
+  RETURNING 1
+)
+INSERT INTO clearance_schema_migrations(version, plan_id, plan_sha256, from_version)
+SELECT :'to_version', :'plan_id', :'plan_sha', :'from_version'
+FROM advanced;
+COMMIT;
+SQL
+
+require_application_release "$URL" "$TO"
+ledger_plan="$(psql_q "$URL" -At -c "SELECT plan_id FROM clearance_schema_migrations WHERE version = '${TO}'")"
+[[ "$ledger_plan" == "$PLAN_ID" ]] || die "migration ledger did not record plan $PLAN_ID"

package/dist/ops/scripts/backup-create.sh

@@ -0,0 +1,197 @@
+#!/usr/bin/env bash
+# Create a real Postgres backup (pg_dump) with checksum + metadata.
+# Never prints credentials. Never overwrites without a new backup id.
+set -Eeuo pipefail
+
+ROOT="$(cd "$(dirname "$0")/.." && pwd)"
+# shellcheck source=lib/ops-common.sh
+source "$ROOT/scripts/lib/ops-common.sh"
+
+# Dumps contain production data. Make every created directory/artifact owner-only.
+umask 077
+
+usage() {
+  cat <<'EOF'
+Usage: backup-create.sh [--dir DIR]
+
+Environment:
+  DATABASE_URL   Required. Postgres connection string for the active database.
+
+Writes:
+  DIR/<id>.sql           plain-format pg_dump
+  DIR/<id>.meta.json     checksum, counts, timestamps, source db name
+  DIR/<id>.verified.json read-only evidence binding the dump and metadata
+  Prints backup id on stdout (last line: BACKUP_ID=<id>)
+EOF
+}
+
+BACKUP_DIR="${CLEARANCE_BACKUP_DIR:-$ROOT/.clearance/backups}"
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --dir) BACKUP_DIR="$2"; shift 2 ;;
+    -h|--help) usage; exit 0 ;;
+    *) die "unknown argument: $1" ;;
+  esac
+done
+
+require_pg_client
+require_cmd node
+require_cmd openssl
+URL="$(resolve_database_url)"
+require_database_url "$URL"
+
+ACTIVE_DB="$(db_name_from_url "$URL")"
+assert_safe_ident "$ACTIVE_DB" "active database name"
+
+mkdir -p "$BACKUP_DIR"
+chmod 700 "$BACKUP_DIR"
+# Unique id: bak_<utc>_<random>
+ID="bak_$(date -u +%Y%m%dT%H%M%SZ)_$(openssl rand -hex 4)"
+DUMP="$BACKUP_DIR/${ID}.sql"
+META="$BACKUP_DIR/${ID}.meta.json"
+VERIFIED="$BACKUP_DIR/${ID}.verified.json"
+EVIDENCE_TMP="$(mktemp "${TMPDIR:-/tmp}/clearance-bak-evidence.XXXXXX")"
+SNAPSHOT_DIR="$(mktemp -d "${TMPDIR:-/tmp}/clearance-bak-snapshot.XXXXXX")"
+SNAPSHOT_IN="$SNAPSHOT_DIR/in"
+SNAPSHOT_OUT="$SNAPSHOT_DIR/out"
+SNAPSHOT_OPEN=0
+SNAPSHOT_PID=""
+cleanup_tmp() {
+  local ec=$?
+  if [[ "$SNAPSHOT_OPEN" -eq 1 ]]; then
+    printf 'ROLLBACK;\n\\q\n' >&8 2>/dev/null || true
+    exec 8>&-
+    exec 9<&-
+    wait "$SNAPSHOT_PID" 2>/dev/null || true
+  fi
+  find "$SNAPSHOT_DIR" -type p -delete 2>/dev/null || true
+  rmdir "$SNAPSHOT_DIR" 2>/dev/null || true
+  find "$EVIDENCE_TMP" -type f -delete 2>/dev/null || true
+  if [[ "$ec" -ne 0 ]]; then
+    find "$DUMP" "$META" "$VERIFIED" -type f -delete 2>/dev/null || true
+  fi
+}
+trap cleanup_tmp EXIT
+
+printf 'backup-create: dumping database=%s (url redacted=%s)\n' "$ACTIVE_DB" "$(redact_url "$URL")" >&2
+
+# Hold one exported repeatable-read snapshot across both pg_dump and the
+# resource/version evidence queries. Without this, ordinary writes between the
+# dump and count collection can make a valid backup permanently unverifiable.
+mkfifo "$SNAPSHOT_IN" "$SNAPSHOT_OUT"
+psql --no-psqlrc -qAt -v ON_ERROR_STOP=1 "$URL" <"$SNAPSHOT_IN" >"$SNAPSHOT_OUT" &
+SNAPSHOT_PID=$!
+exec 8>"$SNAPSHOT_IN"
+exec 9<"$SNAPSHOT_OUT"
+SNAPSHOT_OPEN=1
+printf 'BEGIN ISOLATION LEVEL REPEATABLE READ;\nSELECT pg_export_snapshot();\n' >&8
+IFS= read -r SNAPSHOT_ID <&9 || die "could not export backup snapshot"
+[[ "$SNAPSHOT_ID" =~ ^[0-9A-Fa-f-]+$ ]] || die "postgres returned an invalid backup snapshot id"
+if [[ "${CLEARANCE_OPS_TESTING:-0}" == "1" ]]; then
+  if [[ -n "${CLEARANCE_BACKUP_TEST_SNAPSHOT_READY_FILE:-}" ]]; then
+    : >"$CLEARANCE_BACKUP_TEST_SNAPSHOT_READY_FILE"
+  fi
+  delay="${CLEARANCE_BACKUP_TEST_DELAY_AFTER_SNAPSHOT_SECONDS:-0}"
+  [[ "$delay" =~ ^[0-9]+([.][0-9]+)?$ ]] || die "invalid backup snapshot testing delay"
+  if [[ "$delay" != "0" ]]; then sleep "$delay"; fi
+fi
+
+# Real pg_dump — plain SQL for inspectability. Fail closed on any dump error.
+# Uses matching major client via Docker when local pg_dump is older/newer than server.
+pg_dump_to_file "$URL" "$DUMP" "$SNAPSHOT_ID"
+
+printf '%s\n' \
+  'CREATE TEMP TABLE clearance_backup_counts(table_name text PRIMARY KEY, row_count bigint NOT NULL);' \
+  "SELECT format('INSERT INTO clearance_backup_counts SELECT %L, count(*) FROM %I.%I;', table_name, table_schema, table_name) FROM information_schema.tables WHERE table_schema='public' AND table_type='BASE TABLE' ORDER BY table_name;" \
+  '\gexec' \
+  "SELECT json_build_object('capturedAt', to_char(CURRENT_TIMESTAMP AT TIME ZONE 'UTC','YYYY-MM-DD\"T\"HH24:MI:SS\"Z\"'), 'tables', COALESCE((SELECT json_object_agg(table_name,row_count ORDER BY table_name) FROM clearance_backup_counts),'{}'::json))::text;" \
+  "SELECT CASE WHEN to_regclass('public.clearance_management_snapshot') IS NULL THEN 'SELECT '''' || ''__CLEARANCE_APP_VERSION__'';' ELSE 'SELECT COALESCE((SELECT data->>''releaseVersion'' FROM clearance_management_snapshot WHERE id=1),'''') || ''__CLEARANCE_APP_VERSION__'';' END;" \
+  '\gexec' \
+  "SELECT '__CLEARANCE_SNAPSHOT_END__';" >&8
+
+SNAPSHOT_EVIDENCE=""
+APP_VERSION=""
+while IFS= read -r line <&9; do
+  if [[ "$line" == "__CLEARANCE_SNAPSHOT_END__" ]]; then
+    break
+  elif [[ "$line" == *"__CLEARANCE_APP_VERSION__" ]]; then
+    APP_VERSION="${line%__CLEARANCE_APP_VERSION__}"
+  elif [[ "$line" == \{* ]]; then
+    SNAPSHOT_EVIDENCE="$line"
+  fi
+done
+[[ -n "$SNAPSHOT_EVIDENCE" ]] || die "could not collect evidence from the backup snapshot"
+printf '%s\n' "$SNAPSHOT_EVIDENCE" >"$EVIDENCE_TMP"
+printf 'COMMIT;\n\\q\n' >&8
+exec 8>&-
+exec 9<&-
+wait "$SNAPSHOT_PID"
+SNAPSHOT_OPEN=0
+
+[[ -s "$DUMP" ]] || die "pg_dump produced empty file"
+if ! grep -q 'PostgreSQL database dump' "$DUMP"; then
+  die "pg_dump output missing expected PostgreSQL dump header"
+fi
+grep -q 'PostgreSQL database dump complete' "$DUMP" \
+  || die "pg_dump output missing completion marker"
+
+CHECKSUM="$(sha256_file "$DUMP")"
+BYTES="$(wc -c <"$DUMP" | tr -d ' ')"
+PG_VER="$(grep -m1 'Dumped from database version' "$DUMP" || pg_dump --version | head -n1 || true)"
+
+# Resource counts as flat map from evidence tables
+COUNTS_JSON="$(node -e '
+    const fs=require("fs");
+    const j=JSON.parse(fs.readFileSync(process.argv[1],"utf8"));
+    if (!j.tables || typeof j.tables !== "object" || Array.isArray(j.tables)) process.exit(2);
+    process.stdout.write(JSON.stringify(j.tables));
+  ' "$EVIDENCE_TMP")" || die "failed to serialize required backup evidence"
+
+CREATED="$(iso_now)"
+cat >"$META" <<EOF
+{
+  "id": $(json_escape "$ID"),
+  "format": "pg_dump_plain",
+  "createdAt": $(json_escape "$CREATED"),
+  "dumpFile": $(json_escape "$(basename "$DUMP")"),
+  "metaFile": $(json_escape "$(basename "$META")"),
+  "checksumSha256": $(json_escape "$CHECKSUM"),
+  "bytes": $BYTES,
+  "sourceDatabase": $(json_escape "$ACTIVE_DB"),
+  "applicationVersion": $(if [[ -n "$APP_VERSION" ]]; then json_escape "$APP_VERSION"; else printf 'null'; fi),
+  "pgDumpVersion": $(json_escape "$PG_VER"),
+  "resourceCounts": $COUNTS_JSON,
+  "tool": "scripts/backup-create.sh"
+}
+EOF
+
+# Refuse writing secrets into meta
+if grep -qiE 'password|postgresql://[^"]+:[^"]+@' "$META"; then
+  find "$DUMP" "$META" -type f -delete 2>/dev/null || true
+  die "metadata unexpectedly contained credential material; aborted"
+fi
+
+# Bind both artifacts into immutable local verification evidence. The evidence
+# is owner-readable and non-writable; backup-verify checks every digest.
+META_CHECKSUM="$(sha256_file "$META")"
+RESOURCE_FINGERPRINT="$(schema_fingerprint_from_evidence "$EVIDENCE_TMP")"
+cat >"$VERIFIED" <<EOF
+{
+  "id": $(json_escape "$ID"),
+  "verifiedAt": $(json_escape "$(iso_now)"),
+  "dumpFile": $(json_escape "$(basename "$DUMP")"),
+  "metaFile": $(json_escape "$(basename "$META")"),
+  "dumpSha256": $(json_escape "$CHECKSUM"),
+  "metaSha256": $(json_escape "$META_CHECKSUM"),
+  "resourceEvidenceSha256": $(json_escape "$RESOURCE_FINGERPRINT"),
+  "checks": ["nonempty", "pg_dump_complete", "metadata_bound", "resource_counts_captured"]
+}
+EOF
+chmod 600 "$DUMP" "$META"
+chmod 400 "$VERIFIED"
+
+printf 'backup-create: wrote %s (%s bytes, sha256=%s)\n' "$DUMP" "$BYTES" "$CHECKSUM" >&2
+printf 'BACKUP_ID=%s\n' "$ID"
+printf 'BACKUP_DUMP=%s\n' "$DUMP"
+printf 'BACKUP_META=%s\n' "$META"
+printf 'BACKUP_EVIDENCE=%s\n' "$VERIFIED"

package/dist/ops/scripts/backup-restore-verify.sh

@@ -0,0 +1,129 @@
+#!/usr/bin/env bash
+# Restore a verified backup into an ISOLATED temporary database, compare evidence,
+# and clean up. Never drops or overwrites the active database.
+set -Eeuo pipefail
+
+ROOT="$(cd "$(dirname "$0")/.." && pwd)"
+# shellcheck source=lib/ops-common.sh
+source "$ROOT/scripts/lib/ops-common.sh"
+
+usage() {
+  cat <<'EOF'
+Usage: backup-restore-verify.sh --id ID [--dir DIR]
+       backup-restore-verify.sh --dump PATH --meta PATH
+
+Restores into a temporary database name (clr_rv_*), queries management/runtime
+tables when present, compares row counts to backup metadata evidence, drops the
+temp database, and exits non-zero on mismatch.
+
+Refuses to target the active database name under all circumstances.
+EOF
+}
+
+BACKUP_DIR="${CLEARANCE_BACKUP_DIR:-$ROOT/.clearance/backups}"
+ID=""
+DUMP=""
+META=""
+KEEP_TEMP=0
+
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --id) ID="$2"; shift 2 ;;
+    --dir) BACKUP_DIR="$2"; shift 2 ;;
+    --dump) DUMP="$2"; shift 2 ;;
+    --meta) META="$2"; shift 2 ;;
+    --keep-temp) KEEP_TEMP=1; shift ;;
+    -h|--help) usage; exit 0 ;;
+    *) die "unknown argument: $1" ;;
+  esac
+done
+
+if [[ -n "$ID" ]]; then
+  DUMP="${DUMP:-$BACKUP_DIR/${ID}.sql}"
+  META="${META:-$BACKUP_DIR/${ID}.meta.json}"
+fi
+[[ -n "$DUMP" && -f "$DUMP" ]] || die "dump required"
+[[ -n "$META" && -f "$META" ]] || die "meta required"
+
+require_pg_client
+require_cmd node
+require_cmd openssl
+URL="$(resolve_database_url)"
+require_database_url "$URL"
+ACTIVE_DB="$(db_name_from_url "$URL")"
+ADMIN_URL="$(admin_url_from_url "$URL")"
+
+# Integrity first
+bash "$ROOT/scripts/backup-verify.sh" --dump "$DUMP" --meta "$META" >/dev/null
+
+# Isolated temp DB name
+SUFFIX="$(openssl rand -hex 4)"
+TEMP_DB="clr_rv_${SUFFIX}"
+assert_safe_ident "$TEMP_DB" "temp database"
+assert_not_active_db "$TEMP_DB"
+[[ "$TEMP_DB" != "$ACTIVE_DB" ]] || die "temp db collided with active db"
+
+TEMP_URL="$(url_with_db "$URL" "$TEMP_DB")"
+CREATED=0
+RESTORED_EV=""
+
+drop_temp_database() {
+  assert_not_active_db "$TEMP_DB"
+  psql_q "$ADMIN_URL" -c "SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname = '${TEMP_DB}' AND pid <> pg_backend_pid();" >/dev/null 2>&1 || true
+  psql_q "$ADMIN_URL" -c "DROP DATABASE IF EXISTS \"${TEMP_DB}\";" >/dev/null 2>&1 || return 1
+  local remains
+  remains="$(psql_q "$ADMIN_URL" -At -c "SELECT count(*) FROM pg_database WHERE datname = '${TEMP_DB}'")" || return 1
+  [[ "$remains" == "0" ]]
+}
+
+cleanup() {
+  local ec=$?
+  [[ -z "$RESTORED_EV" ]] || rm -f "$RESTORED_EV"
+  if [[ "$KEEP_TEMP" -eq 1 ]]; then
+    printf 'backup-restore-verify: keeping temp database %s (active remains %s)\n' "$TEMP_DB" "$ACTIVE_DB" >&2
+    exit "$ec"
+  fi
+  if [[ "$CREATED" -eq 1 ]]; then
+    if ! drop_temp_database; then
+      printf 'backup-restore-verify: failed to clean up isolated database %s\n' "$TEMP_DB" >&2
+      ec=1
+    fi
+  fi
+  exit "$ec"
+}
+trap cleanup EXIT INT TERM
+
+printf 'backup-restore-verify: creating isolated database %s (active=%s remains untouched)\n' \
+  "$TEMP_DB" "$ACTIVE_DB" >&2
+
+psql_q "$ADMIN_URL" -c "CREATE DATABASE \"${TEMP_DB}\";"
+CREATED=1
+
+printf 'backup-restore-verify: restoring dump into isolated database\n' >&2
+psql_restore_file "$TEMP_URL" "$DUMP"
+
+# Evidence from restored DB
+RESTORED_EV="$(mktemp "${TMPDIR:-/tmp}/clearance-restore-ev.XXXXXX")"
+collect_db_evidence "$TEMP_URL" "$RESTORED_EV"
+
+# Compare table evidence and the application release captured with the backup.
+compare_backup_evidence "$META" "$RESTORED_EV"
+EXPECTED_APP_VERSION="$(node -e 'const j=require(process.argv[1]); process.stdout.write(j.applicationVersion||"")' "$META")"
+if [[ -n "$EXPECTED_APP_VERSION" ]]; then
+  require_application_release "$TEMP_URL" "$EXPECTED_APP_VERSION"
+fi
+rm -f "$RESTORED_EV"
+RESTORED_EV=""
+
+# Confirm active DB still exists and was not the restore target
+STILL="$(psql_q "$ADMIN_URL" -At -c "SELECT 1 FROM pg_database WHERE datname = '${ACTIVE_DB}'")"
+[[ "$STILL" == "1" ]] || die "active database disappeared — aborting (investigation required)"
+assert_not_active_db "$TEMP_DB"
+
+if [[ "$KEEP_TEMP" -eq 0 ]]; then
+  drop_temp_database || die "isolated restore succeeded but temporary database cleanup failed"
+  CREATED=0
+fi
+
+printf 'backup-restore-verify: OK (isolated restore compared; active db %s untouched; cleanup verified)\n' "$ACTIVE_DB"
+printf 'RESTORE_VERIFIED=1\n'