hecks 3.0.5 → 3.1.1

data/lib/hecks/projections/deploy/box/templates/restore-to-rds.sh.tmpl

@@ -0,0 +1,91 @@
+#!/bin/bash
+# Copy @@STACK@@'s schemas from the old database into the new RDS instance through a bastion that can
+# reach both, then verify the copy. Piped dump | restore, so nothing is written to disk. Never writes to
+# the source.
+#
+#   restore-to-rds.sh <bastion-instance-id> <source-host> <source-secret-arn> <rds-host> <rds-secret-arn>
+#
+#   FORCE=1   drop the target schemas first: a re-load before cutover, or a rollback copy in the other
+#             direction (swap the hosts and secrets, and SRC_DB and DST_DB)
+#   SRC_DB    the source database (default @@SOURCE_DATABASE@@)
+#   DST_DB    the target database (default @@DATABASE@@)
+#
+# Schemas copied: @@SCHEMAS@@
+#
+# Why not a plain pg_restore: a Hecks schema's materialized views call hecks_tr_extract() unqualified, and
+# pg_restore runs with an empty search_path, so the restore's REFRESH step errors on them. This restores
+# tolerating exactly that error, then refreshes the views with the schema on the search_path, and fails on
+# any other error.
+#
+# Needs pg_dump, pg_restore and psql 16 or newer, the SSM plugin, jq and AWS credentials that can read both
+# secrets and start SSM sessions.
+set -euo pipefail
+HERE=$(cd "$(dirname "$0")" && pwd)
+
+BASTION=${1:?bastion instance id}; SRC_HOST=${2:?source host}; SRC_SECRET=${3:?source secret arn}
+DST_HOST=${4:?target host}; DST_SECRET=${5:?target secret arn}
+SCHEMAS="@@SCHEMAS@@"
+SRC_DB=${SRC_DB:-@@SOURCE_DATABASE@@}; DST_DB=${DST_DB:-@@DATABASE@@}
+SRC_PORT=15432; DST_PORT=15433
+
+pg_major() { "$1" --version 2>/dev/null | sed -E 's/.* ([0-9]+)[.].*/\1/'; }
+# The default client on the PATH is often older than the server (Homebrew's is 14); prefer postgresql@16.
+if [ "$(pg_major pg_dump || echo 0)" -lt 16 ] && [ -x /opt/homebrew/opt/postgresql@16/bin/pg_dump ]; then
+  export PATH=/opt/homebrew/opt/postgresql@16/bin:$PATH
+fi
+for t in pg_dump pg_restore psql; do
+  [ "$(pg_major $t || echo 0)" -ge 16 ] || { echo "need $t 16 or newer on the PATH" >&2; exit 1; }
+done
+
+WORK=$(mktemp -d)
+PIDS=()
+cleanup() { for p in "${PIDS[@]:-}"; do [ -n "$p" ] && kill "$p" 2>/dev/null || true; done; rm -rf "$WORK"; }
+trap cleanup EXIT
+
+secret_field() { aws secretsmanager get-secret-value --secret-id "$1" --query SecretString --output text | jq -r "$2"; }
+SRC_PW=$(secret_field "$SRC_SECRET" .password); DST_PW=$(secret_field "$DST_SECRET" .password)
+SRC_USER=$(secret_field "$SRC_SECRET" '.username // "postgres"'); DST_USER=$(secret_field "$DST_SECRET" '.username // "postgres"')
+
+tunnel() { # host local-port
+  aws ssm start-session --target "$BASTION" --document-name AWS-StartPortForwardingSessionToRemoteHost \
+    --parameters "{\"host\":[\"$1\"],\"portNumber\":[\"5432\"],\"localPortNumber\":[\"$2\"]}" >"$WORK/tunnel_$2.log" 2>&1 &
+  PIDS+=($!)
+  disown # so shutting the tunnel down at exit does not print a "Terminated" line
+}
+tunnel "$SRC_HOST" $SRC_PORT; tunnel "$DST_HOST" $DST_PORT
+for i in $(seq 1 20); do
+  grep -q "Waiting for connections" "$WORK/tunnel_$SRC_PORT.log" 2>/dev/null && grep -q "Waiting for connections" "$WORK/tunnel_$DST_PORT.log" 2>/dev/null && break
+  sleep 1
+done
+
+SRC() { PGPASSWORD=$SRC_PW psql -h localhost -p $SRC_PORT -U "$SRC_USER" -d "$SRC_DB" -Atc "$1"; }
+DST() { PGPASSWORD=$DST_PW psql -h localhost -p $DST_PORT -U "$DST_USER" -d "$DST_DB" -Atc "$1"; }
+echo "source $(SRC 'show server_version'), target $(DST 'show server_version')"
+
+# Refuse to clobber a target that already holds the schemas.
+for S in $SCHEMAS; do
+  if [ "$(DST "select count(*) from pg_namespace where nspname='$S'")" != 0 ]; then
+    if [ "${FORCE:-}" = 1 ]; then DST "drop schema \"$S\" cascade" >/dev/null; echo "dropped target schema $S"
+    else echo "target already has schema $S; re-run with FORCE=1 to replace it" >&2; exit 1; fi
+  fi
+done
+
+for S in $SCHEMAS; do
+  echo "== copying schema $S"
+  PGPASSWORD=$SRC_PW pg_dump -h localhost -p $SRC_PORT -U "$SRC_USER" -d "$SRC_DB" --schema="$S" -Fc --no-owner --no-privileges \
+    | PGPASSWORD=$DST_PW pg_restore -h localhost -p $DST_PORT -U "$DST_USER" -d "$DST_DB" --no-owner --no-privileges 2>"$WORK/err_$S.txt" || true
+  TOTAL=$(grep -c 'error:' "$WORK/err_$S.txt" || true)
+  UNEXPECTED=$(grep 'error:' "$WORK/err_$S.txt" | grep -vc 'hecks_tr_extract' || true)
+  echo "   restore errors: $TOTAL (unexpected: $UNEXPECTED)"
+  if [ "$UNEXPECTED" != 0 ]; then grep 'error:' "$WORK/err_$S.txt" | grep -v 'hecks_tr_extract' | head -5 >&2; exit 1; fi
+done
+
+# Refresh what pg_restore could not (with the schema on the search_path).
+for S in $SCHEMAS; do
+  for MV in $(DST "select matviewname from pg_matviews where schemaname='$S' and not ispopulated"); do
+    PGPASSWORD=$DST_PW psql -h localhost -p $DST_PORT -U "$DST_USER" -d "$DST_DB" -qAtc "set search_path=\"$S\",public; refresh materialized view \"$S\".\"$MV\""
+  done
+done
+
+# Same structure, same exact row counts in every table, nothing unpopulated.
+A_DB=$SRC_DB B_DB=$DST_DB bash "$HERE/verify-copy.sh" "$BASTION" "$SRC_HOST" "$SRC_SECRET" "$DST_HOST" "$DST_SECRET"

data/lib/hecks/projections/deploy/box/templates/verify-copy.sh.tmpl

@@ -0,0 +1,58 @@
+#!/bin/bash
+# Compare two databases holding @@STACK@@'s schemas (structure and exact row counts of every table)
+# through a bastion that can reach both. Read-only on both. Used at the end of restore-to-rds.sh, for a
+# backup restore drill (production vs a database restored from its backup) and for checking a rollback copy.
+#
+#   verify-copy.sh <bastion-instance-id> <host-a> <secret-arn-a> <host-b> <secret-arn-b>
+#
+#   A_DB, B_DB   the database on each side (default @@DATABASE@@)
+#
+# Schemas compared: @@SCHEMAS@@
+# Exit 0 only if both are identical and no materialized view is unpopulated in either.
+set -euo pipefail
+
+BASTION=${1:?bastion instance id}; A_HOST=${2:?}; A_SECRET=${3:?}; B_HOST=${4:?}; B_SECRET=${5:?}
+SCHEMAS="@@SCHEMAS_CSV@@"
+A_DB=${A_DB:-@@DATABASE@@}; B_DB=${B_DB:-@@DATABASE@@}
+A_PORT=15434; B_PORT=15435
+
+pg_major() { "$1" --version 2>/dev/null | sed -E 's/.* ([0-9]+)[.].*/\1/'; }
+if [ "$(pg_major psql || echo 0)" -lt 16 ] && [ -x /opt/homebrew/opt/postgresql@16/bin/psql ]; then
+  export PATH=/opt/homebrew/opt/postgresql@16/bin:$PATH
+fi
+
+WORK=$(mktemp -d); PIDS=()
+cleanup() { for p in "${PIDS[@]:-}"; do [ -n "$p" ] && kill "$p" 2>/dev/null || true; done; rm -rf "$WORK"; }
+trap cleanup EXIT
+
+secret_field() { aws secretsmanager get-secret-value --secret-id "$1" --query SecretString --output text | jq -r "$2"; }
+A_PW=$(secret_field "$A_SECRET" .password); B_PW=$(secret_field "$B_SECRET" .password)
+A_USER=$(secret_field "$A_SECRET" '.username // "postgres"'); B_USER=$(secret_field "$B_SECRET" '.username // "postgres"')
+tunnel() {
+  aws ssm start-session --target "$BASTION" --document-name AWS-StartPortForwardingSessionToRemoteHost \
+    --parameters "{\"host\":[\"$1\"],\"portNumber\":[\"5432\"],\"localPortNumber\":[\"$2\"]}" >"$WORK/t_$2.log" 2>&1 &
+  PIDS+=($!); disown
+}
+tunnel "$A_HOST" $A_PORT; tunnel "$B_HOST" $B_PORT
+for i in $(seq 1 20); do
+  grep -q "Waiting for connections" "$WORK/t_$A_PORT.log" 2>/dev/null && grep -q "Waiting for connections" "$WORK/t_$B_PORT.log" 2>/dev/null && break
+  sleep 1
+done
+A() { PGPASSWORD=$A_PW psql -h localhost -p $A_PORT -U "$A_USER" -d "$A_DB" -Atc "$1"; }
+B() { PGPASSWORD=$B_PW psql -h localhost -p $B_PORT -U "$B_USER" -d "$B_DB" -Atc "$1"; }
+
+SHAPE="select n.nspname||' '||c.relkind::text, count(*) from pg_class c join pg_namespace n on n.oid=c.relnamespace where n.nspname = any(string_to_array('$SCHEMAS', ',')) group by 1
+ union all select 'policies', count(*) from pg_policies where schemaname = any(string_to_array('$SCHEMAS', ','))
+ union all select 'functions', count(*) from pg_proc p join pg_namespace n on n.oid=p.pronamespace where n.nspname = any(string_to_array('$SCHEMAS', ',')) order by 1"
+COUNTS="select table_schema||'.'||table_name, (xpath('/row/c/text()', query_to_xml(format('select count(*) as c from %I.%I', table_schema, table_name), false, true, '')))[1]::text from information_schema.tables where table_type='BASE TABLE' and table_schema = any(string_to_array('$SCHEMAS', ',')) order by 1"
+UNPOP="select count(*) from pg_matviews where schemaname = any(string_to_array('$SCHEMAS', ',')) and not ispopulated"
+
+BAD=0
+A "$SHAPE" >"$WORK/shape_a"; B "$SHAPE" >"$WORK/shape_b"
+A "$COUNTS" >"$WORK/counts_a"; B "$COUNTS" >"$WORK/counts_b"
+[ -s "$WORK/counts_a" ] || { echo "FAIL: first database has no tables to compare" >&2; BAD=1; }
+[ "$(A "$UNPOP")" = 0 ] && [ "$(B "$UNPOP")" = 0 ] || { echo "FAIL: unpopulated materialized views" >&2; BAD=1; }
+diff "$WORK/shape_a" "$WORK/shape_b" >/dev/null || { echo "FAIL: structure differs" >&2; diff "$WORK/shape_a" "$WORK/shape_b" >&2 || true; BAD=1; }
+diff "$WORK/counts_a" "$WORK/counts_b" >/dev/null || { echo "FAIL: row counts differ" >&2; diff "$WORK/counts_a" "$WORK/counts_b" >&2 || true; BAD=1; }
+[ "$BAD" = 0 ] || exit 1
+echo "OK: $(wc -l <"$WORK/counts_a" | tr -d ' ') tables, identical row counts and structure"