@biffo/cli 0.212.1 → 0.213.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -0,0 +1,232 @@
|
|
|
1
|
+
#!/usr/bin/env sh
|
|
2
|
+
#
|
|
3
|
+
# Give the Postgres-dependent test lane a database with a CURRENT schema, and
|
|
4
|
+
# print its DSN.
|
|
5
|
+
#
|
|
6
|
+
# ## Why this exists
|
|
7
|
+
#
|
|
8
|
+
# `scripts/verify.sh` grew a `pg-test` check (#1089) because on 2026-08-02 nine
|
|
9
|
+
# of thirteen locally-catchable failing CI steps across the estate were one
|
|
10
|
+
# repo's real-Postgres lane -- a required check with no local counterpart at all.
|
|
11
|
+
# But a gate can only run that lane against a database, and no repo documented
|
|
12
|
+
# how to get one: no compose file, no script, no DSN written down. The container
|
|
13
|
+
# that existed on the workstation had been created ad hoc in some earlier session
|
|
14
|
+
# and held a scatter of scratch databases. That undocumented setup WAS the
|
|
15
|
+
# fail-open, because a gate nobody can run is not a gate.
|
|
16
|
+
#
|
|
17
|
+
# ## Why freshness, not rebuild-every-time
|
|
18
|
+
#
|
|
19
|
+
# The expensive failure is not a slow rebuild, it is a STALE one. Measured on
|
|
20
|
+
# tabsii-platform while writing this: a database built about an hour earlier,
|
|
21
|
+
# before two PRs merged, produced **23 failures** in a module that had nothing to
|
|
22
|
+
# do with the change in hand. Rebuilt from the same tree it passed 336/336, and
|
|
23
|
+
# passed 336 again on an immediate re-run -- so the lane was genuinely
|
|
24
|
+
# re-runnable and every one of those failures was the old schema.
|
|
25
|
+
#
|
|
26
|
+
# That is the worst shape a local gate can have. Twenty-three red tests that are
|
|
27
|
+
# not your fault teach people the gate is unreliable, and an unreliable gate gets
|
|
28
|
+
# bypassed -- which H4 pre-registered as the condition refuting the whole
|
|
29
|
+
# local-gate programme. So the schema inputs are fingerprinted and a rebuild
|
|
30
|
+
# happens only when they actually changed: reuse ~0.3s, rebuild ~4s.
|
|
31
|
+
#
|
|
32
|
+
# ## Why it is generic
|
|
33
|
+
#
|
|
34
|
+
# It adapts to the repo rather than being told about it, for the same reason
|
|
35
|
+
# `verify.sh` does: forks drift, and a per-instance copy of this would drift from
|
|
36
|
+
# the DDL layout it is meant to build. Everything instance-specific is DERIVED --
|
|
37
|
+
# the schema directories from `db/imports/*/`, the engine image from whether the
|
|
38
|
+
# DDL asks for PostGIS, and the did-it-build threshold from the number of
|
|
39
|
+
# policies the DDL itself declares. Nothing here names a product.
|
|
40
|
+
#
|
|
41
|
+
# ## Usage
|
|
42
|
+
#
|
|
43
|
+
# eval "$(sh scripts/pg-test-db.sh --export)" # export BIFFO_TEST_PG_DSN
|
|
44
|
+
# sh scripts/pg-test-db.sh # print the DSN on stdout
|
|
45
|
+
# sh scripts/pg-test-db.sh --recreate # force a rebuild
|
|
46
|
+
#
|
|
47
|
+
# Only the DSN reaches stdout, so it is safe to capture; progress goes to stderr.
|
|
48
|
+
#
|
|
49
|
+
# Overridable: BIFFO_PG_HOST, BIFFO_PG_PORT, BIFFO_PG_USER, BIFFO_PG_PASSWORD,
|
|
50
|
+
# BIFFO_PG_DB, BIFFO_PG_CONTAINER, BIFFO_PG_IMAGE.
|
|
51
|
+
|
|
52
|
+
set -eu
|
|
53
|
+
|
|
54
|
+
HOST="${BIFFO_PG_HOST:-localhost}"
|
|
55
|
+
PORT="${BIFFO_PG_PORT:-55432}"
|
|
56
|
+
USER_="${BIFFO_PG_USER:-postgres}"
|
|
57
|
+
PASS="${BIFFO_PG_PASSWORD:-postgres}"
|
|
58
|
+
DB="${BIFFO_PG_DB:-biffo_test}"
|
|
59
|
+
CONTAINER="${BIFFO_PG_CONTAINER:-biffo-pg-test}"
|
|
60
|
+
|
|
61
|
+
RECREATE=0
|
|
62
|
+
EXPORT=0
|
|
63
|
+
for arg in "$@"; do
|
|
64
|
+
case "$arg" in
|
|
65
|
+
--recreate) RECREATE=1 ;;
|
|
66
|
+
--export) EXPORT=1 ;;
|
|
67
|
+
-h | --help)
|
|
68
|
+
sed -n '2,48p' "$0" | sed 's/^#\{1,2\} \{0,1\}//'
|
|
69
|
+
exit 0
|
|
70
|
+
;;
|
|
71
|
+
*)
|
|
72
|
+
echo "unknown argument: $arg" >&2
|
|
73
|
+
exit 2
|
|
74
|
+
;;
|
|
75
|
+
esac
|
|
76
|
+
done
|
|
77
|
+
|
|
78
|
+
say() { echo "pg-test-db: $*" >&2; }
|
|
79
|
+
|
|
80
|
+
REPO_ROOT=$(cd "$(dirname "$0")/.." && pwd)
|
|
81
|
+
cd "$REPO_ROOT"
|
|
82
|
+
|
|
83
|
+
# --- what this repo's schema is made of --------------------------------------
|
|
84
|
+
#
|
|
85
|
+
# `db/imports/<name>/*.sql` is the Biffo DDL-import convention that the API's own
|
|
86
|
+
# `ddl_import.list_sql_files` reads at startup, so deriving from it means this
|
|
87
|
+
# script and the running app agree by construction rather than by someone
|
|
88
|
+
# remembering to update both.
|
|
89
|
+
DDL_FILES=$(find db/imports -mindepth 2 -maxdepth 2 -name '*.sql' 2>/dev/null | LC_ALL=C sort || true)
|
|
90
|
+
ALEMBIC_DIR=""
|
|
91
|
+
for _d in services/api .; do
|
|
92
|
+
[ -f "$_d/alembic.ini" ] && ALEMBIC_DIR="$_d" && break
|
|
93
|
+
done
|
|
94
|
+
|
|
95
|
+
if [ -z "$DDL_FILES" ] && [ -z "$ALEMBIC_DIR" ]; then
|
|
96
|
+
say "no db/imports/*/ DDL and no alembic.ini - this repo has no schema to build"
|
|
97
|
+
exit 1
|
|
98
|
+
fi
|
|
99
|
+
|
|
100
|
+
# PostGIS or plain, decided by what the DDL asks for. A plain `postgres` image
|
|
101
|
+
# fails on the first `CREATE EXTENSION postgis`, and picking the heavier image
|
|
102
|
+
# unconditionally would slow every repo that does not need it.
|
|
103
|
+
if [ -n "$DDL_FILES" ] && echo "$DDL_FILES" | xargs grep -liE 'EXTENSION[[:space:]]+(IF[[:space:]]+NOT[[:space:]]+EXISTS[[:space:]]+)?postgis' >/dev/null 2>&1; then
|
|
104
|
+
IMAGE="${BIFFO_PG_IMAGE:-postgis/postgis:16-3.4}"
|
|
105
|
+
else
|
|
106
|
+
IMAGE="${BIFFO_PG_IMAGE:-postgres:16}"
|
|
107
|
+
fi
|
|
108
|
+
|
|
109
|
+
export PGPASSWORD="$PASS"
|
|
110
|
+
psql_admin() { psql -q -h "$HOST" -p "$PORT" -U "$USER_" -d postgres "$@"; }
|
|
111
|
+
psql_db() { psql -q -h "$HOST" -p "$PORT" -U "$USER_" -d "$DB" "$@"; }
|
|
112
|
+
|
|
113
|
+
# --- 1. a reachable server ---------------------------------------------------
|
|
114
|
+
#
|
|
115
|
+
# Started here rather than assumed, because "docker run one yourself" is exactly
|
|
116
|
+
# the tribal knowledge this script replaces. An already-running server is reused.
|
|
117
|
+
if ! psql_admin -c 'SELECT 1' >/dev/null 2>&1; then
|
|
118
|
+
if ! command -v docker >/dev/null 2>&1; then
|
|
119
|
+
say "no Postgres at $HOST:$PORT and docker is not installed."
|
|
120
|
+
say "Start one and re-run, or set BIFFO_PG_HOST / BIFFO_PG_PORT."
|
|
121
|
+
exit 1
|
|
122
|
+
fi
|
|
123
|
+
if docker ps -a --format '{{.Names}}' | grep -qx "$CONTAINER"; then
|
|
124
|
+
say "starting existing container $CONTAINER"
|
|
125
|
+
docker start "$CONTAINER" >/dev/null
|
|
126
|
+
else
|
|
127
|
+
say "creating container $CONTAINER ($IMAGE) on port $PORT"
|
|
128
|
+
docker run -d --name "$CONTAINER" \
|
|
129
|
+
-e POSTGRES_PASSWORD="$PASS" -p "$PORT:5432" "$IMAGE" >/dev/null
|
|
130
|
+
fi
|
|
131
|
+
# Polled, not slept: a cold image pull and a warm restart differ by an order of
|
|
132
|
+
# magnitude, and one fixed sleep is wrong for both.
|
|
133
|
+
_waited=0
|
|
134
|
+
until psql_admin -c 'SELECT 1' >/dev/null 2>&1; do
|
|
135
|
+
_waited=$((_waited + 1))
|
|
136
|
+
if [ "$_waited" -gt 90 ]; then
|
|
137
|
+
say "Postgres did not become ready in 90s"
|
|
138
|
+
exit 1
|
|
139
|
+
fi
|
|
140
|
+
sleep 1
|
|
141
|
+
done
|
|
142
|
+
say "Postgres ready after ${_waited}s"
|
|
143
|
+
fi
|
|
144
|
+
|
|
145
|
+
DSN="postgresql+asyncpg://$USER_:$PASS@$HOST:$PORT/$DB"
|
|
146
|
+
emit() {
|
|
147
|
+
if [ "$EXPORT" -eq 1 ]; then
|
|
148
|
+
echo "export BIFFO_TEST_PG_DSN='$DSN'"
|
|
149
|
+
else
|
|
150
|
+
echo "$DSN"
|
|
151
|
+
fi
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
# --- 2. is the existing schema current? --------------------------------------
|
|
155
|
+
#
|
|
156
|
+
# By CONTENT, not mtime: a branch switch changes content and leaves mtime
|
|
157
|
+
# anywhere. Stored inside the database, so it cannot outlive a drop or describe
|
|
158
|
+
# some other database.
|
|
159
|
+
fingerprint() {
|
|
160
|
+
{
|
|
161
|
+
[ -n "$ALEMBIC_DIR" ] && find "$ALEMBIC_DIR" -name '*.py' -path '*alembic*' -type f 2>/dev/null |
|
|
162
|
+
LC_ALL=C sort | xargs cat 2>/dev/null
|
|
163
|
+
[ -n "$DDL_FILES" ] && echo "$DDL_FILES" | xargs cat 2>/dev/null
|
|
164
|
+
} | sha256sum | cut -d' ' -f1
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
WANT=$(fingerprint)
|
|
168
|
+
HAVE=""
|
|
169
|
+
if [ "$RECREATE" -eq 0 ] &&
|
|
170
|
+
psql_admin -tAc "SELECT 1 FROM pg_database WHERE datname='$DB'" 2>/dev/null | grep -q 1; then
|
|
171
|
+
HAVE=$(psql -tAq -h "$HOST" -p "$PORT" -U "$USER_" -d "$DB" \
|
|
172
|
+
-c "SELECT value FROM biffo_pg_test_fingerprint LIMIT 1" 2>/dev/null || true)
|
|
173
|
+
fi
|
|
174
|
+
|
|
175
|
+
if [ -n "$HAVE" ] && [ "$HAVE" = "$WANT" ]; then
|
|
176
|
+
say "schema is current, reusing $DB"
|
|
177
|
+
emit
|
|
178
|
+
exit 0
|
|
179
|
+
fi
|
|
180
|
+
|
|
181
|
+
[ -n "$HAVE" ] && say "schema inputs changed - rebuilding rather than serving a stale schema"
|
|
182
|
+
|
|
183
|
+
# --- 3. rebuild the way the app and CI do ------------------------------------
|
|
184
|
+
say "rebuilding $DB"
|
|
185
|
+
psql_admin -c "DROP DATABASE IF EXISTS $DB WITH (FORCE)" >/dev/null
|
|
186
|
+
psql_admin -c "CREATE DATABASE $DB" >/dev/null
|
|
187
|
+
|
|
188
|
+
if [ -n "$ALEMBIC_DIR" ]; then
|
|
189
|
+
BIFFO_DATABASE_URL="$DSN" uv run --directory "$ALEMBIC_DIR" alembic upgrade head >/dev/null
|
|
190
|
+
say "alembic upgrade head"
|
|
191
|
+
fi
|
|
192
|
+
|
|
193
|
+
if [ -n "$DDL_FILES" ]; then
|
|
194
|
+
# ONE psql session, sorted by filename, mirroring the API's own DDL import.
|
|
195
|
+
# Session state an early module sets -- typically `SET search_path` in the
|
|
196
|
+
# first file -- has to survive into later ones, so a per-file connection would
|
|
197
|
+
# silently change the meaning of every unqualified name after it. LC_ALL=C
|
|
198
|
+
# keeps the shell's sort byte-ordered to match Python's.
|
|
199
|
+
# shellcheck disable=SC2046
|
|
200
|
+
psql -q -v ON_ERROR_STOP=1 -h "$HOST" -p "$PORT" -U "$USER_" -d "$DB" \
|
|
201
|
+
--single-transaction $(echo "$DDL_FILES" | sed 's/^/-f /' | tr '\n' ' ') >/dev/null
|
|
202
|
+
say "$(echo "$DDL_FILES" | wc -l | tr -d ' ') DDL modules applied"
|
|
203
|
+
fi
|
|
204
|
+
|
|
205
|
+
# --- 4. refuse to bless a half-built schema ----------------------------------
|
|
206
|
+
#
|
|
207
|
+
# The threshold is derived, not guessed: count the policies the DDL declares and
|
|
208
|
+
# require the database to hold at least half. Recording a fingerprint against a
|
|
209
|
+
# partial schema is worse than failing, because the NEXT run would trust it and
|
|
210
|
+
# every failure after that would look like the developer's own change.
|
|
211
|
+
if [ -n "$DDL_FILES" ]; then
|
|
212
|
+
_declared=$(echo "$DDL_FILES" | xargs grep -ciE '^[[:space:]]*CREATE[[:space:]]+POLICY' 2>/dev/null |
|
|
213
|
+
awk -F: '{s+=$NF} END {print s+0}')
|
|
214
|
+
if [ "${_declared:-0}" -gt 0 ]; then
|
|
215
|
+
_actual=$(psql -tAq -h "$HOST" -p "$PORT" -U "$USER_" -d "$DB" \
|
|
216
|
+
-c "SELECT count(*) FROM pg_policies" 2>/dev/null || echo 0)
|
|
217
|
+
if [ "${_actual:-0}" -lt $((_declared / 2)) ]; then
|
|
218
|
+
say "only ${_actual:-0} policies present against $_declared declared - the schema did not build."
|
|
219
|
+
say "Not recording a fingerprint; fix the DDL and re-run."
|
|
220
|
+
exit 1
|
|
221
|
+
fi
|
|
222
|
+
say "$_actual RLS policies ($_declared declared)"
|
|
223
|
+
fi
|
|
224
|
+
fi
|
|
225
|
+
|
|
226
|
+
psql_db \
|
|
227
|
+
-c "CREATE TABLE IF NOT EXISTS biffo_pg_test_fingerprint (value text primary key)" \
|
|
228
|
+
-c "TRUNCATE biffo_pg_test_fingerprint" \
|
|
229
|
+
-c "INSERT INTO biffo_pg_test_fingerprint (value) VALUES ('$WANT')" >/dev/null
|
|
230
|
+
|
|
231
|
+
say "ready"
|
|
232
|
+
emit
|
|
@@ -478,6 +478,23 @@ pg_test_run() {
|
|
|
478
478
|
}
|
|
479
479
|
|
|
480
480
|
_pg_modules=$(pg_test_modules)
|
|
481
|
+
|
|
482
|
+
# Provision the database rather than requiring the operator to remember.
|
|
483
|
+
#
|
|
484
|
+
# A gate that only runs when you exported the right variable is a gate that runs
|
|
485
|
+
# on the days you did not need it. `scripts/pg-test-db.sh` is idempotent and
|
|
486
|
+
# cheap when the schema is unchanged (~0.3s; ~4s when it genuinely has to
|
|
487
|
+
# rebuild), so calling it is better than warning about it. Failure is silent
|
|
488
|
+
# BECAUSE the WARN below is the honest report of it -- no Docker, no server, no
|
|
489
|
+
# schema all end in the same place: the lane did not run, and the gate says so.
|
|
490
|
+
if [ -z "$PG_TEST_DSN" ] && [ -n "$_pg_modules" ] && [ -z "$LIST" ] && [ -f scripts/pg-test-db.sh ]; then
|
|
491
|
+
PG_TEST_DSN=$(sh scripts/pg-test-db.sh 2>/dev/null | tail -1) || PG_TEST_DSN=""
|
|
492
|
+
case "$PG_TEST_DSN" in
|
|
493
|
+
postgres*) ;;
|
|
494
|
+
*) PG_TEST_DSN="" ;;
|
|
495
|
+
esac
|
|
496
|
+
fi
|
|
497
|
+
|
|
481
498
|
# Order matters, and getting it wrong made these very tests machine-dependent:
|
|
482
499
|
# with `uv not installed` checked FIRST, a runner without uv skipped quietly and
|
|
483
500
|
# the gap warning never printed -- green on a workstation, red on CI, for a
|
|
@@ -0,0 +1,232 @@
|
|
|
1
|
+
#!/usr/bin/env sh
|
|
2
|
+
#
|
|
3
|
+
# Give the Postgres-dependent test lane a database with a CURRENT schema, and
|
|
4
|
+
# print its DSN.
|
|
5
|
+
#
|
|
6
|
+
# ## Why this exists
|
|
7
|
+
#
|
|
8
|
+
# `scripts/verify.sh` grew a `pg-test` check (#1089) because on 2026-08-02 nine
|
|
9
|
+
# of thirteen locally-catchable failing CI steps across the estate were one
|
|
10
|
+
# repo's real-Postgres lane -- a required check with no local counterpart at all.
|
|
11
|
+
# But a gate can only run that lane against a database, and no repo documented
|
|
12
|
+
# how to get one: no compose file, no script, no DSN written down. The container
|
|
13
|
+
# that existed on the workstation had been created ad hoc in some earlier session
|
|
14
|
+
# and held a scatter of scratch databases. That undocumented setup WAS the
|
|
15
|
+
# fail-open, because a gate nobody can run is not a gate.
|
|
16
|
+
#
|
|
17
|
+
# ## Why freshness, not rebuild-every-time
|
|
18
|
+
#
|
|
19
|
+
# The expensive failure is not a slow rebuild, it is a STALE one. Measured on
|
|
20
|
+
# tabsii-platform while writing this: a database built about an hour earlier,
|
|
21
|
+
# before two PRs merged, produced **23 failures** in a module that had nothing to
|
|
22
|
+
# do with the change in hand. Rebuilt from the same tree it passed 336/336, and
|
|
23
|
+
# passed 336 again on an immediate re-run -- so the lane was genuinely
|
|
24
|
+
# re-runnable and every one of those failures was the old schema.
|
|
25
|
+
#
|
|
26
|
+
# That is the worst shape a local gate can have. Twenty-three red tests that are
|
|
27
|
+
# not your fault teach people the gate is unreliable, and an unreliable gate gets
|
|
28
|
+
# bypassed -- which H4 pre-registered as the condition refuting the whole
|
|
29
|
+
# local-gate programme. So the schema inputs are fingerprinted and a rebuild
|
|
30
|
+
# happens only when they actually changed: reuse ~0.3s, rebuild ~4s.
|
|
31
|
+
#
|
|
32
|
+
# ## Why it is generic
|
|
33
|
+
#
|
|
34
|
+
# It adapts to the repo rather than being told about it, for the same reason
|
|
35
|
+
# `verify.sh` does: forks drift, and a per-instance copy of this would drift from
|
|
36
|
+
# the DDL layout it is meant to build. Everything instance-specific is DERIVED --
|
|
37
|
+
# the schema directories from `db/imports/*/`, the engine image from whether the
|
|
38
|
+
# DDL asks for PostGIS, and the did-it-build threshold from the number of
|
|
39
|
+
# policies the DDL itself declares. Nothing here names a product.
|
|
40
|
+
#
|
|
41
|
+
# ## Usage
|
|
42
|
+
#
|
|
43
|
+
# eval "$(sh scripts/pg-test-db.sh --export)" # export BIFFO_TEST_PG_DSN
|
|
44
|
+
# sh scripts/pg-test-db.sh # print the DSN on stdout
|
|
45
|
+
# sh scripts/pg-test-db.sh --recreate # force a rebuild
|
|
46
|
+
#
|
|
47
|
+
# Only the DSN reaches stdout, so it is safe to capture; progress goes to stderr.
|
|
48
|
+
#
|
|
49
|
+
# Overridable: BIFFO_PG_HOST, BIFFO_PG_PORT, BIFFO_PG_USER, BIFFO_PG_PASSWORD,
|
|
50
|
+
# BIFFO_PG_DB, BIFFO_PG_CONTAINER, BIFFO_PG_IMAGE.
|
|
51
|
+
|
|
52
|
+
set -eu
|
|
53
|
+
|
|
54
|
+
HOST="${BIFFO_PG_HOST:-localhost}"
|
|
55
|
+
PORT="${BIFFO_PG_PORT:-55432}"
|
|
56
|
+
USER_="${BIFFO_PG_USER:-postgres}"
|
|
57
|
+
PASS="${BIFFO_PG_PASSWORD:-postgres}"
|
|
58
|
+
DB="${BIFFO_PG_DB:-biffo_test}"
|
|
59
|
+
CONTAINER="${BIFFO_PG_CONTAINER:-biffo-pg-test}"
|
|
60
|
+
|
|
61
|
+
RECREATE=0
|
|
62
|
+
EXPORT=0
|
|
63
|
+
for arg in "$@"; do
|
|
64
|
+
case "$arg" in
|
|
65
|
+
--recreate) RECREATE=1 ;;
|
|
66
|
+
--export) EXPORT=1 ;;
|
|
67
|
+
-h | --help)
|
|
68
|
+
sed -n '2,48p' "$0" | sed 's/^#\{1,2\} \{0,1\}//'
|
|
69
|
+
exit 0
|
|
70
|
+
;;
|
|
71
|
+
*)
|
|
72
|
+
echo "unknown argument: $arg" >&2
|
|
73
|
+
exit 2
|
|
74
|
+
;;
|
|
75
|
+
esac
|
|
76
|
+
done
|
|
77
|
+
|
|
78
|
+
say() { echo "pg-test-db: $*" >&2; }
|
|
79
|
+
|
|
80
|
+
REPO_ROOT=$(cd "$(dirname "$0")/.." && pwd)
|
|
81
|
+
cd "$REPO_ROOT"
|
|
82
|
+
|
|
83
|
+
# --- what this repo's schema is made of --------------------------------------
|
|
84
|
+
#
|
|
85
|
+
# `db/imports/<name>/*.sql` is the Biffo DDL-import convention that the API's own
|
|
86
|
+
# `ddl_import.list_sql_files` reads at startup, so deriving from it means this
|
|
87
|
+
# script and the running app agree by construction rather than by someone
|
|
88
|
+
# remembering to update both.
|
|
89
|
+
DDL_FILES=$(find db/imports -mindepth 2 -maxdepth 2 -name '*.sql' 2>/dev/null | LC_ALL=C sort || true)
|
|
90
|
+
ALEMBIC_DIR=""
|
|
91
|
+
for _d in services/api .; do
|
|
92
|
+
[ -f "$_d/alembic.ini" ] && ALEMBIC_DIR="$_d" && break
|
|
93
|
+
done
|
|
94
|
+
|
|
95
|
+
if [ -z "$DDL_FILES" ] && [ -z "$ALEMBIC_DIR" ]; then
|
|
96
|
+
say "no db/imports/*/ DDL and no alembic.ini - this repo has no schema to build"
|
|
97
|
+
exit 1
|
|
98
|
+
fi
|
|
99
|
+
|
|
100
|
+
# PostGIS or plain, decided by what the DDL asks for. A plain `postgres` image
|
|
101
|
+
# fails on the first `CREATE EXTENSION postgis`, and picking the heavier image
|
|
102
|
+
# unconditionally would slow every repo that does not need it.
|
|
103
|
+
if [ -n "$DDL_FILES" ] && echo "$DDL_FILES" | xargs grep -liE 'EXTENSION[[:space:]]+(IF[[:space:]]+NOT[[:space:]]+EXISTS[[:space:]]+)?postgis' >/dev/null 2>&1; then
|
|
104
|
+
IMAGE="${BIFFO_PG_IMAGE:-postgis/postgis:16-3.4}"
|
|
105
|
+
else
|
|
106
|
+
IMAGE="${BIFFO_PG_IMAGE:-postgres:16}"
|
|
107
|
+
fi
|
|
108
|
+
|
|
109
|
+
export PGPASSWORD="$PASS"
|
|
110
|
+
psql_admin() { psql -q -h "$HOST" -p "$PORT" -U "$USER_" -d postgres "$@"; }
|
|
111
|
+
psql_db() { psql -q -h "$HOST" -p "$PORT" -U "$USER_" -d "$DB" "$@"; }
|
|
112
|
+
|
|
113
|
+
# --- 1. a reachable server ---------------------------------------------------
|
|
114
|
+
#
|
|
115
|
+
# Started here rather than assumed, because "docker run one yourself" is exactly
|
|
116
|
+
# the tribal knowledge this script replaces. An already-running server is reused.
|
|
117
|
+
if ! psql_admin -c 'SELECT 1' >/dev/null 2>&1; then
|
|
118
|
+
if ! command -v docker >/dev/null 2>&1; then
|
|
119
|
+
say "no Postgres at $HOST:$PORT and docker is not installed."
|
|
120
|
+
say "Start one and re-run, or set BIFFO_PG_HOST / BIFFO_PG_PORT."
|
|
121
|
+
exit 1
|
|
122
|
+
fi
|
|
123
|
+
if docker ps -a --format '{{.Names}}' | grep -qx "$CONTAINER"; then
|
|
124
|
+
say "starting existing container $CONTAINER"
|
|
125
|
+
docker start "$CONTAINER" >/dev/null
|
|
126
|
+
else
|
|
127
|
+
say "creating container $CONTAINER ($IMAGE) on port $PORT"
|
|
128
|
+
docker run -d --name "$CONTAINER" \
|
|
129
|
+
-e POSTGRES_PASSWORD="$PASS" -p "$PORT:5432" "$IMAGE" >/dev/null
|
|
130
|
+
fi
|
|
131
|
+
# Polled, not slept: a cold image pull and a warm restart differ by an order of
|
|
132
|
+
# magnitude, and one fixed sleep is wrong for both.
|
|
133
|
+
_waited=0
|
|
134
|
+
until psql_admin -c 'SELECT 1' >/dev/null 2>&1; do
|
|
135
|
+
_waited=$((_waited + 1))
|
|
136
|
+
if [ "$_waited" -gt 90 ]; then
|
|
137
|
+
say "Postgres did not become ready in 90s"
|
|
138
|
+
exit 1
|
|
139
|
+
fi
|
|
140
|
+
sleep 1
|
|
141
|
+
done
|
|
142
|
+
say "Postgres ready after ${_waited}s"
|
|
143
|
+
fi
|
|
144
|
+
|
|
145
|
+
DSN="postgresql+asyncpg://$USER_:$PASS@$HOST:$PORT/$DB"
|
|
146
|
+
emit() {
|
|
147
|
+
if [ "$EXPORT" -eq 1 ]; then
|
|
148
|
+
echo "export BIFFO_TEST_PG_DSN='$DSN'"
|
|
149
|
+
else
|
|
150
|
+
echo "$DSN"
|
|
151
|
+
fi
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
# --- 2. is the existing schema current? --------------------------------------
|
|
155
|
+
#
|
|
156
|
+
# By CONTENT, not mtime: a branch switch changes content and leaves mtime
|
|
157
|
+
# anywhere. Stored inside the database, so it cannot outlive a drop or describe
|
|
158
|
+
# some other database.
|
|
159
|
+
fingerprint() {
|
|
160
|
+
{
|
|
161
|
+
[ -n "$ALEMBIC_DIR" ] && find "$ALEMBIC_DIR" -name '*.py' -path '*alembic*' -type f 2>/dev/null |
|
|
162
|
+
LC_ALL=C sort | xargs cat 2>/dev/null
|
|
163
|
+
[ -n "$DDL_FILES" ] && echo "$DDL_FILES" | xargs cat 2>/dev/null
|
|
164
|
+
} | sha256sum | cut -d' ' -f1
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
WANT=$(fingerprint)
|
|
168
|
+
HAVE=""
|
|
169
|
+
if [ "$RECREATE" -eq 0 ] &&
|
|
170
|
+
psql_admin -tAc "SELECT 1 FROM pg_database WHERE datname='$DB'" 2>/dev/null | grep -q 1; then
|
|
171
|
+
HAVE=$(psql -tAq -h "$HOST" -p "$PORT" -U "$USER_" -d "$DB" \
|
|
172
|
+
-c "SELECT value FROM biffo_pg_test_fingerprint LIMIT 1" 2>/dev/null || true)
|
|
173
|
+
fi
|
|
174
|
+
|
|
175
|
+
if [ -n "$HAVE" ] && [ "$HAVE" = "$WANT" ]; then
|
|
176
|
+
say "schema is current, reusing $DB"
|
|
177
|
+
emit
|
|
178
|
+
exit 0
|
|
179
|
+
fi
|
|
180
|
+
|
|
181
|
+
[ -n "$HAVE" ] && say "schema inputs changed - rebuilding rather than serving a stale schema"
|
|
182
|
+
|
|
183
|
+
# --- 3. rebuild the way the app and CI do ------------------------------------
|
|
184
|
+
say "rebuilding $DB"
|
|
185
|
+
psql_admin -c "DROP DATABASE IF EXISTS $DB WITH (FORCE)" >/dev/null
|
|
186
|
+
psql_admin -c "CREATE DATABASE $DB" >/dev/null
|
|
187
|
+
|
|
188
|
+
if [ -n "$ALEMBIC_DIR" ]; then
|
|
189
|
+
BIFFO_DATABASE_URL="$DSN" uv run --directory "$ALEMBIC_DIR" alembic upgrade head >/dev/null
|
|
190
|
+
say "alembic upgrade head"
|
|
191
|
+
fi
|
|
192
|
+
|
|
193
|
+
if [ -n "$DDL_FILES" ]; then
|
|
194
|
+
# ONE psql session, sorted by filename, mirroring the API's own DDL import.
|
|
195
|
+
# Session state an early module sets -- typically `SET search_path` in the
|
|
196
|
+
# first file -- has to survive into later ones, so a per-file connection would
|
|
197
|
+
# silently change the meaning of every unqualified name after it. LC_ALL=C
|
|
198
|
+
# keeps the shell's sort byte-ordered to match Python's.
|
|
199
|
+
# shellcheck disable=SC2046
|
|
200
|
+
psql -q -v ON_ERROR_STOP=1 -h "$HOST" -p "$PORT" -U "$USER_" -d "$DB" \
|
|
201
|
+
--single-transaction $(echo "$DDL_FILES" | sed 's/^/-f /' | tr '\n' ' ') >/dev/null
|
|
202
|
+
say "$(echo "$DDL_FILES" | wc -l | tr -d ' ') DDL modules applied"
|
|
203
|
+
fi
|
|
204
|
+
|
|
205
|
+
# --- 4. refuse to bless a half-built schema ----------------------------------
|
|
206
|
+
#
|
|
207
|
+
# The threshold is derived, not guessed: count the policies the DDL declares and
|
|
208
|
+
# require the database to hold at least half. Recording a fingerprint against a
|
|
209
|
+
# partial schema is worse than failing, because the NEXT run would trust it and
|
|
210
|
+
# every failure after that would look like the developer's own change.
|
|
211
|
+
if [ -n "$DDL_FILES" ]; then
|
|
212
|
+
_declared=$(echo "$DDL_FILES" | xargs grep -ciE '^[[:space:]]*CREATE[[:space:]]+POLICY' 2>/dev/null |
|
|
213
|
+
awk -F: '{s+=$NF} END {print s+0}')
|
|
214
|
+
if [ "${_declared:-0}" -gt 0 ]; then
|
|
215
|
+
_actual=$(psql -tAq -h "$HOST" -p "$PORT" -U "$USER_" -d "$DB" \
|
|
216
|
+
-c "SELECT count(*) FROM pg_policies" 2>/dev/null || echo 0)
|
|
217
|
+
if [ "${_actual:-0}" -lt $((_declared / 2)) ]; then
|
|
218
|
+
say "only ${_actual:-0} policies present against $_declared declared - the schema did not build."
|
|
219
|
+
say "Not recording a fingerprint; fix the DDL and re-run."
|
|
220
|
+
exit 1
|
|
221
|
+
fi
|
|
222
|
+
say "$_actual RLS policies ($_declared declared)"
|
|
223
|
+
fi
|
|
224
|
+
fi
|
|
225
|
+
|
|
226
|
+
psql_db \
|
|
227
|
+
-c "CREATE TABLE IF NOT EXISTS biffo_pg_test_fingerprint (value text primary key)" \
|
|
228
|
+
-c "TRUNCATE biffo_pg_test_fingerprint" \
|
|
229
|
+
-c "INSERT INTO biffo_pg_test_fingerprint (value) VALUES ('$WANT')" >/dev/null
|
|
230
|
+
|
|
231
|
+
say "ready"
|
|
232
|
+
emit
|
|
@@ -478,6 +478,23 @@ pg_test_run() {
|
|
|
478
478
|
}
|
|
479
479
|
|
|
480
480
|
_pg_modules=$(pg_test_modules)
|
|
481
|
+
|
|
482
|
+
# Provision the database rather than requiring the operator to remember.
|
|
483
|
+
#
|
|
484
|
+
# A gate that only runs when you exported the right variable is a gate that runs
|
|
485
|
+
# on the days you did not need it. `scripts/pg-test-db.sh` is idempotent and
|
|
486
|
+
# cheap when the schema is unchanged (~0.3s; ~4s when it genuinely has to
|
|
487
|
+
# rebuild), so calling it is better than warning about it. Failure is silent
|
|
488
|
+
# BECAUSE the WARN below is the honest report of it -- no Docker, no server, no
|
|
489
|
+
# schema all end in the same place: the lane did not run, and the gate says so.
|
|
490
|
+
if [ -z "$PG_TEST_DSN" ] && [ -n "$_pg_modules" ] && [ -z "$LIST" ] && [ -f scripts/pg-test-db.sh ]; then
|
|
491
|
+
PG_TEST_DSN=$(sh scripts/pg-test-db.sh 2>/dev/null | tail -1) || PG_TEST_DSN=""
|
|
492
|
+
case "$PG_TEST_DSN" in
|
|
493
|
+
postgres*) ;;
|
|
494
|
+
*) PG_TEST_DSN="" ;;
|
|
495
|
+
esac
|
|
496
|
+
fi
|
|
497
|
+
|
|
481
498
|
# Order matters, and getting it wrong made these very tests machine-dependent:
|
|
482
499
|
# with `uv not installed` checked FIRST, a runner without uv skipped quietly and
|
|
483
500
|
# the gap warning never printed -- green on a workstation, red on CI, for a
|