@biffo/cli 0.176.1 → 0.176.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -100,16 +100,68 @@ SKIPPED=""
|
|
|
100
100
|
# out. An override that only forces on would leave no way to escape a suite that
|
|
101
101
|
# has quietly grown past the threshold.
|
|
102
102
|
PYTEST_BUDGET_SECONDS="${BIFFO_VERIFY_PYTEST_BUDGET:-15}"
|
|
103
|
+
# How long a measurement is trusted before being re-taken. Only ever matters for
|
|
104
|
+
# a `slow` verdict; a `fast` one is re-measured by every run that uses it.
|
|
105
|
+
PYTEST_MAX_AGE_DAYS="${BIFFO_VERIFY_PYTEST_MAX_AGE_DAYS:-7}"
|
|
103
106
|
PYTEST="${BIFFO_VERIFY_PYTEST:-}"
|
|
104
107
|
|
|
105
108
|
# Cached per directory, because timing the suite to decide whether to run the
|
|
106
109
|
# suite would cost exactly what it is trying to save. The cache lives with the
|
|
107
110
|
# repo, not in $HOME, so it cannot leak a fast verdict from one repo to another.
|
|
111
|
+
# Where the measurement lives.
|
|
112
|
+
#
|
|
113
|
+
# NOT in the working tree. The first version wrote `$_d/.pytest-duration` and
|
|
114
|
+
# was gitignored in biffo-template only -- .gitignore is not a synced file, so
|
|
115
|
+
# every other repo in the estate grew an untracked `?? services/api/.pytest-duration`
|
|
116
|
+
# the moment the gate ran. A cache that dirties `git status` in fifteen repos is
|
|
117
|
+
# a defect regardless of what it caches.
|
|
118
|
+
#
|
|
119
|
+
# The git common dir is outside every working tree, shared by all worktrees of a
|
|
120
|
+
# clone (they run the same suite), and cannot be committed by accident. Keyed by
|
|
121
|
+
# the directory measured, so a root suite and services/api do not collide.
|
|
122
|
+
pytest_cache_file() {
|
|
123
|
+
_cd=$(git rev-parse --path-format=absolute --git-common-dir 2>/dev/null)/biffo-verify
|
|
124
|
+
mkdir -p "$_cd" 2>/dev/null || true
|
|
125
|
+
printf '%s/pytest-%s' "$_cd" "$(printf '%s' "$1" | tr '/.' '__')"
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
# Record what a real run actually took. The gate runs pytest whenever it believes
|
|
129
|
+
# the suite is fast, so it observes the true duration every single time -- and
|
|
130
|
+
# throwing that away was the whole bug. A suite that grows past the budget now
|
|
131
|
+
# excludes itself on the next push, for free and exactly.
|
|
132
|
+
pytest_record() {
|
|
133
|
+
[ -n "${2:-}" ] || return 0
|
|
134
|
+
printf '%s\n' "$2" > "$(pytest_cache_file "$1")" 2>/dev/null || true
|
|
135
|
+
}
|
|
136
|
+
|
|
108
137
|
pytest_is_fast() {
|
|
109
138
|
_d="$1"
|
|
110
|
-
_cache
|
|
111
|
-
|
|
139
|
+
_cache=$(pytest_cache_file "$_d")
|
|
140
|
+
# Age matters in ONE direction. A `fast` verdict is re-confirmed by every run
|
|
141
|
+
# (pytest_record), so it cannot go stale. A `slow` verdict is never re-tested,
|
|
142
|
+
# because the whole point of it is that the suite does not run -- so a suite
|
|
143
|
+
# that has since been split or sped up stays excluded for ever. Expiry is what
|
|
144
|
+
# gives it a way back in.
|
|
145
|
+
if [ -f "$_cache" ] && [ -n "$(find "$_cache" -mtime "-$PYTEST_MAX_AGE_DAYS" 2>/dev/null)" ]; then
|
|
146
|
+
_secs=$(cat "$_cache" 2>/dev/null)
|
|
147
|
+
elif [ -f "$_cache" ] && [ -n "$LIST" ]; then
|
|
148
|
+
# Expired, and --list must not run a suite to answer a question. Use the
|
|
149
|
+
# stale value rather than guessing: it is evidence, just old.
|
|
112
150
|
_secs=$(cat "$_cache" 2>/dev/null)
|
|
151
|
+
elif [ -n "$LIST" ]; then
|
|
152
|
+
# --list must not run a test suite to answer a question about the repo, so
|
|
153
|
+
# with no cached measurement it has to guess -- and the direction of the
|
|
154
|
+
# guess is the whole decision.
|
|
155
|
+
#
|
|
156
|
+
# Guessing "fast" makes --list CLAIM a check the gate may not run, which is
|
|
157
|
+
# the fail-open direction and exactly what this tooling exists to eliminate.
|
|
158
|
+
# Guessing "slow" makes it under-report a check the gate does run: visible,
|
|
159
|
+
# conservative, and self-correcting, because the first real run writes the
|
|
160
|
+
# measurement and every --list after that is exact.
|
|
161
|
+
#
|
|
162
|
+
# Both directions were tried. Under-reporting is the one that cannot lie
|
|
163
|
+
# about coverage.
|
|
164
|
+
_secs=99999
|
|
113
165
|
else
|
|
114
166
|
# First run in a repo: time it once, then decide from then on. A timeout
|
|
115
167
|
# means "too slow", which is the correct verdict rather than a hang.
|
|
@@ -120,7 +172,7 @@ pytest_is_fast() {
|
|
|
120
172
|
timeout "$((PYTEST_BUDGET_SECONDS * 4))" uv run --directory "$_d" pytest -q >/dev/null 2>&1 || true
|
|
121
173
|
fi
|
|
122
174
|
_secs=$(($(date +%s) - _start))
|
|
123
|
-
|
|
175
|
+
printf '%s\n' "$_secs" > "$_cache" 2>/dev/null || true
|
|
124
176
|
fi
|
|
125
177
|
[ "${_secs:-9999}" -le "$PYTEST_BUDGET_SECONDS" ]
|
|
126
178
|
}
|
|
@@ -232,9 +284,11 @@ run_check() {
|
|
|
232
284
|
return 0
|
|
233
285
|
fi
|
|
234
286
|
start=$(date +%s)
|
|
287
|
+
LAST_CHECK_SECONDS=""
|
|
235
288
|
if "$@" >"/tmp/biffo-verify.$$" 2>&1; then
|
|
236
289
|
PASSED="$PASSED $name"
|
|
237
|
-
|
|
290
|
+
LAST_CHECK_SECONDS=$(($(date +%s) - start))
|
|
291
|
+
printf ' \033[32mOK\033[0m %-16s %ss\n' "$name" "$LAST_CHECK_SECONDS"
|
|
238
292
|
else
|
|
239
293
|
FAILED="$FAILED $name"
|
|
240
294
|
printf ' \033[31mFAIL\033[0m %-16s %ss\n' "$name" "$(($(date +%s) - start))"
|
|
@@ -290,8 +344,15 @@ if [ -n "$PY_DIRS" ]; then
|
|
|
290
344
|
[ -n "$bandit_paths" ] && ci_has "bandit" && run_check "bandit$suffix" uv run bandit -r $bandit_paths -ll -q
|
|
291
345
|
if [ "$PYTEST" = "0" ]; then
|
|
292
346
|
skip "pytest$suffix" "excluded by BIFFO_VERIFY_PYTEST=0"
|
|
293
|
-
elif [ -n "$PYTEST" ] ||
|
|
294
|
-
ci_has "pytest"
|
|
347
|
+
elif [ -n "$PYTEST" ] || pytest_is_fast "."; then
|
|
348
|
+
if ci_has "pytest"; then
|
|
349
|
+
run_check "pytest$suffix" uv run pytest -q
|
|
350
|
+
# Re-confirm the verdict from what the run actually took. This is the
|
|
351
|
+
# invalidation that costs nothing: the gate has just measured the
|
|
352
|
+
# suite, so a suite that has grown past the budget excludes itself on
|
|
353
|
+
# the next push rather than slowing every push for ever.
|
|
354
|
+
pytest_record "." "$LAST_CHECK_SECONDS"
|
|
355
|
+
fi
|
|
295
356
|
else
|
|
296
357
|
skip "pytest$suffix" "suite is slower than ${PYTEST_BUDGET_SECONDS}s - CI keeps it"
|
|
297
358
|
fi
|
|
@@ -302,8 +363,11 @@ if [ -n "$PY_DIRS" ]; then
|
|
|
302
363
|
ci_has "bandit" && run_check "bandit$suffix" uv run --directory "$d" bandit -r src -ll -q
|
|
303
364
|
if [ "$PYTEST" = "0" ]; then
|
|
304
365
|
skip "pytest$suffix" "excluded by BIFFO_VERIFY_PYTEST=0"
|
|
305
|
-
elif [ -n "$PYTEST" ] ||
|
|
306
|
-
ci_has "pytest"
|
|
366
|
+
elif [ -n "$PYTEST" ] || pytest_is_fast "$d"; then
|
|
367
|
+
if ci_has "pytest"; then
|
|
368
|
+
run_check "pytest$suffix" uv run --directory "$d" pytest -q
|
|
369
|
+
pytest_record "$d" "$LAST_CHECK_SECONDS"
|
|
370
|
+
fi
|
|
307
371
|
else
|
|
308
372
|
skip "pytest$suffix" "suite is slower than ${PYTEST_BUDGET_SECONDS}s - CI keeps it"
|
|
309
373
|
fi
|
|
@@ -100,16 +100,68 @@ SKIPPED=""
|
|
|
100
100
|
# out. An override that only forces on would leave no way to escape a suite that
|
|
101
101
|
# has quietly grown past the threshold.
|
|
102
102
|
PYTEST_BUDGET_SECONDS="${BIFFO_VERIFY_PYTEST_BUDGET:-15}"
|
|
103
|
+
# How long a measurement is trusted before being re-taken. Only ever matters for
|
|
104
|
+
# a `slow` verdict; a `fast` one is re-measured by every run that uses it.
|
|
105
|
+
PYTEST_MAX_AGE_DAYS="${BIFFO_VERIFY_PYTEST_MAX_AGE_DAYS:-7}"
|
|
103
106
|
PYTEST="${BIFFO_VERIFY_PYTEST:-}"
|
|
104
107
|
|
|
105
108
|
# Cached per directory, because timing the suite to decide whether to run the
|
|
106
109
|
# suite would cost exactly what it is trying to save. The cache lives with the
|
|
107
110
|
# repo, not in $HOME, so it cannot leak a fast verdict from one repo to another.
|
|
111
|
+
# Where the measurement lives.
|
|
112
|
+
#
|
|
113
|
+
# NOT in the working tree. The first version wrote `$_d/.pytest-duration` and
|
|
114
|
+
# was gitignored in biffo-template only -- .gitignore is not a synced file, so
|
|
115
|
+
# every other repo in the estate grew an untracked `?? services/api/.pytest-duration`
|
|
116
|
+
# the moment the gate ran. A cache that dirties `git status` in fifteen repos is
|
|
117
|
+
# a defect regardless of what it caches.
|
|
118
|
+
#
|
|
119
|
+
# The git common dir is outside every working tree, shared by all worktrees of a
|
|
120
|
+
# clone (they run the same suite), and cannot be committed by accident. Keyed by
|
|
121
|
+
# the directory measured, so a root suite and services/api do not collide.
|
|
122
|
+
pytest_cache_file() {
|
|
123
|
+
_cd=$(git rev-parse --path-format=absolute --git-common-dir 2>/dev/null)/biffo-verify
|
|
124
|
+
mkdir -p "$_cd" 2>/dev/null || true
|
|
125
|
+
printf '%s/pytest-%s' "$_cd" "$(printf '%s' "$1" | tr '/.' '__')"
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
# Record what a real run actually took. The gate runs pytest whenever it believes
|
|
129
|
+
# the suite is fast, so it observes the true duration every single time -- and
|
|
130
|
+
# throwing that away was the whole bug. A suite that grows past the budget now
|
|
131
|
+
# excludes itself on the next push, for free and exactly.
|
|
132
|
+
pytest_record() {
|
|
133
|
+
[ -n "${2:-}" ] || return 0
|
|
134
|
+
printf '%s\n' "$2" > "$(pytest_cache_file "$1")" 2>/dev/null || true
|
|
135
|
+
}
|
|
136
|
+
|
|
108
137
|
pytest_is_fast() {
|
|
109
138
|
_d="$1"
|
|
110
|
-
_cache
|
|
111
|
-
|
|
139
|
+
_cache=$(pytest_cache_file "$_d")
|
|
140
|
+
# Age matters in ONE direction. A `fast` verdict is re-confirmed by every run
|
|
141
|
+
# (pytest_record), so it cannot go stale. A `slow` verdict is never re-tested,
|
|
142
|
+
# because the whole point of it is that the suite does not run -- so a suite
|
|
143
|
+
# that has since been split or sped up stays excluded for ever. Expiry is what
|
|
144
|
+
# gives it a way back in.
|
|
145
|
+
if [ -f "$_cache" ] && [ -n "$(find "$_cache" -mtime "-$PYTEST_MAX_AGE_DAYS" 2>/dev/null)" ]; then
|
|
146
|
+
_secs=$(cat "$_cache" 2>/dev/null)
|
|
147
|
+
elif [ -f "$_cache" ] && [ -n "$LIST" ]; then
|
|
148
|
+
# Expired, and --list must not run a suite to answer a question. Use the
|
|
149
|
+
# stale value rather than guessing: it is evidence, just old.
|
|
112
150
|
_secs=$(cat "$_cache" 2>/dev/null)
|
|
151
|
+
elif [ -n "$LIST" ]; then
|
|
152
|
+
# --list must not run a test suite to answer a question about the repo, so
|
|
153
|
+
# with no cached measurement it has to guess -- and the direction of the
|
|
154
|
+
# guess is the whole decision.
|
|
155
|
+
#
|
|
156
|
+
# Guessing "fast" makes --list CLAIM a check the gate may not run, which is
|
|
157
|
+
# the fail-open direction and exactly what this tooling exists to eliminate.
|
|
158
|
+
# Guessing "slow" makes it under-report a check the gate does run: visible,
|
|
159
|
+
# conservative, and self-correcting, because the first real run writes the
|
|
160
|
+
# measurement and every --list after that is exact.
|
|
161
|
+
#
|
|
162
|
+
# Both directions were tried. Under-reporting is the one that cannot lie
|
|
163
|
+
# about coverage.
|
|
164
|
+
_secs=99999
|
|
113
165
|
else
|
|
114
166
|
# First run in a repo: time it once, then decide from then on. A timeout
|
|
115
167
|
# means "too slow", which is the correct verdict rather than a hang.
|
|
@@ -120,7 +172,7 @@ pytest_is_fast() {
|
|
|
120
172
|
timeout "$((PYTEST_BUDGET_SECONDS * 4))" uv run --directory "$_d" pytest -q >/dev/null 2>&1 || true
|
|
121
173
|
fi
|
|
122
174
|
_secs=$(($(date +%s) - _start))
|
|
123
|
-
|
|
175
|
+
printf '%s\n' "$_secs" > "$_cache" 2>/dev/null || true
|
|
124
176
|
fi
|
|
125
177
|
[ "${_secs:-9999}" -le "$PYTEST_BUDGET_SECONDS" ]
|
|
126
178
|
}
|
|
@@ -232,9 +284,11 @@ run_check() {
|
|
|
232
284
|
return 0
|
|
233
285
|
fi
|
|
234
286
|
start=$(date +%s)
|
|
287
|
+
LAST_CHECK_SECONDS=""
|
|
235
288
|
if "$@" >"/tmp/biffo-verify.$$" 2>&1; then
|
|
236
289
|
PASSED="$PASSED $name"
|
|
237
|
-
|
|
290
|
+
LAST_CHECK_SECONDS=$(($(date +%s) - start))
|
|
291
|
+
printf ' \033[32mOK\033[0m %-16s %ss\n' "$name" "$LAST_CHECK_SECONDS"
|
|
238
292
|
else
|
|
239
293
|
FAILED="$FAILED $name"
|
|
240
294
|
printf ' \033[31mFAIL\033[0m %-16s %ss\n' "$name" "$(($(date +%s) - start))"
|
|
@@ -290,8 +344,15 @@ if [ -n "$PY_DIRS" ]; then
|
|
|
290
344
|
[ -n "$bandit_paths" ] && ci_has "bandit" && run_check "bandit$suffix" uv run bandit -r $bandit_paths -ll -q
|
|
291
345
|
if [ "$PYTEST" = "0" ]; then
|
|
292
346
|
skip "pytest$suffix" "excluded by BIFFO_VERIFY_PYTEST=0"
|
|
293
|
-
elif [ -n "$PYTEST" ] ||
|
|
294
|
-
ci_has "pytest"
|
|
347
|
+
elif [ -n "$PYTEST" ] || pytest_is_fast "."; then
|
|
348
|
+
if ci_has "pytest"; then
|
|
349
|
+
run_check "pytest$suffix" uv run pytest -q
|
|
350
|
+
# Re-confirm the verdict from what the run actually took. This is the
|
|
351
|
+
# invalidation that costs nothing: the gate has just measured the
|
|
352
|
+
# suite, so a suite that has grown past the budget excludes itself on
|
|
353
|
+
# the next push rather than slowing every push for ever.
|
|
354
|
+
pytest_record "." "$LAST_CHECK_SECONDS"
|
|
355
|
+
fi
|
|
295
356
|
else
|
|
296
357
|
skip "pytest$suffix" "suite is slower than ${PYTEST_BUDGET_SECONDS}s - CI keeps it"
|
|
297
358
|
fi
|
|
@@ -302,8 +363,11 @@ if [ -n "$PY_DIRS" ]; then
|
|
|
302
363
|
ci_has "bandit" && run_check "bandit$suffix" uv run --directory "$d" bandit -r src -ll -q
|
|
303
364
|
if [ "$PYTEST" = "0" ]; then
|
|
304
365
|
skip "pytest$suffix" "excluded by BIFFO_VERIFY_PYTEST=0"
|
|
305
|
-
elif [ -n "$PYTEST" ] ||
|
|
306
|
-
ci_has "pytest"
|
|
366
|
+
elif [ -n "$PYTEST" ] || pytest_is_fast "$d"; then
|
|
367
|
+
if ci_has "pytest"; then
|
|
368
|
+
run_check "pytest$suffix" uv run --directory "$d" pytest -q
|
|
369
|
+
pytest_record "$d" "$LAST_CHECK_SECONDS"
|
|
370
|
+
fi
|
|
307
371
|
else
|
|
308
372
|
skip "pytest$suffix" "suite is slower than ${PYTEST_BUDGET_SECONDS}s - CI keeps it"
|
|
309
373
|
fi
|