@biffo/cli 0.176.2 → 0.176.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -73,6 +73,16 @@ LIST=""
|
|
|
73
73
|
FAILED=""
|
|
74
74
|
PASSED=""
|
|
75
75
|
SKIPPED=""
|
|
76
|
+
# Defined up here, not inside run_check. `run_check` returns EARLY in --list
|
|
77
|
+
# mode, before it would set this -- so `pytest_record "$d" "$LAST_CHECK_SECONDS"`
|
|
78
|
+
# read an unset variable and `set -u` killed the script silently, mid-list.
|
|
79
|
+
#
|
|
80
|
+
# The damage was invisible and downstream: gate-coverage.sh reads --list, so a
|
|
81
|
+
# truncated list looked like MISSING COVERAGE. tabsii-geo dropped from 8/8 to
|
|
82
|
+
# 4/8 -- and only in repos where a pytest measurement already existed, i.e. only
|
|
83
|
+
# after the gate had run there once. A defect that appears on second use is the
|
|
84
|
+
# hardest kind to attribute.
|
|
85
|
+
LAST_CHECK_SECONDS=""
|
|
76
86
|
|
|
77
87
|
# pytest is included where the suite is FAST ENOUGH TO PAY, measured rather than
|
|
78
88
|
# opted into (#869, H5 gap 4).
|
|
@@ -100,15 +110,53 @@ SKIPPED=""
|
|
|
100
110
|
# out. An override that only forces on would leave no way to escape a suite that
|
|
101
111
|
# has quietly grown past the threshold.
|
|
102
112
|
PYTEST_BUDGET_SECONDS="${BIFFO_VERIFY_PYTEST_BUDGET:-15}"
|
|
113
|
+
# How long a measurement is trusted before being re-taken. Only ever matters for
|
|
114
|
+
# a `slow` verdict; a `fast` one is re-measured by every run that uses it.
|
|
115
|
+
PYTEST_MAX_AGE_DAYS="${BIFFO_VERIFY_PYTEST_MAX_AGE_DAYS:-7}"
|
|
103
116
|
PYTEST="${BIFFO_VERIFY_PYTEST:-}"
|
|
104
117
|
|
|
105
118
|
# Cached per directory, because timing the suite to decide whether to run the
|
|
106
119
|
# suite would cost exactly what it is trying to save. The cache lives with the
|
|
107
120
|
# repo, not in $HOME, so it cannot leak a fast verdict from one repo to another.
|
|
121
|
+
# Where the measurement lives.
|
|
122
|
+
#
|
|
123
|
+
# NOT in the working tree. The first version wrote `$_d/.pytest-duration` and
|
|
124
|
+
# was gitignored in biffo-template only -- .gitignore is not a synced file, so
|
|
125
|
+
# every other repo in the estate grew an untracked `?? services/api/.pytest-duration`
|
|
126
|
+
# the moment the gate ran. A cache that dirties `git status` in fifteen repos is
|
|
127
|
+
# a defect regardless of what it caches.
|
|
128
|
+
#
|
|
129
|
+
# The git common dir is outside every working tree, shared by all worktrees of a
|
|
130
|
+
# clone (they run the same suite), and cannot be committed by accident. Keyed by
|
|
131
|
+
# the directory measured, so a root suite and services/api do not collide.
|
|
132
|
+
pytest_cache_file() {
|
|
133
|
+
_cd=$(git rev-parse --path-format=absolute --git-common-dir 2>/dev/null)/biffo-verify
|
|
134
|
+
mkdir -p "$_cd" 2>/dev/null || true
|
|
135
|
+
printf '%s/pytest-%s' "$_cd" "$(printf '%s' "$1" | tr '/.' '__')"
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
# Record what a real run actually took. The gate runs pytest whenever it believes
|
|
139
|
+
# the suite is fast, so it observes the true duration every single time -- and
|
|
140
|
+
# throwing that away was the whole bug. A suite that grows past the budget now
|
|
141
|
+
# excludes itself on the next push, for free and exactly.
|
|
142
|
+
pytest_record() {
|
|
143
|
+
[ -n "${2:-}" ] || return 0
|
|
144
|
+
printf '%s\n' "$2" > "$(pytest_cache_file "$1")" 2>/dev/null || true
|
|
145
|
+
}
|
|
146
|
+
|
|
108
147
|
pytest_is_fast() {
|
|
109
148
|
_d="$1"
|
|
110
|
-
_cache
|
|
111
|
-
|
|
149
|
+
_cache=$(pytest_cache_file "$_d")
|
|
150
|
+
# Age matters in ONE direction. A `fast` verdict is re-confirmed by every run
|
|
151
|
+
# (pytest_record), so it cannot go stale. A `slow` verdict is never re-tested,
|
|
152
|
+
# because the whole point of it is that the suite does not run -- so a suite
|
|
153
|
+
# that has since been split or sped up stays excluded for ever. Expiry is what
|
|
154
|
+
# gives it a way back in.
|
|
155
|
+
if [ -f "$_cache" ] && [ -n "$(find "$_cache" -mtime "-$PYTEST_MAX_AGE_DAYS" 2>/dev/null)" ]; then
|
|
156
|
+
_secs=$(cat "$_cache" 2>/dev/null)
|
|
157
|
+
elif [ -f "$_cache" ] && [ -n "$LIST" ]; then
|
|
158
|
+
# Expired, and --list must not run a suite to answer a question. Use the
|
|
159
|
+
# stale value rather than guessing: it is evidence, just old.
|
|
112
160
|
_secs=$(cat "$_cache" 2>/dev/null)
|
|
113
161
|
elif [ -n "$LIST" ]; then
|
|
114
162
|
# --list must not run a test suite to answer a question about the repo, so
|
|
@@ -134,7 +182,7 @@ pytest_is_fast() {
|
|
|
134
182
|
timeout "$((PYTEST_BUDGET_SECONDS * 4))" uv run --directory "$_d" pytest -q >/dev/null 2>&1 || true
|
|
135
183
|
fi
|
|
136
184
|
_secs=$(($(date +%s) - _start))
|
|
137
|
-
|
|
185
|
+
printf '%s\n' "$_secs" > "$_cache" 2>/dev/null || true
|
|
138
186
|
fi
|
|
139
187
|
[ "${_secs:-9999}" -le "$PYTEST_BUDGET_SECONDS" ]
|
|
140
188
|
}
|
|
@@ -248,7 +296,8 @@ run_check() {
|
|
|
248
296
|
start=$(date +%s)
|
|
249
297
|
if "$@" >"/tmp/biffo-verify.$$" 2>&1; then
|
|
250
298
|
PASSED="$PASSED $name"
|
|
251
|
-
|
|
299
|
+
LAST_CHECK_SECONDS=$(($(date +%s) - start))
|
|
300
|
+
printf ' \033[32mOK\033[0m %-16s %ss\n' "$name" "$LAST_CHECK_SECONDS"
|
|
252
301
|
else
|
|
253
302
|
FAILED="$FAILED $name"
|
|
254
303
|
printf ' \033[31mFAIL\033[0m %-16s %ss\n' "$name" "$(($(date +%s) - start))"
|
|
@@ -305,7 +354,14 @@ if [ -n "$PY_DIRS" ]; then
|
|
|
305
354
|
if [ "$PYTEST" = "0" ]; then
|
|
306
355
|
skip "pytest$suffix" "excluded by BIFFO_VERIFY_PYTEST=0"
|
|
307
356
|
elif [ -n "$PYTEST" ] || pytest_is_fast "."; then
|
|
308
|
-
ci_has "pytest"
|
|
357
|
+
if ci_has "pytest"; then
|
|
358
|
+
run_check "pytest$suffix" uv run pytest -q
|
|
359
|
+
# Re-confirm the verdict from what the run actually took. This is the
|
|
360
|
+
# invalidation that costs nothing: the gate has just measured the
|
|
361
|
+
# suite, so a suite that has grown past the budget excludes itself on
|
|
362
|
+
# the next push rather than slowing every push for ever.
|
|
363
|
+
pytest_record "." "$LAST_CHECK_SECONDS"
|
|
364
|
+
fi
|
|
309
365
|
else
|
|
310
366
|
skip "pytest$suffix" "suite is slower than ${PYTEST_BUDGET_SECONDS}s - CI keeps it"
|
|
311
367
|
fi
|
|
@@ -317,7 +373,10 @@ if [ -n "$PY_DIRS" ]; then
|
|
|
317
373
|
if [ "$PYTEST" = "0" ]; then
|
|
318
374
|
skip "pytest$suffix" "excluded by BIFFO_VERIFY_PYTEST=0"
|
|
319
375
|
elif [ -n "$PYTEST" ] || pytest_is_fast "$d"; then
|
|
320
|
-
ci_has "pytest"
|
|
376
|
+
if ci_has "pytest"; then
|
|
377
|
+
run_check "pytest$suffix" uv run --directory "$d" pytest -q
|
|
378
|
+
pytest_record "$d" "$LAST_CHECK_SECONDS"
|
|
379
|
+
fi
|
|
321
380
|
else
|
|
322
381
|
skip "pytest$suffix" "suite is slower than ${PYTEST_BUDGET_SECONDS}s - CI keeps it"
|
|
323
382
|
fi
|
|
@@ -73,6 +73,16 @@ LIST=""
|
|
|
73
73
|
FAILED=""
|
|
74
74
|
PASSED=""
|
|
75
75
|
SKIPPED=""
|
|
76
|
+
# Defined up here, not inside run_check. `run_check` returns EARLY in --list
|
|
77
|
+
# mode, before it would set this -- so `pytest_record "$d" "$LAST_CHECK_SECONDS"`
|
|
78
|
+
# read an unset variable and `set -u` killed the script silently, mid-list.
|
|
79
|
+
#
|
|
80
|
+
# The damage was invisible and downstream: gate-coverage.sh reads --list, so a
|
|
81
|
+
# truncated list looked like MISSING COVERAGE. tabsii-geo dropped from 8/8 to
|
|
82
|
+
# 4/8 -- and only in repos where a pytest measurement already existed, i.e. only
|
|
83
|
+
# after the gate had run there once. A defect that appears on second use is the
|
|
84
|
+
# hardest kind to attribute.
|
|
85
|
+
LAST_CHECK_SECONDS=""
|
|
76
86
|
|
|
77
87
|
# pytest is included where the suite is FAST ENOUGH TO PAY, measured rather than
|
|
78
88
|
# opted into (#869, H5 gap 4).
|
|
@@ -100,15 +110,53 @@ SKIPPED=""
|
|
|
100
110
|
# out. An override that only forces on would leave no way to escape a suite that
|
|
101
111
|
# has quietly grown past the threshold.
|
|
102
112
|
PYTEST_BUDGET_SECONDS="${BIFFO_VERIFY_PYTEST_BUDGET:-15}"
|
|
113
|
+
# How long a measurement is trusted before being re-taken. Only ever matters for
|
|
114
|
+
# a `slow` verdict; a `fast` one is re-measured by every run that uses it.
|
|
115
|
+
PYTEST_MAX_AGE_DAYS="${BIFFO_VERIFY_PYTEST_MAX_AGE_DAYS:-7}"
|
|
103
116
|
PYTEST="${BIFFO_VERIFY_PYTEST:-}"
|
|
104
117
|
|
|
105
118
|
# Cached per directory, because timing the suite to decide whether to run the
|
|
106
119
|
# suite would cost exactly what it is trying to save. The cache lives with the
|
|
107
120
|
# repo, not in $HOME, so it cannot leak a fast verdict from one repo to another.
|
|
121
|
+
# Where the measurement lives.
|
|
122
|
+
#
|
|
123
|
+
# NOT in the working tree. The first version wrote `$_d/.pytest-duration` and
|
|
124
|
+
# was gitignored in biffo-template only -- .gitignore is not a synced file, so
|
|
125
|
+
# every other repo in the estate grew an untracked `?? services/api/.pytest-duration`
|
|
126
|
+
# the moment the gate ran. A cache that dirties `git status` in fifteen repos is
|
|
127
|
+
# a defect regardless of what it caches.
|
|
128
|
+
#
|
|
129
|
+
# The git common dir is outside every working tree, shared by all worktrees of a
|
|
130
|
+
# clone (they run the same suite), and cannot be committed by accident. Keyed by
|
|
131
|
+
# the directory measured, so a root suite and services/api do not collide.
|
|
132
|
+
pytest_cache_file() {
|
|
133
|
+
_cd=$(git rev-parse --path-format=absolute --git-common-dir 2>/dev/null)/biffo-verify
|
|
134
|
+
mkdir -p "$_cd" 2>/dev/null || true
|
|
135
|
+
printf '%s/pytest-%s' "$_cd" "$(printf '%s' "$1" | tr '/.' '__')"
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
# Record what a real run actually took. The gate runs pytest whenever it believes
|
|
139
|
+
# the suite is fast, so it observes the true duration every single time -- and
|
|
140
|
+
# throwing that away was the whole bug. A suite that grows past the budget now
|
|
141
|
+
# excludes itself on the next push, for free and exactly.
|
|
142
|
+
pytest_record() {
|
|
143
|
+
[ -n "${2:-}" ] || return 0
|
|
144
|
+
printf '%s\n' "$2" > "$(pytest_cache_file "$1")" 2>/dev/null || true
|
|
145
|
+
}
|
|
146
|
+
|
|
108
147
|
pytest_is_fast() {
|
|
109
148
|
_d="$1"
|
|
110
|
-
_cache
|
|
111
|
-
|
|
149
|
+
_cache=$(pytest_cache_file "$_d")
|
|
150
|
+
# Age matters in ONE direction. A `fast` verdict is re-confirmed by every run
|
|
151
|
+
# (pytest_record), so it cannot go stale. A `slow` verdict is never re-tested,
|
|
152
|
+
# because the whole point of it is that the suite does not run -- so a suite
|
|
153
|
+
# that has since been split or sped up stays excluded for ever. Expiry is what
|
|
154
|
+
# gives it a way back in.
|
|
155
|
+
if [ -f "$_cache" ] && [ -n "$(find "$_cache" -mtime "-$PYTEST_MAX_AGE_DAYS" 2>/dev/null)" ]; then
|
|
156
|
+
_secs=$(cat "$_cache" 2>/dev/null)
|
|
157
|
+
elif [ -f "$_cache" ] && [ -n "$LIST" ]; then
|
|
158
|
+
# Expired, and --list must not run a suite to answer a question. Use the
|
|
159
|
+
# stale value rather than guessing: it is evidence, just old.
|
|
112
160
|
_secs=$(cat "$_cache" 2>/dev/null)
|
|
113
161
|
elif [ -n "$LIST" ]; then
|
|
114
162
|
# --list must not run a test suite to answer a question about the repo, so
|
|
@@ -134,7 +182,7 @@ pytest_is_fast() {
|
|
|
134
182
|
timeout "$((PYTEST_BUDGET_SECONDS * 4))" uv run --directory "$_d" pytest -q >/dev/null 2>&1 || true
|
|
135
183
|
fi
|
|
136
184
|
_secs=$(($(date +%s) - _start))
|
|
137
|
-
|
|
185
|
+
printf '%s\n' "$_secs" > "$_cache" 2>/dev/null || true
|
|
138
186
|
fi
|
|
139
187
|
[ "${_secs:-9999}" -le "$PYTEST_BUDGET_SECONDS" ]
|
|
140
188
|
}
|
|
@@ -248,7 +296,8 @@ run_check() {
|
|
|
248
296
|
start=$(date +%s)
|
|
249
297
|
if "$@" >"/tmp/biffo-verify.$$" 2>&1; then
|
|
250
298
|
PASSED="$PASSED $name"
|
|
251
|
-
|
|
299
|
+
LAST_CHECK_SECONDS=$(($(date +%s) - start))
|
|
300
|
+
printf ' \033[32mOK\033[0m %-16s %ss\n' "$name" "$LAST_CHECK_SECONDS"
|
|
252
301
|
else
|
|
253
302
|
FAILED="$FAILED $name"
|
|
254
303
|
printf ' \033[31mFAIL\033[0m %-16s %ss\n' "$name" "$(($(date +%s) - start))"
|
|
@@ -305,7 +354,14 @@ if [ -n "$PY_DIRS" ]; then
|
|
|
305
354
|
if [ "$PYTEST" = "0" ]; then
|
|
306
355
|
skip "pytest$suffix" "excluded by BIFFO_VERIFY_PYTEST=0"
|
|
307
356
|
elif [ -n "$PYTEST" ] || pytest_is_fast "."; then
|
|
308
|
-
ci_has "pytest"
|
|
357
|
+
if ci_has "pytest"; then
|
|
358
|
+
run_check "pytest$suffix" uv run pytest -q
|
|
359
|
+
# Re-confirm the verdict from what the run actually took. This is the
|
|
360
|
+
# invalidation that costs nothing: the gate has just measured the
|
|
361
|
+
# suite, so a suite that has grown past the budget excludes itself on
|
|
362
|
+
# the next push rather than slowing every push for ever.
|
|
363
|
+
pytest_record "." "$LAST_CHECK_SECONDS"
|
|
364
|
+
fi
|
|
309
365
|
else
|
|
310
366
|
skip "pytest$suffix" "suite is slower than ${PYTEST_BUDGET_SECONDS}s - CI keeps it"
|
|
311
367
|
fi
|
|
@@ -317,7 +373,10 @@ if [ -n "$PY_DIRS" ]; then
|
|
|
317
373
|
if [ "$PYTEST" = "0" ]; then
|
|
318
374
|
skip "pytest$suffix" "excluded by BIFFO_VERIFY_PYTEST=0"
|
|
319
375
|
elif [ -n "$PYTEST" ] || pytest_is_fast "$d"; then
|
|
320
|
-
ci_has "pytest"
|
|
376
|
+
if ci_has "pytest"; then
|
|
377
|
+
run_check "pytest$suffix" uv run --directory "$d" pytest -q
|
|
378
|
+
pytest_record "$d" "$LAST_CHECK_SECONDS"
|
|
379
|
+
fi
|
|
321
380
|
else
|
|
322
381
|
skip "pytest$suffix" "suite is slower than ${PYTEST_BUDGET_SECONDS}s - CI keeps it"
|
|
323
382
|
fi
|