@feigi/fleet-ctl 3.47.6 → 3.47.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/skills/run-team/SKILL.md +74 -40
package/package.json
CHANGED
package/skills/run-team/SKILL.md
CHANGED
|
@@ -2173,7 +2173,7 @@ PRs**, because the two causes live at different levels — the budget is one
|
|
|
2173
2173
|
account, the payload is one PR:
|
|
2174
2174
|
|
|
2175
2175
|
```sh
|
|
2176
|
-
budget_out= list_out= cap_out= blind=
|
|
2176
|
+
budget_out= hidden_out= list_out= cap_out= blind= # seed the latches: set -u reads each below before tick 1 ever assigns one
|
|
2177
2177
|
pr_cap=200 # the open-PR read's --limit; gh stops there with exit 0 and no warning
|
|
2178
2178
|
while :; do # one tick
|
|
2179
2179
|
rl=$(gh api rate_limit --jq '.resources.core.remaining' 2>/dev/null || echo ERR)
|
|
@@ -2210,34 +2210,52 @@ while :; do # one tick
|
|
|
2210
2210
|
# ceiling of one ci-state.mjs call on the counter `gh api rate_limit`
|
|
2211
2211
|
# reports, for single-page reads: 2 on the common path, 4 for a PR whose
|
|
2212
2212
|
# head dropped required jobs.
|
|
2213
|
-
#
|
|
2214
|
-
# does not report and this gate does not guard. A shortfall pauses this
|
|
2215
|
-
# tick's pass.
|
|
2213
|
+
# A shortfall here, or at the Actions-bucket gate below, pauses this tick's pass.
|
|
2216
2214
|
if [ "$rl" -lt $((200 + 4 * count)) ]; then
|
|
2217
2215
|
if [ -z "$budget_out" ]; then
|
|
2218
2216
|
echo "WATCHER DEGRADED: core REST budget=$rl < $((200 + 4 * count)) needed for $count PRs — CI polling paused, silence is NOT green"
|
|
2219
2217
|
budget_out=1
|
|
2220
2218
|
fi
|
|
2221
2219
|
else
|
|
2222
|
-
|
|
2223
|
-
|
|
2224
|
-
|
|
2225
|
-
|
|
2226
|
-
|
|
2227
|
-
|
|
2228
|
-
|
|
2229
|
-
|
|
2230
|
-
|
|
2231
|
-
|
|
2232
|
-
|
|
2233
|
-
|
|
2220
|
+
# The workflow and run lookups bill a second core bucket that
|
|
2221
|
+
# `rate_limit` does not report. Its count is only in the response
|
|
2222
|
+
# headers of a lookup that bills it: one 1-unit read per tick, 4 units
|
|
2223
|
+
# per listed PR on the same 200 floor. Recovery for both buckets waits
|
|
2224
|
+
# until this gate clears too, so no line says polling resumed while
|
|
2225
|
+
# this one still pauses it.
|
|
2226
|
+
hrl=$(gh api -i 'repos/{owner}/{repo}/actions/workflows?per_page=1' 2>/dev/null | awk 'tolower($1)=="x-ratelimit-remaining:" {sub(/\r$/, "", $2); print $2; exit}')
|
|
2227
|
+
case "$hrl" in ''|*[!0-9]*) hrl=ERR;; esac # a missing header is an outage, not a budget
|
|
2228
|
+
if [ "$hrl" = ERR ] || [ "$hrl" -lt $((200 + 4 * count)) ]; then
|
|
2229
|
+
if [ -z "$hidden_out" ]; then
|
|
2230
|
+
need=; [ "$hrl" = ERR ] || need=" < $((200 + 4 * count)) needed for $count PRs" # an unreadable header has no amount to compare
|
|
2231
|
+
echo "WATCHER DEGRADED: unreported Actions core REST bucket=$hrl$need — CI polling paused, silence is NOT green"
|
|
2232
|
+
hidden_out=1
|
|
2233
|
+
fi
|
|
2234
|
+
else
|
|
2235
|
+
if [ -n "$budget_out" ]; then
|
|
2236
|
+
echo "WATCHER RECOVERED: core budget=$rl — CI polling resumed"
|
|
2237
|
+
budget_out=
|
|
2234
2238
|
fi
|
|
2235
|
-
|
|
2236
|
-
echo "WATCHER RECOVERED:
|
|
2237
|
-
|
|
2238
|
-
|
|
2239
|
-
|
|
2240
|
-
|
|
2239
|
+
if [ -n "$hidden_out" ]; then
|
|
2240
|
+
echo "WATCHER RECOVERED: unreported Actions core REST bucket=$hrl — CI polling resumed"
|
|
2241
|
+
hidden_out=
|
|
2242
|
+
fi
|
|
2243
|
+
for pr in $(printf '%s\n' "$prs"); do # inline $(...): `for pr in $prs` is ONE iteration under zsh
|
|
2244
|
+
st=$(~/.fleet/bin/fleet-run ci-state.mjs --pr "$pr" 2>/dev/null) || : # not-green exits non-zero; the payload is the verdict
|
|
2245
|
+
if ! printf '%s' "$st" | jq -e '.verdict and .verdict != "rate-limited"' >/dev/null 2>&1; then
|
|
2246
|
+
case " $blind " in *" $pr "*) ;; *) # latch keyed BY PR: this cause is per-PR
|
|
2247
|
+
echo "WATCHER DEGRADED: no usable ci-state reading for #$pr — silence is NOT green"
|
|
2248
|
+
blind="$blind $pr" ;;
|
|
2249
|
+
esac
|
|
2250
|
+
continue # inner continue — still reaches the tick sleep
|
|
2251
|
+
fi
|
|
2252
|
+
case " $blind " in *" $pr "*)
|
|
2253
|
+
echo "WATCHER RECOVERED: ci-state reading #$pr again — CI polling resumed"
|
|
2254
|
+
keep=; for b in $(printf '%s\n' "$blind"); do [ "$b" = "$pr" ] || keep="$keep $b"; done; blind=$keep ;;
|
|
2255
|
+
esac
|
|
2256
|
+
: # your normal handling of $st for this PR
|
|
2257
|
+
done
|
|
2258
|
+
fi
|
|
2241
2259
|
fi
|
|
2242
2260
|
fi
|
|
2243
2261
|
fi
|
|
@@ -2263,8 +2281,8 @@ as a CI reading. Require the field, then compare it.
|
|
|
2263
2281
|
never one per tick. An unlatched line at a 120s poll gets the Monitor
|
|
2264
2282
|
auto-stopped for volume during even a two-minute outage, reintroducing the same
|
|
2265
2283
|
blindness by another route. **Latch each cause at the level its cause lives
|
|
2266
|
-
at**: the budget, the open-PR list and that
|
|
2267
|
-
each — all
|
|
2284
|
+
at**: the budget, the unreported Actions bucket, the open-PR list and that
|
|
2285
|
+
list's cap are one global flag each — all account-level — but the payload latch is keyed by PR, and a
|
|
2268
2286
|
shared scalar for those is an unlatched line wearing a latch's clothes — one
|
|
2269
2287
|
persistently blind PR alongside one healthy PR re-clears the flag on every tick,
|
|
2270
2288
|
emitting a DEGRADED and a false RECOVERED pair forever. The recovery line is
|
|
@@ -2280,7 +2298,7 @@ probe pairs during exactly the outage it is reporting. Every degraded
|
|
|
2280
2298
|
branch above falls through to the same single sleep instead.
|
|
2281
2299
|
|
|
2282
2300
|
**Guard every probe, not just the ones with a verdict field.** The block makes
|
|
2283
|
-
|
|
2301
|
+
several API calls, and `gh pr list` is the one that looks like
|
|
2284
2302
|
plumbing rather than a probe. A non-quota failure there (secondary/abuse limit,
|
|
2285
2303
|
5xx, network blip, expired token) does not lower `.resources.core.remaining`, so
|
|
2286
2304
|
the budget gate clears, and an unguarded `for pr in $(gh pr list …)` then
|
|
@@ -2306,8 +2324,9 @@ enforce. It latches globally like the list failure but does not pause polling:
|
|
|
2306
2324
|
the PRs it did return are real, so the pass still watches them, and the
|
|
2307
2325
|
DEGRADED line names the rest as unwatched until a pass comes back under the cap.
|
|
2308
2326
|
|
|
2309
|
-
**The budget is gated
|
|
2310
|
-
tick-top floor (`200`) covers the
|
|
2327
|
+
**The budget is gated at the floor and again at the pass, and only the pass
|
|
2328
|
+
gates announce recovery.** The tick-top floor (`200`) covers the `rate_limit`
|
|
2329
|
+
and `gh pr list` reads. Once the open-PR list is
|
|
2311
2330
|
read, the pass gate reserves `200 + 4 × count` core REST units — 4 per listed PR
|
|
2312
2331
|
on top of the floor, so the reserve grows with the list instead of staying a
|
|
2313
2332
|
flat 200 that a long list drains mid-pass. The 4 is the measured ceiling of one
|
|
@@ -2316,21 +2335,36 @@ flat 200 that a long list drains mid-pass. The 4 is the measured ceiling of one
|
|
|
2316
2335
|
to 100 jobs and 100 rules — more pages cost more. That counter is not the only
|
|
2317
2336
|
core bucket one call bills. The lookups `actions/workflows`,
|
|
2318
2337
|
`actions/workflows/<id>`, `actions/workflows/<id>/runs` and `actions/runs/<id>`
|
|
2319
|
-
bill a second one that `rate_limit` does not report
|
|
2320
|
-
|
|
2321
|
-
|
|
2322
|
-
|
|
2323
|
-
|
|
2324
|
-
|
|
2325
|
-
|
|
2338
|
+
bill a second one that `rate_limit` does not report; the boundary is per route,
|
|
2339
|
+
so `actions/runs/<id>/jobs` is on the reported counter. Every response says
|
|
2340
|
+
`X-Ratelimit-Resource: core`, so only the reset epoch and the used sequence tell
|
|
2341
|
+
the two apart, and the unreported bucket's remaining count is readable only from
|
|
2342
|
+
the `X-Ratelimit-Remaining` header of a call that bills it. **A second pass gate
|
|
2343
|
+
therefore probes it too**: one `gh api -i` read of
|
|
2344
|
+
`actions/workflows?per_page=1` per tick, 1 unit of that bucket (measured
|
|
2345
|
+
2026-10-13: its remaining count fell by 1 per call while `rate_limit` stood
|
|
2346
|
+
still, and its reset epoch differed from `rate_limit`'s). A `ci-state` call
|
|
2347
|
+
spends 4 units of it on the common path — one `gh run list` and one `gh run view`
|
|
2348
|
+
cost 2 each (measured the same day) — so it is held to the same `200 + 4 × count`
|
|
2349
|
+
reserve. It has its own global latch (`hidden_out`) and its DEGRADED and
|
|
2350
|
+
RECOVERED lines name it, so a pause on this bucket is not misread as the
|
|
2351
|
+
reported budget running low. An unreadable header is an outage here exactly as a
|
|
2352
|
+
non-numeric `rate_limit` read is. The probe runs only when the reported-bucket
|
|
2353
|
+
gate has cleared, so a tick costs at most one probe call. The gate pauses before
|
|
2354
|
+
the per-PR pass, so it never touches the per-PR blind latch: a PR is either not
|
|
2355
|
+
polled — no per-PR line, and one already latched blind stays latched — or polled
|
|
2356
|
+
and reported through that latch alone, never by both. A shortfall at any of
|
|
2357
|
+
these gates pauses the **whole** pass and never polls a prefix of the list: a prefix runs
|
|
2326
2358
|
in list order, so the same tail PRs would go unread every tick with no line at
|
|
2327
2359
|
all (never polled, so nothing marks them blind), while the PRs that were polled
|
|
2328
|
-
would run dry mid-pass and each emit a blind line. The DEGRADED line from
|
|
2329
|
-
pass gate names the amount it needed and the count it needed it for
|
|
2330
|
-
|
|
2331
|
-
|
|
2332
|
-
|
|
2333
|
-
|
|
2360
|
+
would run dry mid-pass and each emit a blind line. The DEGRADED line from each
|
|
2361
|
+
pass gate names the amount it needed and the count it needed it for — except an
|
|
2362
|
+
unreadable header, which has no amount to compare and prints `bucket=ERR` alone
|
|
2363
|
+
— so a healthy-looking `budget=999` is not read as an outage with no cause. Only the
|
|
2364
|
+
pass gates emit RECOVERED lines, and both only once the pass will actually run:
|
|
2365
|
+
announcing recovery at the floor, or for one bucket while the other still pauses
|
|
2366
|
+
the pass, would report resumption on a tick whose other gate then pauses it
|
|
2367
|
+
again — a DEGRADED/RECOVERED pair every tick.
|
|
2334
2368
|
|
|
2335
2369
|
**Judge `ci-state` on its payload, never its exit code** — the same rule as
|
|
2336
2370
|
`merge-gate.mjs` applies on the merge path, applied to the watcher. `not-green` is an
|