forgedock 1.18.0 → 1.20.0

package/scripts/wait-ci-green.sh

@@ -18,6 +18,9 @@
 #   2  ERROR    bad usage, gh unreadable for too long, the PR head moved during the wait, or
 #               FORGE_CI_REQUIRE_CHECKS=1 and no checks were reported
 #   3  TIMEOUT  checks still pending when the timeout expired (listed)
+#   4  RATE_LIMITED  the window ended while GitHub only answered with rate-limit (403/429,
+#               x-ratelimit-remaining: 0, secondary-limit text) or transient 5xx/network errors. Retryable,
+#               never a CI verdict: callers treat it like TIMEOUT (wait and re-run), and it is never a merge.
 #
 # Every result also prints "CI_GATE_HEAD: <sha>" — the exact PR head the decision applies to. Callers
 # MUST merge with `--match-head-commit <that sha>` so a push after the gate cannot land unchecked.
@@ -26,7 +29,10 @@
 #      caller re-runs the gate, it does not raise the timeout), FORGE_CI_INTERVAL (default 20),
 #      FORGE_CI_NO_CHECKS_GRACE (default 120), FORGE_CI_SETTLE (default 45 — never PASS earlier than
 #      this after the wait starts, so workflows that register late are not missed),
-#      FORGE_CI_REQUIRE_CHECKS (default 0).
+#      FORGE_CI_REQUIRE_CHECKS (default 0), FORGE_CI_JITTER_PCT (default 20 — each sleep varies by +/- this
+#      percent so many waiting lanes do not poll in lockstep).
+# Rate-limit and transient gh failures are retried with a growing, jittered sleep and never count toward
+# the 5-error ERROR limit.
 # "No checks" passes only when GitHub Actions also reports no workflow runs for the head commit;
 # if runs exist but their checks have not appeared yet, the gate keeps waiting.
 set -uo pipefail
@@ -44,7 +50,7 @@ while [ $# -gt 0 ]; do
     -R*) REPO="${1#-R}"; REPO="${REPO# }"; shift ;;
     --timeout) [ $# -ge 2 ] || usage; TIMEOUT="$2"; shift 2 ;;
     --interval) [ $# -ge 2 ] || usage; INTERVAL="$2"; shift 2 ;;
-    -h|--help) sed -n '5,25p' "$0"; exit 0 ;;
+    -h|--help) sed -n '5,32p' "$0"; exit 0 ;;
     *) [ -z "$PR" ] || usage; PR="$1"; shift ;;
   esac
 done
@@ -52,17 +58,50 @@ case "$PR" in ''|*[!0-9]*) usage ;; esac
 case "$REPO" in */*) ;; *) usage ;; esac
 for v in "$TIMEOUT" "$INTERVAL" "$GRACE" "$SETTLE"; do case "$v" in ''|*[!0-9]*) usage ;; esac; done
 [ "$INTERVAL" -ge 1 ] || INTERVAL=1
+JITTER_PCT="${FORGE_CI_JITTER_PCT:-20}"; case "$JITTER_PCT" in ''|*[!0-9]*) JITTER_PCT=20 ;; esac
+[ "$JITTER_PCT" -le 90 ] || JITTER_PCT=90
+# jitter <seconds> -> seconds varied by +/- JITTER_PCT percent (never below 1).
+jitter() { awk -v s="$1" -v p="$JITTER_PCT" -v r="$RANDOM" 'BEGIN{ v=s*(1-p/100+2*(p/100)*(r/32767)); if (v<1) v=1; printf "%.2f", v }'; }
+
+# classify_gh <text> -> prints one class, or nothing when the text is unrecognised:
+#   limited    real rate-limit evidence (x-ratelimit-remaining: 0, HTTP 429, "API rate limit exceeded" or GraphQL "API rate limit already exceeded", secondary rate limit,
+#              abuse detection, too many requests): a wait, bounded by the whole window
+#   auth       a bare 401/403 (bad credentials, missing scope): permanent, never waited on
+#   transient  5xx / gateway / network failures: a small bounded retry (FORGE_CI_TRANSIENT_RETRIES), never the long wait
+classify_gh() {
+  if printf '%s' "$1" | grep -Eqi 'api rate limit (already )?exceeded|secondary (rate )?limit|abuse detection|x-ratelimit-remaining: *0|HTTP 429|status 429|too many requests'; then echo limited
+  elif printf '%s' "$1" | grep -Eqi 'HTTP (401|403)|status (401|403)|bad credentials|requires authentication'; then echo auth
+  elif printf '%s' "$1" | grep -Eqi 'HTTP 5[0-9][0-9]|status 5[0-9][0-9]|bad gateway|service unavailable|gateway time-?out|timed? ?out|i/o timeout|connection (reset|refused)|could not resolve host|unexpected EOF|network is unreachable|TLS handshake'; then echo transient
+  fi
+}
 
 emit() {
-  case "$1" in PASS) _rc=0 ;; FAIL) _rc=1 ;; TIMEOUT) _rc=3 ;; *) _rc=2 ;; esac
+  case "$1" in PASS) _rc=0 ;; FAIL) _rc=1 ;; TIMEOUT) _rc=3 ;; RATE_LIMITED) _rc=4 ;; *) _rc=2 ;; esac
   FORGE_REPO="${FORGE_REPO:-$REPO}" FORGE_PR="${FORGE_PR:-$PR}" forge_emit ci_gate result="$1" rc="$_rc" checks="${total:-0}"
   echo "CI_GATE: $1"; echo "CI_GATE_HEAD: ${START_SHA:-unknown}"
 }
 head_sha() { gh pr view "$PR" -R "$REPO" --json headRefOid --jq '.headRefOid' 2>/dev/null; }
-START_SHA="$(head_sha)"
-if [ -z "$START_SHA" ]; then START_SHA=""; emit ERROR; echo "cannot read PR #$PR head in $REPO"; exit 2; fi
+# forge#4037: the initial head read goes through the same classification as the checks poll: a rate limit is a wait
+# (rc 4 once the window is spent), a bare 401/403 is a fast permanent error, anything else a small bounded retry.
+START_SHA=""; _hstart=$(date +%s); _hlim=0; _hfail=0; _herr="$(mktemp)"; trap 'rm -f "$_herr"' EXIT
+while :; do
+  START_SHA="$(gh pr view "$PR" -R "$REPO" --json headRefOid --jq '.headRefOid' 2>"$_herr")" && [ -n "$START_SHA" ] && break
+  START_SHA=""; _hc="$(classify_gh "$(cat "$_herr" 2>/dev/null)")"
+  case "$_hc" in
+    auth) emit ERROR; echo "cannot read PR #$PR head in $REPO (HTTP 401/403 without rate-limit evidence)"; exit 2 ;;
+    limited)
+      _hlim=$((_hlim + 1))
+      if [ $(( $(date +%s) - _hstart )) -ge "$TIMEOUT" ]; then emit RATE_LIMITED; echo "PR #$PR: GitHub rate-limited for the whole ${TIMEOUT}s window reading the PR head; retry later"; exit 4; fi
+      _m=$((_hlim > 4 ? 4 : _hlim)); sleep "$(jitter $((INTERVAL * (1 << (_m - 1)))))" ;;
+    *)
+      _hfail=$((_hfail + 1))
+      if [ "$_hfail" -gt 2 ]; then emit ERROR; echo "cannot read PR #$PR head in $REPO"; exit 2; fi
+      sleep "$(jitter "$INTERVAL")" ;;
+  esac
+done
 
-start=$(date +%s); errors=0
+start=$(date +%s); errors=0; limited=0; transient=0
+TRANSIENT_MAX="${FORGE_CI_TRANSIENT_RETRIES:-4}"; case "$TRANSIENT_MAX" in ''|*[!0-9]*) TRANSIENT_MAX=4 ;; esac
 while :; do
   now=$(date +%s); elapsed=$((now - start))
   if ! JSON="$(gh pr checks "$PR" -R "$REPO" --json name,bucket 2>/dev/null)" || ! printf '%s' "$JSON" | jq -e 'type=="array"' >/dev/null 2>&1; then
@@ -72,14 +111,27 @@ while :; do
       # does precisely in the "no checks reported" case this branch exists to detect.
       NC_OUT="$(gh pr checks "$PR" -R "$REPO" 2>&1 || true)"
       if printf '%s' "$NC_OUT" | grep -qi 'no checks reported'; then JSON='[]'
+      elif GH_CLASS="$(classify_gh "$NC_OUT")"; [ "$GH_CLASS" = auth ]; then
+        # A bare 401/403 is an auth fault that never clears: fail fast instead of waiting out the window (forge#4037).
+        emit ERROR; echo "gh pr checks refused (HTTP 401/403 without rate-limit evidence); fix the token or its scopes"; exit 2
+      elif [ "$GH_CLASS" = transient ]; then
+        # Network timeout / 5xx: a small bounded retry, never the long rate-limit wait.
+        transient=$((transient + 1))
+        if [ "$transient" -gt "$TRANSIENT_MAX" ]; then emit ERROR; echo "gh pr checks unavailable (transient error) ${transient} times in a row"; exit 2; fi
+        sleep "$(jitter "$INTERVAL")"; continue
+      elif [ "$GH_CLASS" = limited ]; then
+        # Real rate limit: a wait, not a verdict. Retry with a growing jittered sleep; never counts toward ERROR.
+        limited=$((limited + 1)); now=$(date +%s)
+        if [ $((now - start)) -ge "$TIMEOUT" ]; then emit RATE_LIMITED; echo "PR #$PR: GitHub rate-limited for the whole ${TIMEOUT}s window (${limited} retries); retry later"; exit 4; fi
+        _m=$((limited > 4 ? 4 : limited)); sleep "$(jitter $((INTERVAL * (1 << (_m - 1)))))"; continue
       else
         errors=$((errors + 1))
         if [ "$errors" -ge 5 ]; then emit ERROR; echo "gh pr checks unreadable 5 times in a row"; exit 2; fi
-        sleep "$INTERVAL"; continue
+        sleep "$(jitter "$INTERVAL")"; continue
       fi
     fi
   fi
-  errors=0
+  errors=0; limited=0; transient=0
   cur="$(head_sha)"
   if [ -n "$cur" ] && [ "$cur" != "$START_SHA" ]; then emit ERROR; echo "PR head moved during the wait ($START_SHA -> $cur); re-review the new head"; exit 2; fi
   total=$(printf '%s' "$JSON" | jq 'length')
@@ -101,5 +153,5 @@ while :; do
     emit PASS; echo "PR #$PR head ${START_SHA:0:7}: $total checks passed or skipped"; exit 0
   fi
   if [ "$elapsed" -ge "$TIMEOUT" ]; then emit TIMEOUT; echo "PR #$PR: still pending after ${TIMEOUT}s:"; echo "$pending"; exit 3; fi
-  sleep "$INTERVAL"
+  sleep "$(jitter "$INTERVAL")"
 done

package/scripts/wait-ci-green.test.sh

@@ -16,6 +16,7 @@ cat > "$T/bin/gh" <<'GH'
 #!/usr/bin/env bash
 D="$FAKE_DIR"
 if [ "$1 $2" = "pr view" ]; then
+  if [ -f "$D/headerr" ]; then hn=$(( $(cat "$D/headerr.n" 2>/dev/null || echo 0) + 1 )); echo "$hn" > "$D/headerr.n"; if [ "$hn" -le "$(cat "$D/headerr.max")" ]; then cat "$D/headerr" >&2; exit 1; fi; fi
   n=$(( $(cat "$D/sha.n" 2>/dev/null || echo 0) + 1 )); echo "$n" > "$D/sha.n"
   total=$(wc -l < "$D/sha"); [ "$n" -gt "$total" ] && n=$total; sed -n "${n}p" "$D/sha"; exit 0
 fi
@@ -24,10 +25,14 @@ if [ "$1" = "api" ]; then
   exit 1
 fi
 if [ "$1 $2" = "pr checks" ]; then
-  case " $* " in *" --json "*) ;; *) [ -f "$D/nochecks" ] && { echo "no checks reported on the 'x' branch" >&2; exit 1; }; exit 1 ;; esac
+  case " $* " in *" --json "*) ;; *)
+    [ -f "$D/nochecks" ] && { echo "no checks reported on the 'x' branch" >&2; exit 1; }
+    cn=$(cat "$D/chk.n" 2>/dev/null || echo 1); while [ ! -f "$D/checks.$cn" ] && [ "$cn" -gt 1 ]; do cn=$((cn-1)); done; cc=$(cat "$D/checks.$cn" 2>/dev/null || true)
+    case "$cc" in RALR) echo "HTTP 403: API rate limit already exceeded for user ID 12345. (https://api.github.com/graphql)" >&2 ;; RATE) echo "HTTP 403: API rate limit exceeded for user (https://api.github.com/graphql)" >&2 ;; SECOND) echo "You have exceeded a secondary rate limit" >&2 ;; ERR) echo "HTTP 502: Bad Gateway" >&2 ;; ERRX) echo "boom: unexpected failure" >&2 ;; A403) echo "HTTP 403: Resource not accessible by personal access token" >&2 ;; A401) echo "HTTP 401: Bad credentials" >&2 ;; R429) echo "HTTP 429: Too Many Requests" >&2 ;; RHDR) printf "HTTP 403: Forbidden\nx-ratelimit-remaining: 0\n" >&2 ;; TMO) echo "dial tcp: i/o timeout" >&2 ;; esac
+    exit 1 ;; esac
   n=$(( $(cat "$D/chk.n" 2>/dev/null || echo 0) + 1 )); echo "$n" > "$D/chk.n"
   f="$D/checks.$n"; while [ ! -f "$f" ] && [ "$n" -gt 1 ]; do n=$((n-1)); f="$D/checks.$n"; done
-  c=$(cat "$f"); [ "$c" = "ERR" ] && { echo "HTTP 502" >&2; exit 1; }
+  c=$(cat "$f"); case "$c" in ERR|ERRX|RALR|RATE|SECOND|A403|A401|R429|RHDR|TMO) exit 1 ;; esac
   [ "$c" = "NONE" ] && { echo "no checks reported on the 'x' branch" >&2; exit 1; }
   echo "$c"; exit 0
 fi
@@ -51,7 +56,40 @@ expect "pending then fail" "1|CI_GATE: FAIL" "$(run "$(mk pendfail '[{"name":"a"
 expect "pending past timeout" "3|CI_GATE: TIMEOUT" "$(run "$(mk slow '[{"name":"a","bucket":"pending"}]')" --timeout 2)"
 expect "fail wins over pending" "1|CI_GATE: FAIL" "$(run "$(mk mixed '[{"name":"a","bucket":"pending"},{"name":"b","bucket":"fail"}]')")"
 expect "transient outage then pass" "0|CI_GATE: PASS" "$(run "$(mk blip ERR "$P")")"
-expect "persistent outage" "2|CI_GATE: ERROR" "$(run "$(mk down ERR)" --timeout 60)"
+expect "persistent unclassified failure is an ERROR" "2|CI_GATE: ERROR" "$(run "$(mk down ERRX)" --timeout 60)"
+# Rate-limit / 5xx classification: retried, never an ERROR; a window that only saw them ends RATE_LIMITED (rc 4).
+expect "rate limited then pass retries" "0|CI_GATE: PASS" "$(run "$(mk rl RATE "$P")")"
+expect "secondary limit then pass retries" "0|CI_GATE: PASS" "$(run "$(mk rl2 SECOND "$P")")"
+expect "transient 5xx then pass retries" "0|CI_GATE: PASS" "$(run "$(mk fivexx ERR "$P")")"
+expect "rate limited for the whole window is RATE_LIMITED rc 4" "4|CI_GATE: RATE_LIMITED" "$(run "$(mk rlall RATE)" --timeout 2)"
+expect "more than 5 rate-limit hits are not an ERROR" "0|CI_GATE: PASS" "$(run "$(mk rlmany RATE RATE RATE RATE RATE RATE RATE "$P")" --timeout 60)"
+# forge#4037: only real rate-limit evidence is RATE_LIMITED; a bare 401/403 fails fast; timeouts/5xx are a small bounded retry.
+expect "bare 403 is a permanent error rc 2" "2|CI_GATE: ERROR" "$(run "$(mk a403 A403 "$P")" --timeout 60)"
+expect "bare 401 is a permanent error rc 2" "2|CI_GATE: ERROR" "$(run "$(mk a401 A401 "$P")" --timeout 60)"
+expect "HTTP 429 is RATE_LIMITED rc 4" "4|CI_GATE: RATE_LIMITED" "$(run "$(mk r429 R429)" --timeout 2)"
+expect "x-ratelimit-remaining: 0 on a 403 is RATE_LIMITED rc 4" "4|CI_GATE: RATE_LIMITED" "$(run "$(mk rhdr RHDR)" --timeout 2)"
+expect "network timeout then pass retries" "0|CI_GATE: PASS" "$(run "$(mk tmo TMO "$P")")"
+expect "persistent network timeout is a bounded ERROR, not the long wait" "2|CI_GATE: ERROR" "$(FORGE_CI_TRANSIENT_RETRIES=2 run "$(mk tmoall TMO)" --timeout 600)"
+expect "persistent 5xx is a bounded ERROR, not the long wait" "2|CI_GATE: ERROR" "$(FORGE_CI_TRANSIENT_RETRIES=2 run "$(mk fivexxall ERR)" --timeout 600)"
+# forge#4037: the initial head read uses the same classification.
+d=$(mk headrl "$P"); echo "HTTP 403: API rate limit exceeded for user" > "$d/headerr"; echo 2 > "$d/headerr.max"
+expect "rate-limited head read waits, then passes" "0|CI_GATE: PASS" "$(run "$d" --timeout 60)"
+d=$(mk headrlall "$P"); echo "HTTP 429: Too Many Requests" > "$d/headerr"; echo 99999 > "$d/headerr.max"
+expect "rate-limited head read for the whole window is rc 4" "4|CI_GATE: RATE_LIMITED" "$(run "$d" --timeout 2)"
+d=$(mk head403 "$P"); echo "HTTP 403: Resource not accessible" > "$d/headerr"; echo 99999 > "$d/headerr.max"
+# forge#4040: the GraphQL wording "API rate limit already exceeded (HTTP 403)" is a rate limit (rc 4), not the auth class (rc 2).
+expect "GraphQL 'rate limit already exceeded' 403 is RATE_LIMITED rc 4" "4|CI_GATE: RATE_LIMITED" "$(run "$(mk ralr RALR)" --timeout 2)"
+expect "rate limit already exceeded, then pass" "0|CI_GATE: PASS" "$(run "$(mk ralr2 RALR "$P")" --timeout 60)"
+d=$(mk headralr "$P"); echo "HTTP 403: API rate limit already exceeded for user ID 12345." > "$d/headerr"; echo 99999 > "$d/headerr.max"
+expect "GraphQL 'already exceeded' head read is RATE_LIMITED rc 4" "4|CI_GATE: RATE_LIMITED" "$(run "$d" --timeout 2)"
+d=$(mk head403b "$P"); echo "HTTP 403: Resource not accessible" > "$d/headerr"; echo 99999 > "$d/headerr.max"
+expect "bare 403 head read is rc 2" "2|CI_GATE: ERROR" "$(run "$d" --timeout 60)"
+# Jitter: sleeps vary within +/-20% of the interval (fake sleep records its argument and returns at once).
+mkdir -p "$T/bin2"; printf '#!/usr/bin/env bash\necho "$1" >> "$FAKE_DIR/sleeps"\n' > "$T/bin2/sleep"; chmod +x "$T/bin2/sleep"
+d=$(mk jit '[{"name":"a","bucket":"pending"}]' '[{"name":"a","bucket":"pending"}]' '[{"name":"a","bucket":"pending"}]' '[{"name":"a","bucket":"pending"}]' '[{"name":"a","bucket":"pending"}]' '[{"name":"a","bucket":"pending"}]' '[{"name":"a","bucket":"pending"}]' '[{"name":"a","bucket":"pending"}]' "$P")
+FAKE_DIR="$d" PATH="$T/bin2:$T/bin:$PATH" FORGE_CI_INTERVAL=100 FORGE_CI_SETTLE=0 bash "$GATE" 7 -R o/r >/dev/null 2>&1
+expect "jittered sleeps stay within +/-20%" "0" "$(awk '$1<80||$1>120{bad++} END{print bad+0}' "$d/sleeps")"
+expect "jitter is applied (not a fixed interval)" "1" "$(awk '{u[$1]=1} END{n=0; for(k in u) n++; print (n>1)?1:0}' "$d/sleeps")"
 d=$(mk none NONE); touch "$d/nochecks"
 expect "no checks after grace passes" "0|CI_GATE: PASS" "$(run "$d")"
 expect "no checks + REQUIRE fails closed" "2|CI_GATE: ERROR" "$(FORGE_CI_REQUIRE_CHECKS=1 run "$d")"

package/scripts/with-run-tmpdir.sh

@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+# SPDX-FileCopyrightText: Copyright (c) RapierCraft Studios
+# SPDX-License-Identifier: AGPL-3.0-or-later
+#
+# with-run-tmpdir.sh — run a command with a per-run TMPDIR that is removed when it exits.
+#
+# Usage: bash scripts/with-run-tmpdir.sh <command> [args...]
+# Children inherit the exported TMPDIR (os.tmpdir() honours it), so temp dirs a test
+# forgets to remove cannot outlive the run. The command's exit code is preserved.
+# Bash 3.2 portable.
+
+set -u
+
+[ "$#" -gt 0 ] || { echo "with-run-tmpdir: no command given" >&2; exit 2; }
+
+RUN_TMP=$(mktemp -d) || { echo "with-run-tmpdir: cannot create per-run TMPDIR" >&2; exit 1; }
+trap 'rm -rf "$RUN_TMP"' EXIT
+export TMPDIR="$RUN_TMP"
+
+"$@"