forgedock 1.32.0 → 1.33.0

package/scripts/hotspots.sh

@@ -7,14 +7,17 @@
 #
 # Usage: hotspots.sh [--base <ref>] [--] <path>...
 # Config: forge.yaml maintenance.hotspot {enabled, file_lines, growth_lines, exclude_globs}; defaults true/2000/150/[]
-#   (also when forge.yaml or yq is missing).
+#   (also when forge.yaml or yq is missing); maintenance.generated_paths (forge#4519): shell globs of generated,
+#   bot-owned files, default [].
 # Output (stdout, one JSON object):
-#   {"enabled","file_lines","growth_lines","files":[{path,lines,commits_90d,hotspot,over,excluded,base_lines,growth}]}
+#   {"enabled","file_lines","growth_lines","generated_paths",
+#    "files":[{path,lines,commits_90d,hotspot,over,generated,excluded,base_lines,growth}]}
 # lines = `wc -l` of the working-tree file (0 when missing); commits_90d = `git rev-list --count --since=90.days HEAD`;
-# hotspot = lines × commits_90d; over = lines > file_lines; excluded = linguist-generated / linguist-vendored set by
-# .gitattributes, or a match of an exclude_glob (shell pattern, `*` crosses `/`). Without --base no blob is read and
+# hotspot = lines × commits_90d; over = lines > file_lines; generated = linguist-generated set by .gitattributes, a
+# match of a generated_paths glob, or `@generated` / `DO NOT EDIT` in the first 5 lines of the working-tree file;
+# excluded = generated, linguist-vendored, or a match of an exclude_glob (shell pattern, `*` crosses `/`). Without --base no blob is read and
 # base_lines/growth are null; with --base they are the line count at git merge-base(<ref>, HEAD) (0 when absent there)
-# and lines - base_lines. enabled:false prints files:[] and does no per-file work.
+# and lines - base_lines. enabled:false prints files:[] (generated_paths still set) and does no per-file work.
 # Exit: 0 ok | 2 usage or bad ref. bash 3.2 portable.
 set -uo pipefail
 
@@ -31,19 +34,21 @@ while [ $# -gt 0 ]; do
 done
 
 # ---- config: maintenance.hotspot as JSON, {} when unreadable ----
-CFG='{}'
+CFG='{}'; GEN_CFG='[]'
 if [ -f forge.yaml ] && command -v yq >/dev/null 2>&1; then
   c=$(yq -o=json '.maintenance.hotspot // {}' forge.yaml 2>/dev/null) || c=""
   printf '%s' "$c" | jq -e 'type == "object"' >/dev/null 2>&1 && CFG="$c"
+  g=$(yq -o=json '.maintenance.generated_paths // []' forge.yaml 2>/dev/null) || g=""
+  printf '%s' "$g" | jq -e 'type == "array"' >/dev/null 2>&1 && GEN_CFG="$g"
 fi
+GEN_CFG=$(printf '%s' "$GEN_CFG" | jq -c '[.[] | strings | select(length > 0)]')
 CFG=$(printf '%s' "$CFG" | jq -c '
   def pos($d): if type == "number" and . > 0 then floor else $d end;
   { enabled: (.enabled != false), file_lines: (.file_lines | pos(2000)), growth_lines: (.growth_lines | pos(150)),
     exclude_globs: ([(.exclude_globs // [])[]? | strings]) }')
 ENABLED=$(printf '%s' "$CFG" | jq -r '.enabled')
 FILE_LINES=$(printf '%s' "$CFG" | jq -r '.file_lines')
-GROWTH_LINES=$(printf '%s' "$CFG" | jq -r '.growth_lines')
-head_json() { printf '%s' "$CFG" | jq -c --argjson files "$1" '{enabled, file_lines, growth_lines, files: $files}'; }
+head_json() { printf '%s' "$CFG" | jq -c --argjson files "$1" --argjson gen "$GEN_CFG" '{enabled, file_lines, growth_lines, generated_paths: $gen, files: $files}'; }
 [ "$ENABLED" = true ] || { head_json '[]'; exit 0; }
 
 MB=""
@@ -53,6 +58,19 @@ if [ -n "$BASE" ]; then
 fi
 HAS_HEAD=0; git rev-parse --verify -q "HEAD^{commit}" >/dev/null && HAS_HEAD=1
 GLOBS=$(printf '%s' "$CFG" | jq -r '.exclude_globs[]')
+GEN_GLOBS=$(printf '%s' "$GEN_CFG" | jq -r '.[]')
+# 0 when $1 matches a newline-separated list of shell patterns in $2 (`*` crosses `/`)
+glob_hit() {
+  [ -n "$2" ] || return 1
+  while IFS= read -r g; do
+    [ -n "$g" ] || continue
+    # shellcheck disable=SC2254
+    case "$1" in $g) return 0 ;; esac
+  done <<EOF
+$2
+EOF
+  return 1
+}
 
 T=$(mktemp -d "${TMPDIR:-/tmp}/hotspots.XXXXXX") || die "mktemp failed"
 trap 'rm -rf "$T"' EXIT
@@ -72,29 +90,23 @@ for p in "$@"; do
   [ -n "$p" ] || continue
   lines=0; [ -f "$p" ] && lines=$(wc -l < "$p" | tr -d ' ')
   commits=$(cat "$T/c.$i" 2>/dev/null); case "$commits" in ''|*[!0-9]*) commits=0 ;; esac
-  excluded=false
-  if git check-attr linguist-generated linguist-vendored -- "$p" 2>/dev/null | grep -qE ': (set|true)$'; then excluded=true; fi
-  if [ "$excluded" = false ] && [ -n "$GLOBS" ]; then
-    while IFS= read -r g; do
-      [ -n "$g" ] || continue
-      # shellcheck disable=SC2254
-      case "$p" in $g) excluded=true; break ;; esac
-    done <<EOF
-$GLOBS
-EOF
-  fi
+  generated=false; excluded=false
+  if git check-attr linguist-generated -- "$p" 2>/dev/null | grep -qE ': (set|true)$' || glob_hit "$p" "$GEN_GLOBS" \
+    || { [ -f "$p" ] && head -n 5 "$p" 2>/dev/null | grep -qE '@generated|DO NOT EDIT'; }; then generated=true; excluded=true; fi
+  if [ "$excluded" = false ] && git check-attr linguist-vendored -- "$p" 2>/dev/null | grep -qE ': (set|true)$'; then excluded=true; fi
+  if [ "$excluded" = false ] && glob_hit "$p" "$GLOBS"; then excluded=true; fi
   base_lines=null; growth=null
   if [ -n "$MB" ]; then
     base_lines=0
     git cat-file -e "$MB:$p" 2>/dev/null && base_lines=$(git show "$MB:$p" | wc -l | tr -d ' ')
     growth=$((lines - base_lines))
   fi
-  printf '%s\t%s\t%s\t%s\t%s\t%s\n' "$p" "$lines" "$commits" "$excluded" "$base_lines" "$growth" >> "$T/rows"
+  printf '%s\t%s\t%s\t%s\t%s\t%s\t%s\n' "$p" "$lines" "$commits" "$excluded" "$base_lines" "$growth" "$generated" >> "$T/rows"
 done
 
 FILES=$(jq -R -s -c --argjson fl "$FILE_LINES" '
   [split("\n")[] | select(length > 0) | split("\t") |
    { path: .[0], lines: (.[1] | tonumber), commits_90d: (.[2] | tonumber) } as $r |
-   $r + { hotspot: ($r.lines * $r.commits_90d), over: ($r.lines > $fl), excluded: (.[3] == "true"),
+   $r + { hotspot: ($r.lines * $r.commits_90d), over: ($r.lines > $fl), generated: (.[6] == "true"), excluded: (.[3] == "true"),
           base_lines: (.[4] | fromjson), growth: (.[5] | fromjson) }]' < "$T/rows")
 head_json "$FILES"

package/scripts/hotspots.test.sh

@@ -27,7 +27,7 @@ echo y >> big.py; lines small.py 10; commit c2
 OUT="$(bash "$S" big.py ./small.py missing/gone.py)"; rc=$?
 eq "rc" "$rc" 0
 eq "defaults" "$(q '[.enabled,.file_lines,.growth_lines]')" '[true,2000,150]'
-eq "big facts" "$(q '.files[0]')" '{"path":"big.py","lines":2102,"commits_90d":2,"hotspot":4204,"over":true,"excluded":false,"base_lines":null,"growth":null}'
+eq "big facts" "$(q '.files[0]')" '{"path":"big.py","lines":2102,"commits_90d":2,"hotspot":4204,"over":true,"generated":false,"excluded":false,"base_lines":null,"growth":null}'
 eq "small facts (./ stripped)" "$(q '.files[1] | [.path,.lines,.commits_90d,.over]')" '["small.py",10,1,false]'
 eq "missing file" "$(q '.files[2] | [.path,.lines,.commits_90d,.over]')" '["missing/gone.py",0,0,false]'
 OUT="$(bash "$S")"; eq "no paths" "$(q '.files')" '[]'
@@ -49,6 +49,13 @@ printf 'gen/** linguist-generated\nvendor/** linguist-vendored=true\n' > .gitatt
 commit gen
 OUT="$(bash "$S" gen/api.py vendor/lib/x.js dist/bundle.js big.py)"
 eq "attr exclusions" "$(q '[.files[] | .excluded]')" '[true,true,false,false]'
+eq "attr generated" "$(q '[.files[] | .generated]')" '[true,false,false,false]'
+# forge#4519: a generated-file header in the first 5 lines marks the file generated
+{ echo '# @generated'; i=0; while [ $i -lt 2500 ]; do echo "l$i"; i=$((i+1)); done; } > hdr.py
+{ i=0; while [ $i -lt 2500 ]; do echo "l$i"; i=$((i+1)); done; echo '# DO NOT EDIT'; } > late.py
+OUT="$(bash "$S" hdr.py late.py)"
+rm -f hdr.py late.py
+eq "header generated" "$(q '[.files[] | [.generated,.excluded]]')" '[[true,true],[false,false]]'
 if [ "$HAVE_YQ" = 1 ]; then
   printf 'maintenance:\n  hotspot:\n    file_lines: 100\n    growth_lines: 5\n    exclude_globs: ["dist/*", "*.lock"]\n' > forge.yaml
   OUT="$(bash "$S" dist/bundle.js small.py big.py)"
@@ -60,7 +67,19 @@ if [ "$HAVE_YQ" = 1 ]; then
   eq "opt-out" "$(q '[.enabled,.files]')" '[false,[]]'
   printf 'maintenance:\n  hotspot:\n    file_lines: nope\n' > forge.yaml
   OUT="$(bash "$S" big.py)"; eq "bad value falls back" "$(q '.file_lines')" 2000
-  rm -f forge.yaml
+  # forge#4519: maintenance.generated_paths marks bot-owned files generated (and so excluded), even over the threshold
+  mkdir -p web/public/openapi-versions
+  lines web/public/openapi.json 2500; lines web/public/openapi-versions/3.29.2.json 2500
+  printf 'maintenance:\n  generated_paths: ["web/public/openapi*.json", "web/public/openapi-versions/*"]\n' > forge.yaml
+  OUT="$(bash "$S" web/public/openapi.json web/public/openapi-versions/3.29.2.json big.py gen/api.py)"
+  eq "generated_paths reported" "$(q '.generated_paths')" '["web/public/openapi*.json","web/public/openapi-versions/*"]'
+  eq "generated_paths: generated" "$(q '[.files[] | .generated]')" '[true,true,false,true]'
+  eq "generated_paths: excluded" "$(q '[.files[] | .excluded]')" '[true,true,false,true]'
+  printf 'maintenance:\n  generated_paths: ["*.json"]\n  hotspot:\n    enabled: false\n' > forge.yaml
+  OUT="$(bash "$S" big.py)"; eq "generated_paths with the gate off" "$(q '[.enabled,.generated_paths,.files]')" '[false,["*.json"],[]]'
+  printf 'maintenance:\n  generated_paths: nope\n' > forge.yaml
+  OUT="$(bash "$S" big.py)"; eq "bad generated_paths falls back" "$(q '.generated_paths')" '[]'
+  rm -rf forge.yaml web
 else
   echo "SKIP: forge.yaml cases need mikefarah yq v4"
 fi
@@ -93,7 +112,7 @@ done
 eq "30k exact commits" "$(q '[.files[] | .commits_90d]')" '[300,60,60,60]'
 eq "30k exact lines" "$(q '[.files[] | .lines]')" '[1,1,1,1]'
 if [ "$best" -lt 1000 ]; then ok; else bad "30k-commit run took ${best} ms (limit 1000)"; fi
-grep -q '"rev-list"' "$TMP/trace.json" && ok || bad "git trace not captured"
+if grep -q '"rev-list"' "$TMP/trace.json"; then ok; else bad "git trace not captured"; fi
 if grep -qE '"(cat-file|show)"' "$TMP/trace.json"; then bad "blob read without --base"; else ok; fi
 if [ -e "$TMP/net.log" ]; then bad "network tool invoked: $(tr '\n' ' ' < "$TMP/net.log")"; else ok; fi
 

package/scripts/refresh-bot-token.test.sh

@@ -0,0 +1,56 @@
+#!/usr/bin/env bash
+# SPDX-FileCopyrightText: Copyright (c) RapierCraft Studios
+# SPDX-License-Identifier: AGPL-3.0-or-later
+#
+# Fixtures for scripts/refresh-bot-token.sh (#4507): hosts.yml is replaced by rename, never rewritten in place.
+# Run: bash scripts/refresh-bot-token.test.sh (bash 3.2 compatible). Stubs `node` and `gh`; no network.
+
+set -u
+DIR=$(cd "$(dirname "$0")" && pwd)
+S="$DIR/refresh-bot-token.sh"
+T=$(mktemp -d "${TMPDIR:-/tmp}/refresh-bot-token-test.XXXXXX")
+trap 'rm -rf "$T"' EXIT
+pass=0; fail=0
+ok() { pass=$((pass+1)); }
+no() { fail=$((fail+1)); echo "FAIL: $1"; }
+
+mkdir -p "$T/bin" "$T/cfg"
+echo key > "$T/key.pem"
+cat > "$T/bin/node" <<'STUB'
+#!/usr/bin/env bash
+printf 'tok-new'
+STUB
+cat > "$T/bin/gh" <<'STUB'
+#!/usr/bin/env bash
+echo "$GH_CONFIG_DIR $*" >> "$GH_LOG"
+case "$1 $2" in
+  "auth login") [ -n "${GH_FAIL:-}" ] && exit 1; cat > /dev/null; printf 'github.com:\n    oauth_token: tok-new\n' > "$GH_CONFIG_DIR/hosts.yml" ;;
+  "auth status") echo "Logged in to github.com account rapiercraft-forgedock[bot]" ;;
+esac
+exit 0
+STUB
+chmod +x "$T/bin/node" "$T/bin/gh"
+inode() { ls -i "$1" | awk '{print $1}'; }
+
+# (a) success: gh writes in a staging dir; hosts.yml is renamed into place
+printf 'github.com:\n    oauth_token: tok-old\n' > "$T/cfg/hosts.yml"
+before=$(inode "$T/cfg/hosts.yml")
+OUT=$(PATH="$T/bin:$PATH" GH_CONFIG_DIR="$T/cfg" GH_LOG="$T/log" bash "$S" --pem "$T/key.pem" 2>&1); RC=$?
+[ "$RC" = 0 ] && ok || no "(a) exit $RC: $OUT"
+grep -q 'tok-new' "$T/cfg/hosts.yml" && ok || no "(a) hosts.yml not updated"
+[ "$(inode "$T/cfg/hosts.yml")" != "$before" ] && ok || no "(a) hosts.yml rewritten in place (same inode)"
+if awk '{print $1}' "$T/log" | grep -qx "$T/cfg"; then no "(a) gh ran against the live config dir"; else ok; fi
+grep -q "auth login" "$T/log" && ok || no "(a) gh auth login not run"
+ls -a "$T/cfg" | grep -q '^\.hosts-stage\.' && no "(a) staging dir left behind" || ok
+case "$OUT" in *"token refreshed"*) ok ;; *) no "(a) success message missing: $OUT" ;; esac
+
+# (b) failed login: the live hosts.yml stays byte-identical, exit != 0
+printf 'github.com:\n    oauth_token: tok-keep\n' > "$T/cfg/hosts.yml"
+cp "$T/cfg/hosts.yml" "$T/expect"
+PATH="$T/bin:$PATH" GH_CONFIG_DIR="$T/cfg" GH_LOG="$T/log" GH_FAIL=1 bash "$S" --pem "$T/key.pem" >/dev/null 2>&1; RC=$?
+[ "$RC" != 0 ] && ok || no "(b) failed login exited 0"
+cmp -s "$T/expect" "$T/cfg/hosts.yml" && ok || no "(b) hosts.yml changed after a failed login"
+ls -a "$T/cfg" | grep -q '^\.hosts-stage\.' && no "(b) staging dir left behind" || ok
+
+echo "refresh-bot-token: $pass passed, $fail failed"
+[ "$fail" = 0 ]