forgedock 1.27.0 → 1.28.0

package/scripts/maintenance-scan.sh

@@ -0,0 +1,185 @@
+#!/usr/bin/env bash
+# SPDX-FileCopyrightText: Copyright (c) RapierCraft Studios
+# SPDX-License-Identifier: AGPL-3.0-or-later
+#
+# maintenance-scan.sh — deterministic health signals → maintenance issue candidates (forge#4197). No LLM: git, jq,
+# wc and awk, plus the repo's own dead-code tools when installed. Run from inside the repo.
+#
+# Usage: maintenance-scan.sh [--repo owner/repo] [--file-lines N] [--commits-days D] [--doc-kb K]
+#                            [--open-issues FILE] [--out-dir DIR] [--print-issue-args]
+# Findings (one candidate each, labels `maintenance` + the kind):
+#   refactor:split  tracked file over --file-lines lines (default forge.yaml maintenance.file_lines, else 2000) with
+#                   commits in the last D days (default 90); linguist-generated/-vendored and vendored dirs excluded
+#   refactor        a workspace package dependency both ways (A->B and B->A), from scripts/structure-delta.sh
+#   dead-code       vulture / knip / ts-prune on PATH (and relevant to the repo) reporting findings, one per tool
+#   docs            tracked *.md over --doc-kb KB (default forge.yaml maintenance.doc_kb, else 64)
+# Dedup: a candidate is dropped when an open issue body carries its `<!-- FORGE:MAINT:<kind>:<key> -->` marker, or its
+# title is a near-duplicate (scripts/issue-dedup.sh) of an open issue naming the same key. --open-issues FILE (JSON array
+# of {number,title,body}) replaces the one `gh issue list` call; it then matches markers and exact titles only.
+# Output: a JSON array of {kind,key,title,labels,body_file,marker,numbers}, `[]` when everything is under threshold.
+# Body files (with one FORGE:BODY-INTEGRITY marker, per /issue's programmatic contract) go to --out-dir (default a
+# mktemp dir under $TMPDIR). --print-issue-args prints one `/issue --title ... --body-file ... --label ...` line per
+# candidate instead, for `Skill(skill="{FORGE_SKILL_PREFIX}issue", args=<line>)`.
+# Exit: 0 ok | 2 usage, not a git repo, or open issues unreadable. bash 3.2 portable.
+set -uo pipefail
+
+die() { echo "maintenance-scan: $*" >&2; exit 2; }
+command -v jq >/dev/null 2>&1 || die "jq is required"
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+
+REPO="" FILE_LINES="" DAYS=90 DOC_KB="" OPEN_FILE="" OUT_DIR="" PRINT_ARGS=false
+while [ $# -gt 0 ]; do
+  case "$1" in
+    --repo) REPO="${2:-}"; shift 2 ;;
+    --file-lines) FILE_LINES="${2:-}"; shift 2 ;;
+    --commits-days) DAYS="${2:-}"; shift 2 ;;
+    --doc-kb) DOC_KB="${2:-}"; shift 2 ;;
+    --open-issues) OPEN_FILE="${2:-}"; shift 2 ;;
+    --out-dir) OUT_DIR="${2:-}"; shift 2 ;;
+    --print-issue-args) PRINT_ARGS=true; shift ;;
+    -h|--help) sed -n '5,24p' "$0"; exit 0 ;;
+    *) die "unknown argument: $1" ;;
+  esac
+done
+
+ROOT=$(git rev-parse --show-toplevel 2>/dev/null) || die "not inside a git repository"
+cd "$ROOT" || die "cannot cd to $ROOT"
+
+# forge.yaml `maintenance:` block (block form only, like bin/forge-utils.mjs parseForgeYaml): direct children only, so
+# a sub-map's keys (`  hotspot:` → `    file_lines:`, forge#4195) never stand in for the section's own
+yaml_maint() {
+  [ -f forge.yaml ] || return 0
+  awk -v k="$1" '
+    /^[A-Za-z_][A-Za-z0-9_]*:[[:space:]]*(#.*)?$/ { on = ($0 ~ /^maintenance:/); ci = -1; next }
+    /^[^[:space:]#]/ { on = 0 }
+    on && /^[[:space:]]*(#|$)/ { next }
+    on { match($0, /^[[:space:]]*/); if (ci >= 0 && RLENGTH > ci) next; ci = RLENGTH }
+    on && $0 ~ "^[[:space:]]+" k ":" { sub("^[[:space:]]+" k ":[[:space:]]*", ""); sub("[[:space:]]*#.*$", ""); gsub(/"/, ""); print; exit }
+  ' forge.yaml
+}
+[ -n "$FILE_LINES" ] || FILE_LINES=$(yaml_maint file_lines)
+[ -n "$DOC_KB" ] || DOC_KB=$(yaml_maint doc_kb)
+FILE_LINES=${FILE_LINES:-2000} DOC_KB=${DOC_KB:-64}
+for v in "$FILE_LINES" "$DAYS" "$DOC_KB"; do case "$v" in ''|*[!0-9]*) die "thresholds must be positive integers" ;; esac; done
+
+T=$(mktemp -d "${TMPDIR:-/tmp}/maintenance-scan.XXXXXX") || die "mktemp failed"
+trap 'rm -rf "$T"' EXIT
+if [ -z "$OUT_DIR" ]; then OUT_DIR=$(mktemp -d "${TMPDIR:-/tmp}/maintenance-scan-out.XXXXXX") || die "mktemp failed"; fi
+mkdir -p "$OUT_DIR" || die "cannot create $OUT_DIR"
+OUT_DIR=$(cd "$OUT_DIR" && pwd)
+
+# ---- open issues, once ----
+if [ -n "$OPEN_FILE" ]; then
+  jq -e 'type == "array"' "$OPEN_FILE" >/dev/null 2>&1 || die "--open-issues must be a JSON array"
+  cp "$OPEN_FILE" "$T/open.json"
+else
+  gh issue list ${REPO:+-R "$REPO"} --state open --limit 1000 --json number,title,body > "$T/open.json" 2>/dev/null || die "cannot list open issues (gh)"
+fi
+
+# ---- tracked files, minus generated/vendored ----
+git ls-files > "$T/all"
+git check-attr --stdin linguist-generated linguist-vendored < "$T/all" \
+  | awk -F': ' '$3 == "set" || $3 == "true" { print $1 }' | sort -u > "$T/excluded"
+grep -vE '(^|/)(node_modules|vendor|third_party|dist|build)/' "$T/all" | sort | comm -23 - "$T/excluded" > "$T/files"
+
+# candidates: "kind<TAB>key<TAB>title<TAB>numbers-json<TAB>evidence"
+: > "$T/cands"
+add() { printf '%s\t%s\t%s\t%s\t%s\n' "$1" "$2" "$3" "$4" "$5" >> "$T/cands"; }
+
+# ---- hotspots: over-threshold files with commits in the window ----
+git log --since="$DAYS days ago" --format= --name-only 2>/dev/null | awk 'NF' | sort | uniq -c \
+  | awk '{ n = $1; sub(/^ *[0-9]+ /, ""); print $0 "\t" n }' > "$T/churn"
+tr '\n' '\0' < "$T/files" | xargs -0 wc -l 2>/dev/null \
+  | awk -v max="$FILE_LINES" '{ n = $1; p = $0; sub(/^ *[0-9]+ /, "", p); if (n > max && p != "total") print p "\t" n }' > "$T/big"
+while IFS="$(printf '\t')" read -r path lines; do
+  [ -f "$path" ] || continue
+  grep -Iq . "$path" 2>/dev/null || continue # binary
+  commits=$(awk -F'\t' -v p="$path" '$1 == p { print $2; exit }' "$T/churn")
+  [ -n "$commits" ] && [ "$commits" -gt 0 ] || continue
+  add "refactor:split" "$path" "split $path ($lines loc, $commits commits/${DAYS}d)" \
+    "{\"lines\":$lines,\"commits\":$commits,\"days\":$DAYS,\"threshold\":$FILE_LINES}" \
+    "\`$path\` has $lines lines (budget $FILE_LINES) and $commits commits in the last $DAYS days."
+done < "$T/big"
+
+# ---- backwards edges: workspace packages depending on each other both ways ----
+if git show HEAD:package.json 2>/dev/null | jq -e '.workspaces' >/dev/null 2>&1 || git cat-file -e HEAD:pnpm-workspace.yaml 2>/dev/null; then
+  first=$(git rev-list --max-parents=0 HEAD | tail -1)
+  if sd=$("$SCRIPT_DIR/structure-delta.sh" "$first" HEAD 2>/dev/null); then
+    printf '%s' "$sd" | jq -r '[(.package_edges_added // [])[] | [.from, .to]] | unique as $e
+      | $e[] | . as $p | select($p[0] < $p[1] and ($e | index([[$p[1], $p[0]]])) != null) | "\($p[0])\t\($p[1])"' 2>/dev/null \
+      | while IFS="$(printf '\t')" read -r a b; do
+          add "refactor" "$a->$b" "untangle $a <-> $b (pkg cycle)" "{\"edges\":2}" \
+            "Workspace package \`$a\` depends on \`$b\` and \`$b\` depends on \`$a\`."
+        done
+  fi
+fi
+
+# ---- dead code: the repo's own tools, only when installed and relevant ----
+count_lines() { awk 'NF' | wc -l | tr -d ' '; }
+dead() { # tool, count
+  [ "$2" -gt 0 ] 2>/dev/null || return 0
+  add "dead-code" "$1" "remove $2 unused ($1)" "{\"findings\":$2}" "\`$1\` reports $2 unused items."
+}
+if command -v vulture >/dev/null 2>&1 && grep -q '\.py$' "$T/files"; then
+  dead vulture "$(grep '\.py$' "$T/files" | tr '\n' '\0' | xargs -0 vulture 2>/dev/null | count_lines)"
+fi
+if [ -f package.json ]; then
+  command -v knip >/dev/null 2>&1 && dead knip "$(knip --no-progress --reporter compact 2>/dev/null | count_lines)"
+  command -v ts-prune >/dev/null 2>&1 && dead ts-prune "$(ts-prune 2>/dev/null | grep -v '(used in module)' | count_lines)"
+fi
+
+# ---- oversized docs ----
+grep -E '\.md$' "$T/files" | tr '\n' '\0' | xargs -0 wc -c 2>/dev/null \
+  | awk -v max="$((DOC_KB * 1024))" '{ n = $1; p = $0; sub(/^ *[0-9]+ /, "", p); if (n > max && p != "total") print p "\t" n }' \
+  | while IFS="$(printf '\t')" read -r path bytes; do
+      kb=$(( (bytes + 1023) / 1024 ))
+      add "docs" "$path" "trim $path ($kb KB > $DOC_KB KB)" "{\"kb\":$kb,\"threshold_kb\":$DOC_KB}" \
+        "\`$path\` is $kb KB; agents load docs under $DOC_KB KB whole."
+    done
+
+# ---- dedup, body files, output ----
+slug() { printf '%s' "$1" | tr -c 'A-Za-z0-9._-' '_' | cut -c1-80; }
+title_dup() { # title, key: an open near-duplicate title that names the same key
+  if [ -n "$OPEN_FILE" ]; then
+    jq -e --arg t "$1" 'any(.[]; .title == $t)' "$T/open.json" >/dev/null
+  else
+    local hit
+    hit=$("$SCRIPT_DIR/issue-dedup.sh" "$1" ${REPO:+-R "$REPO"} 2>/dev/null) && return 1
+    case "$hit" in *"$2"*) return 0 ;; *) return 1 ;; esac
+  fi
+}
+: > "$T/out.jsonl"
+sort -t "$(printf '\t')" -k1,1 -k2,2 -u "$T/cands" | while IFS="$(printf '\t')" read -r kind key title numbers evidence; do
+  marker="FORGE:MAINT:$kind:$key"
+  jq -e --arg m "<!-- $marker -->" 'any(.[]; (.body // "") | contains($m))' "$T/open.json" >/dev/null && continue
+  title_dup "$title" "$key" && continue
+  id="maint_$(slug "$kind")_$(printf '%s' "$kind:$key" | cksum | awk '{print $1}')"
+  body="$OUT_DIR/${id}_$(slug "${key%.md}").md"
+  cat > "$body" <<EOF
+## Problem
+
+$evidence Found by \`scripts/maintenance-scan.sh\`; it is dispatched within the maintenance budget (forge.yaml \`maintenance.share\`).
+
+## Evidence
+
+\`\`\`json
+$numbers
+\`\`\`
+
+## Acceptance Criteria
+
+- [ ] \`scripts/maintenance-scan.sh\` no longer reports \`$kind\` for \`$key\`
+- [ ] Behaviour is unchanged (existing tests pass)
+
+<!-- $marker -->
+<!-- FORGE:BODY-INTEGRITY:$id -->
+EOF
+  jq -n -c --arg kind "$kind" --arg key "$key" --arg title "$title" --arg body "$body" --arg marker "$marker" --argjson numbers "$numbers" \
+    '{kind: $kind, key: $key, title: $title, labels: ["maintenance", $kind], body_file: $body, marker: $marker, numbers: $numbers}' >> "$T/out.jsonl"
+done
+
+if [ "$PRINT_ARGS" = true ]; then
+  jq -r '"/issue --title \"" + (.title | gsub("\""; "\\\"")) + "\" --body-file \"" + .body_file + "\" --label maintenance --label " + .kind' "$T/out.jsonl"
+else
+  jq -s -c '.' "$T/out.jsonl"
+fi

package/scripts/maintenance-scan.test.sh

@@ -0,0 +1,89 @@
+#!/usr/bin/env bash
+# maintenance-scan.test.sh — cases for scripts/maintenance-scan.sh (forge#4197). Throwaway repos under $TMPDIR, no network.
+# SPDX-FileCopyrightText: Copyright (c) RapierCraft Studios
+# SPDX-License-Identifier: AGPL-3.0-or-later
+set -uo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+SCAN="$SCRIPT_DIR/maintenance-scan.sh"
+TMP="$(mktemp -d "${TMPDIR:-/tmp}/maintenance-scan-test.XXXXXX")"; trap 'rm -rf "$TMP"' EXIT
+PASS=0 FAILN=0
+ok() { PASS=$((PASS + 1)); }
+fail() { FAILN=$((FAILN + 1)); echo "FAIL: $*"; }
+check() { if [ "$2" = "$3" ]; then ok; else fail "$1: expected [$3] got [$2]"; fi; }
+
+# A PATH without any dead-code tool (a host install must not leak into the cases).
+BIN="$TMP/bin"; mkdir -p "$BIN"
+for t in git jq awk sed tr wc sort comm uniq cut grep cksum mktemp xargs cat cp mkdir rm dirname head tail env bash; do
+  p=$(command -v "$t") && ln -sf "$p" "$BIN/$t"
+done
+export GIT_AUTHOR_NAME=t GIT_AUTHOR_EMAIL=t@t GIT_COMMITTER_NAME=t GIT_COMMITTER_EMAIL=t@t
+NONE="$TMP/none.json"; echo '[]' > "$NONE"
+scan() { PATH="$BIN" bash "$SCAN" --open-issues "${OPEN:-$NONE}" --out-dir "$TMP/out" "$@"; }
+
+R="$TMP/repo"; mkdir -p "$R"; cd "$R" || exit 1
+git init -q . && echo "# r" > README.md && git add . && git commit -qm init
+
+# ---- 1. below thresholds: [] ----
+seq 1 50 > small.py; git add . && git commit -qm small
+check "below thresholds" "$(scan --file-lines 100)" "[]"
+
+# ---- 2. one over-threshold file: one refactor:split candidate with numbers ----
+seq 1 300 > big.py; git add . && git commit -qm big1; echo x >> big.py; git commit -qam big2
+seq 1 400 > gen.py; echo "gen.py linguist-generated" > .gitattributes; git add . && git commit -qm gen
+out=$(scan --file-lines 100)
+check "one hotspot" "$(printf '%s' "$out" | jq -c '[.[] | [.kind, .key]]')" '[["refactor:split","big.py"]]'
+check "title numbers" "$(printf '%s' "$out" | jq -r '.[0].title')" "split big.py (301 loc, 2 commits/90d)"
+check "numbers" "$(printf '%s' "$out" | jq -c '.[0].numbers')" '{"lines":301,"commits":2,"days":90,"threshold":100}'
+check "labels" "$(printf '%s' "$out" | jq -c '.[0].labels')" '["maintenance","refactor:split"]'
+bf=$(printf '%s' "$out" | jq -r '.[0].body_file')
+check "body marker" "$(grep -c 'FORGE:MAINT:refactor:split:big.py' "$bf")" "1"
+check "integrity marker" "$(grep -cE '<!-- FORGE:BODY-INTEGRITY:[^>]+ -->' "$bf")" "1"
+# thresholds from forge.yaml (block form)
+printf 'maintenance:\n  enabled: true\n  file_lines: 100\n' > forge.yaml; git add . && git commit -qm cfg
+check "forge.yaml threshold" "$(scan | jq -c '[.[].key]')" '["big.py"]'
+# a sub-map's key (maintenance.hotspot.file_lines, forge#4195) is not the section's own threshold
+printf 'maintenance:\n  enabled: true\n  hotspot:\n    file_lines: 5000\n  file_lines: 100\n' > forge.yaml; git add . && git commit -qm cfg2
+check "sub-map key ignored" "$(scan | jq -c '[.[].key]')" '["big.py"]'
+
+# ---- 3. dedup: an open issue carrying the marker, or the exact title ----
+jq -n --arg b "x <!-- FORGE:MAINT:refactor:split:big.py --> y" '[{number: 1, title: "other", body: $b}]' > "$TMP/open1.json"
+check "dedup by marker" "$(OPEN="$TMP/open1.json" scan)" "[]"
+jq -n '[{number: 2, title: "split big.py (301 loc, 2 commits/90d)", body: ""}]' > "$TMP/open2.json"
+check "dedup by title" "$(OPEN="$TMP/open2.json" scan)" "[]"
+
+# ---- 4. oversized doc ----
+awk 'BEGIN { for (i = 0; i < 3000; i++) print "doc line padding padding padding" }' > ARCH.md; git add . && git commit -qm doc
+check "doc candidate" "$(scan --doc-kb 64 | jq -c '[.[] | select(.kind == "docs") | [.key, .numbers.kb]]')" '[["ARCH.md",97]]'
+check "doc under budget" "$(scan --doc-kb 200 | jq -c '[.[] | select(.kind == "docs")]')" "[]"
+
+# ---- 5. dead code from a stubbed knip (only when package.json exists) ----
+printf '#!/usr/bin/env bash\nprintf "a.ts: x\\nb.ts: y\\nc.ts: z\\n"\n' > "$BIN/knip"; chmod +x "$BIN/knip"
+check "knip needs package.json" "$(scan | jq -c '[.[] | select(.kind == "dead-code")]')" "[]"
+echo '{"name":"r"}' > package.json; git add . && git commit -qm pkg
+check "knip candidate" "$(scan | jq -c '[.[] | select(.kind == "dead-code") | [.key, .numbers.findings, .title]]')" '[["knip",3,"remove 3 unused (knip)"]]'
+rm -f "$BIN/knip"
+
+# ---- 6. determinism and /issue argument lines ----
+a=$(scan); b=$(scan)
+check "deterministic" "$a" "$b"
+args=$(scan --print-issue-args)
+check "issue-arg lines" "$(printf '%s\n' "$args" | grep -c '^/issue --title ".*" --body-file ".*" --label maintenance --label ')" "$(printf '%s' "$a" | jq length)"
+
+# ---- 7. a workspace package cycle ----
+W="$TMP/ws"; mkdir -p "$W"; cd "$W" || exit 1
+git init -q . && echo "# w" > README.md && git add . && git commit -qm init
+echo '{"name":"root","private":true,"workspaces":["packages/*"]}' > package.json
+mkdir -p packages/a packages/b packages/c
+echo '{"name":"@s/a","dependencies":{"@s/b":"*"}}' > packages/a/package.json
+echo '{"name":"@s/b","dependencies":{"@s/a":"*"}}' > packages/b/package.json
+echo '{"name":"@s/c","dependencies":{"@s/a":"*"}}' > packages/c/package.json
+git add . && git commit -qm ws
+check "package cycle" "$(scan | jq -c '[.[] | select(.kind == "refactor") | [.key, .title]]')" '[["@s/a->@s/b","untangle @s/a <-> @s/b (pkg cycle)"]]'
+
+# ---- 8. usage and no model call ----
+PATH="$BIN" bash "$SCAN" --bogus >/dev/null 2>&1; check "usage exit" "$?" "2"
+check "no LLM invocation" "$(grep -nEi '\b(claude|codex|anthropic|openai|ollama)\b' "$SCAN" | grep -v '^[0-9]*:#' || true)" ""
+
+echo "maintenance-scan.test.sh: $PASS passed, $FAILN failed"
+[ "$FAILN" -eq 0 ]

package/scripts/milestone-drift.sh

@@ -0,0 +1,206 @@
+#!/usr/bin/env bash
+# SPDX-FileCopyrightText: Copyright (c) RapierCraft Studios
+# SPDX-License-Identifier: AGPL-3.0-or-later
+#
+# milestone-drift.sh — deterministic structural drift of a milestone branch since its fork point (forge#4161). No LLM,
+# no network: git + jq + awk, plus `jscpd` only when it is on PATH. Run from inside the repo (fetch the refs first).
+#
+# Usage: milestone-drift.sh <slug> [--base <ref>] [--head <ref>] [--summary FILE] [--top N] [--format json|text]
+#                           [--pair-growth N] [--package-lines N] [--file-lines N]
+#   --base     default origin/<forge.yaml branches.staging, else staging> (the local branch when origin/ is absent)
+#   --head     default origin/milestone/<slug> (the local branch when origin/ is absent)
+#   --summary  the orchestrate FORGE:DRIFT summary JSON (bin/lean/drift.mjs); adds the PRs behind each import pair
+#   thresholds default to the forge.yaml `orchestrate:` block (`drift:` keys pair_growth, package_lines, file_lines);
+#   none set means `crossings: []`. A value crosses when it is greater than its threshold.
+# Facts, from the fork point F = git merge-base base head to head:
+#   pairs       import-pair growth per from->to (files_added - files_removed), one scripts/structure-delta.sh F head run
+#   new_edges   workspace package dependency edges added (same run)
+#   files       top-N files by line growth (added - removed), one `git diff --numstat F head`; linguist-generated /
+#               linguist-vendored paths (.gitattributes at head) and forge.yaml `orchestrate.generated_globs`
+#               (comma-separated shell patterns, `*` crosses `/`) are excluded and listed in `excluded`
+#   packages    top-N workspace packages by line growth (longest package dir prefix); [] without workspaces
+#   duplication jscpd statistics over the changed files when jscpd is installed and head is checked out, else null
+# Output: one JSON object {milestone,base,head,fork_point,pairs,new_edges,files,packages,excluded,duplication,
+#   thresholds,crossings[{kind:pair_growth|package_lines|file_lines,key,value,threshold,package_path}]}, or with
+#   --format text a short report of the same facts.
+# Exit: 0 ok | 2 usage or bad ref. bash 3.2 portable.
+set -uo pipefail
+
+die() { echo "milestone-drift: $*" >&2; exit 2; }
+command -v jq >/dev/null 2>&1 || die "jq is required"
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+
+SLUG="" BASE="" HEAD="" SUMMARY="" TOP=10 FORMAT=json PAIR_T="" PKG_T="" FILE_T=""
+while [ $# -gt 0 ]; do
+  case "$1" in
+    --base) BASE="${2:-}"; shift 2 ;;
+    --head) HEAD="${2:-}"; shift 2 ;;
+    --summary) SUMMARY="${2:-}"; shift 2 ;;
+    --top) TOP="${2:-}"; shift 2 ;;
+    --format) FORMAT="${2:-}"; shift 2 ;;
+    --pair-growth) PAIR_T="${2:-}"; shift 2 ;;
+    --package-lines) PKG_T="${2:-}"; shift 2 ;;
+    --file-lines) FILE_T="${2:-}"; shift 2 ;;
+    -h|--help) sed -n '5,27p' "$0"; exit 0 ;;
+    -*) die "unknown argument: $1" ;;
+    *) [ -z "$SLUG" ] || die "one slug only"; SLUG="$1"; shift ;;
+  esac
+done
+SLUG="${SLUG#milestone/}"
+[ -n "$SLUG" ] || die "usage: milestone-drift.sh <slug> [--base <ref>] [--head <ref>] [--summary FILE] [--format json|text]"
+case "$FORMAT" in json|text) ;; *) die "--format must be json or text" ;; esac
+case "$TOP" in ''|*[!0-9]*) die "--top must be a positive integer" ;; esac
+
+ROOT=$(git rev-parse --show-toplevel 2>/dev/null) || die "not inside a git repository"
+cd "$ROOT" || die "cannot cd to $ROOT"
+
+# forge.yaml: key under a top-level block (any depth below it), block form only
+yaml_key() { # block key
+  [ -f forge.yaml ] || return 0
+  awk -v b="$1" -v k="$2" '
+    /^[A-Za-z_][A-Za-z0-9_]*:[[:space:]]*(#.*)?$/ { on = ($0 ~ "^" b ":"); next }
+    /^[^[:space:]#]/ { on = 0 }
+    on && $0 ~ "^[[:space:]]+" k ":" { sub("^[[:space:]]+" k ":[[:space:]]*", ""); sub("[[:space:]]*#.*$", ""); gsub(/["\047]/, ""); print; exit }
+  ' forge.yaml
+}
+[ -n "$PAIR_T" ] || PAIR_T=$(yaml_key orchestrate pair_growth)
+[ -n "$PKG_T" ] || PKG_T=$(yaml_key orchestrate package_lines)
+[ -n "$FILE_T" ] || FILE_T=$(yaml_key orchestrate file_lines)
+GLOBS=$(yaml_key orchestrate generated_globs)
+for v in "$PAIR_T" "$PKG_T" "$FILE_T"; do case "$v" in ''|*[!0-9]*) [ -z "$v" ] || die "thresholds must be positive integers" ;; esac; done
+
+ref_or_local() { # origin/<b> when it resolves, else <b>
+  if git rev-parse --verify -q "origin/$1^{commit}" >/dev/null; then echo "origin/$1"; else echo "$1"; fi
+}
+if [ -z "$BASE" ]; then s=$(yaml_key branches staging); BASE=$(ref_or_local "${s:-staging}"); fi
+[ -n "$HEAD" ] || HEAD=$(ref_or_local "milestone/$SLUG")
+git rev-parse --verify -q "$BASE^{commit}" >/dev/null || die "bad base ref: $BASE"
+git rev-parse --verify -q "$HEAD^{commit}" >/dev/null || die "bad head ref: $HEAD"
+HEAD_SHA=$(git rev-parse "$HEAD^{commit}")
+FORK=$(git merge-base "$BASE" "$HEAD") || die "no merge base for $BASE and $HEAD"
+if [ -n "$SUMMARY" ]; then jq -e 'type == "object"' "$SUMMARY" >/dev/null 2>&1 || die "--summary must be a JSON object"; fi
+
+T=$(mktemp -d "${TMPDIR:-/tmp}/milestone-drift.XXXXXX") || die "mktemp failed"
+trap 'rm -rf "$T"' EXIT
+TAB="$(printf '\t')"
+
+# ---- pairs and edges: the existing structure delta from the fork point ----
+SD=$("$SCRIPT_DIR/structure-delta.sh" "$FORK" "$HEAD_SHA" 2>/dev/null) || SD='{}'
+printf '%s' "$SD" | jq -e 'type == "object"' >/dev/null 2>&1 || SD='{}'
+printf '%s' "$SD" > "$T/sd.json"
+if [ -n "$SUMMARY" ]; then cp "$SUMMARY" "$T/summary.json"; else echo '{}' > "$T/summary.json"; fi
+
+# ---- workspace package dirs at head: "dir<TAB>name", longest dir first ----
+{
+  git show "$HEAD_SHA:package.json" 2>/dev/null | jq -r '(.workspaces // empty) | if type == "array" then .[] elif type == "object" then (.packages // [])[] else empty end | strings' 2>/dev/null
+  git show "$HEAD_SHA:pnpm-workspace.yaml" 2>/dev/null | awk '
+    /^packages:/ { on = 1; next }
+    on && /^[^ \t#-]/ { on = 0 }
+    on && /^[ \t]*-[ \t]*/ { sub(/^[ \t]*-[ \t]*/, ""); sub(/[ \t]+#.*$/, ""); gsub(/["\047]/, ""); sub(/[ \t]+$/, ""); if ($0 != "") print }'
+} | sed 's#^\./##; s#/$##' | sort -u > "$T/globs"
+to_re() { printf '%s' "$1" | sed 's/[.+^$()|{}[]/\\&/g; s/\*\*/%%/g; s/\*/[^\/]*/g; s/%%/.*/g; s/?/[^\/]/g'; }
+: > "$T/inc"; : > "$T/exc"; : > "$T/dirs"
+while IFS= read -r g; do
+  case "$g" in \!*) echo "^$(to_re "${g#!}")\$" >> "$T/exc" ;; *) echo "^$(to_re "$g")\$" >> "$T/inc" ;; esac
+done < "$T/globs"
+if [ -s "$T/inc" ]; then
+  git ls-tree -r --name-only "$HEAD_SHA" | grep -E '/package\.json$' | grep -vE '(^|/)node_modules/' | while IFS= read -r m; do
+    d=${m%/package.json}
+    printf '%s\n' "$d" | grep -qE -f "$T/inc" || continue
+    if [ -s "$T/exc" ] && printf '%s\n' "$d" | grep -qE -f "$T/exc"; then continue; fi
+    n=$(git show "$HEAD_SHA:$m" 2>/dev/null | jq -r '.name // empty | strings' 2>/dev/null)
+    [ -n "$n" ] && printf '%s\t%s\t%s\n' "${#d}" "$d" "$n"
+  done | sort -t "$TAB" -k1,1nr | cut -f2- > "$T/dirs"
+fi
+
+# ---- size growth: one numstat, generated/vendored excluded ----
+git diff --numstat --no-renames "$FORK" "$HEAD_SHA" 2>/dev/null | awk -F'\t' 'NF >= 3 && $1 ~ /^[0-9]+$/ && $2 ~ /^[0-9]+$/' > "$T/numstat"
+cut -f3 "$T/numstat" > "$T/paths"
+: > "$T/excluded"
+if [ -s "$T/paths" ]; then
+  # attributes as of head (--source); a git without it reads the working tree's .gitattributes
+  { git check-attr --source="$HEAD_SHA" --stdin linguist-generated linguist-vendored < "$T/paths" 2>/dev/null \
+      || git check-attr --stdin linguist-generated linguist-vendored < "$T/paths"; } \
+    | awk -F': ' '$3 == "set" || $3 == "true" { print $1 }' >> "$T/excluded"
+  if [ -n "$GLOBS" ]; then
+    while IFS= read -r p; do
+      oldifs=$IFS; IFS=,
+      for g in $GLOBS; do
+        g=$(printf '%s' "$g" | sed 's/^[[:space:]]*//; s/[[:space:]]*$//')
+        [ -n "$g" ] || continue
+        # shellcheck disable=SC2254 # the glob is the pattern
+        case "$p" in $g) echo "$p" >> "$T/excluded"; break ;; esac
+      done
+      IFS=$oldifs
+    done < "$T/paths"
+  fi
+fi
+sort -u "$T/excluded" -o "$T/excluded"
+# kept rows: "path<TAB>added<TAB>removed<TAB>package-dir<TAB>package-name"
+awk -F'\t' '
+  FILENAME == ARGV[1] { ex[$0] = 1; next }
+  FILENAME == ARGV[2] { nd++; dir[nd] = $1; nm[nd] = $2; next }
+  ($3 in ex) { next }
+  { d = ""; n = ""; for (i = 1; i <= nd; i++) if (index($3, dir[i] "/") == 1) { d = dir[i]; n = nm[i]; break }
+    print $3 "\t" $1 "\t" $2 "\t" d "\t" n }
+' "$T/excluded" "$T/dirs" "$T/numstat" > "$T/kept"
+
+jq -R -s -c 'split("\n") | map(select(length > 0) | split("\t") | {path: .[0], added: (.[1] | tonumber), removed: (.[2] | tonumber), pkg_dir: .[3], pkg: .[4]})' "$T/kept" > "$T/kept.json"
+jq -R -s -c 'split("\n") | map(select(length > 0))' "$T/excluded" > "$T/excluded.json"
+jq -R -s -c 'split("\n") | map(select(length > 0) | split("\t") | {path: .[0], name: .[1]})' "$T/dirs" > "$T/dirs.json"
+
+# ---- duplication: the repo's own jscpd, only when installed and head is the working tree ----
+DUP=null
+if command -v jscpd >/dev/null 2>&1 && [ "$(git rev-parse HEAD 2>/dev/null)" = "$HEAD_SHA" ] && [ -s "$T/kept" ]; then
+  cut -f1 "$T/kept" | while IFS= read -r p; do [ -f "$p" ] && printf '%s\0' "$p"; done > "$T/dupfiles"
+  if [ -s "$T/dupfiles" ]; then
+    xargs -0 jscpd --silent --reporters json --output "$T/jscpd" < "$T/dupfiles" >/dev/null 2>&1
+    if [ -f "$T/jscpd/jscpd-report.json" ]; then
+      DUP=$(jq -c '(.statistics.total // {}) | {clones: (.clones // 0), duplicated_lines: (.duplicatedLines // 0), percentage: (.percentage // 0)}' "$T/jscpd/jscpd-report.json" 2>/dev/null) || DUP=null
+      [ -n "$DUP" ] || DUP=null
+    fi
+  fi
+fi
+
+num_or_null() { if [ -n "$1" ]; then printf '%s' "$1"; else printf 'null'; fi; }
+REPORT=$(jq -n -c \
+  --arg milestone "$SLUG" --arg base "$BASE" --arg head "$HEAD" --arg fork "$FORK" --argjson top "$TOP" \
+  --slurpfile sd "$T/sd.json" --slurpfile summary "$T/summary.json" --slurpfile kept "$T/kept.json" \
+  --slurpfile excluded "$T/excluded.json" --slurpfile dirs "$T/dirs.json" --argjson dup "$DUP" \
+  --argjson pt "$(num_or_null "$PAIR_T")" --argjson kt "$(num_or_null "$PKG_T")" --argjson ft "$(num_or_null "$FILE_T")" '
+  ($sd[0]) as $sd | ($summary[0]) as $sum | ($kept[0]) as $kept | ($dirs[0]) as $dirs
+  | ($dirs | map({key: .name, value: .path}) | from_entries) as $dirOf
+  | [($sd.import_pairs // [])[]
+      | {from, to, growth: ((.files_added // 0) - (.files_removed // 0)), files_added: (.files_added // 0),
+         files_removed: (.files_removed // 0), files: (.files // []),
+         prs: (($sum.pairs // {})["\(.from):\(.to)"].prs // [])}]
+    | sort_by(-.growth, .from, .to) as $pairs
+  | [$kept[] | {path, added, removed, growth: (.added - .removed), package: (if .pkg == "" then null else .pkg end)}]
+    | sort_by(-.growth, .path) as $files
+  | [$kept[] | select(.pkg != "")] | group_by(.pkg)
+    | map({name: .[0].pkg, path: .[0].pkg_dir, added: (map(.added) | add), removed: (map(.removed) | add)} | .growth = (.added - .removed))
+    | sort_by(-.growth, .name) as $packages
+  | {milestone: $milestone, base: $base, head: $head, fork_point: $fork,
+     pairs: $pairs, new_edges: ($sd.package_edges_added // []),
+     files: $files[:$top], packages: $packages[:$top], excluded: $excluded[0], duplication: $dup,
+     thresholds: {pair_growth: $pt, package_lines: $kt, file_lines: $ft},
+     crossings: (
+       (if $pt == null then [] else [$pairs[] | select(.growth > $pt)
+          | {kind: "pair_growth", key: "\(.from)->\(.to)", value: .growth, threshold: $pt, package_path: ($dirOf[.from] // null)}] end)
+       + (if $kt == null then [] else [$packages[] | select(.growth > $kt)
+          | {kind: "package_lines", key: .name, value: .growth, threshold: $kt, package_path: .path}] end)
+       + (if $ft == null then [] else [$files[] | select(.growth > $ft)
+          | {kind: "file_lines", key: .path, value: .growth, threshold: $ft, package_path: (if .package == null then null else $dirOf[.package] end)}] end))}
+') || die "report failed"
+
+if [ "$FORMAT" = json ]; then printf '%s\n' "$REPORT"; exit 0; fi
+printf '%s' "$REPORT" | jq -r '
+  def line(f): if length == 0 then "  (none)" else map("  " + f) | join("\n") end;
+  "milestone/\(.milestone) drift since \(.fork_point[:12]) (base \(.base), head \(.head))",
+  "import pairs (net files):", (.pairs | line("\(.from) -> \(.to)  +\(.growth)\(if (.prs | length) > 0 then "  PRs " + (.prs | map("#\(.)") | join(", ")) else "" end)")),
+  "new package edges:", (.new_edges | line("\(.from) -> \(.to)  (\(.kind), \(.manifest))")),
+  "top files by growth:", (.files | line("\(.path)  +\(.added) -\(.removed)")),
+  "top packages by growth:", (.packages | line("\(.name) (\(.path))  +\(.added) -\(.removed)")),
+  "excluded (generated/vendored): \(.excluded | length)",
+  (if .duplication == null then "duplication: n/a (jscpd not run)" else "duplication: \(.duplication.clones) clones, \(.duplication.duplicated_lines) lines (\(.duplication.percentage)%)" end),
+  (if (.crossings | length) == 0 then "thresholds: none crossed" else "thresholds crossed:", (.crossings | line("\(.kind) \(.key): \(.value) > \(.threshold)")) end)'

package/scripts/milestone-drift.test.sh

@@ -0,0 +1,120 @@
+#!/usr/bin/env bash
+# milestone-drift.test.sh — cases for scripts/milestone-drift.sh (forge#4161).
+# SPDX-FileCopyrightText: Copyright (c) RapierCraft Studios
+# SPDX-License-Identifier: AGPL-3.0-or-later
+set -uo pipefail
+S="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/milestone-drift.sh"
+PASS=0; FAILN=0
+TMP="$(mktemp -d "${TMPDIR:-/tmp}/milestone-drift-test.XXXXXX")"; trap 'rm -rf "$TMP"' EXIT
+ok() { PASS=$((PASS+1)); }
+bad() { FAILN=$((FAILN+1)); echo "FAIL: $1"; }
+eq() { if [ "$2" = "$3" ]; then ok; else bad "$1 (got: $2 want: $3)"; fi }
+q() { printf '%s' "$OUT" | jq -c "$1"; }
+
+new_repo() {
+  rm -rf "$TMP/w"; mkdir -p "$TMP/w"; cd "$TMP/w" || exit 1
+  git init -q -b staging; git config user.email t@example.com; git config user.name t; git config commit.gpgsign false
+}
+commit() { git add -A; git commit -qm "$1"; }
+pj() { mkdir -p "$1"; printf '%s\n' "$2" > "$1/package.json"; }
+lines() { i=0; while [ "$i" -lt "$2" ]; do echo "export const v$i = $i;"; i=$((i+1)); done >> "$1"; }
+
+# ---- fixture milestone: two packages, an app, generated files ----
+new_repo
+printf '{"name":"root","private":true,"workspaces":["apps/*","packages/*"]}\n' > package.json
+pj apps/web '{"name":"@s/web","dependencies":{"@s/db":"workspace:*"}}'
+pj packages/db '{"name":"@s/db"}'
+pj packages/core '{"name":"@s/core"}'
+mkdir -p apps/web/src packages/db/src packages/core/src
+echo "export const a = 1;" > apps/web/src/a.ts
+echo "export const d = 1;" > packages/db/src/d.ts
+echo "export const c = 1;" > packages/core/src/c.ts
+printf 'packages/db/src/schema.gen.ts linguist-generated\n' > .gitattributes
+commit base
+FORK_SHA=$(git rev-parse HEAD)
+git checkout -q -b milestone/m1
+# lane 1: two app files import the db package, db grows a lot
+echo "import { d } from '@s/db';" > apps/web/src/one.ts
+echo "import { d } from '@s/db';" > apps/web/src/two.ts
+lines packages/db/src/big.ts 40
+commit "lane 1"
+# lane 2: a new package edge web -> core, a third db import, generated and globbed files grow
+pj apps/web '{"name":"@s/web","dependencies":{"@s/db":"workspace:*","@s/core":"workspace:*"}}'
+echo "import { d } from '@s/db';" > apps/web/src/three.ts
+echo "import { c } from '@s/core';" > apps/web/src/four.ts
+lines packages/db/src/schema.gen.ts 500
+mkdir -p data; lines data/fixtures.json 300
+lines apps/web/src/a.ts 5
+commit "lane 2"
+# staging moves on after the fork: never part of the milestone's drift
+git checkout -q staging; lines packages/core/src/c.ts 900; commit "staging only"; git checkout -q milestone/m1
+
+OUT="$(bash "$S" m1)"; rc=$?
+eq "rc" "$rc" 0
+eq "fork point" "$(printf '%s' "$OUT" | jq -r '.fork_point')" "$FORK_SHA"
+eq "refs default to the local staging and milestone branches" "$(q '[.base, .head]')" '["staging","milestone/m1"]'
+eq "pair growth" "$(q '[.pairs[] | {from, to, growth}]')" '[{"from":"@s/web","to":"@s/db","growth":3},{"from":"@s/web","to":"@s/core","growth":1}]'
+eq "pair files" "$(q '.pairs[0].files')" '["apps/web/src/one.ts","apps/web/src/three.ts","apps/web/src/two.ts"]'
+eq "new package edges" "$(q '[.new_edges[] | [.from, .to, .kind]]')" '[["@s/web","@s/core","dependencies"]]'
+eq "top files (generated excluded, staging-only change absent)" "$(q '[.files[] | [.path, .growth]] | .[:2]')" '[["data/fixtures.json",300],["packages/db/src/big.ts",40]]'
+eq "generated file excluded" "$(q '.excluded')" '["packages/db/src/schema.gen.ts"]'
+eq "packages by growth" "$(q '[.packages[] | [.name, .path, .growth]]')" '[["@s/db","packages/db",40],["@s/web","apps/web",9]]'
+eq "file outside packages has no package" "$(q '.files[0].package')" 'null'
+eq "no thresholds, no crossings" "$(q '[.crossings, .thresholds]')" '[[],{"pair_growth":null,"package_lines":null,"file_lines":null}]'
+eq "no jscpd, no duplication" "$(PATH="/usr/bin:/bin" bash "$S" m1 | jq -c '.duplication')" 'null'
+
+# forge.yaml glob exclusion and thresholds from the orchestrate.drift block
+cat > forge.yaml <<'EOF'
+branches:
+  staging: staging
+orchestrate:
+  drift:
+    pair_growth: 2
+    package_lines: 30
+    file_lines: 250
+    generated_globs: data/*, *.lock
+EOF
+OUT="$(bash "$S" m1)"
+eq "glob excluded" "$(q '.excluded')" '["data/fixtures.json","packages/db/src/schema.gen.ts"]'
+eq "thresholds read from forge.yaml" "$(q '.thresholds')" '{"pair_growth":2,"package_lines":30,"file_lines":250}'
+eq "crossings" "$(q '[.crossings[] | [.kind, .key, .value, .threshold, .package_path]]')" '[["pair_growth","@s/web->@s/db",3,2,"apps/web"],["package_lines","@s/db",40,30,"packages/db"]]'
+# flags override forge.yaml; below every threshold nothing crosses
+OUT="$(bash "$S" m1 --pair-growth 3 --package-lines 40 --file-lines 40)"
+eq "at the threshold is not a crossing" "$(q '.crossings')" '[]'
+rm forge.yaml
+
+# --summary attributes PRs to pairs
+printf '{"v":1,"pairs":{"@s/web:@s/db":{"files_added":3,"files_removed":0,"prs":[11,12]}}}\n' > "$TMP/summary.json"
+OUT="$(bash "$S" m1 --summary "$TMP/summary.json")"
+eq "pair PRs from the summary" "$(q '.pairs[0].prs')" '[11,12]'
+
+# text format
+TXT="$(bash "$S" milestone/m1 --format text --file-lines 100)"
+case "$TXT" in *"@s/web -> @s/db  +3"*"file_lines data/fixtures.json: 300 > 100"*) ok ;; *) bad "text report: $TXT" ;; esac
+
+# optional jscpd: used only when on PATH and head is the working tree
+mkdir -p "$TMP/bin"
+cat > "$TMP/bin/jscpd" <<'EOF'
+#!/usr/bin/env bash
+while [ $# -gt 0 ]; do case "$1" in --output) mkdir -p "$2"; printf '{"statistics":{"total":{"clones":2,"duplicatedLines":14,"percentage":3.5}}}' > "$2/jscpd-report.json"; shift 2 ;; *) shift ;; esac; done
+EOF
+chmod +x "$TMP/bin/jscpd"
+eq "jscpd statistics" "$(PATH="$TMP/bin:$PATH" bash "$S" m1 | jq -c '.duplication')" '{"clones":2,"duplicated_lines":14,"percentage":3.5}'
+git checkout -q staging
+eq "jscpd skipped when head is not checked out" "$(PATH="$TMP/bin:$PATH" bash "$S" m1 | jq -c '.duplication')" 'null'
+
+# ---- repo without workspaces: no packages, pairs empty ----
+new_repo
+echo "x" > a.txt; commit base
+git checkout -q -b milestone/plain; lines a.txt 3; commit change
+OUT="$(bash "$S" plain)"
+eq "plain repo" "$(q '[.pairs, .new_edges, .packages, [.files[] | .path]]')" '[[],[],[],["a.txt"]]'
+
+# ---- usage errors exit 2 ----
+bash "$S" >/dev/null 2>&1; eq "no slug" "$?" 2
+bash "$S" nope >/dev/null 2>&1; eq "missing milestone branch" "$?" 2
+bash "$S" plain --pair-growth x >/dev/null 2>&1; eq "bad threshold" "$?" 2
+bash "$S" plain --format yaml >/dev/null 2>&1; eq "bad format" "$?" 2
+
+echo "milestone-drift: $PASS passed, $FAILN failed"
+[ "$FAILN" -eq 0 ]