forgedock 1.27.0 → 1.29.0

package/scripts/repo-facts.sh

@@ -0,0 +1,146 @@
+#!/usr/bin/env bash
+# SPDX-FileCopyrightText: Copyright (c) RapierCraft Studios
+# SPDX-License-Identifier: AGPL-3.0-or-later
+#
+# repo-facts.sh — deterministic repo facts the builder reads instead of re-learning them each lane (forge#4272). No LLM,
+# no network, runs none of the repo's commands: git + jq + awk + node. Codebase-agnostic.
+#
+# Usage: repo-facts.sh [--repo-path DIR] [--output FILE | --stdout]
+#   default output <repo>/.forge/repo-facts.json, rewritten on every run (an optional cache; the builder prompt cites
+#   it by path when present, deleting it restores today's prompts).
+# Output: {"version":1,
+#   "toolchain": bin/lean/toolchain.mjs detectPins(repo) {pm, node, notes} (forge#4163),
+#   "slow_suites": [{command, source, why}]: test-map entries (.forge/test-map.json when valid, else test-map.sh) whose
+#     command names e2e|playwright|cypress|integration|docker, or whose CI job sets `services:` or timeout-minutes >= 20,
+#   "layer_rules": {packages:[{name,dir}], edges:[{from,to,kind,manifest}], boundary_configs:[path]}: the workspace
+#     package graph at HEAD, same workspace and edge definitions as scripts/structure-delta.sh (forge#4159); boundary
+#     configs are eslint/dependency-cruiser/Nx files naming boundary rules,
+#   "hotspots": scripts/hotspots.sh over the tracked files, over-threshold and not excluded, by hotspot desc, max 20
+#     (forge#4195)}
+# Exit: 0 ok | 2 usage or bad repo path. bash 3.2 portable.
+set -uo pipefail
+
+die() { echo "repo-facts: $*" >&2; exit 2; }
+command -v jq >/dev/null 2>&1 || die "jq is required"
+HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+REPO="."; OUT=""; STDOUT=0
+while [ $# -gt 0 ]; do
+  case "$1" in
+    --repo-path) [ $# -ge 2 ] && [ -n "$2" ] || die "usage: repo-facts.sh [--repo-path DIR] [--output FILE | --stdout]"; REPO="$2"; shift 2 ;;
+    --output) [ $# -ge 2 ] && [ -n "$2" ] || die "usage: repo-facts.sh [--repo-path DIR] [--output FILE | --stdout]"; OUT="$2"; shift 2 ;;
+    --stdout) STDOUT=1; shift ;;
+    *) die "unknown option: $1" ;;
+  esac
+done
+[ -d "$REPO" ] || die "repo path not found: $REPO"
+case "$OUT" in ""|/*) ;; *) OUT="$PWD/$OUT" ;; esac
+cd "$REPO" || die "cannot enter $REPO"
+
+T=$(mktemp -d "${TMPDIR:-/tmp}/repo-facts.XXXXXX") || die "mktemp failed"
+trap 'rm -rf "$T"' EXIT
+HAS_HEAD=0; git rev-parse --verify -q "HEAD^{commit}" >/dev/null 2>&1 && HAS_HEAD=1
+# -z and quotepath=off (#4298): non-ASCII and spaced paths come out verbatim, never C-quoted
+if [ "$HAS_HEAD" = 1 ]; then git -c core.quotepath=off ls-tree -r -z --name-only HEAD | tr '\0' '\n' > "$T/files"; else : > "$T/files"; fi
+# file content at HEAD
+at_head() { git show "HEAD:$1" 2>/dev/null; }
+
+# ---- toolchain (reuses #4163 detection) ----
+TOOLCHAIN=$(node --input-type=module -e '
+  const { detectPins } = await import(process.argv[1]);
+  console.log(JSON.stringify(await detectPins(process.argv[2])));
+' "$HERE/../bin/lean/toolchain.mjs" "$PWD" 2>/dev/null) || TOOLCHAIN=""
+printf '%s' "$TOOLCHAIN" | jq -e 'type == "object"' >/dev/null 2>&1 || TOOLCHAIN='{"pm":null,"node":null,"notes":["toolchain detection failed"]}'
+
+# ---- slow suites ----
+MAP=""
+[ -f .forge/test-map.json ] && MAP=$(jq -c 'select(.version == 1 and (.entries | type == "array"))' .forge/test-map.json 2>/dev/null)
+[ -n "$MAP" ] || MAP=$(bash "$HERE/test-map.sh" --stdout 2>/dev/null)
+[ -n "$MAP" ] || MAP='{"version":1,"entries":[]}'
+# CI jobs: "workflow#jobs.<job> TAB services(0|1) TAB timeout-minutes(0 when unset)"
+: > "$T/jobs"
+grep -E '^\.github/workflows/[^/]+\.(yml|yaml)$' "$T/files" | while IFS= read -r wf; do
+  [ -f "$wf" ] || continue
+  awk -v wf="$wf" '
+    function indent(s) { match(s, /^ */); return RLENGTH }
+    function flush() { if (job != "") print wf "#jobs." job "\t" svc "\t" tmo }
+    /^jobs:[ \t]*$/ { on = 1; next }
+    on && /^[^ \t#]/ { flush(); job = ""; on = 0; next }
+    on && /^ +[A-Za-z0-9_-]+:[ \t]*$/ && (jind == 0 || indent($0) == jind) { flush(); jind = indent($0); job = $0; sub(/^ +/, "", job); sub(/:.*$/, "", job); svc = 0; tmo = 0; next }
+    on && job != "" && indent($0) == jind + 2 && /^ *services:/ { svc = 1 }
+    on && job != "" && indent($0) == jind + 2 && /^ *timeout-minutes:[ \t]*[0-9]+/ { t = $0; sub(/^ *timeout-minutes:[ \t]*/, "", t); tmo = t + 0 }
+    END { flush() }
+  ' "$wf" >> "$T/jobs"
+done
+JOBS=$(jq -R -s -c '[split("\n")[] | select(length > 0) | split("\t") | {key: .[0], value: {services: (.[1] == "1"), timeout: (.[2] | tonumber)}}] | from_entries' < "$T/jobs")
+SLOW=$(printf '%s' "$MAP" | jq -c --argjson jobs "$JOBS" '
+  [.entries[] | . as $e
+   | ($e.command | ascii_downcase | [scan("e2e|playwright|cypress|integration|docker")] | first) as $w
+   | ($jobs[$e.source] // null) as $j
+   | (if $w then "command names \($w)"
+      elif $j and $j.services then "CI job \($e.source) uses services"
+      elif $j and $j.timeout >= 20 then "CI job \($e.source) sets timeout-minutes \($j.timeout)"
+      else null end) as $why
+   | select($why) | {command: $e.command, source: $e.source, why: $why}]')
+
+# ---- layer rules: workspace packages and their edges at HEAD (structure-delta.sh definitions) ----
+{
+  at_head package.json | jq -r '(.workspaces // empty) | if type == "array" then .[] elif type == "object" then (.packages // [])[] else empty end | strings' 2>/dev/null
+  at_head pnpm-workspace.yaml | awk '
+    /^packages:/ { on = 1; next }
+    on && /^[^ \t#-]/ { on = 0 }
+    on && /^[ \t]*-[ \t]*/ { sub(/^[ \t]*-[ \t]*/, ""); sub(/[ \t]+#.*$/, ""); gsub(/["\047]/, ""); sub(/[ \t]+$/, ""); if ($0 != "") print }'
+} | sed 's#^\./##; s#/$##' | sort -u > "$T/globs"
+to_re() { printf '%s' "$1" | sed 's/[.+^$()|{}[]/\\&/g; s/\*\*/%%/g; s/\*/[^\/]*/g; s/%%/.*/g; s/?/[^\/]/g'; }
+: > "$T/inc"; : > "$T/exc"; : > "$T/names"; : > "$T/deps"
+while IFS= read -r g; do
+  case "$g" in \!*) echo "^$(to_re "${g#!}")\$" >> "$T/exc" ;; *) echo "^$(to_re "$g")\$" >> "$T/inc" ;; esac
+done < "$T/globs"
+if [ -s "$T/inc" ]; then
+  grep -E '/package\.json$' "$T/files" | grep -v -E '(^|/)node_modules/' | while IFS= read -r m; do
+    d=${m%/package.json}
+    grep -qE -f "$T/inc" <<<"$d" || continue
+    if [ -s "$T/exc" ] && grep -qE -f "$T/exc" <<<"$d"; then continue; fi
+    at_head "$m" > "$T/pj" || continue
+    n=$(jq -r '.name // empty | strings' "$T/pj" 2>/dev/null)
+    [ -n "$n" ] || continue
+    printf '%s\t%s\t%s\n' "$d" "$m" "$n" >> "$T/names"
+    jq -r --arg m "$m" '["dependencies","devDependencies","peerDependencies"][] as $k | (.[$k] // {}) | objects | keys[] | [$m, $k, .] | @tsv' "$T/pj" 2>/dev/null >> "$T/deps"
+  done
+fi
+PACKAGES=$(jq -R -s -c '[split("\n")[] | select(length > 0) | split("\t") | {name: .[2], dir: .[0]}] | sort_by(.name)' < "$T/names")
+EDGES=$(awk -F'\t' '
+  FILENAME == ARGV[1] { name[$2] = $3; ws[$3] = 1; next }
+  ($3 in ws) && ($1 in name) && name[$1] != $3 { print name[$1] "\t" $3 "\t" $2 "\t" $1 }
+' "$T/names" "$T/deps" | sort -u | jq -R -s -c '[split("\n")[] | select(length > 0) | split("\t") | {from: .[0], to: .[1], kind: .[2], manifest: .[3]}]')
+: > "$T/bfiles"
+while IFS= read -r f; do
+  case "${f##*/}" in
+    .eslintrc|.eslintrc.*|eslint.config.*|.dependency-cruiser.*|nx.json|project.json)
+      # here-string, not `at_head | grep -q`: under pipefail an early grep exit SIGPIPEs at_head and reads as no match (#4298)
+      c=$(at_head "$f"); grep -qE 'boundaries|forbidden|depConstraints|DependOnLibsWithTags|no-restricted-paths' <<<"$c" && printf '%s\n' "$f" >> "$T/bfiles" ;;
+  esac
+done < "$T/files"
+BOUNDARY=$(jq -R -s -c 'split("\n") | map(select(length > 0))' < "$T/bfiles")
+
+# ---- hotspots (reuses #4195 facts) ----
+# hotspots.sh with no paths reports its config; only files over file_lines can be hotspots, so only those are passed
+HOT='[]'
+FL=$(bash "$HERE/hotspots.sh" 2>/dev/null | jq -r 'select(.enabled) | .file_lines' 2>/dev/null)
+if [ -s "$T/files" ] && [ -n "$FL" ]; then
+  while IFS= read -r f; do
+    [ -f "$f" ] || continue
+    n=$(wc -l < "$f" | tr -d ' '); [ "$n" -gt "$FL" ] && printf '%s\n' "$f"
+  done < "$T/files" > "$T/big"
+  H=""; [ -s "$T/big" ] && { H=$(tr '\n' '\0' < "$T/big" | xargs -0 -n 200 bash "$HERE/hotspots.sh" -- 2>/dev/null) || H=""; }
+  HOT=$(printf '%s' "$H" | jq -s -c '[.[].files[]? | select(.over and (.excluded | not)) | {path, lines, commits_90d, hotspot}] | sort_by(-.hotspot, .path) | .[:20]' 2>/dev/null) || HOT='[]'
+  [ -n "$HOT" ] || HOT='[]'
+fi
+
+FACTS=$(jq -n -c --argjson tc "$TOOLCHAIN" --argjson slow "$SLOW" --argjson pk "$PACKAGES" --argjson ed "$EDGES" --argjson bc "$BOUNDARY" --argjson hot "$HOT" \
+  '{version: 1, toolchain: $tc, slow_suites: $slow, layer_rules: {packages: $pk, edges: $ed, boundary_configs: $bc}, hotspots: $hot}') || die "jq failed"
+
+if [ "$STDOUT" = 1 ]; then printf '%s\n' "$FACTS"; exit 0; fi
+[ -n "$OUT" ] || OUT=".forge/repo-facts.json"
+mkdir -p "$(dirname "$OUT")" || die "cannot create $(dirname "$OUT")"
+printf '%s\n' "$FACTS" > "$OUT.tmp.$$" && mv "$OUT.tmp.$$" "$OUT" || die "cannot write $OUT"
+echo "$OUT"

package/scripts/repo-facts.test.sh

@@ -0,0 +1,106 @@
+#!/usr/bin/env bash
+# repo-facts.test.sh — cases for scripts/repo-facts.sh (forge#4272).
+# SPDX-FileCopyrightText: Copyright (c) RapierCraft Studios
+# SPDX-License-Identifier: AGPL-3.0-or-later
+set -uo pipefail
+S="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/repo-facts.sh"
+PASS=0; FAILN=0
+TMP="$(mktemp -d "${TMPDIR:-/tmp}/repo-facts-test.XXXXXX")"; trap 'rm -rf "$TMP"' EXIT
+ok() { PASS=$((PASS+1)); }
+bad() { FAILN=$((FAILN+1)); echo "FAIL: $1"; }
+eq() { if [ "$2" = "$3" ]; then ok; else bad "$1 (got: $2 want: $3)"; fi }
+q() { printf '%s' "$OUT" | jq -c "$1"; }
+
+new_repo() {
+  rm -rf "$TMP/$1"; mkdir -p "$TMP/$1"; cd "$TMP/$1" || exit 1
+  git init -q -b main; git config user.email t@example.com; git config user.name t; git config commit.gpgsign false
+}
+commit() { git add -A; git commit -qm "$1"; }
+lines() { i=0; : > "$1"; while [ $i -lt "$2" ]; do echo "l$i" >> "$1"; i=$((i+1)); done; }
+
+# ---- 1. pins, slow suites, package edge, boundary config, hotspot ----
+new_repo full
+mkdir -p packages/a packages/b .github/workflows src
+cat > package.json <<'EOF'
+{"name":"root","private":true,"packageManager":"pnpm@9.1.0","workspaces":["packages/*"],
+ "scripts":{"test":"vitest run","lint":"eslint .","test:e2e":"playwright test"}}
+EOF
+echo "20.11.1" > .nvmrc
+echo '{"name":"@x/a","dependencies":{"@x/b":"workspace:*","left-pad":"1.0.0"}}' > packages/a/package.json
+echo '{"name":"@x/b","scripts":{"test":"vitest run"}}' > packages/b/package.json
+echo "module.exports = { rules: { 'boundaries/element-types': ['error'] } };" > .eslintrc.js
+echo "module.exports = { rules: { semi: 'error' } };" > packages/b/.eslintrc.js
+cat > .github/workflows/ci.yml <<'EOF'
+on: push
+jobs:
+  db:
+    runs-on: ubuntu-latest
+    services:
+      postgres:
+        image: postgres
+    steps:
+      - run: pnpm run test
+  long:
+    runs-on: ubuntu-latest
+    timeout-minutes: 45
+    steps:
+      - run: make lint-all
+  quick:
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    steps:
+      - run: make fmt
+EOF
+lines src/big.ts 2100; commit c1
+echo x >> src/big.ts; lines src/small.ts 10; commit c2
+OUT="$(bash "$S" --stdout)"; rc=$?
+eq "rc" "$rc" 0
+eq "version" "$(q '.version')" 1
+eq "pm pin" "$(q '.toolchain.pm')" '{"name":"pnpm","version":"9.1.0","source":"package.json packageManager"}'
+eq "node pin" "$(q '.toolchain.node')" '{"spec":"20.11.1","source":".nvmrc"}'
+eq "packages" "$(q '.layer_rules.packages')" '[{"name":"@x/a","dir":"packages/a"},{"name":"@x/b","dir":"packages/b"}]'
+eq "edge" "$(q '.layer_rules.edges')" '[{"from":"@x/a","to":"@x/b","kind":"dependencies","manifest":"packages/a/package.json"}]'
+eq "boundary configs" "$(q '.layer_rules.boundary_configs')" '[".eslintrc.js"]'
+eq "services job" "$(q '[.slow_suites[] | select(.command == "pnpm run test") | .why]')" '["CI job .github/workflows/ci.yml#jobs.db uses services"]'
+eq "timeout job" "$(q '[.slow_suites[] | select(.command == "make lint-all") | .why]')" '["CI job .github/workflows/ci.yml#jobs.long sets timeout-minutes 45"]'
+eq "quick job not slow" "$(q '[.slow_suites[] | select(.command == "make fmt")] | length')" 0
+eq "hotspot" "$(q '.hotspots')" '[{"path":"src/big.ts","lines":2101,"commits_90d":2,"hotspot":4202}]'
+
+# ---- 2. a cached test map is read: its e2e command is a slow suite ----
+mkdir -p .forge
+echo '{"version":1,"entries":[{"globs":["e2e/**"],"command":"pnpm exec playwright test","cwd":".","source":"package.json#scripts.test:e2e"}]}' > .forge/test-map.json
+OUT="$(bash "$S" --stdout)"
+eq "map e2e" "$(q '.slow_suites')" '[{"command":"pnpm exec playwright test","source":"package.json#scripts.test:e2e","why":"command names playwright"}]'
+
+# ---- 3. default output file, rerun stable ----
+P="$(bash "$S")"; eq "output path" "$P" ".forge/repo-facts.json"
+cp .forge/repo-facts.json "$TMP/first.json"; bash "$S" >/dev/null
+if cmp -s .forge/repo-facts.json "$TMP/first.json"; then ok; else bad "rerun not identical"; fi
+
+# ---- 4. a repo with none of these: empty sections, exit 0 ----
+new_repo bare
+echo hi > README.md; commit init
+OUT="$(bash "$S" --stdout)"; rc=$?
+eq "bare rc" "$rc" 0
+eq "bare" "$(q '[.toolchain.pm, .toolchain.node, .slow_suites, .layer_rules, .hotspots]')" '[null,null,[],{"packages":[],"edges":[],"boundary_configs":[]},[]]'
+new_repo empty
+OUT="$(bash "$S" --stdout)"; eq "no commits rc" "$?" 0
+eq "no commits" "$(q '.hotspots')" '[]'
+
+# ---- 4b. #4298: a big boundary config matching early (pipefail + grep -q SIGPIPE) and non-ASCII / spaced paths ----
+new_repo paths
+mkdir -p "packages/é b" "apps/ü"
+echo '{"name":"root","private":true,"workspaces":["packages/*"]}' > package.json
+echo '{"name":"@x/c"}' > "packages/é b/package.json"
+{ echo "export default [{ rules: { 'boundaries/element-types': ['error'] } }];"; i=0; while [ $i -lt 20000 ]; do echo "// padding line $i to overflow the pipe buffer ......................................"; i=$((i+1)); done; } > "apps/ü/eslint.config.js"
+commit init
+OUT="$(bash "$S" --stdout)"; eq "paths rc" "$?" 0
+eq "big boundary config in a non-ASCII dir" "$(q '.layer_rules.boundary_configs')" '["apps/ü/eslint.config.js"]'
+eq "spaced non-ASCII package dir" "$(q '.layer_rules.packages')" '[{"name":"@x/c","dir":"packages/é b"}]'
+
+# ---- 5. usage ----
+bash "$S" --repo-path "$TMP/missing" >/dev/null 2>&1; eq "bad repo path" "$?" 2
+bash "$S" --bogus >/dev/null 2>&1; eq "unknown option" "$?" 2
+
+echo "repo-facts.test.sh: $PASS passed, $FAILN failed"
+[ "$FAILN" -eq 0 ]

package/scripts/structure-delta.sh

@@ -9,11 +9,12 @@
 # Output (stdout, one JSON object):
 #   {} when the repo declares no workspace packages (root package.json `workspaces` / pnpm-workspace.yaml), else
 #   {"package_edges_added":[{from,to,kind,manifest}], "package_edges_removed":[...],
-#    "import_pairs":[{from,to,files_added,files_removed}],
+#    "import_pairs":[{from,to,files_added,files_removed,files[]}],
 #    "boundary_config_changed":{changed,files[],loosens,tightens}}
 # A package edge is a dependencies/devDependencies/peerDependencies key naming another workspace package. An import
 # pair counts changed non-test source files whose added (removed) lines quote an exact workspace name, optionally with
-# a subpath ('@s/x' or '@s/x/sub'); `import type` / `export type` lines are ignored. Counts only report.
+# a subpath ('@s/x' or '@s/x/sub'); `import type` / `export type` lines are ignored. Counts only report. `files` names the
+# files that added the import (sorted, at most 20; forge#4165 telemetry and trajectory).
 # Exit: 0 ok | 2 usage or bad ref. bash 3.2 portable.
 set -uo pipefail
 
@@ -122,14 +123,18 @@ IMPORTS=$(awk -F'\t' '
   END {
     for (k in hit) {
       split(k, a, SUBSEP); f = a[1]; to = a[2]; s = a[3]
-      if (s == "+" && !((f SUBSEP to SUBSEP "-") in hit)) add[fromof[f] SUBSEP to]++
+      if (s == "+" && !((f SUBSEP to SUBSEP "-") in hit)) { add[fromof[f] SUBSEP to]++; files[fromof[f] SUBSEP to, add[fromof[f] SUBSEP to]] = f }
       if (s == "-" && !((f SUBSEP to SUBSEP "+") in hit)) rem[fromof[f] SUBSEP to]++
     }
     for (k in add) pair[k] = 1
     for (k in rem) pair[k] = 1
-    for (k in pair) { split(k, a, SUBSEP); print a[1] "\t" a[2] "\t" (add[k] + 0) "\t" (rem[k] + 0) }
+    for (k in pair) {
+      split(k, a, SUBSEP); fl = ""
+      for (i = 1; i <= add[k] + 0; i++) fl = fl (i > 1 ? "\037" : "") files[k, i]
+      print a[1] "\t" a[2] "\t" (add[k] + 0) "\t" (rem[k] + 0) "\t" fl
+    }
   }
-' "$T/dirs" "$T/wsnames" "$T/diff" | sort | jq -R -s -c 'split("\n") | map(select(length > 0) | split("\t") | {from: .[0], to: .[1], files_added: (.[2] | tonumber), files_removed: (.[3] | tonumber)})')
+' "$T/dirs" "$T/wsnames" "$T/diff" | sort | jq -R -s -c 'split("\n") | map(select(length > 0) | split("\t") | {from: .[0], to: .[1], files_added: (.[2] | tonumber), files_removed: (.[3] | tonumber), files: ((.[4] // "") | split("\u001f") | map(select(length > 0)) | sort | .[:20])})')
 
 # ---- boundary config (eslint-plugin-boundaries, dependency-cruiser, Nx depConstraints) ----
 : > "$T/bfiles"; LOOSE=false; TIGHT=false

package/scripts/structure-delta.test.sh

@@ -68,7 +68,7 @@ eq "monorepo rc" "$rc" 0
 eq "edges added" "$(q '.package_edges_added | sort_by(.from)')" '[{"from":"@scope/db","to":"@scope/core","kind":"dependencies","manifest":"packages/db/package.json"},{"from":"@scope/x","to":"@scope/db","kind":"devDependencies","manifest":"packages/x/package.json"}]'
 eq "edges removed" "$(q '.package_edges_removed')" '[{"from":"@scope/web","to":"@scope/old","kind":"dependencies","manifest":"apps/web/package.json"}]'
 eq "non-workspace dep ignored" "$(q '[.package_edges_added[] | select(.to == "zod" or .to == "pg")] | length')" 0
-eq "import pairs" "$(q '.import_pairs | sort_by(.to)')" '[{"from":"@scope/web","to":"@scope/db","files_added":1,"files_removed":0},{"from":"@scope/web","to":"@scope/old","files_added":0,"files_removed":1},{"from":"@scope/web","to":"@scope/x","files_added":2,"files_removed":0}]'
+eq "import pairs" "$(q '.import_pairs | sort_by(.to)')" '[{"from":"@scope/web","to":"@scope/db","files_added":1,"files_removed":0,"files":["apps/web/src/value.ts"]},{"from":"@scope/web","to":"@scope/old","files_added":0,"files_removed":1,"files":[]},{"from":"@scope/web","to":"@scope/x","files_added":2,"files_removed":0,"files":["apps/web/src/dyn.ts","apps/web/src/sub.ts"]}]'
 eq "import type never counted" "$(q '[.import_pairs[] | select(.to == "@scope/core")] | length')" 0
 eq "no boundary change" "$(q '.boundary_config_changed')" '{"changed":false,"files":[],"loosens":false,"tightens":false}'
 # the two-arg form agrees
@@ -127,7 +127,8 @@ for t in gh curl wget npm npx pnpm yarn nc ssh; do printf '#!/bin/sh\necho "%s"
 start=$(date +%s%N 2>/dev/null)
 OUT="$(PATH="$TMP/bin:$PATH" bash "$S" master...big 2>/dev/null || PATH="$TMP/bin:$PATH" bash "$S" main...big)"
 end=$(date +%s%N 2>/dev/null)
-eq "300-file count" "$(q '.import_pairs')" '[{"from":"@s/a","to":"@s/b","files_added":300,"files_removed":0}]'
+eq "300-file count" "$(q '.import_pairs | map(del(.files))')" '[{"from":"@s/a","to":"@s/b","files_added":300,"files_removed":0}]'
+eq "files capped at 20, sorted (forge#4165)" "$(q '.import_pairs[0].files | [length, .[0]]')" '[20,"packages/a/f0.ts"]'
 case "$start$end" in *N*|"") ok ;; *) ms=$(( (end - start) / 1000000 )); if [ "$ms" -lt 2000 ]; then ok; else bad "300-file diff took ${ms} ms (limit 2000)"; fi ;; esac
 if [ -e "$TMP/net.log" ]; then bad "network tool invoked: $(tr '\n' ' ' < "$TMP/net.log")"; else ok; fi
 

package/scripts/symbol-index.mjs

@@ -0,0 +1,253 @@
+#!/usr/bin/env node
+// SPDX-FileCopyrightText: Copyright (c) RapierCraft Studios
+// SPDX-License-Identifier: AGPL-3.0-or-later
+//
+// symbol-index.mjs: JS/TS twin of scripts/symbol-index.py (forge#4208), same output schema.
+// No new dependencies: the `typescript` package is loaded from the target repo (resolved from the target path, then
+// cwd; NODE_PATH is honoured), else the TypeScript bundled with ts-morph (forge#4212). When neither is installed the
+// run prints only {"unsupported":"ts"|"js","reason":"no-parser"} and exits 0.
+// .py targets are handed to symbol-index.py. Never emits source text or literal values.
+//
+// Usage: node scripts/symbol-index.mjs <file | dir>...
+// Output: {"symbols":[{name,kind,start,end,lines,public[,file]}], "edges":[{from,to}],
+//          "imports":[{module,names,line[,file]}], "cycles":[[module,...],...]}
+//   kind: function|async_function|class|method|const|interface|type|enum. public = exported (methods: exported class
+//   and not private/#private). One file: bare names; otherwise `file` on entries and "<module>:<symbol>" edge ends,
+//   module = path relative to its root without extension (index files name their directory). cycles = strongly
+//   connected components of the relative-import graph between indexed modules (`import type` ignored).
+// Exit: 0 ok or unsupported | 2 usage or missing path.
+
+import { spawnSync } from "node:child_process";
+import { existsSync, readdirSync, readFileSync, statSync } from "node:fs";
+import { createRequire } from "node:module";
+import path from "node:path";
+import { fileURLToPath } from "node:url";
+
+const TS_EXT = [".ts", ".tsx", ".mts", ".cts"];
+const JS_EXT = [".js", ".jsx", ".mjs", ".cjs"];
+const SKIP_DIRS = new Set(["node_modules", "dist", "build", "coverage"]);
+const OTHER = { ".go": "go", ".rs": "rust", ".rb": "ruby", ".java": "java", ".sh": "shell", ".php": "php" };
+
+const emit = (o) => process.stdout.write(JSON.stringify(sortKeys(o)) + "\n");
+function sortKeys(v) {
+  if (Array.isArray(v)) return v.map(sortKeys);
+  if (v && typeof v === "object") return Object.fromEntries(Object.keys(v).sort().map((k) => [k, sortKeys(v[k])]));
+  return v;
+}
+const langOf = (ext) => (TS_EXT.includes(ext) ? "ts" : JS_EXT.includes(ext) ? "js" : ext === ".py" ? "python" : OTHER[ext] || ext.slice(1) || "unknown");
+const cmp = (a, b) => (a < b ? -1 : a > b ? 1 : 0);
+
+function loadTs(targets) {
+  const bases = [...targets.map((t) => path.resolve(statSync(t).isDirectory() ? t : path.dirname(t))), process.cwd()];
+  for (const b of bases) {
+    // typescript >= 7 (native port) ships no in-process compiler API at the package root: skip it.
+    try { const ts = createRequire(path.join(b, "noop.js"))("typescript"); if (typeof ts?.createSourceFile === "function") return ts; } catch { /* try next */ }
+  }
+  // forge#4212: a repo with only ts-morph (the refactor:split TS mover) has TypeScript bundled in @ts-morph/common.
+  for (const b of bases) {
+    try { const ts = createRequire(path.join(b, "noop.js"))("ts-morph").ts; if (typeof ts?.createSourceFile === "function") return ts; } catch { /* try next */ }
+  }
+  return null;
+}
+
+function collect(roots) {
+  const found = [];
+  for (const root of roots) {
+    if (statSync(root).isFile()) {
+      const ext = path.extname(root).toLowerCase();
+      found.push({ file: root, module: path.basename(root, path.extname(root)), ext });
+      continue;
+    }
+    const walk = (dir) => {
+      for (const ent of readdirSync(dir, { withFileTypes: true }).sort((a, b) => cmp(a.name, b.name))) {
+        const p = path.join(dir, ent.name);
+        if (ent.isDirectory()) { if (!ent.name.startsWith(".") && !SKIP_DIRS.has(ent.name)) walk(p); continue; }
+        const ext = path.extname(ent.name).toLowerCase();
+        if (![...TS_EXT, ...JS_EXT].includes(ext) || /\.d\.[cm]?ts$/.test(ent.name)) continue;
+        let mod = path.relative(root, p).slice(0, -ext.length).split(path.sep).join("/");
+        if (mod === "index" || mod.endsWith("/index")) mod = mod.slice(0, -"index".length).replace(/\/$/, "") || ".";
+        found.push({ file: p, module: mod, ext });
+      }
+    };
+    walk(root);
+  }
+  return found.sort((a, b) => cmp(a.file, b.file));
+}
+
+function tarjan(graph) {
+  let i = 0;
+  const index = new Map(), low = new Map(), on = new Set(), stack = [], comps = [];
+  const visit = (v) => {
+    index.set(v, i); low.set(v, i); i++; stack.push(v); on.add(v);
+    for (const w of [...(graph.get(v) || [])].sort()) {
+      if (!index.has(w)) { visit(w); low.set(v, Math.min(low.get(v), low.get(w))); }
+      else if (on.has(w)) low.set(v, Math.min(low.get(v), index.get(w)));
+    }
+    if (low.get(v) === index.get(v)) {
+      const comp = [];
+      let w;
+      do { w = stack.pop(); on.delete(w); comp.push(w); } while (w !== v);
+      comps.push(comp.sort());
+    }
+  };
+  for (const v of [...graph.keys()].sort()) if (!index.has(v)) visit(v);
+  return comps;
+}
+
+function index(ts, entries, multi) {
+  const K = ts.SyntaxKind;
+  const byFile = new Map(entries.map((e) => [path.resolve(e.file), e]));
+  const symbols = [], imports = [], edges = new Set(), graph = new Map(entries.map((e) => [e.module, new Set()]));
+  const tables = new Map(); // module -> Map(name -> node)
+  const parsed = entries.map((e) => {
+    const src = ts.createSourceFile(e.file, readFileSync(e.file, "utf8"), ts.ScriptTarget.Latest, true,
+      e.ext.endsWith("x") ? ts.ScriptKind.TSX : undefined);
+    if (src.parseDiagnostics?.length) throw { lang: langOf(e.ext), reason: `parse-error: ${e.file}` };
+    return { e, src };
+  });
+  const line = (src, pos) => src.getLineAndCharacterOfPosition(pos).line + 1;
+  const exported = (n) => (ts.getCombinedModifierFlags?.(n) ?? 0) & ts.ModifierFlags.Export;
+  const ref = (e, name) => (multi ? `${e.module}:${name}` : name);
+
+  for (const { e, src } of parsed) {
+    const table = new Map();
+    tables.set(e.module, table);
+    const add = (name, kind, node, pub, owner = null) => {
+      const start = line(src, node.getStart(src)), end = line(src, node.getEnd());
+      symbols.push({ name, kind, start, end, lines: end - start + 1, public: !!pub, ...(multi ? { file: e.file } : {}) });
+      table.set(name, { node, owner });
+    };
+    const exportNames = new Set();
+    for (const st of src.statements) {
+      if (ts.isExportDeclaration(st) && !st.moduleSpecifier && st.exportClause && ts.isNamedExports(st.exportClause)) {
+        for (const el of st.exportClause.elements) exportNames.add((el.propertyName || el.name).text);
+      } else if (ts.isExportAssignment(st) && ts.isIdentifier(st.expression)) exportNames.add(st.expression.text);
+    }
+    const pub = (n, name) => exported(n) || exportNames.has(name);
+    for (const st of src.statements) {
+      if (ts.isFunctionDeclaration(st) && st.name) {
+        const isAsync = (st.modifiers || []).some((m) => m.kind === K.AsyncKeyword);
+        add(st.name.text, isAsync ? "async_function" : "function", st, pub(st, st.name.text));
+      } else if (ts.isClassDeclaration(st) && st.name) {
+        const cname = st.name.text, cpub = pub(st, cname);
+        add(cname, "class", st, cpub);
+        for (const m of st.members) {
+          if (!(ts.isMethodDeclaration(m) || ts.isGetAccessor(m) || ts.isSetAccessor(m) || ts.isConstructorDeclaration(m))) continue;
+          const mname = ts.isConstructorDeclaration(m) ? "constructor" : m.name && (m.name.text ?? m.name.getText(src));
+          if (!mname || table.has(`${cname}.${mname}`)) continue;
+          const priv = (m.modifiers || []).some((x) => x.kind === K.PrivateKeyword) || String(mname).startsWith("#");
+          add(`${cname}.${mname}`, "method", m, cpub && !priv, cname);
+        }
+      } else if (ts.isVariableStatement(st)) {
+        for (const d of st.declarationList.declarations) {
+          if (ts.isIdentifier(d.name)) add(d.name.text, "const", d, exported(st) || exportNames.has(d.name.text));
+        }
+      } else if (ts.isInterfaceDeclaration(st)) add(st.name.text, "interface", st, pub(st, st.name.text));
+      else if (ts.isTypeAliasDeclaration(st)) add(st.name.text, "type", st, pub(st, st.name.text));
+      else if (ts.isEnumDeclaration(st)) add(st.name.text, "enum", st, pub(st, st.name.text));
+    }
+  }
+
+  const resolveSpec = (fromFile, spec) => {
+    if (!spec.startsWith(".")) return null;
+    const base = path.resolve(path.dirname(fromFile), spec);
+    const stem = base.replace(/\.(m|c)?js$/, "");
+    const cands = [base, ...[...TS_EXT, ...JS_EXT].flatMap((x) => [stem + x, path.join(base, "index" + x)])];
+    for (const c of cands) if (byFile.has(c)) return byFile.get(c).module;
+    return null;
+  };
+
+  for (const { e, src } of parsed) {
+    const table = tables.get(e.module), bindings = new Map();
+    for (const st of src.statements) {
+      const isImp = ts.isImportDeclaration(st), isRe = ts.isExportDeclaration(st) && st.moduleSpecifier;
+      if (!isImp && !isRe) continue;
+      const spec = st.moduleSpecifier.text, names = [];
+      const typeOnly = isImp ? !!st.importClause?.isTypeOnly : !!st.isTypeOnly;
+      const target = resolveSpec(e.file, spec);
+      const clause = isImp ? st.importClause : null;
+      if (clause?.name) { names.push("default"); bindings.set(clause.name.text, [target, "default"]); }
+      const nb = isImp ? clause?.namedBindings : st.exportClause;
+      if (nb && ts.isNamespaceImport(nb)) { names.push("*"); bindings.set(nb.name.text, [target, null]); }
+      else if (nb && (ts.isNamedImports(nb) || ts.isNamedExports(nb))) {
+        for (const el of nb.elements) {
+          const orig = (el.propertyName || el.name).text;
+          names.push(orig);
+          if (isImp) bindings.set(el.name.text, [target, orig]);
+        }
+      }
+      imports.push({ module: target ?? spec, names: names.sort(), line: line(src, st.getStart(src)), ...(multi ? { file: e.file } : {}) });
+      if (target !== null && !typeOnly) graph.get(e.module).add(target);
+    }
+    for (const [name, { node, owner }] of table) {
+      const from = ref(e, name);
+      const roots = ts.isClassDeclaration(node)
+        ? [...(node.heritageClauses || []), ...node.members.filter((m) => !(ts.isMethodDeclaration(m) || ts.isGetAccessor(m) || ts.isSetAccessor(m) || ts.isConstructorDeclaration(m)))]
+        : [node];
+      const visit = (n) => {
+        let to = null;
+        if (ts.isPropertyAccessExpression(n) && ts.isIdentifier(n.name)) {
+          const obj = n.expression;
+          if (owner && obj.kind === K.ThisKeyword && table.has(`${owner}.${n.name.text}`)) to = ref(e, `${owner}.${n.name.text}`);
+          else if (ts.isIdentifier(obj) && bindings.has(obj.text) && bindings.get(obj.text)[1] === null) {
+            const [mod] = bindings.get(obj.text);
+            if (mod !== null && tables.get(mod)?.has(n.name.text)) to = `${mod}:${n.name.text}`;
+          }
+        } else if (ts.isIdentifier(n) && !(n.parent && ts.isPropertyAccessExpression(n.parent) && n.parent.name === n)) {
+          if (n !== node.name && table.has(n.text)) to = ref(e, n.text);
+          else if (bindings.has(n.text)) {
+            const [mod, sym] = bindings.get(n.text);
+            if (mod !== null && sym !== null && tables.get(mod)?.has(sym)) to = `${mod}:${sym}`;
+          }
+        }
+        if (to && to !== from) edges.add(JSON.stringify([from, to]));
+        ts.forEachChild(n, visit);
+      };
+      for (const r of roots) visit(r);
+    }
+  }
+
+  const cycles = tarjan(graph).filter((c) => c.length > 1 || graph.get(c[0]).has(c[0])).sort((a, b) => cmp(a.join("\0"), b.join("\0")));
+  symbols.sort((a, b) => cmp(a.file || "", b.file || "") || a.start - b.start || cmp(a.name, b.name));
+  imports.sort((a, b) => cmp(a.file || "", b.file || "") || a.line - b.line || cmp(a.module, b.module));
+  return {
+    symbols,
+    edges: [...edges].sort().map((s) => { const [from, to] = JSON.parse(s); return { from, to }; }),
+    imports,
+    cycles,
+  };
+}
+
+function main(argv) {
+  if (!argv.length || argv[0] === "-h" || argv[0] === "--help") {
+    process.stderr.write("usage: symbol-index.mjs <file | dir>...\n");
+    return 2;
+  }
+  for (const p of argv) if (!existsSync(p)) { process.stderr.write(`symbol-index: no such path: ${p}\n`); return 2; }
+  const files = argv.filter((p) => statSync(p).isFile());
+  const exts = files.map((f) => path.extname(f).toLowerCase());
+  if (exts.includes(".py")) {
+    if (exts.some((x) => x !== ".py")) return emit({ unsupported: "mixed", reason: "mixed-languages" }), 0;
+    const py = path.join(path.dirname(fileURLToPath(import.meta.url)), "symbol-index.py");
+    const r = spawnSync("python3", [py, ...argv], { stdio: "inherit" });
+    if (r.error) return emit({ unsupported: "python", reason: "no-python3" }), 0;
+    return r.status ?? 2;
+  }
+  const bad = exts.find((x) => ![...TS_EXT, ...JS_EXT].includes(x));
+  if (bad !== undefined) return emit({ unsupported: langOf(bad), reason: "unknown-language" }), 0;
+  const entries = collect(argv);
+  if (!entries.length) return emit({ unsupported: "unknown", reason: "no-js-ts-files" }), 0;
+  const lang = entries.some((e) => TS_EXT.includes(e.ext)) ? "ts" : "js";
+  const ts = loadTs(argv);
+  if (!ts) return emit({ unsupported: lang, reason: "no-parser" }), 0;
+  const multi = !(argv.length === 1 && files.length === 1);
+  let out;
+  try { out = index(ts, entries, multi); } catch (err) {
+    if (err && err.reason) return emit({ unsupported: err.lang, reason: err.reason }), 0;
+    throw err;
+  }
+  emit(out);
+  return 0;
+}
+
+process.exitCode = main(process.argv.slice(2));