@mmerterden/multi-agent-pipeline 11.4.0 → 11.5.0

package/pipeline/scripts/phase-tracker.sh

@@ -289,6 +289,61 @@ save_state() {
   mv "${TRACKER_FILE}.tmp" "$TRACKER_FILE"
 }
 
+# --- read-modify-write lock ---------------------------------------------------
+# tmp+rename makes each save atomic, but two concurrent invocations (e.g. two
+# parallel Phase 4 reviewers reporting `tokens`) both load the same state and
+# the second save silently drops the first delta. Guard every load->save
+# sequence with a portable mkdir spinlock (macOS bash 3.2: no flock builtin).
+# The wait is bounded and the lock FAILS OPEN with a warning so a stuck lock
+# can never hang the pipeline; a dead owner pid or a lock older than 30s is
+# reclaimed.
+TRACKER_LOCK_DIR=""
+TRACKER_LOCK_HELD=0
+
+state_lock_stale() {
+  local pid mtime now age
+  pid=$(cat "$TRACKER_LOCK_DIR/pid" 2>/dev/null || true)
+  if [ -n "$pid" ]; then
+    kill -0 "$pid" 2>/dev/null && return 1
+    return 0
+  fi
+  mtime=$(stat -f %m "$TRACKER_LOCK_DIR" 2>/dev/null || stat -c %Y "$TRACKER_LOCK_DIR" 2>/dev/null || echo "")
+  [ -z "$mtime" ] && return 1
+  now=$(date +%s)
+  age=$((now - mtime))
+  [ "$age" -gt 30 ]
+}
+
+acquire_state_lock() {
+  TRACKER_LOCK_DIR="${TRACKER_FILE}.lock"
+  mkdir -p "$TRACKER_DIR" 2>/dev/null
+  local tries=0
+  # ~5s bound: 50 tries x 0.1s.
+  while ! mkdir "$TRACKER_LOCK_DIR" 2>/dev/null; do
+    if state_lock_stale; then
+      rm -rf "$TRACKER_LOCK_DIR" 2>/dev/null
+      continue
+    fi
+    tries=$((tries + 1))
+    if [ "$tries" -ge 50 ]; then
+      echo "phase-tracker: WARN  -  state lock busy ($TRACKER_LOCK_DIR); proceeding without lock" >&2
+      return 0
+    fi
+    sleep 0.1
+  done
+  echo "$$" > "$TRACKER_LOCK_DIR/pid" 2>/dev/null
+  TRACKER_LOCK_HELD=1
+}
+
+release_state_lock() {
+  if [ "$TRACKER_LOCK_HELD" = "1" ]; then
+    rm -rf "$TRACKER_LOCK_DIR" 2>/dev/null
+    TRACKER_LOCK_HELD=0
+  fi
+}
+# Never leave a lock behind if a mutating action dies mid-critical-section.
+trap release_state_lock EXIT
+
 render() {
   need_jq
   local state
@@ -509,8 +564,10 @@ case "$ACTION" in
     TRACKER_DIR="$(dirname "$TRACKER_FILE")"
     mkdir -p "$TRACKER_DIR" 2>/dev/null
     NOW=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
+    acquire_state_lock
     save_state "$(jq -nc --arg id "$TASK_ID" --arg ts "$NOW" \
       '{task_id:$id,started_at:$ts,phases:[]}')"
+    release_state_lock
     # Update pointer so subsequent calls without env can find this tracker.
     # NOTE: the pointer is global; for concurrent runs on the same user account,
     # callers should set MULTI_AGENT_TASK_ID in their shell or prefix every
@@ -528,6 +585,7 @@ case "$ACTION" in
     need_jq
     [ "$#" -ge 2 ] || { echo "add needs <phase_id> <name>" >&2; exit 64; }
     PID="$1"; PNAME="$2"
+    acquire_state_lock
     state=$(load_state)
     # Idempotent: a phase id that already exists is a no-op (name, status, and
     # token history are preserved). This is what lets continuation commands
@@ -538,6 +596,7 @@ case "$ACTION" in
       else .phases += [{"id":$id,"name":$name,"status":"pending","subs":[]}]
       end')
     save_state "$new"
+    release_state_lock
     render
     ;;
 
@@ -550,6 +609,7 @@ case "$ACTION" in
       *) echo "bad status: $STATUS" >&2; exit 64 ;;
     esac
     NOW_ISO=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
+    acquire_state_lock
     state=$(load_state)
     # Set status, stamp started_at on first transition to in_progress,
     # stamp completed_at on transition to terminal states (completed/failed/skipped).
@@ -563,6 +623,7 @@ case "$ACTION" in
       )
     ')
     save_state "$new"
+    release_state_lock
     emit_otel_span "phase.update" "$PID" "$STATUS" "{\"multi_agent.status\": \"$STATUS\"}"
     render
     ;;
@@ -575,6 +636,7 @@ case "$ACTION" in
       pending|in_progress|completed|failed|skipped) ;;
       *) echo "bad status: $SSTATUS" >&2; exit 64 ;;
     esac
+    acquire_state_lock
     state=$(load_state)
     # If sub with this id exists under this phase, update; else append.
     new=$(echo "$state" | jq \
@@ -590,6 +652,7 @@ case "$ACTION" in
         else . end
       )')
     save_state "$new"
+    release_state_lock
     emit_otel_span "phase.sub" "$PID" "$SNAME" "{\"multi_agent.sub_id\": \"$SID\", \"multi_agent.status\": \"$SSTATUS\"}"
     render
     ;;
@@ -602,6 +665,7 @@ case "$ACTION" in
     case "$T_IN$T_OUT$T_CACHED" in
       *[!0-9]*) echo "tokens: in/out/cached must be non-negative integers" >&2; exit 64 ;;
     esac
+    acquire_state_lock
     state=$(load_state)
     # Additive: add to existing totals so multiple LLM calls accumulate.
     new=$(echo "$state" | jq --arg id "$PID" --argjson ti "$T_IN" --argjson to "$T_OUT" --argjson tc "$T_CACHED" '
@@ -614,6 +678,7 @@ case "$ACTION" in
       )
     ')
     save_state "$new"
+    release_state_lock
     emit_otel_span "phase.tokens" "$PID" "" "{\"multi_agent.tokens_in_delta\": $T_IN, \"multi_agent.tokens_out_delta\": $T_OUT, \"multi_agent.tokens_cached_delta\": $T_CACHED}"
     # Re-render so live token count is visible on the active phase. Callers
     # that want silent accumulation (e.g. tight loops) can suppress with
@@ -627,6 +692,7 @@ case "$ACTION" in
     need_jq
     [ "$#" -ge 3 ] || { echo "meta needs <phase_id> <key> <value>" >&2; exit 64; }
     PID="$1"; KEY="$2"; VALUE="$3"
+    acquire_state_lock
     state=$(load_state)
     new=$(echo "$state" | jq --arg id "$PID" --arg k "$KEY" --arg v "$VALUE" '
       .phases |= map(
@@ -635,6 +701,7 @@ case "$ACTION" in
         else . end
       )')
     save_state "$new"
+    release_state_lock
     emit_otel_span "phase.meta" "$PID" "$KEY" "{\"multi_agent.meta_key\": \"$KEY\", \"multi_agent.meta_value\": \"$VALUE\"}"
     if [ "${TRACKER_QUIET:-0}" != "1" ]; then
       render
@@ -645,12 +712,14 @@ case "$ACTION" in
     need_jq
     [ "$#" -ge 2 ] || { echo "model needs <phase_id> <model_name>" >&2; exit 64; }
     PID="$1"; MODEL="$2"
+    acquire_state_lock
     state=$(load_state)
     new=$(echo "$state" | jq --arg id "$PID" --arg m "$MODEL" '
       .phases |= map(
         if .id == $id then .model = $m else . end
       )')
     save_state "$new"
+    release_state_lock
     emit_otel_span "phase.model" "$PID" "" "{\"multi_agent.model\": \"$MODEL\"}"
     if [ "${TRACKER_QUIET:-0}" != "1" ]; then
       render
@@ -667,12 +736,14 @@ case "$ACTION" in
     if [ "${#TEXT}" -gt 60 ]; then
       TEXT="${TEXT:0:59}…"
     fi
+    acquire_state_lock
     state=$(load_state)
     new=$(echo "$state" | jq --arg id "$PID" --arg v "$TEXT" '
       .phases |= map(
         if .id == $id then .meta = ((.meta // {}) + {"Now": $v}) else . end
       )')
     save_state "$new"
+    release_state_lock
     emit_otel_span "phase.now" "$PID" "$TEXT" "{}"
     ;;
 

package/pipeline/scripts/pre-commit-check.sh

@@ -5,6 +5,45 @@
 
 set -uo pipefail
 
+# Hook-mode fast exit. The Claude Code PreToolUse matcher is the tool name
+# ("Bash"), so this script now fires for EVERY Bash call and receives JSON on
+# stdin: {"tool_name":"Bash","tool_input":{"command":"..."}}. Extract the
+# command and exit 0 immediately unless it actually invokes `git commit`.
+# Direct invocations (empty/non-JSON stdin, e.g. smoke tests or manual runs)
+# fall through to the full scan. Extraction failure also falls through - the
+# only cost is an unnecessary scan, never a skipped one on a real commit.
+if [ ! -t 0 ]; then
+  HOOK_INPUT="$(cat 2>/dev/null || true)"
+  if [ -n "$HOOK_INPUT" ]; then
+    HOOK_COMMAND=""
+    if command -v python3 >/dev/null 2>&1; then
+      HOOK_COMMAND="$(printf '%s' "$HOOK_INPUT" | python3 -c '
+import json, sys
+try:
+    print(json.load(sys.stdin).get("tool_input", {}).get("command", ""))
+except Exception:
+    pass
+' 2>/dev/null || true)"
+    elif command -v node >/dev/null 2>&1; then
+      HOOK_COMMAND="$(printf '%s' "$HOOK_INPUT" | node -e '
+let raw = "";
+process.stdin.on("data", (c) => (raw += c));
+process.stdin.on("end", () => {
+  try {
+    process.stdout.write(String(JSON.parse(raw)?.tool_input?.command ?? ""));
+  } catch { /* fall through to full scan */ }
+});
+' 2>/dev/null || true)"
+    fi
+    if [ -n "$HOOK_COMMAND" ]; then
+      if ! printf '%s\n' "$HOOK_COMMAND" \
+        | grep -qE '(^|[^[:alnum:]_./-])git[[:space:]]+([^|&;]*[[:space:]])?commit([^[:alnum:]_-]|$)'; then
+        exit 0
+      fi
+    fi
+  fi
+fi
+
 if [ -z "$(git diff --cached --name-only 2>/dev/null)" ]; then
   exit 0
 fi

package/pipeline/scripts/scan-skills.sh

@@ -85,12 +85,24 @@ add_finding() {
   FINDINGS+=("$sev|$file|$line|$pattern|$message")
 }
 
-# --- Pattern scanning ---------------------------------------------------
+# --- Pattern scanning (single-pass tree-wide) ----------------------------
+#
+# Each pattern family runs as ONE grep over the whole file list instead of
+# one grep per file (the per-file design spawned thousands of processes and
+# dominated install time). Findings are tagged with (fileIdx, familyIdx,
+# seq) sort keys and re-ordered afterwards so the output is byte-identical
+# to the historical per-file scan order:
+#   per file: critical(1 shell-pipe, 2 base64-pipe, 3 eval-net, 4 bidi)
+#             high(5 js-eval / 6 py-exec, 7 credential, 8 pastebin, 9 chmod)
+#             medium(10 long-base64)  low(11 unknown-endpoint, 12 SKILL.md)
 
 # Files to consider: *.md, *.sh, *.py, *.mjs, *.js, *.ts (skill content)
 # Exclude: binary, images, large fixtures, hidden
-SCAN_LIST=$(mktemp)
-trap 'rm -f "$SCAN_LIST"' EXIT
+WORK="$(mktemp -d)"
+trap 'rm -rf "$WORK"' EXIT
+SCAN_LIST="$WORK/files"
+RAW="$WORK/raw"
+: > "$RAW"
 
 find "$ROOT" -type f \
   \( -name "*.md" -o -name "*.sh" -o -name "*.py" -o -name "*.mjs" -o -name "*.js" -o -name "*.ts" \) \
@@ -98,156 +110,230 @@ find "$ROOT" -type f \
   -not -path "*/.git/*" \
   2>/dev/null > "$SCAN_LIST"
 
-scan_critical() {
-  local f="$1"
-  # curl|wget piping to shell
-  while IFS= read -r hit; do
-    [ -z "$hit" ] && continue
-    local ln="${hit%%:*}"
-    add_finding critical "$f" "$ln" "shell-pipe-exec" "curl/wget piped to shell interpreter"
-  done < <(grep -nE '(curl|wget)[^|]*\|[[:space:]]*(sh|bash|zsh|ksh|ash|dash)([[:space:]]|$)' "$f" 2>/dev/null)
+FILE_COUNT=$(wc -l < "$SCAN_LIST" | tr -d '[:space:]')
 
-  # base64 -d | shell
-  while IFS= read -r hit; do
-    [ -z "$hit" ] && continue
-    local ln="${hit%%:*}"
-    add_finding critical "$f" "$ln" "base64-pipe-exec" "base64 decoded output piped to shell"
-  done < <(grep -nE '(base64|openssl[[:space:]]+base64)[^|]*-d[^|]*\|[[:space:]]*(sh|bash|zsh|eval|sudo)' "$f" 2>/dev/null)
+# One grep invocation for a whole pattern family. xargs preserves argument
+# order, so hits come out grouped per file in SCAN_LIST order.
+tree_grep() {
+  tr '\n' '\0' < "$SCAN_LIST" | xargs -0 grep -nHE -- "$1" 2>/dev/null
+  return 0
+}
 
-  # eval $(curl|wget ...) or eval `curl ...`
-  while IFS= read -r hit; do
-    [ -z "$hit" ] && continue
-    local ln="${hit%%:*}"
-    add_finding critical "$f" "$ln" "eval-of-network" "eval of network-fetched content"
-  done < <(grep -nE 'eval[[:space:]]+(\$\([[:space:]]*(curl|wget|fetch)|`[[:space:]]*(curl|wget|fetch))' "$f" 2>/dev/null)
+# stdin: "path:line:content" grep hits -> stdout: "fileIdx|path|line|content"
+index_hits() {
+  awk -v listfile="$SCAN_LIST" '
+    BEGIN {
+      i = 0
+      while ((getline l < listfile) > 0) { i++; idx[l] = i }
+      close(listfile)
+    }
+    {
+      p = index($0, ":"); f = substr($0, 1, p - 1)
+      rest = substr($0, p + 1)
+      q = index(rest, ":"); ln = substr(rest, 1, q - 1)
+      printf "%d|%s|%s|%s\n", (f in idx ? idx[f] : 999999), f, ln, substr(rest, q + 1)
+    }'
+}
 
-  # Unicode bidi override characters  -  invisible injection attack
-  if LC_ALL=C grep -l $'\xe2\x80\xad\|\xe2\x80\xae\|\xe2\x81\xa6\|\xe2\x81\xa7\|\xe2\x81\xa8\|\xe2\x81\xa9' "$f" 2>/dev/null >/dev/null; then
-    add_finding critical "$f" "0" "unicode-bidi" "bidirectional control chars (U+202D-202E, U+2066-2069)  -  trojan source risk"
-  fi
+# stdin: bare file paths -> stdout: "fileIdx|path"
+index_files() {
+  awk -v listfile="$SCAN_LIST" '
+    BEGIN {
+      i = 0
+      while ((getline l < listfile) > 0) { i++; idx[l] = i }
+      close(listfile)
+    }
+    { printf "%d|%s\n", ($0 in idx ? idx[$0] : 999999), $0 }'
 }
 
-scan_high() {
-  local f="$1"
-  local ext="${f##*.}"
+emit_raw() {
+  # $1 fileIdx  $2 familyIdx  $3 seq  $4 sev  $5 file  $6 line  $7 pattern  $8 message
+  printf '%s|%s|%s|%s|%s|%s|%s|%s\n' "$1" "$2" "$3" "$4" "$5" "$6" "$7" "$8" >> "$RAW"
+}
+
+# Split an indexed hit "fileIdx|path|line|content" into HIT_IDX/HIT_FILE/
+# HIT_LINE/HIT_CONTENT (content may itself contain pipes  -  it is the rest).
+parse_hit() {
+  HIT_IDX="${1%%|*}"; local rest="${1#*|}"
+  HIT_FILE="${rest%%|*}"; rest="${rest#*|}"
+  HIT_LINE="${rest%%|*}"
+  HIT_CONTENT="${rest#*|}"
+}
 
+# --- critical families (rank 0  -  always at/above threshold) ------------
+
+seq=0
+while IFS= read -r hit; do
+  [ -z "$hit" ] && continue
+  parse_hit "$hit"
+  seq=$((seq+1))
+  emit_raw "$HIT_IDX" 1 "$seq" critical "$HIT_FILE" "$HIT_LINE" "shell-pipe-exec" "curl/wget piped to shell interpreter"
+done < <(tree_grep '(curl|wget)[^|]*\|[[:space:]]*(sh|bash|zsh|ksh|ash|dash)([[:space:]]|$)' | index_hits)
+
+seq=0
+while IFS= read -r hit; do
+  [ -z "$hit" ] && continue
+  parse_hit "$hit"
+  seq=$((seq+1))
+  emit_raw "$HIT_IDX" 2 "$seq" critical "$HIT_FILE" "$HIT_LINE" "base64-pipe-exec" "base64 decoded output piped to shell"
+done < <(tree_grep '(base64|openssl[[:space:]]+base64)[^|]*-d[^|]*\|[[:space:]]*(sh|bash|zsh|eval|sudo)' | index_hits)
+
+seq=0
+while IFS= read -r hit; do
+  [ -z "$hit" ] && continue
+  parse_hit "$hit"
+  seq=$((seq+1))
+  emit_raw "$HIT_IDX" 3 "$seq" critical "$HIT_FILE" "$HIT_LINE" "eval-of-network" "eval of network-fetched content"
+done < <(tree_grep 'eval[[:space:]]+(\$\([[:space:]]*(curl|wget|fetch)|`[[:space:]]*(curl|wget|fetch))' | index_hits)
+
+# Unicode bidi override characters  -  invisible injection attack
+seq=0
+while IFS= read -r row; do
+  [ -z "$row" ] && continue
+  seq=$((seq+1))
+  emit_raw "${row%%|*}" 4 "$seq" critical "${row#*|}" "0" "unicode-bidi" "bidirectional control chars (U+202D-202E, U+2066-2069)  -  trojan source risk"
+done < <(tr '\n' '\0' < "$SCAN_LIST" \
+  | LC_ALL=C xargs -0 grep -l $'\xe2\x80\xad\|\xe2\x80\xae\|\xe2\x81\xa6\|\xe2\x81\xa7\|\xe2\x81\xa8\|\xe2\x81\xa9' 2>/dev/null \
+  | index_files; true)
+
+# --- high families (rank 1) ----------------------------------------------
+
+if [ "$THRESHOLD_RANK" -ge 1 ]; then
   # JavaScript/TypeScript: eval(, new Function(, Function(
-  if [ "$ext" = "js" ] || [ "$ext" = "mjs" ] || [ "$ext" = "ts" ]; then
-    while IFS= read -r hit; do
-      [ -z "$hit" ] && continue
-      local ln="${hit%%:*}"
-      add_finding high "$f" "$ln" "js-dynamic-eval" "JavaScript dynamic code execution"
-    done < <(grep -nE '\b(eval|new[[:space:]]+Function|Function)\s*\(' "$f" 2>/dev/null)
-  fi
+  seq=0
+  while IFS= read -r hit; do
+    [ -z "$hit" ] && continue
+    parse_hit "$hit"
+    case "$HIT_FILE" in *.js|*.mjs|*.ts) ;; *) continue ;; esac
+    seq=$((seq+1))
+    emit_raw "$HIT_IDX" 5 "$seq" high "$HIT_FILE" "$HIT_LINE" "js-dynamic-eval" "JavaScript dynamic code execution"
+  done < <(tree_grep '\b(eval|new[[:space:]]+Function|Function)\s*\(' | index_hits)
 
   # Python: exec(, eval( on non-literal  -  exclude re.compile (regex) and subprocess
-  if [ "$ext" = "py" ]; then
-    while IFS= read -r hit; do
-      [ -z "$hit" ] && continue
-      local ln="${hit%%:*}"
-      local content="${hit#*:}"
-      # Skip re.compile (standard regex) and subprocess.* (legitimate process invocation)
-      echo "$content" | grep -qE '(\bre\.compile|\bre2\.compile|subprocess\.|typing\.)' && continue
-      add_finding high "$f" "$ln" "py-dynamic-exec" "Python dynamic code execution"
-    done < <(grep -nE '(^|[^a-zA-Z0-9_.])(exec|eval)[[:space:]]*\(' "$f" 2>/dev/null)
-  fi
+  seq=0
+  while IFS= read -r hit; do
+    [ -z "$hit" ] && continue
+    parse_hit "$hit"
+    case "$HIT_FILE" in *.py) ;; *) continue ;; esac
+    # Skip re.compile (standard regex) and subprocess.* (legitimate process invocation)
+    echo "$HIT_CONTENT" | grep -qE '(\bre\.compile|\bre2\.compile|subprocess\.|typing\.)' && continue
+    seq=$((seq+1))
+    emit_raw "$HIT_IDX" 6 "$seq" high "$HIT_FILE" "$HIT_LINE" "py-dynamic-exec" "Python dynamic code execution"
+  done < <(tree_grep '(^|[^a-zA-Z0-9_.])(exec|eval)[[:space:]]*\(' | index_hits)
 
   # Hardcoded API keys  -  AWS, OpenAI, GitHub, generic sk-* with length
   # Skip lines in FORBIDDEN/NEVER/example blocks (false positives from docs)
+  seq=0
   while IFS= read -r hit; do
     [ -z "$hit" ] && continue
-    local ln="${hit%%:*}"
-    # Check ±3 lines around match for documentation context markers
-    local ctx_start=$((ln - 3))
+    parse_hit "$hit"
+    # Check +/-3 lines around match for documentation context markers
+    ctx_start=$((HIT_LINE - 3))
     [ "$ctx_start" -lt 1 ] && ctx_start=1
-    local ctx_end=$((ln + 3))
-    local ctx
-    ctx=$(sed -n "${ctx_start},${ctx_end}p" "$f" 2>/dev/null)
+    ctx_end=$((HIT_LINE + 3))
+    ctx=$(sed -n "${ctx_start},${ctx_end}p" "$HIT_FILE" 2>/dev/null)
     if echo "$ctx" | grep -qiE '(FORBIDDEN|NEVER do|don.t do|example:|placeholder|sample credential|DO NOT|✗|XXX|YYY|dummy|fake.?key)'; then
       continue
     fi
-    add_finding high "$f" "$ln" "hardcoded-credential" "possible hardcoded API credential"
-  done < <(grep -nE '(AKIA[0-9A-Z]{16}|sk-(live|test|proj|ant|or)-[A-Za-z0-9_-]{20,}|ghp_[A-Za-z0-9]{36}|github_pat_[A-Za-z0-9_]{82}|gho_[A-Za-z0-9]{36}|xox[bp]-[A-Za-z0-9-]{10,})' "$f" 2>/dev/null)
+    seq=$((seq+1))
+    emit_raw "$HIT_IDX" 7 "$seq" high "$HIT_FILE" "$HIT_LINE" "hardcoded-credential" "possible hardcoded API credential"
+  done < <(tree_grep '(AKIA[0-9A-Z]{16}|sk-(live|test|proj|ant|or)-[A-Za-z0-9_-]{20,}|ghp_[A-Za-z0-9]{36}|github_pat_[A-Za-z0-9_]{82}|gho_[A-Za-z0-9]{36}|xox[bp]-[A-Za-z0-9-]{10,})' | index_hits)
 
   # Pastebin / URL shortener raw content fetching
+  seq=0
   while IFS= read -r hit; do
     [ -z "$hit" ] && continue
-    local ln="${hit%%:*}"
-    add_finding high "$f" "$ln" "pastebin-fetch" "fetch from ephemeral/obscured content host"
-  done < <(grep -nE 'https?://(pastebin\.com/raw|paste\.ee|ghostbin|hastebin|bit\.ly|tinyurl\.com|goo\.gl|t\.co|is\.gd|ow\.ly|rebrand\.ly|gist\.github\.com/[^/]+/[a-f0-9]+/raw)' "$f" 2>/dev/null)
+    parse_hit "$hit"
+    seq=$((seq+1))
+    emit_raw "$HIT_IDX" 8 "$seq" high "$HIT_FILE" "$HIT_LINE" "pastebin-fetch" "fetch from ephemeral/obscured content host"
+  done < <(tree_grep 'https?://(pastebin\.com/raw|paste\.ee|ghostbin|hastebin|bit\.ly|tinyurl\.com|goo\.gl|t\.co|is\.gd|ow\.ly|rebrand\.ly|gist\.github\.com/[^/]+/[a-f0-9]+/raw)' | index_hits)
 
   # chmod +x followed by execution on same file in same block
+  seq=0
   while IFS= read -r hit; do
     [ -z "$hit" ] && continue
-    local ln="${hit%%:*}"
-    add_finding high "$f" "$ln" "chmod-then-exec" "script made executable and immediately invoked"
-  done < <(grep -nE 'chmod[[:space:]]+\+x[[:space:]]+[^&;]+[[:space:]]*(&&|;)[[:space:]]*\./' "$f" 2>/dev/null)
-}
+    parse_hit "$hit"
+    seq=$((seq+1))
+    emit_raw "$HIT_IDX" 9 "$seq" high "$HIT_FILE" "$HIT_LINE" "chmod-then-exec" "script made executable and immediately invoked"
+  done < <(tree_grep 'chmod[[:space:]]+\+x[[:space:]]+[^&;]+[[:space:]]*(&&|;)[[:space:]]*\./' | index_hits)
+fi
 
-scan_medium() {
-  local f="$1"
+# --- medium families (rank 2) ---------------------------------------------
 
+if [ "$THRESHOLD_RANK" -ge 2 ]; then
   # Long base64 blobs  -  obfuscation indicator.
   # Skip .md files (documentation often shows example b64)  -  only scan executable content types.
-  case "$f" in
-    *.md) ;;
-    *)
-      while IFS= read -r hit; do
-        [ -z "$hit" ] && continue
-        local ln="${hit%%:*}"
-        add_finding medium "$f" "$ln" "long-base64" "base64-looking blob >200 chars (possible obfuscation)"
-      done < <(grep -nE '[A-Za-z0-9+/]{200,}={0,2}' "$f" 2>/dev/null)
-      ;;
-  esac
-}
+  seq=0
+  while IFS= read -r hit; do
+    [ -z "$hit" ] && continue
+    parse_hit "$hit"
+    case "$HIT_FILE" in *.md) continue ;; esac
+    seq=$((seq+1))
+    emit_raw "$HIT_IDX" 10 "$seq" medium "$HIT_FILE" "$HIT_LINE" "long-base64" "base64-looking blob >200 chars (possible obfuscation)"
+  done < <(tree_grep '[A-Za-z0-9+/]{200,}={0,2}' | index_hits)
+fi
 
-scan_info() {
-  local f="$1"
+# --- low families (rank 3) -------------------------------------------------
 
+if [ "$THRESHOLD_RANK" -ge 3 ]; then
   # Unknown network endpoints  -  LOW severity (informational). URLs in skills
   # are usually doc references, not exfil channels. Only the critical
   # shell-pipe-exec patterns detect active network abuse.
+  # Historical behavior: at most the first 20 URL-bearing lines per file.
+  url_re='https?://[a-zA-Z0-9._-]+'
+  allow_re="^${ALLOW_DOMAINS}\$"
+  seq=0
+  prev_file=""
+  url_lines=0
   while IFS= read -r hit; do
     [ -z "$hit" ] && continue
-    local ln="${hit%%:*}"
-    local url_line="${hit#*:}"
-    local url
-    url=$(echo "$url_line" | grep -oE 'https?://[a-zA-Z0-9._-]+' | head -1)
-    [ -z "$url" ] && continue
-    local host="${url#*://}"
+    parse_hit "$hit"
+    if [ "$HIT_FILE" != "$prev_file" ]; then
+      prev_file="$HIT_FILE"
+      url_lines=0
+    fi
+    url_lines=$((url_lines+1))
+    [ "$url_lines" -gt 20 ] && continue
+    [[ "$HIT_CONTENT" =~ $url_re ]] || continue
+    url="${BASH_REMATCH[0]}"
+    host="${url#*://}"
     host="${host%%/*}"
-    if ! echo "$host" | grep -qE "^${ALLOW_DOMAINS}$" 2>/dev/null; then
-      add_finding low "$f" "$ln" "unknown-endpoint" "network endpoint not in allow-list: $host"
+    if ! [[ "$host" =~ $allow_re ]]; then
+      seq=$((seq+1))
+      emit_raw "$HIT_IDX" 11 "$seq" low "$HIT_FILE" "$HIT_LINE" "unknown-endpoint" "network endpoint not in allow-list: $host"
     fi
-  done < <(grep -nE 'https?://[a-zA-Z0-9._-]+' "$f" 2>/dev/null | head -20)
-}
+  done < <(tree_grep 'https?://[a-zA-Z0-9._-]+' | index_hits)
 
-scan_low() {
-  local f="$1"
   # Missing SKILL.md frontmatter (only check files named SKILL.md)
-  case "$f" in
-    */SKILL.md)
-      if ! head -1 "$f" | grep -Fxq -- "---"; then
-        add_finding low "$f" "1" "missing-frontmatter" "SKILL.md without YAML frontmatter"
-      elif ! grep -qE '^description:' "$f"; then
-        add_finding low "$f" "0" "missing-description" "SKILL.md frontmatter missing 'description'"
-      fi
-      ;;
-  esac
-}
-
-# --- Run scan -----------------------------------------------------------
+  seq=0
+  file_idx=0
+  while IFS= read -r file; do
+    file_idx=$((file_idx+1))
+    [ -z "$file" ] && continue
+    case "$file" in
+      */SKILL.md)
+        if ! head -1 "$file" | grep -Fxq -- "---"; then
+          seq=$((seq+1))
+          emit_raw "$file_idx" 12 "$seq" low "$file" "1" "missing-frontmatter" "SKILL.md without YAML frontmatter"
+        elif ! grep -qE '^description:' "$file"; then
+          seq=$((seq+1))
+          emit_raw "$file_idx" 12 "$seq" low "$file" "0" "missing-description" "SKILL.md frontmatter missing 'description'"
+        fi
+        ;;
+    esac
+  done < "$SCAN_LIST"
+fi
 
-FILE_COUNT=0
-while IFS= read -r file; do
-  [ -z "$file" ] && continue
-  FILE_COUNT=$((FILE_COUNT+1))
-  scan_critical "$file"
-  scan_high "$file"
-  scan_medium "$file"
-  scan_info "$file"
-  scan_low "$file"
-done < "$SCAN_LIST"
+# Re-order into the historical per-file scan order and load into FINDINGS.
+while IFS= read -r row; do
+  [ -z "$row" ] && continue
+  rest="${row#*|}"; rest="${rest#*|}"; rest="${rest#*|}"  # drop idx|fam|seq|
+  sev="${rest%%|*}"; rest="${rest#*|}"
+  file="${rest%%|*}"; rest="${rest#*|}"
+  line="${rest%%|*}"; rest="${rest#*|}"
+  pat="${rest%%|*}"; msg="${rest#*|}"
+  add_finding "$sev" "$file" "$line" "$pat" "$msg"
+done < <(sort -t'|' -k1,1n -k2,2n -k3,3n "$RAW")
 
 # --- Report -------------------------------------------------------------
 
@@ -261,20 +347,24 @@ if [ "$JSON" -eq 1 ]; then
   printf '  "findings": [\n'
   local_i=0
   total=${#FINDINGS[@]}
-  for f in "${FINDINGS[@]:-}"; do
-    local_i=$((local_i+1))
-    sev="${f%%|*}"; rest="${f#*|}"
-    file="${rest%%|*}"; rest="${rest#*|}"
-    line="${rest%%|*}"; rest="${rest#*|}"
-    pat="${rest%%|*}"; msg="${rest#*|}"
-    # json-escape file/msg minimally
-    jf=$(printf '%s' "$file" | sed 's/"/\\"/g')
-    jm=$(printf '%s' "$msg" | sed 's/"/\\"/g')
-    printf '    { "severity": "%s", "file": "%s", "line": "%s", "pattern": "%s", "message": "%s" }' \
-      "$sev" "$jf" "$line" "$pat" "$jm"
-    [ "$local_i" -lt "$total" ] && printf ','
-    printf '\n'
-  done
+  # Guard the expansion: on an empty array "${FINDINGS[@]:-}" yields one empty
+  # word, which used to emit a spurious all-empty findings element.
+  if [ "$total" -gt 0 ]; then
+    for f in "${FINDINGS[@]}"; do
+      local_i=$((local_i+1))
+      sev="${f%%|*}"; rest="${f#*|}"
+      file="${rest%%|*}"; rest="${rest#*|}"
+      line="${rest%%|*}"; rest="${rest#*|}"
+      pat="${rest%%|*}"; msg="${rest#*|}"
+      # json-escape file/msg minimally
+      jf=$(printf '%s' "$file" | sed 's/"/\\"/g')
+      jm=$(printf '%s' "$msg" | sed 's/"/\\"/g')
+      printf '    { "severity": "%s", "file": "%s", "line": "%s", "pattern": "%s", "message": "%s" }' \
+        "$sev" "$jf" "$line" "$pat" "$jm"
+      [ "$local_i" -lt "$total" ] && printf ','
+      printf '\n'
+    done
+  fi
   printf '  ]\n}\n'
 else
   # Colored text report
@@ -295,7 +385,7 @@ else
     printf '  ✓ clean (0 findings)\n'
   else
     printf '  %sfound %d%s: critical=%d high=%d medium=%d low=%d\n\n' "$C_BLD" "$total" "$C_RST" "$CRIT" "$HIGH" "$MED" "$LOW"
-    for f in "${FINDINGS[@]:-}"; do
+    for f in "${FINDINGS[@]}"; do
       sev="${f%%|*}"; rest="${f#*|}"
       file="${rest%%|*}"; rest="${rest#*|}"
       line="${rest%%|*}"; rest="${rest#*|}"
@@ -307,7 +397,7 @@ else
         low)      ico="· "; col="$C_DIM" ;;
         *)        ico="? "; col="" ;;
       esac
-      rel="${file#$REPO_ROOT/}"
+      rel="${file#"$REPO_ROOT"/}"
       printf '  %s%s%s %-8s %s:%s  [%s]\n' "$col" "$ico" "$C_RST" "$sev" "$rel" "$line" "$pat"
       printf '     %s%s%s\n' "$C_DIM" "$msg" "$C_RST"
     done