forgedock 1.19.0 → 1.21.0

package/scripts/ci-local.test.sh

@@ -915,5 +915,194 @@ OUT=$(PATH="$T/bin:$PATH" bash "$S" run --repo "$r" --base main --base-branch ma
 eq "FIX-3: the run passes with the job deferred" 0 "$RC"
 hasnt "FIX-3: no container was started" "$(cat "$FAKE_DOCKER_LOG")" "run -d"
 
+# ------------------------------------------------------------------------------------------------ forge#4057: concurrency
+# SEC-3 universe: a single-test step is judged against the repo-wide tests, not the one test it runs
+r=$(blank); mkdir -p "$r/scripts"
+echo 'echo a' > "$r/scripts/a.sh"; echo 'bash scripts/a.sh' > "$r/scripts/a.test.sh"
+echo 'echo x' > "$r/scripts/x.test.sh"; echo 'echo z' > "$r/scripts/zz.sh"
+commit_base "$r"
+_scope() { # _scope <command> <changed files...>
+  node -e "
+import('$ROOT/scripts/ci-local/scoping.mjs').then(m=>{const [cmd,...c]=process.argv.slice(1);const s=m.makeScopeCtx({repo:'$r',baseRef:'main',from:'main',changed:c});
+const r=m.scopeCommand(cmd,'',s);console.log(JSON.stringify({action:r.action,widen:r.widen_reason||''}))})" "$@"; }
+echo '# c' >> "$r/scripts/a.sh"
+R=$(_scope 'bash scripts/x.test.sh' scripts/a.sh)
+eq "4057: an unrelated x.test.sh is skipped when a.sh has a sibling a.test.sh" skip "$(printf '%s' "$R" | jq -r '.action')"  # edge_e2_unrelated_test_skipped
+R=$(_scope 'bash scripts/a.test.sh' scripts/a.sh)
+eq "4057: the sibling a.test.sh still runs" run "$(printf '%s' "$R" | jq -r '.action')"
+echo '# c' >> "$r/scripts/zz.sh"
+R=$(_scope 'bash scripts/x.test.sh' scripts/zz.sh)
+has "4057: SEC-3 widen still fires when a changed source reaches no test anywhere" "$(printf '%s' "$R" | jq -r '.widen')" "empty-selection"  # edge_e1_sec3_widen_still_fires
+eq "4057: ...and the step runs" run "$(printf '%s' "$R" | jq -r '.action')"
+R=$(_scope 'bash scripts/x.test.sh' scripts/a.sh scripts/zz.sh)
+has "4057: one unreached source file among several still widens a single-test step" "$(printf '%s' "$R" | jq -r '.widen')" "scripts/zz.sh"
+# SEC-3 universe is per ecosystem: a test in another language never covers a changed source (polyglot under-block)
+r=$(blank); mkdir -p "$r/pkg" "$r/tests"
+echo 'def f(): return 1' > "$r/pkg/utils.py"; echo 'def test_core(): assert True' > "$r/tests/test_core.py"
+echo 'test("u", () => {})' > "$r/tests/utils.test.ts"
+commit_base "$r"
+echo '# c' >> "$r/pkg/utils.py"
+R=$(_scope 'pytest tests/test_core.py' pkg/utils.py)
+eq "SEC-3 ecosystem: a pytest step is not skipped because an unrelated .ts test matches the changed .py" run "$(printf '%s' "$R" | jq -r '.action')"
+has "SEC-3 ecosystem: ...it widens and names the unreached source" "$(printf '%s' "$R" | jq -r '.widen')" "pkg/utils.py"
+R=$(_scope 'pytest' pkg/utils.py)
+eq "SEC-3 ecosystem: a bare pytest runner step widens too" run "$(printf '%s' "$R" | jq -r '.action')"
+R=$(_scope 'python3 tests/test_core.py' pkg/utils.py)
+eq "SEC-3 ecosystem: a single python test step widens (no py test reaches the change)" run "$(printf '%s' "$R" | jq -r '.action')"
+# DRR-1: a single-language change never widens another ecosystem's runner step (it skips/narrows as at a0d45ae)
+r=$(blank); mkdir -p "$r/pkg" "$r/tests"
+echo 'def f(): return 1' > "$r/pkg/utils.py"; printf 'from pkg import utils\ndef test_core(): assert utils.f()\n' > "$r/tests/test_core.py"
+echo 'import test from "node:test"; test("a", () => {})' > "$r/tests/a.test.mjs"
+commit_base "$r"
+echo '# c' >> "$r/pkg/utils.py"
+R=$(_scope 'node --test tests/a.test.mjs' pkg/utils.py)
+eq "DRR-1: a node step is not widened by a Python-only change" skip "$(printf '%s' "$R" | jq -r '.action')"
+eq "DRR-1: ...and carries no widen reason" "" "$(printf '%s' "$R" | jq -r '.widen')"
+R=$(_scope 'pytest tests/test_core.py' pkg/utils.py)
+eq "DRR-1: the pytest step still runs the reaching test" run "$(printf '%s' "$R" | jq -r '.action')"
+eq "DRR-1: ...scoped, not widened" "" "$(printf '%s' "$R" | jq -r '.widen')"
+
+# FIFO slots (tickets), dead-owner reaping, deadline while queued
+_slot() { FORGE_CI_LOCAL_HOME="$SLOT_HOME" FORGE_CI_LOCAL_SLOTS=1 node --input-type=module -e "
+import('$ROOT/scripts/ci-local/run.mjs').then(async (m) => {
+  const [id, hold, budget] = process.argv.slice(1).map(Number);
+  const t0 = Date.now();
+  const rel = await m.acquireSlot('t' + id, t0 + budget);
+  if (!rel) { console.log('timeout ' + id); return; }
+  console.log('acq ' + id + (process.env.SLOT_SHOW_WAIT === '1' ? ' wait=' + (Date.now() - t0) : '')); await new Promise((r) => setTimeout(r, hold)); rel();
+})" "$@"; }
+SLOT_HOME="$T/fifo-home"; mkdir -p "$SLOT_HOME"; LOG="$T/fifo.log"; : > "$LOG"
+# hold (1500ms) exceeds the cumulative spawn spacing (3 x 0.3s), so invocations 2-4 all queue behind invocation 1
+for i in 1 2 3 4; do SLOT_SHOW_WAIT=1 _slot "$i" 1500 30000 >> "$LOG" 2>&1 & sleep 0.3; done; wait
+eq "4057: 4 invocations, 1 slot: acquired strictly in ticket order, none starved" "acq 1 acq 2 acq 3 acq 4" "$(sed 's/ wait=.*//' "$LOG" | tr '\n' ' ' | sed 's/ $//')"  # edge_e4_fifo_order
+W1=$(sed -n 's/^acq 1 wait=//p' "$LOG")
+case "$W1" in ''|*[!0-9]*) bad "4059: invocation 1 wait not recorded ('$W1')" ;; *) [ "$W1" -lt 500 ] && ok || bad "4059: invocation 1 acquired immediately (wait ${W1}ms, want < 500)" ;; esac
+for i in 2 3 4; do  # edge_e2_queued_waits
+  Wi=$(sed -n "s/^acq $i wait=//p" "$LOG")
+  case "$Wi" in ''|*[!0-9]*) bad "4059: invocation $i wait not recorded ('$Wi')" ;; *) [ "$Wi" -ge 500 ] && ok || bad "4059: invocation $i queued behind the holder (wait ${Wi}ms, want >= 500)" ;; esac
+done
+eq "4057: no ticket is left behind" 0 "$(ls "$SLOT_HOME/ci-local.lock/queue" | wc -l | tr -d ' ')"
+SLOT_HOME="$T/fifo-home2"; mkdir -p "$SLOT_HOME/ci-local.lock/slot-0" "$SLOT_HOME/ci-local.lock/queue"
+( : ) & DEAD=$!; wait "$DEAD" 2>/dev/null
+printf '%s 1 ghost\n' "$DEAD" > "$SLOT_HOME/ci-local.lock/slot-0/owner"
+: > "$SLOT_HOME/ci-local.lock/queue/00000000000001-000001-$DEAD-abcdef"
+eq "4057: a dead slot owner and a dead earlier ticket are reaped, the queue is not blocked" "acq 9" "$(_slot 9 0 5000)"  # edge_e3_dead_owner_reaped
+eq "4057: no ticket left after a reaped acquire" 0 "$(ls "$SLOT_HOME/ci-local.lock/queue" | wc -l | tr -d ' ')"
+mkdir -p "$SLOT_HOME/ci-local.lock/slot-0"; printf '%s %s000 live\n' "$$" "$(date +%s)" > "$SLOT_HOME/ci-local.lock/slot-0/owner"
+eq "4057: the deadline while queued returns no slot" "timeout 8" "$(_slot 8 0 700)"  # edge_e6_deadline_ticket_removed
+eq "4057: ...and removes its ticket (no leak)" 0 "$(ls "$SLOT_HOME/ci-local.lock/queue" | wc -l | tr -d ' ')"
+
+# single-flight dedup: identical check in two jobs runs once; a different env must not dedup
+r=$(blank); mkdir -p "$r/.github/workflows"
+cat > "$r/.github/workflows/ci.yml" <<'YML'
+on: pull_request
+jobs:
+  a:
+    runs-on: ubuntu-latest
+    steps:
+      - run: echo x >> counter.txt; sleep 1
+  b:
+    runs-on: ubuntu-latest
+    steps:
+      - run: echo x >> counter.txt; sleep 1
+  c:
+    runs-on: ubuntu-latest
+    env:
+      VARIANT: "1"
+    steps:
+      - run: echo x >> counter.txt; sleep 1
+YML
+echo s > "$r/s.txt"; commit_base "$r"; echo x >> "$r/s.txt"
+OUT=$(FORGE_CI_LOCAL_NO_CACHE=1 bash "$S" run --repo "$r" --base main --base-branch main --head-branch feat 2>&1); RC=$?
+eq "4057: dedup run passes" 0 "$RC"
+eq "4057: identical step ran once, the different-env step ran too (2 executions of 3 jobs)" 2 "$(wc -l < "$r/counter.txt" | tr -d ' ')"  # edge_e8_different_env_not_deduped
+has "4057: the duplicate job reports dedup" "$OUT" "dedup=1"
+has "4057: the result line carries the dedup count and wait_ms" "$OUT" "wait_ms="
+r=$(blank); mkdir -p "$r/.github/workflows"
+cat > "$r/.github/workflows/ci.yml" <<'YML'
+on: pull_request
+jobs:
+  a:
+    runs-on: ubuntu-latest
+    steps:
+      - run: echo y >> fail.txt; sleep 1; exit 3
+  b:
+    runs-on: ubuntu-latest
+    steps:
+      - run: echo y >> fail.txt; sleep 1; exit 3
+YML
+echo s > "$r/s.txt"; commit_base "$r"; echo x >> "$r/s.txt"
+OUT=$(FORGE_CI_LOCAL_NO_CACHE=1 bash "$S" run --repo "$r" --base main --base-branch main --head-branch feat 2>&1); RC=$?
+eq "4057: a failing check shared by two jobs fails the gate" 1 "$RC"
+eq "4057: ...and was executed once, not re-run by the duplicate" 1 "$(wc -l < "$r/fail.txt" | tr -d ' ')"  # edge_e5_failure_shared_once
+eq "4057: ...both jobs report the failure" 2 "$(printf '%s\n' "$OUT" | grep -c '^CI_LOCAL_JOB: fail ')"
+
+# forge#4060: jobs with the same command but different service containers must not share a dedup result or a cached pass
+mksvc() { # $1 = repo, $2 = image of job b's postgres, $3 = job c's second-service ordering flag
+  mkdir -p "$1/.github/workflows"
+  cat > "$1/.github/workflows/ci.yml" <<YML
+on: pull_request
+jobs:
+  a:
+    runs-on: ubuntu-latest
+    services:
+      postgres:
+        image: postgres:14
+    steps:
+      - run: echo x >> counter.txt; sleep 1
+  b:
+    runs-on: ubuntu-latest
+    services:
+      postgres:
+        image: $2
+    steps:
+      - run: echo x >> counter.txt; sleep 1
+YML
+}
+r=$(blank); mksvc "$r" postgres:16
+echo s > "$r/s.txt"; commit_base "$r"; echo x >> "$r/s.txt"
+: > "$FAKE_DOCKER_LOG"
+OUT=$(PATH="$T/bin:$PATH" FORGE_CI_LOCAL_NO_CACHE=1 bash "$S" run --repo "$r" --base main --base-branch main --head-branch feat 2>&1); RC=$?
+eq "4060: different-services run passes" 0 "$RC"
+eq "4060: same command with different services ran in both jobs, no shared flight result" 2 "$(wc -l < "$r/counter.txt" | tr -d ' ')"  # edge_e4_different_services_not_deduped
+hasnt "4060: no job reports dedup" "$OUT" "dedup=1"
+r=$(blank); mksvc "$r" postgres:14
+echo s > "$r/s.txt"; commit_base "$r"; echo x >> "$r/s.txt"
+OUT=$(PATH="$T/bin:$PATH" FORGE_CI_LOCAL_NO_CACHE=1 bash "$S" run --repo "$r" --base main --base-branch main --head-branch feat 2>&1); RC=$?
+eq "4060: identical services still dedup (one execution)" 1 "$(wc -l < "$r/counter.txt" | tr -d ' ')"
+has "4060: the identical-services duplicate reports dedup" "$OUT" "dedup=1"
+# persisted cache: a pass for services X is not a hit for services Y
+r=$(blank); mksvc "$r" postgres:16
+echo s > "$r/s.txt"; commit_base "$r"; echo x >> "$r/s.txt"
+OUT=$(PATH="$T/bin:$PATH" FORGE_CI_LOCAL_NO_CACHE=0 FORGE_CI_LOCAL_SLOTS=2 bash "$S" run --repo "$r" --base main --base-branch main --head-branch feat --max-parallel 1 2>&1)
+eq "4060: persisted cache is not shared across different service sets" 2 "$(wc -l < "$r/counter.txt" | tr -d ' ')"  # edge_e5_cache_not_shared
+
+# wait_ms is reported apart from duration; cached jobs wait for nothing; cheap jobs go first once durations are known
+r=$(blank); mkdir -p "$r/.github/workflows"
+cat > "$r/.github/workflows/ci.yml" <<'YML'
+on: pull_request
+jobs:
+  big:
+    runs-on: ubuntu-latest
+    steps:
+      - run: sleep 1.5 # ord-big
+  small:
+    runs-on: ubuntu-latest
+    steps:
+      - run: echo ord-small
+YML
+echo s > "$r/s.txt"; commit_base "$r"; echo x >> "$r/s.txt"
+OUT=$(FORGE_CI_LOCAL_NO_CACHE=0 FORGE_CI_LOCAL_SLOTS=1 bash "$S" run --repo "$r" --base main --base-branch main --head-branch feat --max-parallel 2 2>&1); RC=$?
+eq "4057: wait run passes" 0 "$RC"
+WM=$(printf '%s\n' "$OUT" | sed -n 's/^CI_LOCAL_RESULT:.* wait_ms=\([0-9]*\).*/\1/p')
+[ "${WM:-0}" -ge 500 ] && ok || bad "4057: one job queued for the single slot, so wait_ms is reported (got '${WM:-}')" "$OUT"
+has "4057: job lines carry wait=" "$OUT" " wait="
+OUT=$(FORGE_CI_LOCAL_NO_CACHE=0 FORGE_CI_LOCAL_SLOTS=1 bash "$S" run --repo "$r" --base main --base-branch main --head-branch feat --max-parallel 2 2>&1)
+has "4057: a fully cached job waits for no slot (wait=0.0s)" "$OUT" "cached=1 wait=0.0s"  # edge_e7_cached_job_no_wait
+OUT=$(FORGE_CI_LOCAL_NO_CACHE=1 FORGE_CI_LOCAL_SLOTS=1 bash "$S" run --repo "$r" --base main --base-branch main --head-branch feat --max-parallel 1 2>&1)
+FIRST=$(printf '%s\n' "$OUT" | grep '^CI_LOCAL_JOB: pass' | head -n 1)
+has "4057: the cheapest known job runs first (small before big)" "$FIRST" "gha:ci:small"
+FORGE_TELEMETRY=off bash -c 'grep -q "wait_ms" "$0"' "$S" && ok || bad "4057: ci-local.sh forwards wait_ms to telemetry"
+
 echo "ci-local.test.sh: $pass passed, $fail failed"
 [ "$fail" -eq 0 ]

package/scripts/classify-finding.sh

@@ -56,7 +56,7 @@
 #                 lists the deferred issue numbers confirmed open, one per line; a deferred item
 #                 demotes only when its number is listed (closed, unreadable or absent: no demotion).
 #                 The caller lists a number only after verifying it is an open issue (not a PR) whose
-#                 body carries `FORGE:DEFERRED_FROM: #<merge issue>` and the path.
+#                 body carries `FORGE:DEFERRED_FROM: #<merge issue>` and a checklist line for the exact path.
 #                 `--merge-issue <N>` is the PR's linked issue: `deferred → #N` never demotes (that
 #                 issue is always open during review, so it is no follow-up).
 #                 Never demoted: CRITICAL/HIGH, `not-affected` items, safety-exempt findings

package/scripts/file-deferred-followups.sh

@@ -4,8 +4,9 @@
 # SPDX-License-Identifier: AGPL-3.0-or-later
 #
 # The builder decides fold-vs-defer (judgment, build.md B2.0); this script does the mechanics for the items
-# that stay deferred: trusted lookup of an open follow-up carrying <!-- FORGE:DEFERRED_FROM: #N -->, append the
-# missing items to it as checklist entries (exact path match), or create ONE issue with a checklist.
+# that stay deferred: trusted lookup (claim marker <!-- FORGE:DEFERRED_ISSUE: #M --> on the parent, then a state-all search) of a follow-up carrying
+# <!-- FORGE:DEFERRED_FROM: #N -->, append the missing items (exact path match; a closed one is reopened), or create ONE issue
+# with a checklist, claim it on the parent and re-check once for a concurrent lower-numbered duplicate.
 #
 # Usage: file-deferred-followups.sh --parent N --repo OWNER/REPO [--items FILE|-] [--folded N]
 #   items TSV, one per line: <path> TAB <why> TAB <known-flaw flag: known-flaw|1|yes|true, else empty>
@@ -71,27 +72,60 @@ finish() { # $1 = batched_into number or empty, $2 = filed count
 }
 
 # --- Trusted lookup (author predicate applied before path match), one retry ---------------------------------
-do_lookup() {
+# Sources, in order: (1) claim markers `<!-- FORGE:DEFERRED_ISSUE: #M -->` on the parent's REST comments (read-after-write
+# consistent), (2) a search over ALL issue states (eventually consistent, so only a fallback). Every candidate is then
+# verified by number over REST (trusted author, DEFERRED_FROM marker, not a PR); the lowest number is canonical.
+JQ_PRE='def lst: split(",") | map(gsub("^\\s+|\\s+$"; "")) | map(select(length > 0));
+  def trusted: ((.author_association // "") as $x | ($assoc | lst) | index($x) != null)
+    or ((.user.type // "") == "Bot")
+    or ((.user.login // "") as $x | ($logins | lst) | index($x) != null);'
+CLAIM_RE='^\s*<!-- FORGE:DEFERRED_ISSUE: #(?<n>[0-9]+) -->'
+claim_nums() {
+  gh api --paginate "repos/${REPO}/issues/${PARENT}/comments" 2>/dev/null \
+  | jq -s -r --arg assoc "$TRUSTED_ASSOC" --arg logins "$TRUSTED_LOGINS" --arg re "$CLAIM_RE" "$JQ_PRE"'
+      [.[][] | select(trusted) | (.body // "") | capture($re) | .n | tonumber] | unique | .[]' 2>/dev/null
+}
+search_nums() {
   gh api -X GET search/issues -f per_page=50 \
-    -f q="\"FORGE:DEFERRED_FROM: #${PARENT}\" in:body repo:${REPO} is:issue is:open" 2>/dev/null \
-  | jq -c --arg assoc "$TRUSTED_ASSOC" --arg logins "$TRUSTED_LOGINS" --arg m "$MARKER" '
-      ($assoc | split(",") | map(gsub("^\\s+|\\s+$"; "")) | map(select(length > 0))) as $A
-      | ($logins | split(",") | map(gsub("^\\s+|\\s+$"; "")) | map(select(length > 0))) as $L
-      | [.items[]
-          | select(((.author_association // "") as $x | $A | index($x) != null)
-                   or ((.user.type // "") == "Bot")
-                   or ((.user.login // "") as $x | $L | index($x) != null))
-          | select((.body // "") | contains($m))
-          | {number, body: (.body // "")}]
-      | sort_by(.number)' 2>/dev/null
+    -f q="\"FORGE:DEFERRED_FROM: #${PARENT}\" in:body repo:${REPO} is:issue" 2>/dev/null \
+  | jq -r --arg assoc "$TRUSTED_ASSOC" --arg logins "$TRUSTED_LOGINS" --arg m "$MARKER" "$JQ_PRE"'
+      [.items[] | select(trusted) | select((.body // "") | contains($m)) | .number] | sort | .[]' 2>/dev/null
+}
+verify_num() { # number -> `number<TAB>state` when a trusted, non-PR issue carries the marker; empty when not (a 404 is "not it")
+  local out rc
+  out=$(gh api "repos/${REPO}/issues/$1" 2>"$TMP/verr"); rc=$?
+  if [ "$rc" -ne 0 ]; then grep -qE '404|Not Found|410' "$TMP/verr" 2>/dev/null && return 0; return 1; fi
+  printf '%s' "$out" | jq -r --arg assoc "$TRUSTED_ASSOC" --arg logins "$TRUSTED_LOGINS" --arg m "$MARKER" "$JQ_PRE"'
+      select(.pull_request == null) | select(trusted) | select((.body // "") | contains($m))
+      | "\(.number)\t\(.state)"' 2>/dev/null
+}
+CANON=""; CSTATE=""
+pick_verified() { # numbers... ; sets CANON/CSTATE from the first verified one; non-zero only when a fetch failed
+  local n r
+  for n in "$@"; do
+    r=$(verify_num "$n") || return 1
+    if [ -n "$r" ]; then CANON="${r%%$'\t'*}"; CSTATE="${r#*$'\t'}"; return 0; fi
+  done
+  return 0
 }
-LOOKUP=""
+find_canonical() {
+  local cands
+  CANON=""; CSTATE=""
+  cands=$(claim_nums) || return 1
+  # shellcheck disable=SC2086
+  pick_verified $cands || return 1
+  [ -n "$CANON" ] && return 0
+  cands=$(search_nums) || return 1
+  # shellcheck disable=SC2086
+  pick_verified $cands || return 1
+  return 0
+}
+LOOKUP_OK=""
 for attempt in 1 2; do
-  LOOKUP=$(do_lookup) && [ -n "$LOOKUP" ] && break
-  LOOKUP=""
+  if find_canonical; then LOOKUP_OK=1; break; fi
   [ "$attempt" = 1 ] && sleep "${FORGE_DEFERRED_RETRY_SLEEP:-2}"
 done
-if [ -z "$LOOKUP" ]; then
+if [ -z "$LOOKUP_OK" ]; then
   echo "DEFERRED_LOOKUP=FAILED"
   exit 3
 fi
@@ -108,27 +142,35 @@ has_entry() { # body-file path : exact checklist-line match, not substring
   grep -F -- "\`$2\`" "$1" | grep -qE "^- \[[ xX]\] \`$(printf '%s' "$2" | sed 's/[][\.*^$+?(){}|]/\\&/g')\`( |\$)"
 }
 
-NUM=$(printf '%s' "$LOOKUP" | jq -r 'first | .number // empty' 2>/dev/null)
-
-if [ -n "$NUM" ]; then
+# Append the missing items to follow-up $1 (state $2), then reopen it when closed (review-finalize needs it open).
+append_items() { # number state ; non-zero = failure
+  local num="$1" state="$2" note=""
   # Re-read the body immediately before editing (parallel builds of the same parent), then append what is missing.
-  if ! gh api "repos/${REPO}/issues/${NUM}" --jq '.body // ""' > "$TMP/body" 2>/dev/null; then
-    echo "DEFERRED_FILE=FAILED"; exit 4
-  fi
+  gh api "repos/${REPO}/issues/${num}" --jq '.body // ""' > "$TMP/body" 2>/dev/null || return 1
   : > "$TMP/add"
   while IFS="$SEP" read -r p w k; do
     has_entry "$TMP/body" "$p" || entry "$p" "$w" "$k" >> "$TMP/add"
   done < "$TMP/items"
+  [ "$state" = "closed" ] && note=" (follow-up was closed; reopened)"
   if [ -s "$TMP/add" ]; then
-    { cat "$TMP/body"; printf '\n### Added by a later build of #%s\n\n' "$PARENT"; cat "$TMP/add"; } > "$TMP/newbody"
+    { cat "$TMP/body"; printf '\n### Added by a later build of #%s%s\n\n' "$PARENT" "$note"; cat "$TMP/add"; } > "$TMP/newbody"
     if [ -n "${DRY_RUN:-}" ]; then
-      echo "DRY_RUN: gh issue edit $NUM (append $(wc -l < "$TMP/add" | tr -d ' ') item(s))" >&2
-    elif ! gh issue edit "$NUM" -R "$REPO" --body-file "$TMP/newbody" >/dev/null 2>&1; then
-      echo "DEFERRED_FILE=FAILED"; exit 4
+      echo "DRY_RUN: gh issue edit $num (append $(wc -l < "$TMP/add" | tr -d ' ') item(s))" >&2
+    else
+      gh issue edit "$num" -R "$REPO" --body-file "$TMP/newbody" >/dev/null 2>&1 || return 1
     fi
   fi
-  while IFS="$SEP" read -r p w k; do printf '%s\t%s\n' "$p" "$NUM"; done < "$TMP/items"
-  finish "$NUM" 0
+  if [ "$state" = "closed" ]; then
+    if [ -n "${DRY_RUN:-}" ]; then echo "DRY_RUN: gh issue reopen $num" >&2
+    else gh issue reopen "$num" -R "$REPO" >/dev/null 2>&1 || return 1; fi
+  fi
+  return 0
+}
+
+if [ -n "$CANON" ]; then
+  append_items "$CANON" "$CSTATE" || { echo "DEFERRED_FILE=FAILED"; exit 4; }
+  while IFS="$SEP" read -r p w k; do printf '%s\t%s\n' "$p" "$CANON"; done < "$TMP/items"
+  finish "$CANON" 0
   exit 0
 fi
 
@@ -154,6 +196,25 @@ else
   NUM=$(printf '%s' "$URL" | grep -oE '[0-9]+$' | tail -n 1)
   [ -n "$NUM" ] || { echo "DEFERRED_FILE=FAILED"; exit 4; }
 fi
+if [ -z "${DRY_RUN:-}" ]; then
+  # Claim the follow-up on the parent (REST comments are read-after-write consistent; best effort, the search still finds it).
+  gh issue comment "$PARENT" -R "$REPO" --body "<!-- FORGE:DEFERRED_ISSUE: #${NUM} -->" >/dev/null 2>&1 \
+    || echo "file-deferred-followups: warning: could not post claim marker for #${NUM} on #${PARENT}" >&2
+  # Post-create re-check: a concurrent build may have filed first. The lowest trusted number is canonical; the newer one is closed.
+  if find_canonical; then
+    if [ -n "$CANON" ] && [ "$CANON" -lt "$NUM" ]; then
+      if append_items "$CANON" "$CSTATE"; then
+        gh issue close "$NUM" -R "$REPO" --reason "not planned" --comment "Duplicate of #${CANON}: deferred items of #${PARENT} are tracked there." >/dev/null 2>&1 \
+          || echo "file-deferred-followups: warning: could not close duplicate #${NUM}" >&2
+        NUM="$CANON"
+      else
+        echo "file-deferred-followups: warning: could not move items to #${CANON}; keeping #${NUM}" >&2
+      fi
+    fi
+  else
+    echo "file-deferred-followups: warning: post-create re-check failed; keeping #${NUM}" >&2
+  fi
+fi
 while IFS="$SEP" read -r p w k; do printf '%s\t%s\n' "$p" "$NUM"; done < "$TMP/items"
 finish "$NUM" 1
 exit 0

package/scripts/file-deferred-followups.test.sh

@@ -8,19 +8,27 @@ S="$HERE/file-deferred-followups.sh"
 T=$(mktemp -d); trap 'rm -rf "$T"' EXIT
 mkdir "$T/bin"
 export FORGE_DEFERRED_RETRY_SLEEP=0
-export GH_LOG="$T/gh.log" SEARCH_JSON="$T/search.json" ISSUE_BODY="$T/issue.body"
+export GH_LOG="$T/gh.log" SEARCH_JSON="$T/search.json" ISSUE_BODY="$T/issue.body" COMMENTS_JSON="$T/comments.json" T_DIR="$T"
 
 cat > "$T/bin/gh" <<'MOCK'
 #!/usr/bin/env bash
 echo "$*" >> "$GH_LOG"
 case "$1 $2" in
   "api -X") [ -n "${LOOKUP_FAIL:-}" ] && exit 1; cat "$SEARCH_JSON"; exit 0 ;;
+  "api --paginate") [ -n "${CLAIM_FAIL:-}" ] && exit 1; cat "$COMMENTS_JSON"; exit 0 ;;
   "issue create") [ -n "${CREATE_FAIL:-}" ] && exit 1; echo "https://github.com/o/r/issues/901"; exit 0 ;;
+  "issue comment") [ -n "${COMMENT_FAIL:-}" ] && exit 1; exit 0 ;;
+  "issue reopen") [ -n "${REOPEN_FAIL:-}" ] && exit 1; exit 0 ;;
+  "issue close") [ -n "${CLOSE_FAIL:-}" ] && exit 1; exit 0 ;;
   "issue edit")
     [ -n "${EDIT_FAIL:-}" ] && exit 1
     while [ $# -gt 0 ]; do [ "$1" = "--body-file" ] && cp "$2" "$ISSUE_BODY.edited"; shift; done; exit 0 ;;
 esac
-case "$1" in api) cat "$ISSUE_BODY"; exit 0 ;; esac
+if [ "$1" = api ]; then
+  case "$*" in *--jq*) cat "$ISSUE_BODY"; exit 0 ;; esac
+  n="${2##*/}"; [ -f "$T_DIR/issue.$n.json" ] && { cat "$T_DIR/issue.$n.json"; exit 0; }
+  echo "HTTP 404: Not Found" >&2; exit 1
+fi
 exit 1
 MOCK
 chmod +x "$T/bin/gh"; export PATH="$T/bin:$PATH"
@@ -29,7 +37,12 @@ PASS=0; FAIL=0
 ok() { PASS=$((PASS+1)); echo "PASS: $1"; }
 bad() { FAIL=$((FAIL+1)); echo "FAIL: $1"; }
 check() { if eval "$2"; then ok "$1"; else bad "$1"; fi; }
-reset() { : > "$GH_LOG"; rm -f "$ISSUE_BODY.edited"; unset LOOKUP_FAIL CREATE_FAIL EDIT_FAIL DRY_RUN; echo '{"items":[]}' > "$SEARCH_JSON"; }
+reset() { : > "$GH_LOG"; rm -f "$ISSUE_BODY.edited" "$T"/issue.*.json; unset LOOKUP_FAIL CREATE_FAIL EDIT_FAIL CLAIM_FAIL COMMENT_FAIL REOPEN_FAIL CLOSE_FAIL DRY_RUN
+  echo '{"items":[]}' > "$SEARCH_JSON"; echo '[]' > "$COMMENTS_JSON"; }
+# issue_json NUMBER STATE ASSOCIATION [PARENT]: REST view of a follow-up carrying the DEFERRED_FROM marker
+issue_json() { printf '{"number":%s,"state":"%s","author_association":"%s","user":{"type":"User"},"body":"<!-- FORGE:DEFERRED_FROM: #%s -->"}\n' "$1" "$2" "$3" "${4:-10}" > "$T/issue.$1.json"; }
+# claim_json ASSOCIATION NUMBER...: parent comments carrying claim markers
+claim_json() { local a="$1" out="" n; shift; for n in "$@"; do out="${out:+$out,}{\"author_association\":\"$a\",\"user\":{\"type\":\"User\"},\"body\":\"<!-- FORGE:DEFERRED_ISSUE: #$n -->\"}"; done; echo "[$out]" > "$COMMENTS_JSON"; }
 printf 'a/x.sh\twhy a\t\nb/y.sh\twhy b\tknown-flaw\nc/z.sh\twhy c\t\n' > "$T/items.tsv"
 
 # 1. N items -> exactly one create
@@ -43,7 +56,7 @@ check "create: telemetry" "printf '%s' \"\$OUT\" | grep -q 'DEFERRED_TELEMETRY d
 # 2. append to existing trusted open issue; exact path match
 reset
 printf '## Affected Files\n- `a/x.sh`\n\n- [ ] `a/x.sh.bak` — other\n<!-- FORGE:DEFERRED_FROM: #10 -->\n' > "$ISSUE_BODY"
-echo '{"items":[{"number":77,"author_association":"OWNER","user":{"type":"User"},"body":"<!-- FORGE:DEFERRED_FROM: #10 -->"}]}' > "$SEARCH_JSON"
+echo '{"items":[{"number":77,"author_association":"OWNER","user":{"type":"User"},"body":"<!-- FORGE:DEFERRED_FROM: #10 -->"}]}' > "$SEARCH_JSON"; issue_json 77 open OWNER
 OUT=$(bash "$S" --parent 10 --repo o/r --items "$T/items.tsv"); RC=$?
 check "append: rc 0, no create" "[ $RC -eq 0 ] && ! grep -q '^issue create' '$GH_LOG'"
 check "append: telemetry batched into 77 filed=0" "printf '%s' \"\$OUT\" | grep -q 'batched_into=#77 filed=0'"
@@ -55,7 +68,7 @@ reset
 printf 'd/w.sh\t\tknown-flaw\n' > "$T/emptywhy.tsv"
 bash "$S" --parent 10 --repo o/r --items "$T/emptywhy.tsv" >/dev/null; RC=$?
 rm -f "$ISSUE_BODY.edited"; printf 'x\n' > "$ISSUE_BODY"
-echo '{"items":[{"number":77,"author_association":"OWNER","user":{"type":"User"},"body":"<!-- FORGE:DEFERRED_FROM: #10 -->"}]}' > "$SEARCH_JSON"
+echo '{"items":[{"number":77,"author_association":"OWNER","user":{"type":"User"},"body":"<!-- FORGE:DEFERRED_FROM: #10 -->"}]}' > "$SEARCH_JSON"; issue_json 77 open OWNER
 OUT=$(bash "$S" --parent 10 --repo o/r --items "$T/emptywhy.tsv"); RC=$?
 check "empty-why: append keeps known-flaw flag" "[ $RC -eq 0 ] && grep -q 'd/w.sh. \*\*known-flaw\*\*' '$ISSUE_BODY.edited'"
 
@@ -66,13 +79,13 @@ check "missing option value: rc 2" "[ $RC -eq 2 ]"
 # 3. already-listed path is not duplicated; nothing to add -> no edit
 reset
 printf -- '- [ ] `a/x.sh` — w\n- [x] `b/y.sh`\n- [ ] `c/z.sh`\n' > "$ISSUE_BODY"
-echo '{"items":[{"number":77,"author_association":"MEMBER","user":{"type":"User"},"body":"<!-- FORGE:DEFERRED_FROM: #10 -->"}]}' > "$SEARCH_JSON"
+echo '{"items":[{"number":77,"author_association":"MEMBER","user":{"type":"User"},"body":"<!-- FORGE:DEFERRED_FROM: #10 -->"}]}' > "$SEARCH_JSON"; issue_json 77 open OWNER
 bash "$S" --parent 10 --repo o/r --items "$T/items.tsv" >/dev/null; RC=$?
 check "idempotent: no edit, no create on resume" "[ $RC -eq 0 ] && ! grep -qE '^issue (edit|create)' '$GH_LOG'"
 
 # 4. untrusted match ignored -> new issue
 reset
-echo '{"items":[{"number":66,"author_association":"NONE","user":{"type":"User"},"body":"<!-- FORGE:DEFERRED_FROM: #10 -->"}]}' > "$SEARCH_JSON"
+echo '{"items":[{"number":66,"author_association":"NONE","user":{"type":"User"},"body":"<!-- FORGE:DEFERRED_FROM: #10 -->"}]}' > "$SEARCH_JSON"; issue_json 66 open NONE; issue_json 77 open OWNER
 OUT=$(bash "$S" --parent 10 --repo o/r --items "$T/items.tsv")
 check "untrusted: ignored, create filed" "grep -q '^issue create' '$GH_LOG' && printf '%s' \"\$OUT\" | grep -q 'filed=1'"
 
@@ -87,7 +100,7 @@ OUT=$(bash "$S" --parent 10 --repo o/r --items "$T/items.tsv"); RC=$?
 check "create failure: rc 4, no telemetry" "[ $RC -eq 4 ] && ! printf '%s' \"\$OUT\" | grep -q DEFERRED_TELEMETRY"
 reset; export EDIT_FAIL=1
 printf 'x\n' > "$ISSUE_BODY"
-echo '{"items":[{"number":77,"author_association":"OWNER","user":{"type":"User"},"body":"<!-- FORGE:DEFERRED_FROM: #10 -->"}]}' > "$SEARCH_JSON"
+echo '{"items":[{"number":77,"author_association":"OWNER","user":{"type":"User"},"body":"<!-- FORGE:DEFERRED_FROM: #10 -->"}]}' > "$SEARCH_JSON"; issue_json 77 open OWNER
 OUT=$(bash "$S" --parent 10 --repo o/r --items "$T/items.tsv"); RC=$?
 check "edit failure: rc 4, no telemetry" "[ $RC -eq 4 ] && ! printf '%s' \"\$OUT\" | grep -q DEFERRED_TELEMETRY"
 
@@ -102,5 +115,85 @@ OUT=$(printf '' | bash "$S" --parent 10 --repo o/r)
 check "empty: no create, deferred_items=0" "! grep -q '^issue create' '$GH_LOG' && printf '%s' \"\$OUT\" | grep -q 'deferred_items=0'"
 bash "$S" --repo o/r >/dev/null 2>&1; check "usage: missing parent rc 2" "[ $? -eq 2 ]"
 
+# 9. e1: search index lags (returns nothing) but the claim marker on the parent finds the follow-up
+reset
+printf 'x\n' > "$ISSUE_BODY"; issue_json 77 open OWNER; claim_json OWNER 77
+OUT=$(bash "$S" --parent 10 --repo o/r --items "$T/items.tsv"); RC=$?
+check "index-lag: claim found, no duplicate create" "[ $RC -eq 0 ] && ! grep -q '^issue create' '$GH_LOG' && printf '%s' \"\$OUT\" | grep -q 'batched_into=#77 filed=0'"
+check "index-lag: search never needed" "! grep -q search/issues '$GH_LOG'"
+
+# 10. e2: closed follow-up (found only by a state-all search) is reused and reopened
+reset
+printf 'x\n' > "$ISSUE_BODY"; issue_json 77 closed OWNER
+echo '{"items":[{"number":77,"author_association":"OWNER","user":{"type":"User"},"body":"<!-- FORGE:DEFERRED_FROM: #10 -->"}]}' > "$SEARCH_JSON"
+OUT=$(bash "$S" --parent 10 --repo o/r --items "$T/items.tsv"); RC=$?
+check "closed: reused, reopened, no create" "[ $RC -eq 0 ] && grep -q '^issue reopen 77' '$GH_LOG' && ! grep -q '^issue create' '$GH_LOG'"
+check "closed: search query covers all states" "grep -q 'is:issue' '$GH_LOG' && ! grep 'api -X' '$GH_LOG' | grep -q 'is:open'"
+reset; export REOPEN_FAIL=1
+printf 'x\n' > "$ISSUE_BODY"; issue_json 77 closed OWNER; claim_json OWNER 77
+bash "$S" --parent 10 --repo o/r --items "$T/items.tsv" >/dev/null; RC=$?
+check "closed: reopen failure is rc 4" "[ $RC -eq 4 ]"
+
+# 11. create posts the claim marker on the parent
+reset
+OUT=$(bash "$S" --parent 10 --repo o/r --items "$T/items.tsv"); RC=$?
+check "claim: posted on parent after create" "[ $RC -eq 0 ] && grep -q '^issue comment 10 .*FORGE:DEFERRED_ISSUE: #901' '$GH_LOG'"
+reset; export COMMENT_FAIL=1
+OUT=$(bash "$S" --parent 10 --repo o/r --items "$T/items.tsv" 2>/dev/null); RC=$?
+check "claim: post failure only warns" "[ $RC -eq 0 ] && printf '%s' \"\$OUT\" | grep -q 'batched_into=#901 filed=1'"
+
+# 12. e3: a concurrent build filed #800 first; the first claim read (before our create) misses it, the post-create re-check sees it
+race_setup() { # first comments read returns [] (the other build has not claimed yet), later reads return the real listing
+  CALLS="$T/claim.calls"; : > "$CALLS"; export CALLS
+  mv "$T/bin/gh" "$T/bin/gh.real"
+  cat > "$T/bin/gh" <<'WRAP'
+#!/usr/bin/env bash
+if [ "$1 $2" = "api --paginate" ]; then
+  echo x >> "$CALLS"
+  if [ "$(wc -l < "$CALLS" | tr -d ' ')" -le 1 ]; then echo "$*" >> "$GH_LOG"; echo '[]'; exit 0; fi
+fi
+exec "$(dirname "$0")/gh.real" "$@"
+WRAP
+  chmod +x "$T/bin/gh"
+}
+race_teardown() { mv "$T/bin/gh.real" "$T/bin/gh"; }
+reset
+printf 'x\n' > "$ISSUE_BODY"; issue_json 800 open OWNER; claim_json OWNER 800
+race_setup
+OUT=$(bash "$S" --parent 10 --repo o/r --items "$T/items.tsv"); RC=$?
+race_teardown
+check "race: newer duplicate closed, items moved to lowest" "[ $RC -eq 0 ] && grep -q '^issue close 901' '$GH_LOG' && grep -q '^issue edit 800' '$GH_LOG' && printf '%s' \"\$OUT\" | grep -q 'batched_into=#800 filed=1'"
+check "race: rows point at canonical issue" "[ \$(printf '%s\n' \"\$OUT\" | grep -c '800\$') -ge 3 ]"
+# e5: duplicate-close failure only warns
+reset; export CLOSE_FAIL=1
+printf 'x\n' > "$ISSUE_BODY"; issue_json 800 open OWNER; claim_json OWNER 800
+race_setup
+OUT=$(bash "$S" --parent 10 --repo o/r --items "$T/items.tsv" 2>/dev/null); RC=$?
+race_teardown
+check "race: duplicate-close failure still rc 0" "[ $RC -eq 0 ] && printf '%s' \"\$OUT\" | grep -q 'batched_into=#800'"
+
+# 13. e4: claim listing failure is a failed lookup, never "none found"
+reset; export CLAIM_FAIL=1
+OUT=$(bash "$S" --parent 10 --repo o/r --items "$T/items.tsv"); RC=$?
+check "claim-failure: rc 3, no create" "[ $RC -eq 3 ] && printf '%s' \"\$OUT\" | grep -q DEFERRED_LOOKUP=FAILED && ! grep -q '^issue create' '$GH_LOG'"
+
+# 15. e6: untrusted claim comment is ignored (falls through to create)
+reset
+issue_json 77 open OWNER; claim_json NONE 77
+OUT=$(bash "$S" --parent 10 --repo o/r --items "$T/items.tsv" 2>/dev/null); RC=$?
+check "untrusted-claim: ignored, create filed" "[ $RC -eq 0 ] && grep -q '^issue create' '$GH_LOG'"
+
+# 16. e7: several claims, lowest valid wins; a claim to a missing issue falls through
+reset
+printf 'x\n' > "$ISSUE_BODY"; issue_json 80 open OWNER; issue_json 90 open OWNER; claim_json OWNER 70 90 80
+OUT=$(bash "$S" --parent 10 --repo o/r --items "$T/items.tsv"); RC=$?
+check "multi-claim: lowest valid (80) wins, missing 70 skipped" "[ $RC -eq 0 ] && printf '%s' \"\$OUT\" | grep -q 'batched_into=#80 filed=0'"
+
+# 17. DRY_RUN: no claim, reopen or close
+reset; export DRY_RUN=1
+printf 'x\n' > "$ISSUE_BODY"; issue_json 77 closed OWNER; claim_json OWNER 77
+bash "$S" --parent 10 --repo o/r --items "$T/items.tsv" >/dev/null 2>&1
+check "dry-run: no comment/reopen/close/edit" "! grep -qE '^issue (comment|reopen|close|edit|create)' '$GH_LOG'"
+
 echo "passed=$PASS failed=$FAIL"
 [ "$FAIL" -eq 0 ]

package/scripts/lib/telemetry.sh

@@ -51,6 +51,8 @@ _forge_tel_record() { # event k=v... -> one JSON line on stdout
        run: ($run | str), phase: ($phase | str), event: $ev, eid: $eid} + $p' --args "${args[@]}"
 }
 
+# build_selfreview (forge#3971: tier, agents, findings, fixed, declined, skipped=<reason>; ids and counts only) has no review-side
+# routing: it defaults to the issue target below, like diff_size.
 # finding_disposition payload is generic key=value (incl. scope=introduced|pre-existing|unknown); no per-field allowlist applies.
 # Prints "<number> <repo>" of the comment target (issue or PR) for this event, or nothing when none is usable.
 _forge_tel_target() {