forgedock 1.18.0 → 1.20.0

package/scripts/file-deferred-followups.sh

@@ -4,8 +4,9 @@
 # SPDX-License-Identifier: AGPL-3.0-or-later
 #
 # The builder decides fold-vs-defer (judgment, build.md B2.0); this script does the mechanics for the items
-# that stay deferred: trusted lookup of an open follow-up carrying <!-- FORGE:DEFERRED_FROM: #N -->, append the
-# missing items to it as checklist entries (exact path match), or create ONE issue with a checklist.
+# that stay deferred: trusted lookup (claim marker <!-- FORGE:DEFERRED_ISSUE: #M --> on the parent, then a state-all search) of a follow-up carrying
+# <!-- FORGE:DEFERRED_FROM: #N -->, append the missing items (exact path match; a closed one is reopened), or create ONE issue
+# with a checklist, claim it on the parent and re-check once for a concurrent lower-numbered duplicate.
 #
 # Usage: file-deferred-followups.sh --parent N --repo OWNER/REPO [--items FILE|-] [--folded N]
 #   items TSV, one per line: <path> TAB <why> TAB <known-flaw flag: known-flaw|1|yes|true, else empty>
@@ -71,27 +72,60 @@ finish() { # $1 = batched_into number or empty, $2 = filed count
 }
 
 # --- Trusted lookup (author predicate applied before path match), one retry ---------------------------------
-do_lookup() {
+# Sources, in order: (1) claim markers `<!-- FORGE:DEFERRED_ISSUE: #M -->` on the parent's REST comments (read-after-write
+# consistent), (2) a search over ALL issue states (eventually consistent, so only a fallback). Every candidate is then
+# verified by number over REST (trusted author, DEFERRED_FROM marker, not a PR); the lowest number is canonical.
+JQ_PRE='def lst: split(",") | map(gsub("^\\s+|\\s+$"; "")) | map(select(length > 0));
+  def trusted: ((.author_association // "") as $x | ($assoc | lst) | index($x) != null)
+    or ((.user.type // "") == "Bot")
+    or ((.user.login // "") as $x | ($logins | lst) | index($x) != null);'
+CLAIM_RE='^\s*<!-- FORGE:DEFERRED_ISSUE: #(?<n>[0-9]+) -->'
+claim_nums() {
+  gh api --paginate "repos/${REPO}/issues/${PARENT}/comments" 2>/dev/null \
+  | jq -s -r --arg assoc "$TRUSTED_ASSOC" --arg logins "$TRUSTED_LOGINS" --arg re "$CLAIM_RE" "$JQ_PRE"'
+      [.[][] | select(trusted) | (.body // "") | capture($re) | .n | tonumber] | unique | .[]' 2>/dev/null
+}
+search_nums() {
   gh api -X GET search/issues -f per_page=50 \
-    -f q="\"FORGE:DEFERRED_FROM: #${PARENT}\" in:body repo:${REPO} is:issue is:open" 2>/dev/null \
-  | jq -c --arg assoc "$TRUSTED_ASSOC" --arg logins "$TRUSTED_LOGINS" --arg m "$MARKER" '
-      ($assoc | split(",") | map(gsub("^\\s+|\\s+$"; "")) | map(select(length > 0))) as $A
-      | ($logins | split(",") | map(gsub("^\\s+|\\s+$"; "")) | map(select(length > 0))) as $L
-      | [.items[]
-          | select(((.author_association // "") as $x | $A | index($x) != null)
-                   or ((.user.type // "") == "Bot")
-                   or ((.user.login // "") as $x | $L | index($x) != null))
-          | select((.body // "") | contains($m))
-          | {number, body: (.body // "")}]
-      | sort_by(.number)' 2>/dev/null
+    -f q="\"FORGE:DEFERRED_FROM: #${PARENT}\" in:body repo:${REPO} is:issue" 2>/dev/null \
+  | jq -r --arg assoc "$TRUSTED_ASSOC" --arg logins "$TRUSTED_LOGINS" --arg m "$MARKER" "$JQ_PRE"'
+      [.items[] | select(trusted) | select((.body // "") | contains($m)) | .number] | sort | .[]' 2>/dev/null
+}
+verify_num() { # number -> `number<TAB>state` when a trusted, non-PR issue carries the marker; empty when not (a 404 is "not it")
+  local out rc
+  out=$(gh api "repos/${REPO}/issues/$1" 2>"$TMP/verr"); rc=$?
+  if [ "$rc" -ne 0 ]; then grep -qE '404|Not Found|410' "$TMP/verr" 2>/dev/null && return 0; return 1; fi
+  printf '%s' "$out" | jq -r --arg assoc "$TRUSTED_ASSOC" --arg logins "$TRUSTED_LOGINS" --arg m "$MARKER" "$JQ_PRE"'
+      select(.pull_request == null) | select(trusted) | select((.body // "") | contains($m))
+      | "\(.number)\t\(.state)"' 2>/dev/null
+}
+CANON=""; CSTATE=""
+pick_verified() { # numbers... ; sets CANON/CSTATE from the first verified one; non-zero only when a fetch failed
+  local n r
+  for n in "$@"; do
+    r=$(verify_num "$n") || return 1
+    if [ -n "$r" ]; then CANON="${r%%$'\t'*}"; CSTATE="${r#*$'\t'}"; return 0; fi
+  done
+  return 0
 }
-LOOKUP=""
+find_canonical() {
+  local cands
+  CANON=""; CSTATE=""
+  cands=$(claim_nums) || return 1
+  # shellcheck disable=SC2086
+  pick_verified $cands || return 1
+  [ -n "$CANON" ] && return 0
+  cands=$(search_nums) || return 1
+  # shellcheck disable=SC2086
+  pick_verified $cands || return 1
+  return 0
+}
+LOOKUP_OK=""
 for attempt in 1 2; do
-  LOOKUP=$(do_lookup) && [ -n "$LOOKUP" ] && break
-  LOOKUP=""
+  if find_canonical; then LOOKUP_OK=1; break; fi
   [ "$attempt" = 1 ] && sleep "${FORGE_DEFERRED_RETRY_SLEEP:-2}"
 done
-if [ -z "$LOOKUP" ]; then
+if [ -z "$LOOKUP_OK" ]; then
   echo "DEFERRED_LOOKUP=FAILED"
   exit 3
 fi
@@ -108,27 +142,35 @@ has_entry() { # body-file path : exact checklist-line match, not substring
   grep -F -- "\`$2\`" "$1" | grep -qE "^- \[[ xX]\] \`$(printf '%s' "$2" | sed 's/[][\.*^$+?(){}|]/\\&/g')\`( |\$)"
 }
 
-NUM=$(printf '%s' "$LOOKUP" | jq -r 'first | .number // empty' 2>/dev/null)
-
-if [ -n "$NUM" ]; then
+# Append the missing items to follow-up $1 (state $2), then reopen it when closed (review-finalize needs it open).
+append_items() { # number state ; non-zero = failure
+  local num="$1" state="$2" note=""
   # Re-read the body immediately before editing (parallel builds of the same parent), then append what is missing.
-  if ! gh api "repos/${REPO}/issues/${NUM}" --jq '.body // ""' > "$TMP/body" 2>/dev/null; then
-    echo "DEFERRED_FILE=FAILED"; exit 4
-  fi
+  gh api "repos/${REPO}/issues/${num}" --jq '.body // ""' > "$TMP/body" 2>/dev/null || return 1
   : > "$TMP/add"
   while IFS="$SEP" read -r p w k; do
     has_entry "$TMP/body" "$p" || entry "$p" "$w" "$k" >> "$TMP/add"
   done < "$TMP/items"
+  [ "$state" = "closed" ] && note=" (follow-up was closed; reopened)"
   if [ -s "$TMP/add" ]; then
-    { cat "$TMP/body"; printf '\n### Added by a later build of #%s\n\n' "$PARENT"; cat "$TMP/add"; } > "$TMP/newbody"
+    { cat "$TMP/body"; printf '\n### Added by a later build of #%s%s\n\n' "$PARENT" "$note"; cat "$TMP/add"; } > "$TMP/newbody"
     if [ -n "${DRY_RUN:-}" ]; then
-      echo "DRY_RUN: gh issue edit $NUM (append $(wc -l < "$TMP/add" | tr -d ' ') item(s))" >&2
-    elif ! gh issue edit "$NUM" -R "$REPO" --body-file "$TMP/newbody" >/dev/null 2>&1; then
-      echo "DEFERRED_FILE=FAILED"; exit 4
+      echo "DRY_RUN: gh issue edit $num (append $(wc -l < "$TMP/add" | tr -d ' ') item(s))" >&2
+    else
+      gh issue edit "$num" -R "$REPO" --body-file "$TMP/newbody" >/dev/null 2>&1 || return 1
     fi
   fi
-  while IFS="$SEP" read -r p w k; do printf '%s\t%s\n' "$p" "$NUM"; done < "$TMP/items"
-  finish "$NUM" 0
+  if [ "$state" = "closed" ]; then
+    if [ -n "${DRY_RUN:-}" ]; then echo "DRY_RUN: gh issue reopen $num" >&2
+    else gh issue reopen "$num" -R "$REPO" >/dev/null 2>&1 || return 1; fi
+  fi
+  return 0
+}
+
+if [ -n "$CANON" ]; then
+  append_items "$CANON" "$CSTATE" || { echo "DEFERRED_FILE=FAILED"; exit 4; }
+  while IFS="$SEP" read -r p w k; do printf '%s\t%s\n' "$p" "$CANON"; done < "$TMP/items"
+  finish "$CANON" 0
   exit 0
 fi
 
@@ -154,6 +196,25 @@ else
   NUM=$(printf '%s' "$URL" | grep -oE '[0-9]+$' | tail -n 1)
   [ -n "$NUM" ] || { echo "DEFERRED_FILE=FAILED"; exit 4; }
 fi
+if [ -z "${DRY_RUN:-}" ]; then
+  # Claim the follow-up on the parent (REST comments are read-after-write consistent; best effort, the search still finds it).
+  gh issue comment "$PARENT" -R "$REPO" --body "<!-- FORGE:DEFERRED_ISSUE: #${NUM} -->" >/dev/null 2>&1 \
+    || echo "file-deferred-followups: warning: could not post claim marker for #${NUM} on #${PARENT}" >&2
+  # Post-create re-check: a concurrent build may have filed first. The lowest trusted number is canonical; the newer one is closed.
+  if find_canonical; then
+    if [ -n "$CANON" ] && [ "$CANON" -lt "$NUM" ]; then
+      if append_items "$CANON" "$CSTATE"; then
+        gh issue close "$NUM" -R "$REPO" --reason "not planned" --comment "Duplicate of #${CANON}: deferred items of #${PARENT} are tracked there." >/dev/null 2>&1 \
+          || echo "file-deferred-followups: warning: could not close duplicate #${NUM}" >&2
+        NUM="$CANON"
+      else
+        echo "file-deferred-followups: warning: could not move items to #${CANON}; keeping #${NUM}" >&2
+      fi
+    fi
+  else
+    echo "file-deferred-followups: warning: post-create re-check failed; keeping #${NUM}" >&2
+  fi
+fi
 while IFS="$SEP" read -r p w k; do printf '%s\t%s\n' "$p" "$NUM"; done < "$TMP/items"
 finish "$NUM" 1
 exit 0

package/scripts/file-deferred-followups.test.sh

@@ -8,19 +8,27 @@ S="$HERE/file-deferred-followups.sh"
 T=$(mktemp -d); trap 'rm -rf "$T"' EXIT
 mkdir "$T/bin"
 export FORGE_DEFERRED_RETRY_SLEEP=0
-export GH_LOG="$T/gh.log" SEARCH_JSON="$T/search.json" ISSUE_BODY="$T/issue.body"
+export GH_LOG="$T/gh.log" SEARCH_JSON="$T/search.json" ISSUE_BODY="$T/issue.body" COMMENTS_JSON="$T/comments.json" T_DIR="$T"
 
 cat > "$T/bin/gh" <<'MOCK'
 #!/usr/bin/env bash
 echo "$*" >> "$GH_LOG"
 case "$1 $2" in
   "api -X") [ -n "${LOOKUP_FAIL:-}" ] && exit 1; cat "$SEARCH_JSON"; exit 0 ;;
+  "api --paginate") [ -n "${CLAIM_FAIL:-}" ] && exit 1; cat "$COMMENTS_JSON"; exit 0 ;;
   "issue create") [ -n "${CREATE_FAIL:-}" ] && exit 1; echo "https://github.com/o/r/issues/901"; exit 0 ;;
+  "issue comment") [ -n "${COMMENT_FAIL:-}" ] && exit 1; exit 0 ;;
+  "issue reopen") [ -n "${REOPEN_FAIL:-}" ] && exit 1; exit 0 ;;
+  "issue close") [ -n "${CLOSE_FAIL:-}" ] && exit 1; exit 0 ;;
   "issue edit")
     [ -n "${EDIT_FAIL:-}" ] && exit 1
     while [ $# -gt 0 ]; do [ "$1" = "--body-file" ] && cp "$2" "$ISSUE_BODY.edited"; shift; done; exit 0 ;;
 esac
-case "$1" in api) cat "$ISSUE_BODY"; exit 0 ;; esac
+if [ "$1" = api ]; then
+  case "$*" in *--jq*) cat "$ISSUE_BODY"; exit 0 ;; esac
+  n="${2##*/}"; [ -f "$T_DIR/issue.$n.json" ] && { cat "$T_DIR/issue.$n.json"; exit 0; }
+  echo "HTTP 404: Not Found" >&2; exit 1
+fi
 exit 1
 MOCK
 chmod +x "$T/bin/gh"; export PATH="$T/bin:$PATH"
@@ -29,7 +37,12 @@ PASS=0; FAIL=0
 ok() { PASS=$((PASS+1)); echo "PASS: $1"; }
 bad() { FAIL=$((FAIL+1)); echo "FAIL: $1"; }
 check() { if eval "$2"; then ok "$1"; else bad "$1"; fi; }
-reset() { : > "$GH_LOG"; rm -f "$ISSUE_BODY.edited"; unset LOOKUP_FAIL CREATE_FAIL EDIT_FAIL DRY_RUN; echo '{"items":[]}' > "$SEARCH_JSON"; }
+reset() { : > "$GH_LOG"; rm -f "$ISSUE_BODY.edited" "$T"/issue.*.json; unset LOOKUP_FAIL CREATE_FAIL EDIT_FAIL CLAIM_FAIL COMMENT_FAIL REOPEN_FAIL CLOSE_FAIL DRY_RUN
+  echo '{"items":[]}' > "$SEARCH_JSON"; echo '[]' > "$COMMENTS_JSON"; }
+# issue_json NUMBER STATE ASSOCIATION [PARENT]: REST view of a follow-up carrying the DEFERRED_FROM marker
+issue_json() { printf '{"number":%s,"state":"%s","author_association":"%s","user":{"type":"User"},"body":"<!-- FORGE:DEFERRED_FROM: #%s -->"}\n' "$1" "$2" "$3" "${4:-10}" > "$T/issue.$1.json"; }
+# claim_json ASSOCIATION NUMBER...: parent comments carrying claim markers
+claim_json() { local a="$1" out="" n; shift; for n in "$@"; do out="${out:+$out,}{\"author_association\":\"$a\",\"user\":{\"type\":\"User\"},\"body\":\"<!-- FORGE:DEFERRED_ISSUE: #$n -->\"}"; done; echo "[$out]" > "$COMMENTS_JSON"; }
 printf 'a/x.sh\twhy a\t\nb/y.sh\twhy b\tknown-flaw\nc/z.sh\twhy c\t\n' > "$T/items.tsv"
 
 # 1. N items -> exactly one create
@@ -43,7 +56,7 @@ check "create: telemetry" "printf '%s' \"\$OUT\" | grep -q 'DEFERRED_TELEMETRY d
 # 2. append to existing trusted open issue; exact path match
 reset
 printf '## Affected Files\n- `a/x.sh`\n\n- [ ] `a/x.sh.bak` — other\n<!-- FORGE:DEFERRED_FROM: #10 -->\n' > "$ISSUE_BODY"
-echo '{"items":[{"number":77,"author_association":"OWNER","user":{"type":"User"},"body":"<!-- FORGE:DEFERRED_FROM: #10 -->"}]}' > "$SEARCH_JSON"
+echo '{"items":[{"number":77,"author_association":"OWNER","user":{"type":"User"},"body":"<!-- FORGE:DEFERRED_FROM: #10 -->"}]}' > "$SEARCH_JSON"; issue_json 77 open OWNER
 OUT=$(bash "$S" --parent 10 --repo o/r --items "$T/items.tsv"); RC=$?
 check "append: rc 0, no create" "[ $RC -eq 0 ] && ! grep -q '^issue create' '$GH_LOG'"
 check "append: telemetry batched into 77 filed=0" "printf '%s' \"\$OUT\" | grep -q 'batched_into=#77 filed=0'"
@@ -55,7 +68,7 @@ reset
 printf 'd/w.sh\t\tknown-flaw\n' > "$T/emptywhy.tsv"
 bash "$S" --parent 10 --repo o/r --items "$T/emptywhy.tsv" >/dev/null; RC=$?
 rm -f "$ISSUE_BODY.edited"; printf 'x\n' > "$ISSUE_BODY"
-echo '{"items":[{"number":77,"author_association":"OWNER","user":{"type":"User"},"body":"<!-- FORGE:DEFERRED_FROM: #10 -->"}]}' > "$SEARCH_JSON"
+echo '{"items":[{"number":77,"author_association":"OWNER","user":{"type":"User"},"body":"<!-- FORGE:DEFERRED_FROM: #10 -->"}]}' > "$SEARCH_JSON"; issue_json 77 open OWNER
 OUT=$(bash "$S" --parent 10 --repo o/r --items "$T/emptywhy.tsv"); RC=$?
 check "empty-why: append keeps known-flaw flag" "[ $RC -eq 0 ] && grep -q 'd/w.sh. \*\*known-flaw\*\*' '$ISSUE_BODY.edited'"
 
@@ -66,13 +79,13 @@ check "missing option value: rc 2" "[ $RC -eq 2 ]"
 # 3. already-listed path is not duplicated; nothing to add -> no edit
 reset
 printf -- '- [ ] `a/x.sh` — w\n- [x] `b/y.sh`\n- [ ] `c/z.sh`\n' > "$ISSUE_BODY"
-echo '{"items":[{"number":77,"author_association":"MEMBER","user":{"type":"User"},"body":"<!-- FORGE:DEFERRED_FROM: #10 -->"}]}' > "$SEARCH_JSON"
+echo '{"items":[{"number":77,"author_association":"MEMBER","user":{"type":"User"},"body":"<!-- FORGE:DEFERRED_FROM: #10 -->"}]}' > "$SEARCH_JSON"; issue_json 77 open OWNER
 bash "$S" --parent 10 --repo o/r --items "$T/items.tsv" >/dev/null; RC=$?
 check "idempotent: no edit, no create on resume" "[ $RC -eq 0 ] && ! grep -qE '^issue (edit|create)' '$GH_LOG'"
 
 # 4. untrusted match ignored -> new issue
 reset
-echo '{"items":[{"number":66,"author_association":"NONE","user":{"type":"User"},"body":"<!-- FORGE:DEFERRED_FROM: #10 -->"}]}' > "$SEARCH_JSON"
+echo '{"items":[{"number":66,"author_association":"NONE","user":{"type":"User"},"body":"<!-- FORGE:DEFERRED_FROM: #10 -->"}]}' > "$SEARCH_JSON"; issue_json 66 open NONE; issue_json 77 open OWNER
 OUT=$(bash "$S" --parent 10 --repo o/r --items "$T/items.tsv")
 check "untrusted: ignored, create filed" "grep -q '^issue create' '$GH_LOG' && printf '%s' \"\$OUT\" | grep -q 'filed=1'"
 
@@ -87,7 +100,7 @@ OUT=$(bash "$S" --parent 10 --repo o/r --items "$T/items.tsv"); RC=$?
 check "create failure: rc 4, no telemetry" "[ $RC -eq 4 ] && ! printf '%s' \"\$OUT\" | grep -q DEFERRED_TELEMETRY"
 reset; export EDIT_FAIL=1
 printf 'x\n' > "$ISSUE_BODY"
-echo '{"items":[{"number":77,"author_association":"OWNER","user":{"type":"User"},"body":"<!-- FORGE:DEFERRED_FROM: #10 -->"}]}' > "$SEARCH_JSON"
+echo '{"items":[{"number":77,"author_association":"OWNER","user":{"type":"User"},"body":"<!-- FORGE:DEFERRED_FROM: #10 -->"}]}' > "$SEARCH_JSON"; issue_json 77 open OWNER
 OUT=$(bash "$S" --parent 10 --repo o/r --items "$T/items.tsv"); RC=$?
 check "edit failure: rc 4, no telemetry" "[ $RC -eq 4 ] && ! printf '%s' \"\$OUT\" | grep -q DEFERRED_TELEMETRY"
 
@@ -102,5 +115,85 @@ OUT=$(printf '' | bash "$S" --parent 10 --repo o/r)
 check "empty: no create, deferred_items=0" "! grep -q '^issue create' '$GH_LOG' && printf '%s' \"\$OUT\" | grep -q 'deferred_items=0'"
 bash "$S" --repo o/r >/dev/null 2>&1; check "usage: missing parent rc 2" "[ $? -eq 2 ]"
 
+# 9. e1: search index lags (returns nothing) but the claim marker on the parent finds the follow-up
+reset
+printf 'x\n' > "$ISSUE_BODY"; issue_json 77 open OWNER; claim_json OWNER 77
+OUT=$(bash "$S" --parent 10 --repo o/r --items "$T/items.tsv"); RC=$?
+check "index-lag: claim found, no duplicate create" "[ $RC -eq 0 ] && ! grep -q '^issue create' '$GH_LOG' && printf '%s' \"\$OUT\" | grep -q 'batched_into=#77 filed=0'"
+check "index-lag: search never needed" "! grep -q search/issues '$GH_LOG'"
+
+# 10. e2: closed follow-up (found only by a state-all search) is reused and reopened
+reset
+printf 'x\n' > "$ISSUE_BODY"; issue_json 77 closed OWNER
+echo '{"items":[{"number":77,"author_association":"OWNER","user":{"type":"User"},"body":"<!-- FORGE:DEFERRED_FROM: #10 -->"}]}' > "$SEARCH_JSON"
+OUT=$(bash "$S" --parent 10 --repo o/r --items "$T/items.tsv"); RC=$?
+check "closed: reused, reopened, no create" "[ $RC -eq 0 ] && grep -q '^issue reopen 77' '$GH_LOG' && ! grep -q '^issue create' '$GH_LOG'"
+check "closed: search query covers all states" "grep -q 'is:issue' '$GH_LOG' && ! grep 'api -X' '$GH_LOG' | grep -q 'is:open'"
+reset; export REOPEN_FAIL=1
+printf 'x\n' > "$ISSUE_BODY"; issue_json 77 closed OWNER; claim_json OWNER 77
+bash "$S" --parent 10 --repo o/r --items "$T/items.tsv" >/dev/null; RC=$?
+check "closed: reopen failure is rc 4" "[ $RC -eq 4 ]"
+
+# 11. create posts the claim marker on the parent
+reset
+OUT=$(bash "$S" --parent 10 --repo o/r --items "$T/items.tsv"); RC=$?
+check "claim: posted on parent after create" "[ $RC -eq 0 ] && grep -q '^issue comment 10 .*FORGE:DEFERRED_ISSUE: #901' '$GH_LOG'"
+reset; export COMMENT_FAIL=1
+OUT=$(bash "$S" --parent 10 --repo o/r --items "$T/items.tsv" 2>/dev/null); RC=$?
+check "claim: post failure only warns" "[ $RC -eq 0 ] && printf '%s' \"\$OUT\" | grep -q 'batched_into=#901 filed=1'"
+
+# 12. e3: a concurrent build filed #800 first; the first claim read (before our create) misses it, the post-create re-check sees it
+race_setup() { # first comments read returns [] (the other build has not claimed yet), later reads return the real listing
+  CALLS="$T/claim.calls"; : > "$CALLS"; export CALLS
+  mv "$T/bin/gh" "$T/bin/gh.real"
+  cat > "$T/bin/gh" <<'WRAP'
+#!/usr/bin/env bash
+if [ "$1 $2" = "api --paginate" ]; then
+  echo x >> "$CALLS"
+  if [ "$(wc -l < "$CALLS" | tr -d ' ')" -le 1 ]; then echo "$*" >> "$GH_LOG"; echo '[]'; exit 0; fi
+fi
+exec "$(dirname "$0")/gh.real" "$@"
+WRAP
+  chmod +x "$T/bin/gh"
+}
+race_teardown() { mv "$T/bin/gh.real" "$T/bin/gh"; }
+reset
+printf 'x\n' > "$ISSUE_BODY"; issue_json 800 open OWNER; claim_json OWNER 800
+race_setup
+OUT=$(bash "$S" --parent 10 --repo o/r --items "$T/items.tsv"); RC=$?
+race_teardown
+check "race: newer duplicate closed, items moved to lowest" "[ $RC -eq 0 ] && grep -q '^issue close 901' '$GH_LOG' && grep -q '^issue edit 800' '$GH_LOG' && printf '%s' \"\$OUT\" | grep -q 'batched_into=#800 filed=1'"
+check "race: rows point at canonical issue" "[ \$(printf '%s\n' \"\$OUT\" | grep -c '800\$') -ge 3 ]"
+# e5: duplicate-close failure only warns
+reset; export CLOSE_FAIL=1
+printf 'x\n' > "$ISSUE_BODY"; issue_json 800 open OWNER; claim_json OWNER 800
+race_setup
+OUT=$(bash "$S" --parent 10 --repo o/r --items "$T/items.tsv" 2>/dev/null); RC=$?
+race_teardown
+check "race: duplicate-close failure still rc 0" "[ $RC -eq 0 ] && printf '%s' \"\$OUT\" | grep -q 'batched_into=#800'"
+
+# 13. e4: claim listing failure is a failed lookup, never "none found"
+reset; export CLAIM_FAIL=1
+OUT=$(bash "$S" --parent 10 --repo o/r --items "$T/items.tsv"); RC=$?
+check "claim-failure: rc 3, no create" "[ $RC -eq 3 ] && printf '%s' \"\$OUT\" | grep -q DEFERRED_LOOKUP=FAILED && ! grep -q '^issue create' '$GH_LOG'"
+
+# 15. e6: untrusted claim comment is ignored (falls through to create)
+reset
+issue_json 77 open OWNER; claim_json NONE 77
+OUT=$(bash "$S" --parent 10 --repo o/r --items "$T/items.tsv" 2>/dev/null); RC=$?
+check "untrusted-claim: ignored, create filed" "[ $RC -eq 0 ] && grep -q '^issue create' '$GH_LOG'"
+
+# 16. e7: several claims, lowest valid wins; a claim to a missing issue falls through
+reset
+printf 'x\n' > "$ISSUE_BODY"; issue_json 80 open OWNER; issue_json 90 open OWNER; claim_json OWNER 70 90 80
+OUT=$(bash "$S" --parent 10 --repo o/r --items "$T/items.tsv"); RC=$?
+check "multi-claim: lowest valid (80) wins, missing 70 skipped" "[ $RC -eq 0 ] && printf '%s' \"\$OUT\" | grep -q 'batched_into=#80 filed=0'"
+
+# 17. DRY_RUN: no claim, reopen or close
+reset; export DRY_RUN=1
+printf 'x\n' > "$ISSUE_BODY"; issue_json 77 closed OWNER; claim_json OWNER 77
+bash "$S" --parent 10 --repo o/r --items "$T/items.tsv" >/dev/null 2>&1
+check "dry-run: no comment/reopen/close/edit" "! grep -qE '^issue (comment|reopen|close|edit|create)' '$GH_LOG'"
+
 echo "passed=$PASS failed=$FAIL"
 [ "$FAIL" -eq 0 ]

package/scripts/fixtures/ci-local/azure/azure-pipelines.yml

@@ -0,0 +1,11 @@
+pr:
+  branches:
+    include: [main]
+pool:
+  vmImage: ubuntu-latest
+variables:
+  greeting: hi
+steps:
+  - script: echo "$(greeting) azure"
+    displayName: say
+  - task: SomeUnknownTask@1

package/scripts/fixtures/ci-local/bitbucket/bitbucket-pipelines.yml

@@ -0,0 +1,12 @@
+image: atlassian/default-image:4
+pipelines:
+  default:
+    - step:
+        name: Build and test
+        script:
+          - echo bitbucket-test
+    - step:
+        name: Deploy
+        deployment: production
+        script:
+          - echo deploy

package/scripts/fixtures/ci-local/buildkite/.buildkite/pipeline.yml

@@ -0,0 +1,10 @@
+steps:
+  - label: ":hammer: test"
+    command: echo bk-test
+  - wait
+  - label: "docker plugin"
+    command: echo bk-docker
+    plugins:
+      - docker#v5.0.0:
+          image: node:20
+  - block: ":rocket: release"

package/scripts/fixtures/ci-local/circleci/.circleci/config.yml

@@ -0,0 +1,29 @@
+version: 2.1
+jobs:
+  test:
+    docker:
+      - image: cimg/base:stable
+    steps:
+      - checkout
+      - run:
+          name: unit
+          command: echo circle-unit
+  secured:
+    docker:
+      - image: cimg/base:stable
+    steps:
+      - run: echo secured
+  deploy-prod:
+    docker:
+      - image: cimg/base:stable
+    steps:
+      - run: echo deploy
+workflows:
+  main:
+    jobs:
+      - test
+      - secured:
+          context: org-secrets
+          requires: [test]
+      - deploy-prod:
+          requires: [test]

package/scripts/fixtures/ci-local/gha-basic/.github/workflows/ci.yml

@@ -0,0 +1,59 @@
+name: CI
+on:
+  pull_request:
+    branches: [main]
+    paths:
+      - 'src/**'
+      - 'tests/**'
+      - '.github/workflows/ci.yml'
+jobs:
+  unit:
+    name: Unit
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-node@v4
+        with:
+          node-version: 20
+      - name: unit tests
+        run: bash tests/greet.test.sh
+  matrix:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [ubuntu-latest, macos-14, windows-latest]
+    steps:
+      - uses: actions/checkout@v4
+      - run: echo matrix-ok
+  withservice:
+    runs-on: ubuntu-latest
+    services:
+      postgres:
+        image: postgres:16
+        env:
+          POSTGRES_PASSWORD: example
+        ports:
+          - 5432:5432
+        options: --health-cmd pg_isready --health-interval 5s
+    steps:
+      - run: echo service-ok
+  needsecret:
+    runs-on: ubuntu-latest
+    env:
+      API_TOKEN: ${{ secrets.API_TOKEN }}
+    steps:
+      - run: echo uses-secret
+  unknownaction:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: some-org/mystery-action@v9
+  deployprod:
+    runs-on: ubuntu-latest
+    environment: production
+    steps:
+      - run: echo deploy
+  conditional:
+    runs-on: ubuntu-latest
+    if: github.event_name == 'push'
+    steps:
+      - run: echo push-only

package/scripts/fixtures/ci-local/gha-basic/.github/workflows/docs.yml

@@ -0,0 +1,10 @@
+name: Docs
+on:
+  pull_request:
+    paths:
+      - 'docs/**'
+jobs:
+  docs:
+    runs-on: ubuntu-latest
+    steps:
+      - run: echo docs-ok

package/scripts/fixtures/ci-local/gha-basic/.github/workflows/nightly.yml

@@ -0,0 +1,10 @@
+name: Nightly
+on:
+  schedule:
+    - cron: '0 3 * * *'
+  workflow_dispatch:
+jobs:
+  nightly:
+    runs-on: ubuntu-latest
+    steps:
+      - run: echo nightly

package/scripts/fixtures/ci-local/gha-basic/docs/index.md

@@ -0,0 +1 @@
+# docs

package/scripts/fixtures/ci-local/gha-basic/src/greet.sh

@@ -0,0 +1,2 @@
+#!/usr/bin/env bash
+greet() { echo "hello $1"; }

package/scripts/fixtures/ci-local/gha-basic/tests/greet.test.sh

@@ -0,0 +1,3 @@
+#!/usr/bin/env bash
+. "$(dirname "$0")/../src/greet.sh"
+[ "$(greet x)" = "hello x" ] || exit 1

package/scripts/fixtures/ci-local/gha-dorny/.github/workflows/ci.yml

@@ -0,0 +1,51 @@
+name: CI
+on:
+  pull_request:
+    branches: [staging, main]
+jobs:
+  changes:
+    runs-on: ubuntu-latest
+    outputs:
+      backend: ${{ steps.filter.outputs.backend }}
+      frontend: ${{ steps.filter.outputs.frontend }}
+    steps:
+      - uses: actions/checkout@v4
+      - uses: dorny/paths-filter@v3
+        id: filter
+        with:
+          filters: |
+            backend:
+              - 'backend/**'
+            frontend:
+              - 'frontend/**'
+  backend:
+    needs: changes
+    if: needs.changes.outputs.backend == 'true'
+    runs-on: ubuntu-latest
+    services:
+      redis:
+        image: redis:7
+        ports:
+          - 6379:6379
+    steps:
+      - run: echo backend-ok
+  frontend:
+    needs: changes
+    if: needs.changes.outputs.frontend == 'true'
+    runs-on: ubuntu-latest
+    steps:
+      - run: echo frontend-ok
+  integration:
+    needs: [changes]
+    if: needs.changes.outputs.backend == 'true'
+    runs-on: ubuntu-latest
+    env:
+      DB_PASSWORD: ${{ secrets.DB_PASSWORD }}
+    steps:
+      - run: echo integration
+  deploy-staging:
+    needs: [backend]
+    runs-on: ubuntu-latest
+    environment: staging
+    steps:
+      - run: echo deploy

package/scripts/fixtures/ci-local/gha-dorny/backend/app.py

@@ -0,0 +1 @@
+print("a")

package/scripts/fixtures/ci-local/gha-dorny/docs/a.md

@@ -0,0 +1 @@
+# d

package/scripts/fixtures/ci-local/gha-dorny/frontend/app.js

@@ -0,0 +1 @@
+console.log(1)

package/scripts/fixtures/ci-local/gha-run/.github/workflows/ci.yml

@@ -0,0 +1,39 @@
+name: CI
+on:
+  pull_request:
+jobs:
+  first:
+    runs-on: ubuntu-latest
+    steps:
+      - run: echo first >> "$RUN_LOG"
+      - name: reads GITHUB_ENV
+        run: |
+          echo "FROM_STEP=carried" >> "$GITHUB_ENV"
+      - run: test "$FROM_STEP" = carried
+    env:
+      RUN_LOG: run.log
+  second:
+    needs: first
+    runs-on: ubuntu-latest
+    steps:
+      - run: echo second
+  failing:
+    runs-on: ubuntu-latest
+    steps:
+      - run: |
+          echo "about to fail"
+          exit 3
+  skippedbyneed:
+    needs: failing
+    runs-on: ubuntu-latest
+    steps:
+      - run: echo never
+  slow:
+    runs-on: ubuntu-latest
+    timeout-minutes: 1
+    steps:
+      - run: echo slow-ok
+  broken-install:
+    runs-on: ubuntu-latest
+    steps:
+      - run: npm ci --no-audit --registry=http://127.0.0.1:9

package/scripts/fixtures/ci-local/gha-run/src/x.txt

@@ -0,0 +1 @@
+ok

package/scripts/fixtures/ci-local/gitlab/.gitlab-ci.yml

@@ -0,0 +1,43 @@
+stages: [build, test, deploy]
+variables:
+  GREETING: hello
+.template:
+  script:
+    - echo template
+lint:
+  stage: test
+  image: node:20
+  rules:
+    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
+      changes:
+        - src/**/*
+  script:
+    - echo "$GREETING lint"
+unit:
+  stage: test
+  extends: .template
+  only:
+    - merge_requests
+withdb:
+  stage: test
+  services:
+    - postgres:16
+  script:
+    - echo db
+manualjob:
+  stage: test
+  when: manual
+  script:
+    - echo manual
+release:
+  stage: deploy
+  environment: production
+  script:
+    - echo release
+docsonly:
+  stage: test
+  rules:
+    - changes:
+        - docs/**/*
+  script:
+    - echo docs