forgedock 1.32.1 → 1.34.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/engine/outcome-codes.mjs +7 -3
- package/bin/engine/phases.mjs +3 -3
- package/bin/engine-cli.mjs +56 -11
- package/bin/forgedock.mjs +21 -1
- package/bin/lean/basesync.mjs +98 -0
- package/bin/lean/basesync.test.mjs +151 -0
- package/bin/lean/batch.test.mjs +103 -2
- package/bin/lean/budget.mjs +7 -1
- package/bin/lean/budget.test.mjs +9 -1
- package/bin/lean/conformance.mjs +46 -10
- package/bin/lean/conformance.test.mjs +50 -2
- package/bin/lean/depcache.mjs +23 -3
- package/bin/lean/depcache.test.mjs +42 -1
- package/bin/lean/effects.mjs +1 -1
- package/bin/lean/execution.mjs +102 -12
- package/bin/lean/execution.test.mjs +101 -1
- package/bin/lean/hostkill.mjs +8 -0
- package/bin/lean/lean.mjs +49 -21
- package/bin/lean/lean.test.mjs +133 -7
- package/bin/lean/lightlane.mjs +38 -13
- package/bin/lean/lightlane.test.mjs +35 -1
- package/bin/lean/orchestrate.mjs +63 -17
- package/bin/lean/prompts.mjs +19 -9
- package/bin/lean/recovery.test.mjs +26 -0
- package/bin/lean/sandbox-provider.contract.test.mjs +74 -1
- package/bin/lean/sandbox-remote.mjs +40 -7
- package/bin/lean/seqlease.mjs +49 -2
- package/bin/lean/seqlease.test.mjs +14 -0
- package/bin/lean/sequence.mjs +54 -6
- package/bin/lean/sequence.test.mjs +83 -1
- package/bin/pool/agent.mjs +435 -0
- package/bin/pool/cli.mjs +110 -0
- package/bin/pool/docker.mjs +56 -0
- package/bin/pool/http.mjs +55 -0
- package/bin/pool/router.mjs +299 -0
- package/bin/pool/store.mjs +25 -0
- package/bin/pool/version.mjs +44 -0
- package/bin/review-staging.mjs +14 -2
- package/bin/runner.mjs +1 -0
- package/bin/stats/aggregate.mjs +6 -0
- package/bin/stats/efficiency.mjs +103 -17
- package/bin/stats/efficiency.test.mjs +103 -4
- package/bin/stats/index.mjs +40 -6
- package/bin/stats/local-source.mjs +1 -1
- package/bin/stats/pool.mjs +51 -0
- package/bin/stats/pool.test.mjs +39 -0
- package/bin/stats/query.mjs +2 -2
- package/bin/stats/stats.test.mjs +15 -1
- package/commands/orchestrate.md +3 -3
- package/commands/review-pr-agents/protocols.md +1 -1
- package/commands/review-pr-staging.md +1 -1
- package/docs/CONFIG.md +23 -9
- package/docs/OUTCOME-CODES.md +5 -3
- package/docs/STATS.md +36 -1
- package/package.json +2 -1
- package/pool/README.md +13 -0
- package/pool/image/Dockerfile +46 -0
- package/pool/image/claude-shim +8 -0
- package/pool/image/cs-cred +12 -0
- package/pool/image/entrypoint.sh +61 -0
- package/pool/image/gh-shim +6 -0
- package/pool/install/install-agent-linux.sh +53 -0
- package/pool/install/install-agent.ps1 +225 -0
- package/pool/install/install-agent.sh +101 -0
- package/pool/install/install-router.sh +67 -0
- package/pool/install/keepalive.ps1 +36 -0
- package/pool/install/uninstall-agent.ps1 +20 -0
- package/scripts/hotspots.sh +33 -21
- package/scripts/hotspots.test.sh +22 -3
- package/scripts/review-finalize.sh +17 -1
- package/scripts/review-finalize.test.sh +41 -0
- package/scripts/review-pr-domain-map.test.sh +2 -2
- package/scripts/review-pr-staging-scope.test.sh +4 -2
- package/scripts/review-prepare.sh +62 -27
- package/scripts/review-prepare.test.sh +53 -0
- package/scripts/review-staging-pre.sh +29 -9
- package/scripts/review-staging-pre.test.sh +18 -0
package/scripts/hotspots.test.sh
CHANGED
|
@@ -27,7 +27,7 @@ echo y >> big.py; lines small.py 10; commit c2
|
|
|
27
27
|
OUT="$(bash "$S" big.py ./small.py missing/gone.py)"; rc=$?
|
|
28
28
|
eq "rc" "$rc" 0
|
|
29
29
|
eq "defaults" "$(q '[.enabled,.file_lines,.growth_lines]')" '[true,2000,150]'
|
|
30
|
-
eq "big facts" "$(q '.files[0]')" '{"path":"big.py","lines":2102,"commits_90d":2,"hotspot":4204,"over":true,"excluded":false,"base_lines":null,"growth":null}'
|
|
30
|
+
eq "big facts" "$(q '.files[0]')" '{"path":"big.py","lines":2102,"commits_90d":2,"hotspot":4204,"over":true,"generated":false,"excluded":false,"base_lines":null,"growth":null}'
|
|
31
31
|
eq "small facts (./ stripped)" "$(q '.files[1] | [.path,.lines,.commits_90d,.over]')" '["small.py",10,1,false]'
|
|
32
32
|
eq "missing file" "$(q '.files[2] | [.path,.lines,.commits_90d,.over]')" '["missing/gone.py",0,0,false]'
|
|
33
33
|
OUT="$(bash "$S")"; eq "no paths" "$(q '.files')" '[]'
|
|
@@ -49,6 +49,13 @@ printf 'gen/** linguist-generated\nvendor/** linguist-vendored=true\n' > .gitatt
|
|
|
49
49
|
commit gen
|
|
50
50
|
OUT="$(bash "$S" gen/api.py vendor/lib/x.js dist/bundle.js big.py)"
|
|
51
51
|
eq "attr exclusions" "$(q '[.files[] | .excluded]')" '[true,true,false,false]'
|
|
52
|
+
eq "attr generated" "$(q '[.files[] | .generated]')" '[true,false,false,false]'
|
|
53
|
+
# forge#4519: a generated-file header in the first 5 lines marks the file generated
|
|
54
|
+
{ echo '# @generated'; i=0; while [ $i -lt 2500 ]; do echo "l$i"; i=$((i+1)); done; } > hdr.py
|
|
55
|
+
{ i=0; while [ $i -lt 2500 ]; do echo "l$i"; i=$((i+1)); done; echo '# DO NOT EDIT'; } > late.py
|
|
56
|
+
OUT="$(bash "$S" hdr.py late.py)"
|
|
57
|
+
rm -f hdr.py late.py
|
|
58
|
+
eq "header generated" "$(q '[.files[] | [.generated,.excluded]]')" '[[true,true],[false,false]]'
|
|
52
59
|
if [ "$HAVE_YQ" = 1 ]; then
|
|
53
60
|
printf 'maintenance:\n hotspot:\n file_lines: 100\n growth_lines: 5\n exclude_globs: ["dist/*", "*.lock"]\n' > forge.yaml
|
|
54
61
|
OUT="$(bash "$S" dist/bundle.js small.py big.py)"
|
|
@@ -60,7 +67,19 @@ if [ "$HAVE_YQ" = 1 ]; then
|
|
|
60
67
|
eq "opt-out" "$(q '[.enabled,.files]')" '[false,[]]'
|
|
61
68
|
printf 'maintenance:\n hotspot:\n file_lines: nope\n' > forge.yaml
|
|
62
69
|
OUT="$(bash "$S" big.py)"; eq "bad value falls back" "$(q '.file_lines')" 2000
|
|
63
|
-
|
|
70
|
+
# forge#4519: maintenance.generated_paths marks bot-owned files generated (and so excluded), even over the threshold
|
|
71
|
+
mkdir -p web/public/openapi-versions
|
|
72
|
+
lines web/public/openapi.json 2500; lines web/public/openapi-versions/3.29.2.json 2500
|
|
73
|
+
printf 'maintenance:\n generated_paths: ["web/public/openapi*.json", "web/public/openapi-versions/*"]\n' > forge.yaml
|
|
74
|
+
OUT="$(bash "$S" web/public/openapi.json web/public/openapi-versions/3.29.2.json big.py gen/api.py)"
|
|
75
|
+
eq "generated_paths reported" "$(q '.generated_paths')" '["web/public/openapi*.json","web/public/openapi-versions/*"]'
|
|
76
|
+
eq "generated_paths: generated" "$(q '[.files[] | .generated]')" '[true,true,false,true]'
|
|
77
|
+
eq "generated_paths: excluded" "$(q '[.files[] | .excluded]')" '[true,true,false,true]'
|
|
78
|
+
printf 'maintenance:\n generated_paths: ["*.json"]\n hotspot:\n enabled: false\n' > forge.yaml
|
|
79
|
+
OUT="$(bash "$S" big.py)"; eq "generated_paths with the gate off" "$(q '[.enabled,.generated_paths,.files]')" '[false,["*.json"],[]]'
|
|
80
|
+
printf 'maintenance:\n generated_paths: nope\n' > forge.yaml
|
|
81
|
+
OUT="$(bash "$S" big.py)"; eq "bad generated_paths falls back" "$(q '.generated_paths')" '[]'
|
|
82
|
+
rm -rf forge.yaml web
|
|
64
83
|
else
|
|
65
84
|
echo "SKIP: forge.yaml cases need mikefarah yq v4"
|
|
66
85
|
fi
|
|
@@ -93,7 +112,7 @@ done
|
|
|
93
112
|
eq "30k exact commits" "$(q '[.files[] | .commits_90d]')" '[300,60,60,60]'
|
|
94
113
|
eq "30k exact lines" "$(q '[.files[] | .lines]')" '[1,1,1,1]'
|
|
95
114
|
if [ "$best" -lt 1000 ]; then ok; else bad "30k-commit run took ${best} ms (limit 1000)"; fi
|
|
96
|
-
grep -q '"rev-list"' "$TMP/trace.json"
|
|
115
|
+
if grep -q '"rev-list"' "$TMP/trace.json"; then ok; else bad "git trace not captured"; fi
|
|
97
116
|
if grep -qE '"(cat-file|show)"' "$TMP/trace.json"; then bad "blob read without --base"; else ok; fi
|
|
98
117
|
if [ -e "$TMP/net.log" ]; then bad "network tool invoked: $(tr '\n' ' ' < "$TMP/net.log")"; else ok; fi
|
|
99
118
|
|
|
@@ -942,10 +942,12 @@ do_file_prepare() {
|
|
|
942
942
|
# Title (defense in depth: strip command substitution so the raw title stays readable if it round-trips through an eval-based consumer).
|
|
943
943
|
ISSUE_TITLE="fix: ${F_TITLE} (review finding — PR #${PR_NUMBER})"
|
|
944
944
|
ISSUE_TITLE=$(printf '%s' "$ISSUE_TITLE" | tr '\t\n`' " '" | sed 's/\$(/$ (/g')
|
|
945
|
+
# The title is passed WITHOUT the constant "(review finding — PR #N)" suffix: its tokens (review, finding, the PR number) would make
|
|
946
|
+
# any two findings of one PR a >=3-token match and map a finding to its neighbour's issue (#4569).
|
|
945
947
|
# Step 0: deterministic title dedup (scripts/issue-dedup.sh; also excludes the issue this PR fixes: it is about to be closed, so a
|
|
946
948
|
# finding matched to it is tracked nowhere). Unresolved helper: "not deduped"; NEVER exit 1 (which means "duplicate, skip").
|
|
947
949
|
if [ -n "$DEDUP_SCRIPT" ]; then
|
|
948
|
-
DEDUP_RESULT=$(bash "$DEDUP_SCRIPT" "$
|
|
950
|
+
DEDUP_RESULT=$(bash "$DEDUP_SCRIPT" "fix: ${F_TITLE}" "-R $REPO" ${MERGE_ISSUE:+--exclude "$MERGE_ISSUE"} 2>&1); DEDUP_EXIT=$?
|
|
949
951
|
else
|
|
950
952
|
DEDUP_RESULT=""; DEDUP_EXIT=0
|
|
951
953
|
fi
|
|
@@ -965,6 +967,20 @@ do_file_prepare() {
|
|
|
965
967
|
else
|
|
966
968
|
# A CLOSED review-finding at the same location +-5 lines is a regression: create with a warning and elevate to priority:P1.
|
|
967
969
|
[ -n "$F_FILE" ] && REGRESSION=$(line_match "$CLOSED_JSON" "$F_FILE" "$F_LINE" "$F_TITLE" 0)
|
|
970
|
+
# DELTA: the earlier pass's finding is credited and its own issue is closed (fixed): resolved, never re-filed (#4569).
|
|
971
|
+
# Only an issue this PR's review filed counts (title suffix `PR #N)` or a `**Source**: PR #N` line); a closed finding of
|
|
972
|
+
# another PR at the same location stays a regression, or any closed issue nearby could drop a new blocker from the verdict.
|
|
973
|
+
OWN_CLOSED=""
|
|
974
|
+
if [ -n "$REGRESSION" ] && [ "$REVIEW_MODE" = "DELTA" ]; then
|
|
975
|
+
OWN_CLOSED=$(printf '%s' "$CLOSED_JSON" | jq -r --argjson n "$REGRESSION" --arg pr "$PR_NUMBER" '
|
|
976
|
+
.[] | select(.number == $n)
|
|
977
|
+
| select((.title // "" | endswith("PR #" + $pr + ")")) or ((.body // "") | test("(^|\\n)\\*\\*Source\\*\\*: PR #" + $pr + "([^0-9]|$)")))
|
|
978
|
+
| .number' 2>/dev/null | head -1)
|
|
979
|
+
fi
|
|
980
|
+
if [ -n "$OWN_CLOSED" ]; then
|
|
981
|
+
ACTION="skip-resolved"; DEDUP_OF="$REGRESSION"; REGRESSION=""
|
|
982
|
+
echo "RESOLVED: ${F_FILE}:${F_LINE} was fixed in closed issue #${DEDUP_OF} (delta pass)"
|
|
983
|
+
fi
|
|
968
984
|
[ -z "$REGRESSION" ] || echo "REGRESSION: Previously fixed in #${REGRESSION} — elevating priority"
|
|
969
985
|
fi
|
|
970
986
|
fi
|
|
@@ -738,6 +738,47 @@ mkfind G-1 CONFIRMED MEDIUM a.py 10 "Duplicate of the linked issue"; putfind 1
|
|
|
738
738
|
jq '.findings |= map(. + {class: "ISSUE", disposition: "ISSUE x"})' "$WORK/out/findings.json" > "$WORK/out/classified.json"
|
|
739
739
|
run "$GOOD" file-prepare; want "linked issue excluded" '.items[0].action' worklist.json "file"
|
|
740
740
|
|
|
741
|
+
# #4569: the title suffix never makes two findings of one PR dedup to each other; a DELTA closed match is resolved, a FULL one a regression
|
|
742
|
+
fp_setup
|
|
743
|
+
echo '[{"number":60,"title":"fix: credential helper hands token to any host (review finding — PR #7)","body":"see z.py:90"}]' > "$FIX/issues-open.json"
|
|
744
|
+
echo '[{"number":61,"title":"fix: DCO exemption only on staging (review finding — PR #7)","body":"see c.py:5"}]' > "$FIX/issues-closed.json"
|
|
745
|
+
mkfind K-1 CONFIRMED HIGH c.py 5 "DCO exemption only on staging"
|
|
746
|
+
putfind 1
|
|
747
|
+
jq '.findings |= map(. + {class: "ISSUE", disposition: "ISSUE x"})' "$WORK/out/findings.json" > "$WORK/out/classified.json"
|
|
748
|
+
run "$GOOD" file-prepare
|
|
749
|
+
want "full mode: closed own issue is a regression, not skipped" '.items[0] | [.action, (.regression_of | tostring)] | join(",")' worklist.json "file,61"
|
|
750
|
+
fp_setup; mkctx 1 DELTA "$OLD_SHA" 12
|
|
751
|
+
echo '[{"number":60,"title":"fix: credential helper hands token to any host (review finding — PR #7)","body":"see z.py:90"}]' > "$FIX/issues-open.json"
|
|
752
|
+
echo '[{"number":61,"title":"fix: DCO exemption only on staging (review finding — PR #7)","body":"see c.py:5"}]' > "$FIX/issues-closed.json"
|
|
753
|
+
mkfind K-1 CONFIRMED HIGH c.py 5 "DCO exemption only on staging"; putfind 1
|
|
754
|
+
jq '.findings |= map(. + {class: "ISSUE", disposition: "ISSUE x"})' "$WORK/out/findings.json" > "$WORK/out/classified.json"
|
|
755
|
+
run "$GOOD" file-prepare
|
|
756
|
+
want "delta mode: closed own issue is resolved by that issue" '.items[0] | [.action, (.dedup_of | tostring), (.regression_of | tostring)] | join(",")' worklist.json "skip-resolved,61,null"
|
|
757
|
+
# a closed finding of ANOTHER PR at the same location is no proof this finding was fixed: it stays a regression and blocks
|
|
758
|
+
fp_setup; mkctx 1 DELTA "$OLD_SHA" 12
|
|
759
|
+
echo '[]' > "$FIX/issues-open.json"
|
|
760
|
+
echo '[{"number":62,"title":"fix: unrelated old defect (review finding — PR #3)","body":"see c.py:7"}]' > "$FIX/issues-closed.json"
|
|
761
|
+
mkfind K-3 CONFIRMED HIGH c.py 5 "Brand new defect"; putfind 1
|
|
762
|
+
jq '.findings |= map(. + {class: "ISSUE", disposition: "ISSUE x"})' "$WORK/out/findings.json" > "$WORK/out/classified.json"
|
|
763
|
+
run "$GOOD" file-prepare
|
|
764
|
+
want "delta mode: another PR's closed issue is a regression, never resolved" '.items[0] | [.action, (.regression_of | tostring)] | join(",")' worklist.json "file,62"
|
|
765
|
+
# a **Source**: PR #N line marks the issue as this PR's own
|
|
766
|
+
fp_setup; mkctx 1 DELTA "$OLD_SHA" 12
|
|
767
|
+
echo '[]' > "$FIX/issues-open.json"
|
|
768
|
+
printf '[{"number":63,"title":"fix: renamed title","body":"**Source**: PR #7 — x\\nsee c.py:5"}]' > "$FIX/issues-closed.json"
|
|
769
|
+
mkfind K-4 CONFIRMED HIGH c.py 5 "Renamed defect"; putfind 1
|
|
770
|
+
jq '.findings |= map(. + {class: "ISSUE", disposition: "ISSUE x"})' "$WORK/out/findings.json" > "$WORK/out/classified.json"
|
|
771
|
+
run "$GOOD" file-prepare
|
|
772
|
+
want "delta mode: Source line marks own closed issue" '.items[0] | [.action, (.dedup_of | tostring)] | join(",")' worklist.json "skip-resolved,63"
|
|
773
|
+
# the dedup helper receives the title without the shared suffix
|
|
774
|
+
DEDUP_LOG="$WORK/dedup-args"; : > "$DEDUP_LOG"
|
|
775
|
+
printf '#!/usr/bin/env bash\nprintf "%%s\\n" "$1" >> "%s"\nexit 0\n' "$DEDUP_LOG" > "$GOOD/scripts/issue-dedup.sh"
|
|
776
|
+
fp_setup; mkfind K-2 CONFIRMED MEDIUM d.py 7 "Some other defect"; putfind 1
|
|
777
|
+
jq '.findings |= map(. + {class: "ISSUE", disposition: "ISSUE x"})' "$WORK/out/findings.json" > "$WORK/out/classified.json"
|
|
778
|
+
run "$GOOD" file-prepare
|
|
779
|
+
[ "$(cat "$DEDUP_LOG")" = "fix: Some other defect" ] && ok || bad "dedup helper gets the suffix-less title: '$(cat "$DEDUP_LOG")'"
|
|
780
|
+
mkroot "$GOOD"
|
|
781
|
+
|
|
741
782
|
# pattern recurrence: exempt severities skip, a prior instance in the same subsystem consolidates
|
|
742
783
|
fp_setup
|
|
743
784
|
echo '[{"number":50,"body":"x commands/work-on/z.md <!-- FORGE:PATTERN: slug-one -->","labels":[],"stateReason":"COMPLETED"}]' > "$FIX/issues-all.json"
|
|
@@ -93,7 +93,7 @@ done
|
|
|
93
93
|
if grep -qF 'marker domain' "$PERSONAS/protocols.md"; then ok; else bad "protocols.md does not define the marker domain"; fi
|
|
94
94
|
|
|
95
95
|
# Test 7: the roster regexes in delta_roster must equal the per-domain file regexes of the diff slices (both in scripts/review-prepare.sh since #3841),
|
|
96
|
-
# and the slice function must fall back to the full file list
|
|
96
|
+
# and the slice function must NOT fall back to the full file list or full diff for a domain with no hit (#4573)
|
|
97
97
|
for pair in auth:auth billing:billing concurrency:concurrency database:database infra:infra scraper:scraper frontend:frontend api:api; do
|
|
98
98
|
var=${pair%%:*}; dom=${pair#*:}
|
|
99
99
|
a=$(grep -E "^ slice $var " "$PREPARE" | head -n 1 | sed -n 's/.* "\([^"]*\)"$/\1/p')
|
|
@@ -101,7 +101,7 @@ for pair in auth:auth billing:billing concurrency:concurrency database:database
|
|
|
101
101
|
if [ -z "$a" ] || [ -z "$b" ]; then bad "could not extract $var pattern (slice='$a' delta_roster='$b')"
|
|
102
102
|
elif [ "$a" = "$b" ]; then ok; else bad "$var regex drifted: slice='$a' delta_roster='$b'"; fi
|
|
103
103
|
done
|
|
104
|
-
if grep -qF '[ -n "$paths" ] || paths="$SLICE_FILES_SRC"' "$PREPARE"
|
|
104
|
+
if grep -qF '[ -n "$paths" ] || paths="$SLICE_FILES_SRC"' "$PREPARE"; then bad "prepare slice() must not fall back to the full file list"; elif grep -qF '[ -n "$body" ] || body="$FULL_DIFF"' "$PREPARE"; then bad "prepare slice() must not fall back to the full diff"; elif grep -qF '[ -n "$body" ] || body="# No diff blocks' "$PREPARE"; then ok; else bad "prepare slice() lacks the empty-slice note"; fi
|
|
105
105
|
|
|
106
106
|
echo "review-pr-domain-map: $pass passed, $fail failed"
|
|
107
107
|
[ "$fail" -eq 0 ]
|
|
@@ -287,14 +287,16 @@ gh11() {
|
|
|
287
287
|
esac
|
|
288
288
|
}
|
|
289
289
|
[ "$(grep -c '^# BEGIN bundle-pr-kind$' "$PRE")" -eq 2 ] && [ "$(awk '/^# BEGIN bundle-pr-kind$/{f=1;n++} f{print > ("'"$TMP"'/kind" n ".sh")} /^# END bundle-pr-kind$/{f=0}' "$PRE"; cmp -s "$TMP/kind1.sh" "$TMP/kind2.sh" && echo same)" = same ] && ok || bad "0A and 0B.1 must carry identical bundle_pr_kind blocks"
|
|
290
|
+
awk '/^# BEGIN gh-retry$/{f=1;next} /^# END gh-retry$/{f=0} f' "$PRE" > "$TMP/ghretry.sh" # bundle_pr_kind calls gh_retry (forge#4572)
|
|
291
|
+
[ -s "$TMP/ghretry.sh" ] && ok || bad "gh-retry block not found"
|
|
290
292
|
awk '/^# BEGIN bundle-pr-set$/{f=1;next} /^# END bundle-pr-set$/{f=0} f' "$PRE" | sed 's/{GH_REPO}/o\/r/g' > "$TMP/prset.sh"
|
|
291
293
|
[ -s "$TMP/prset.sh" ] && ok || bad "bundle-pr-set block not found in Phase 0A"
|
|
292
|
-
OUT11=$( gh() { gh11 "$@"; }; GH_REPO=o/r; BUNDLE_PRS="5 6 28668 77 90"; MERGE_PRS="90"; . "$TMP/prset.sh"; echo $ALL_PR_NUMBERS "|$SKIPPED_ISSUES" )
|
|
294
|
+
OUT11=$( gh() { gh11 "$@"; }; GH_REPO=o/r; BUNDLE_PRS="5 6 28668 77 90"; MERGE_PRS="90"; FORGE_GH_RETRY_SLEEP=0; . "$TMP/ghretry.sh"; . "$TMP/prset.sh"; echo $ALL_PR_NUMBERS "|$SKIPPED_ISSUES" )
|
|
293
295
|
[ "$OUT11" = "5 6 77 90 | 28668" ] && ok || bad "0A must drop the confirmed issue and keep PRs, merge PRs and unclassifiable numbers (got '$OUT11')"
|
|
294
296
|
awk '/^# BEGIN bundle-scope-loop$/{f=1;next} /^# END bundle-scope-loop$/{f=0} f' "$PRE" > "$TMP/loop.sh"
|
|
295
297
|
[ -s "$TMP/loop.sh" ] && ok || bad "bundle-scope-loop block not found in 0B.1"
|
|
296
298
|
scope11() { # $1 = ALL_PR_NUMBERS; prints "<SCOPE_MODE> <REVIEWED_PR_COUNT>" then the loop output
|
|
297
|
-
( gh() { gh11 "$@"; }; bundle_delta_credit() { DELTA_COMMITS=""; DELTA_FULL=0; }; . "$TMP/kind1.sh"
|
|
299
|
+
( gh() { gh11 "$@"; }; bundle_delta_credit() { DELTA_COMMITS=""; DELTA_FULL=0; }; FORGE_GH_RETRY_SLEEP=0; . "$TMP/ghretry.sh"; . "$TMP/kind1.sh"
|
|
298
300
|
SCOPE_MODE=full; REVIEWED_COMMITS=""; REVIEWED_MERGES=""; REVIEWED_PR_COUNT=0; _PF="$TMP/pf11"; : > "$_PF"
|
|
299
301
|
TRUSTED_SCRIPT="$TC" ALL_PR_NUMBERS="$1" PR_NUMBER=9 GH_REPO=o/r _SD="$TMP" REVIEW_DELTA_SCRIPT=""
|
|
300
302
|
LOG11=$(. "$TMP/loop.sh"; echo "@@$SCOPE_MODE $REVIEWED_PR_COUNT"); echo "${LOG11##*@@}"; echo "${LOG11%@@*}" )
|
|
@@ -473,9 +473,18 @@ if [ -n "$DELTA_FROM_OPT" ]; then
|
|
|
473
473
|
# an agent body with a findings block for <from>, a post-panel-guard proof naming <from> (staging summary, synthesized
|
|
474
474
|
# block, or a # PR Review Summary whose Agents count matches) and no degraded marker after its first agent body.
|
|
475
475
|
_dv="error"
|
|
476
|
+
# forge#4573: a panel that delivered every launched domain at <from> but was never finalized (the head moved) is complete too.
|
|
477
|
+
# The launched list is the PR-keyed file the dispatch persisted; empty or unreadable means no such proof. It counts only when
|
|
478
|
+
# its .head sidecar names <from>: a later dispatch at another head rewrites the list, so it no longer describes the panel at <from>.
|
|
479
|
+
_dlf="${FORGE_SCRATCHPAD:-$PWD/.forge-scratch}/${PR_NUMBER}_launched_review_agents"
|
|
480
|
+
_dl="[]"
|
|
481
|
+
if [ "$(sed -n 1p "${_dlf}.head" 2>/dev/null)" = "$DELTA_FROM_OPT" ]; then
|
|
482
|
+
_dl=$(grep -E '^[a-z0-9-]+$' "$_dlf" 2>/dev/null | sort -u | jq -R -s -c 'split("\n") | map(select(length > 0))' 2>/dev/null) || _dl="[]"
|
|
483
|
+
[ -n "$_dl" ] || _dl="[]"
|
|
484
|
+
fi
|
|
476
485
|
if [ -n "$TRUSTED_SCRIPT" ] && [ -n "$COMMENTS_JSON" ]; then
|
|
477
486
|
_db=$(printf '%s' "$COMMENTS_JSON" | bash "$TRUSTED_SCRIPT" bodies '^<!-- FORGE:REVIEW-AGENT:[a-z0-9-]+ -->|^<!-- REVIEW-FINDINGS-SYNTHESIZED-START -->|# PR Review Summary|^<!-- FORGE:GATE_(PASS|FAILURE) -->|<!-- FORGE:GATE_FAILURE:TYPE=review-panel-integrity|<!-- FORGE:REVIEW_DEGRADED|<!-- FORGE:REVIEW_BLOCKED' 2>/dev/null) \
|
|
478
|
-
&& _dv=$(printf '%s\n' "$_db" | jq -s -r --arg f "$DELTA_FROM_OPT" '
|
|
487
|
+
&& _dv=$(printf '%s\n' "$_db" | jq -s -r --arg f "$DELTA_FROM_OPT" --argjson L "$_dl" '
|
|
479
488
|
def shas($t; $re): [ $t | scan($re) | .[0] ];
|
|
480
489
|
def one($a): if ($a | length) == 1 then $a[0] else null end;
|
|
481
490
|
def rsha: "(?:^|\\n)Reviewed-SHA: ([0-9a-f]{40}|[0-9a-f]{64})\\r?(?:\\n|$)";
|
|
@@ -494,13 +503,15 @@ if [ -n "$DELTA_FROM_OPT" ]; then
|
|
|
494
503
|
and one(shas($t; "\\*\\*Agents\\*\\*: ([0-9]+)")) == ($nd | tostring))) ] | length) as $proof
|
|
495
504
|
| ([ $E[] | select(.key > $first)
|
|
496
505
|
| select(.value | test("<!-- FORGE:GATE_FAILURE:TYPE=review-panel-integrity|<!-- FORGE:REVIEW_DEGRADED|<!-- FORGE:REVIEW_BLOCKED")) ] | length) as $deg
|
|
497
|
-
| if $deg > 0 then "degraded" elif $proof
|
|
506
|
+
| if $deg > 0 then "degraded" elif $proof > 0 then "ok"
|
|
507
|
+
elif ($L | length) > 0 and ([ $L[] | . as $d | select($AG | any(.d == $d)) ] | length) == ($L | length) then "ok"
|
|
508
|
+
else "no-proof" end
|
|
498
509
|
end' 2>/dev/null) || _dv="error"
|
|
499
510
|
fi
|
|
500
511
|
case "$_dv" in
|
|
501
512
|
ok) REVIEW_MODE="DELTA"; DELTA_FROM="$DELTA_FROM_OPT" ;;
|
|
502
513
|
no-agent) block "--delta-from ${DELTA_FROM_OPT}: no trusted FORGE:REVIEW-AGENT comment with a findings block for that head, no earlier pass to credit: run a full review" ;;
|
|
503
|
-
no-proof) block "--delta-from ${DELTA_FROM_OPT}: no trusted proof that the earlier panel completed (staging summary, synthesized block or PR Review Summary for that head): run a full review" ;;
|
|
514
|
+
no-proof) block "--delta-from ${DELTA_FROM_OPT}: no trusted proof that the earlier panel completed (staging summary, synthesized block or PR Review Summary for that head, nor every launched domain delivered at it): run a full review" ;;
|
|
504
515
|
degraded) block "--delta-from ${DELTA_FROM_OPT}: the earlier pass is degraded (review-panel-integrity / REVIEW_DEGRADED / REVIEW_BLOCKED after its agents): run a full review" ;;
|
|
505
516
|
*) block "--delta-from ${DELTA_FROM_OPT}: could not read the trusted review comments, no earlier pass to credit: run a full review" ;;
|
|
506
517
|
esac
|
|
@@ -587,19 +598,31 @@ if [ "$REVIEW_MODE" != "BASE_SYNC_ONLY" ]; then
|
|
|
587
598
|
# slice <domain> <diff-header regex|ALL> <file-path regex|ALL>
|
|
588
599
|
slice() {
|
|
589
600
|
local dom="$1" dre="$2" fre="$3" body paths tmp df="$OUT_DIR/slice-${REVIEW_SHA_SHORT:-unresolved}-$1.diff"
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
|
|
593
|
-
|
|
594
|
-
|
|
601
|
+
# forge#4573: whole `diff --git` blocks only. A block over the per-file cap, or one that would push the slice over the total
|
|
602
|
+
# cap, is listed as omitted (never cut mid-file). No fallback to the full diff: an unmatched domain gets a note, not a copy.
|
|
603
|
+
# The security slice (ALL) lists .github/ blocks last so CI files cannot crowd out source.
|
|
604
|
+
body=$(printf '%s\n' "$FULL_DIFF" | awk -v re="$dre" -v cap="${REVIEW_SLICE_CAP:-102400}" -v fcap="${REVIEW_SLICE_FILE_CAP:-20000}" '
|
|
605
|
+
/^diff --git/ { n++; hdr[n] = $0; blk[n] = "" }
|
|
606
|
+
n { blk[n] = blk[n] $0 "\n" }
|
|
607
|
+
END {
|
|
608
|
+
for (pass = 1; pass <= 2; pass++) for (i = 1; i <= n; i++) {
|
|
609
|
+
ci = (hdr[i] ~ /^diff --git a\/\.github\//) ? 2 : 1
|
|
610
|
+
if (re == "ALL" ? ci != pass : (pass == 2 || hdr[i] !~ re)) continue
|
|
611
|
+
l = length(blk[i])
|
|
612
|
+
if (l > fcap || used + l > cap) { split(hdr[i], h, " "); om = om "# OMITTED (too large for this slice, read it from the head checkout): " substr(h[3], 3) "\n"; continue }
|
|
613
|
+
out = out blk[i]; used += l
|
|
614
|
+
}
|
|
615
|
+
printf "%s%s", om, out
|
|
616
|
+
}')
|
|
617
|
+
[ -n "$body" ] || body="# No diff blocks for this domain in the review range."
|
|
595
618
|
if [ "$fre" = "ALL" ]; then paths="$SLICE_FILES_SRC"
|
|
596
|
-
else paths=$(printf '%s\n' "$SLICE_FILES_SRC" | grep -iE "$fre" || echo "");
|
|
597
|
-
# 100K-char cap, same as the runner's tool-result cap
|
|
598
|
-
if [ "${#body}" -gt 102400 ]; then body="$(printf '%s' "$body" | head -c 102400)"$'\n'"... [TRUNCATED — diff exceeded 100K chars; focus on the files listed]"; fi
|
|
619
|
+
else paths=$(printf '%s\n' "$SLICE_FILES_SRC" | grep -iE "$fre" || echo ""); fi
|
|
599
620
|
# Publish atomically: temp file in $OUT_DIR (same filesystem) then rename, so a concurrent reader sees nothing or the whole file.
|
|
600
621
|
tmp=$(mktemp "$OUT_DIR/.slice-XXXXXX") || { echo "review-prepare: cannot create temp slice in $OUT_DIR" >&2; exit 2; }
|
|
601
622
|
if printf '%s\n' "$body" > "$tmp" && chmod 644 "$tmp"; then mv -f "$tmp" "$df"; else rm -f "$tmp"; echo "review-prepare: cannot write slice $df" >&2; exit 2; fi
|
|
602
|
-
|
|
623
|
+
# paths can exceed the 128 KiB per-argument limit on a huge PR: pass it as a file, not argv
|
|
624
|
+
printf '%s' "$paths" > "$TMP_DIR/slice-paths" || { echo "review-prepare: cannot write $TMP_DIR/slice-paths" >&2; exit 2; }
|
|
625
|
+
SLICES_JSON=$(printf '%s' "$SLICES_JSON" | jq -c --arg d "$dom" --arg df "$df" --rawfile p "$TMP_DIR/slice-paths" '. + {($d): {paths: ($p | split("\n") | map(select(length > 0))), diff_file: $df}}')
|
|
603
626
|
}
|
|
604
627
|
slice security ALL ALL
|
|
605
628
|
slice auth '^diff --git.*/(auth|session|jwt|oauth|permission|middleware|login|token)' "auth|session|login|logout|jwt|oauth|token|user|permission|middleware"
|
|
@@ -617,43 +640,55 @@ fi
|
|
|
617
640
|
STATUS="OK"; [ -n "$BLOCKERS" ] && STATUS="BLOCKED"
|
|
618
641
|
j_lines() { printf '%s' "${1}" | jq -R -s -c 'split("\n") | map(select(length > 0))'; }
|
|
619
642
|
j_words() { printf '%s' "${1}" | jq -R -s -c 'split(" ") | map(select(length > 0))'; }
|
|
643
|
+
# Everything that scales with the PR goes to the final jq through a file (--slurpfile/--rawfile): argv caps one argument at 128 KiB.
|
|
644
|
+
jf_lines() { j_lines "$2" > "$TMP_DIR/$1.json"; }
|
|
645
|
+
jf_words() { j_words "$2" > "$TMP_DIR/$1.json"; }
|
|
646
|
+
jf_raw() { printf '%s' "$2" > "$TMP_DIR/$1.txt"; }
|
|
647
|
+
{ jf_lines blockers "$BLOCKERS" && jf_lines p2 "$P2_FINDINGS" && jf_lines asm "$ASSUMPTIONS" && jf_lines dfiles "$DELTA_FILES" && jf_lines full_files "$FILES" \
|
|
648
|
+
&& printf '%s' "$SLICES_JSON" > "$TMP_DIR/slices.json" \
|
|
649
|
+
&& jf_raw churn "$CHURN_CONTEXT" && jf_raw pctx "$PROJECT_CONTEXT" && jf_raw tmono "$MONOLITHIC_CATALOG" \
|
|
650
|
+
&& jf_raw d_auth "$DOMAIN_CONTEXT_AUTH" && jf_raw d_billing "$DOMAIN_CONTEXT_BILLING" && jf_raw d_conc "$DOMAIN_CONTEXT_CONCURRENCY" \
|
|
651
|
+
&& jf_raw d_infra "$DOMAIN_CONTEXT_INFRA" && jf_raw d_db "$DOMAIN_CONTEXT_DATABASE" && jf_raw d_fe "$DOMAIN_CONTEXT_FRONTEND" \
|
|
652
|
+
&& jf_raw d_sec "$DOMAIN_CONTEXT_SECURITY" && jf_raw d_api "$DOMAIN_CONTEXT_API" && jf_raw d_scr "$DOMAIN_CONTEXT_SCRAPING" \
|
|
653
|
+
&& jf_raw i_auth "$IDX_AUTH" && jf_raw i_billing "$IDX_BILLING" && jf_raw i_db "$IDX_DATABASE" && jf_raw i_api "$IDX_API" \
|
|
654
|
+
&& jf_raw i_fe "$IDX_FRONTEND" && jf_raw i_infra "$IDX_INFRA"; } || { echo "review-prepare: cannot write context inputs under $TMP_DIR" >&2; exit 2; }
|
|
620
655
|
CTX=$(jq -n \
|
|
621
|
-
--arg status "$STATUS" --
|
|
656
|
+
--arg status "$STATUS" --slurpfile blockers "$TMP_DIR/blockers.json" \
|
|
622
657
|
--argjson pr_number "$PR_NUMBER" --arg repo "$REPO" --arg title "$TITLE" --arg author "$PR_AUTHOR" --arg head "$HEAD" --arg base "$BASE" \
|
|
623
658
|
--arg sha "$REVIEW_SHA" --arg sha7 "$REVIEW_SHA_SHORT" --argjson adds "$ADDITIONS" --argjson dels "$DELETIONS" --arg nfiles "${NFILES:-}" --arg first "$IS_FIRST_TIME_CONTRIBUTOR" \
|
|
624
659
|
--arg s2m "$IS_STAGING_TO_MAIN" --arg m2s "$IS_MILESTONE_TO_STAGING" --arg rfb "$REQUIRES_FULL_BUILD" --arg thorough "$THOROUGH" \
|
|
625
660
|
--arg mh "$MERGE_HEALTH" --arg mhs "$MERGE_HEALTH_STATE" --argjson mhr "$MERGE_RETRY" \
|
|
626
661
|
--argjson services "$(j_words "$SERVICES")" --argjson domains "$(j_words "$DOMAINS")" \
|
|
627
|
-
--arg tsb "$TS_BUILD" --arg pyf "$PY_FORMAT" --arg bcs "$BUILD_CONFIG_STATUS" --
|
|
628
|
-
--
|
|
662
|
+
--arg tsb "$TS_BUILD" --arg pyf "$PY_FORMAT" --arg bcs "$BUILD_CONFIG_STATUS" --slurpfile p2 "$TMP_DIR/p2.json" \
|
|
663
|
+
--slurpfile asm "$TMP_DIR/asm.json" --argjson risk "$(j_words "$RISK")" --rawfile churn "$TMP_DIR/churn.txt" --arg churn_esc "$CHURN_ESCALATION" --arg danger "$DANGER" \
|
|
629
664
|
--argjson sa "$SCORE_AUTH" --argjson sb "$SCORE_BILLING" --argjson sc "$SCORE_CONCURRENCY" --argjson sd "$SCORE_DATABASE" --argjson si "$SCORE_INFRA" \
|
|
630
665
|
--argjson ss "$SCORE_SECURITY" --argjson ssc "$SCORE_SCRAPING" --argjson sf "$SCORE_FRONTEND" --argjson sap "$SCORE_API" \
|
|
631
666
|
--arg cissue "$ISSUE_NUM" --arg cflags "$CONTRACT_RISK_FLAGS" \
|
|
632
667
|
--arg rdepth "$REVIEW_DEPTH" --arg rweight "$REVIEW_WEIGHT" --arg rtier "$ROSTER_TIER" --arg rreason "$ROSTER_REASON" --argjson agents "$(j_words "$SELECTED_AGENTS")" \
|
|
633
|
-
--arg dmode "$REVIEW_MODE" --arg dfrom "$DELTA_FROM" --arg dcarried "$CARRIED_FROM" --argjson drounds "$FULL_ROUNDS" --
|
|
634
|
-
--arg pname "$PROJECT_NAME" --
|
|
635
|
-
--
|
|
636
|
-
--
|
|
637
|
-
--arg tsrc "$TEMPLATE_SOURCE" --arg tbase "$TEMPLATE_BASE" --
|
|
638
|
-
--
|
|
639
|
-
--
|
|
640
|
-
{schema_version: 1, status: $status, blockers: $blockers,
|
|
668
|
+
--arg dmode "$REVIEW_MODE" --arg dfrom "$DELTA_FROM" --arg dcarried "$CARRIED_FROM" --argjson drounds "$FULL_ROUNDS" --slurpfile dfiles "$TMP_DIR/dfiles.json" \
|
|
669
|
+
--arg pname "$PROJECT_NAME" --rawfile pctx "$TMP_DIR/pctx.txt" --arg scraping_on "$SCRAPING_ON" --arg rpath "$REPO_PATH" \
|
|
670
|
+
--rawfile d_auth "$TMP_DIR/d_auth.txt" --rawfile d_billing "$TMP_DIR/d_billing.txt" --rawfile d_conc "$TMP_DIR/d_conc.txt" --rawfile d_infra "$TMP_DIR/d_infra.txt" \
|
|
671
|
+
--rawfile d_db "$TMP_DIR/d_db.txt" --rawfile d_fe "$TMP_DIR/d_fe.txt" --rawfile d_sec "$TMP_DIR/d_sec.txt" --rawfile d_api "$TMP_DIR/d_api.txt" --rawfile d_scr "$TMP_DIR/d_scr.txt" \
|
|
672
|
+
--arg tsrc "$TEMPLATE_SOURCE" --arg tbase "$TEMPLATE_BASE" --rawfile tmono "$TMP_DIR/tmono.txt" \
|
|
673
|
+
--rawfile i_auth "$TMP_DIR/i_auth.txt" --rawfile i_billing "$TMP_DIR/i_billing.txt" --rawfile i_db "$TMP_DIR/i_db.txt" --rawfile i_api "$TMP_DIR/i_api.txt" --rawfile i_fe "$TMP_DIR/i_fe.txt" --rawfile i_infra "$TMP_DIR/i_infra.txt" \
|
|
674
|
+
--slurpfile slices "$TMP_DIR/slices.json" --slurpfile full_files "$TMP_DIR/full_files.json" '
|
|
675
|
+
{schema_version: 1, status: $status, blockers: $blockers[0],
|
|
641
676
|
pr: {number: $pr_number, repo: $repo, title: $title, author: $author, head_ref: $head, base_ref: $base, review_sha: $sha, review_sha_short: $sha7,
|
|
642
677
|
additions: $adds, deletions: $dels, changed_files: ($nfiles | tonumber? // null), first_time_contributor: ($first == "true")},
|
|
643
678
|
lane: {is_staging_to_main: ($s2m == "true"), is_milestone_to_staging: ($m2s == "true"), requires_full_build: ($rfb == "true"), thorough: ($thorough == "true")},
|
|
644
679
|
merge_health: {mergeable: $mh, state: $mhs, retries: $mhr},
|
|
645
680
|
classification: {services: $services, domains: $domains},
|
|
646
|
-
phase2: {build_config: {typescript_build: $tsb, python_format: $pyf, status: $bcs}, findings: $p2},
|
|
647
|
-
broken_assumptions: $asm, risk_signals: $risk, churn_context: $churn, churn_escalation: ($churn_esc == "true"), danger_zone: ($danger == "true"),
|
|
681
|
+
phase2: {build_config: {typescript_build: $tsb, python_format: $pyf, status: $bcs}, findings: $p2[0]},
|
|
682
|
+
broken_assumptions: $asm[0], risk_signals: $risk, churn_context: $churn, churn_escalation: ($churn_esc == "true"), danger_zone: ($danger == "true"),
|
|
648
683
|
scores: {AUTH: $sa, BILLING: $sb, CONCURRENCY: $sc, DATABASE: $sd, INFRA: $si, SECURITY: $ss, SCRAPING: $ssc, FRONTEND: $sf, API: $sap},
|
|
649
684
|
contract: {issue: ($cissue | tonumber? // null), flags: $cflags},
|
|
650
685
|
roster: {tier: $rtier, review_depth: $rdepth, review_weight: $rweight, reason: $rreason, agents: $agents},
|
|
651
|
-
delta: {mode: $dmode, from: $dfrom, carried_from: $dcarried, full_rounds: $drounds, files: $dfiles},
|
|
686
|
+
delta: {mode: $dmode, from: $dfrom, carried_from: $dcarried, full_rounds: $drounds, files: $dfiles[0]},
|
|
652
687
|
config: {project_name: $pname, project_context: $pctx, scraping_configured: ($scraping_on == "true"), repo_path: $rpath,
|
|
653
688
|
domains: {auth: $d_auth, billing: $d_billing, concurrency: $d_conc, infra: $d_infra, database: $d_db, frontend: $d_fe, security: $d_sec, api: $d_api, scraping: $d_scr}},
|
|
654
689
|
templates: {source: $tsrc, base: $tbase, monolithic_catalog: $tmono},
|
|
655
690
|
index_slices: {auth: $i_auth, billing: $i_billing, database: $i_db, api: $i_api, frontend: $i_fe, infrastructure: $i_infra},
|
|
656
|
-
slices: $slices, full_files: $full_files}') || { echo "review-prepare: could not assemble the context JSON" >&2; exit 2; }
|
|
691
|
+
slices: $slices[0], full_files: $full_files[0]}') || { echo "review-prepare: could not assemble the context JSON" >&2; exit 2; }
|
|
657
692
|
if [ -n "$OUT" ]; then printf '%s\n' "$CTX" > "$OUT" || { echo "review-prepare: cannot write $OUT" >&2; exit 2; }; else printf '%s\n' "$CTX"; fi
|
|
658
693
|
[ "$STATUS" = "OK" ] && exit 0
|
|
659
694
|
exit 3
|
|
@@ -116,6 +116,28 @@ prep "$ROOT"; eq "garbage state normalized" .merge_health.state UNKNOWN
|
|
|
116
116
|
setup feat/x staging "README.md" "$DIFF_PLAIN"; echo 0 > "$FIX/prior"
|
|
117
117
|
prep "$ROOT"; eq "first-time contributor" .pr.first_time_contributor true
|
|
118
118
|
|
|
119
|
+
# --- 2b. slices are whole-file blocks, bounded, never the full-diff fallback (#4573) ---
|
|
120
|
+
BIG_LINES=$(awk 'BEGIN { for (i = 0; i < 1500; i++) print "-old line " i " of the huge generated file" }')
|
|
121
|
+
DIFF_BIG="diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
|
|
122
|
+
+ci: 1
|
|
123
|
+
diff --git a/services/api/big_scrape.py b/services/api/big_scrape.py
|
|
124
|
+
$BIG_LINES
|
|
125
|
+
diff --git a/services/api/auth.py b/services/api/auth.py
|
|
126
|
+
+def f(): jwt = 1"
|
|
127
|
+
setup feat/x staging "services/api/big_scrape.py
|
|
128
|
+
services/api/auth.py
|
|
129
|
+
.github/workflows/ci.yml" "$DIFF_BIG"
|
|
130
|
+
prep "$ROOT"
|
|
131
|
+
SEC_SLICE=$(jqv '.slices.security.diff_file')
|
|
132
|
+
if grep -q '^# OMITTED.*services/api/big_scrape.py' "$SEC_SLICE" && ! grep -q 'huge generated file' "$SEC_SLICE"; then ok; else bad "oversize file must be listed as omitted, not cut into the slice"; fi
|
|
133
|
+
if grep -q '^+def f(): jwt = 1' "$SEC_SLICE"; then ok; else bad "small blocks stay whole next to an omitted one"; fi
|
|
134
|
+
if [ "$(grep -n 'services/api/auth.py b/' "$SEC_SLICE" | head -1 | cut -d: -f1)" -lt "$(grep -n '\.github/workflows/ci.yml b/' "$SEC_SLICE" | head -1 | cut -d: -f1)" ]; then ok; else bad "security slice lists .github blocks last"; fi
|
|
135
|
+
BILL_SLICE=$(jqv '.slices.billing.diff_file')
|
|
136
|
+
if grep -q 'auth.py\|big_scrape' "$BILL_SLICE"; then bad "unmatched domain must not fall back to the full diff"; else ok; fi
|
|
137
|
+
eq "unmatched domain has no paths" '.slices.billing.paths | length' 0
|
|
138
|
+
REVIEW_SLICE_CAP=100 prep "$ROOT"
|
|
139
|
+
if [ "$(wc -c < "$(jqv '.slices.security.diff_file')")" -lt 600 ]; then ok; else bad "total cap bounds the slice"; fi
|
|
140
|
+
|
|
119
141
|
# --- 3. classification + scores ---
|
|
120
142
|
setup feat/x staging "services/api/auth.py
|
|
121
143
|
services/api/billing.py" "$DIFF_AUTH_BILL"
|
|
@@ -252,6 +274,26 @@ prep "$WORK/root-nodelta" --no-delta --delta-from "$SHA_A"; rc_is "degraded mark
|
|
|
252
274
|
{ cmt "$(agent_body "$SHA_A")"; } | comments_of
|
|
253
275
|
prep "$WORK/root-nodelta" --no-delta --delta-from "$SHA_A"; rc_is "partial earlier pass (no post-guard proof) blocks" 3
|
|
254
276
|
eq "partial earlier pass blocker" '.blockers | join(" ") | test("no trusted proof that the earlier panel completed")' true
|
|
277
|
+
# forge#4573: an unfinalized panel is complete when every launched domain delivered findings at <from> and nothing is degraded
|
|
278
|
+
export FORGE_SCRATCHPAD="$WORK/sd"; mkdir -p "$FORGE_SCRATCHPAD"
|
|
279
|
+
printf 'security\n' > "$FORGE_SCRATCHPAD/7_launched_review_agents"
|
|
280
|
+
printf '%s\n' "$SHA_A" > "$FORGE_SCRATCHPAD/7_launched_review_agents.head"
|
|
281
|
+
{ cmt "$(agent_body "$SHA_A")"; } | comments_of
|
|
282
|
+
prep "$WORK/root-nodelta" --no-delta --delta-from "$SHA_A"; rc_is "complete but unfinalized panel is accepted (#4573)" 0; eq "unfinalized panel mode" .delta.mode DELTA
|
|
283
|
+
printf '%s\n' "$SHA_B" > "$FORGE_SCRATCHPAD/7_launched_review_agents.head"
|
|
284
|
+
prep "$WORK/root-nodelta" --no-delta --delta-from "$SHA_A"; rc_is "a launched list written for another head is no proof" 3
|
|
285
|
+
rm -f "$FORGE_SCRATCHPAD/7_launched_review_agents.head"
|
|
286
|
+
prep "$WORK/root-nodelta" --no-delta --delta-from "$SHA_A"; rc_is "a launched list with no head sidecar is no proof" 3
|
|
287
|
+
printf '%s\n' "$SHA_A" > "$FORGE_SCRATCHPAD/7_launched_review_agents.head"
|
|
288
|
+
printf 'security\nauth\n' > "$FORGE_SCRATCHPAD/7_launched_review_agents"
|
|
289
|
+
prep "$WORK/root-nodelta" --no-delta --delta-from "$SHA_A"; rc_is "a launched domain with no findings keeps the delta blocked" 3
|
|
290
|
+
printf 'security\n' > "$FORGE_SCRATCHPAD/7_launched_review_agents"
|
|
291
|
+
{ cmt "$(agent_body "$SHA_A")"; cmt "<!-- FORGE:REVIEW_DEGRADED -->"; } | comments_of
|
|
292
|
+
prep "$WORK/root-nodelta" --no-delta --delta-from "$SHA_A"; rc_is "degraded unfinalized panel stays blocked" 3
|
|
293
|
+
rm -f "$FORGE_SCRATCHPAD/7_launched_review_agents"
|
|
294
|
+
{ cmt "$(agent_body "$SHA_A")"; } | comments_of
|
|
295
|
+
prep "$WORK/root-nodelta" --no-delta --delta-from "$SHA_A"; rc_is "no launched list: unfinalized panel stays blocked" 3
|
|
296
|
+
unset FORGE_SCRATCHPAD
|
|
255
297
|
{ cmt "$(agent_body "$SHA_A")"; cmt "$(staging_summary "$SHA_A")" NONE; } | comments_of
|
|
256
298
|
prep "$WORK/root-nodelta" --no-delta --delta-from "$SHA_A"; rc_is "untrusted staging summary is no proof" 3
|
|
257
299
|
{ cmt "$(staging_summary "$SHA_B")"; cmt "$(agent_body "$SHA_A")"; } | comments_of
|
|
@@ -435,5 +477,16 @@ if command -v timeout >/dev/null 2>&1 || command -v gtimeout >/dev/null 2>&1; th
|
|
|
435
477
|
if grep -q "code index timed out after 1s" "$WORK/stderr"; then ok; else bad "index timeout must be reported on stderr"; fi
|
|
436
478
|
fi
|
|
437
479
|
|
|
480
|
+
# --- 12. oversized slices JSON (>128 KiB, argv per-argument limit) still assembles ---
|
|
481
|
+
BIG_FILES=$(awk 'BEGIN{for(i=1;i<=1100;i++) printf "services/api/auth/very/long/directory/name/to/pad/the/path/out/for/the/test/module_%05d_handler.py\n", i}')
|
|
482
|
+
BIG_DIFF='diff --git a/services/api/auth/x.py b/services/api/auth/x.py
|
|
483
|
+
+jwt = 1'
|
|
484
|
+
setup feat/x staging "$BIG_FILES" "$BIG_DIFF"
|
|
485
|
+
prep "$ROOT"
|
|
486
|
+
rc_is "oversized slices JSON" 0
|
|
487
|
+
eq "oversized slices JSON is over 128 KiB" '.slices | tojson | length > 131072' true
|
|
488
|
+
eq "oversized security slice keeps every path" '.slices.security.paths | length' 1100
|
|
489
|
+
eq "oversized full_files keeps every path" '.full_files | length' 1100
|
|
490
|
+
|
|
438
491
|
echo "review-prepare tests: $pass passed, $fail failed"
|
|
439
492
|
[ "$fail" -eq 0 ]
|
|
@@ -50,6 +50,31 @@ if [ "$CMD" != gate ]; then
|
|
|
50
50
|
export FORGE_SCRATCHPAD="$SD"
|
|
51
51
|
fi
|
|
52
52
|
_SD="${SD:-${FORGE_SCRATCHPAD:-$PWD/.forge-scratch}}"
|
|
53
|
+
|
|
54
|
+
# BEGIN gh-retry
|
|
55
|
+
# gh_retry [--search] CMD...: run CMD up to FORGE_GH_RETRY_ATTEMPTS (default 6) times with exponential backoff from
|
|
56
|
+
# FORGE_GH_RETRY_SLEEP seconds (default 2, capped at 16 per wait; --search triples the base for the 30 req/min search API).
|
|
57
|
+
# Prints stdout of the first success and returns 0; returns 1 once every attempt failed, so callers still fail closed/open as before (forge#4572).
|
|
58
|
+
gh_retry() {
|
|
59
|
+
local mult=1 n=0 max="${FORGE_GH_RETRY_ATTEMPTS:-6}" base="${FORGE_GH_RETRY_SLEEP:-2}" w out
|
|
60
|
+
[ "${1:-}" = --search ] && { mult=3; shift; }
|
|
61
|
+
case "$max" in ''|*[!0-9]*|0) max=6 ;; esac
|
|
62
|
+
case "$base" in ''|*[!0-9]*) base=2 ;; esac
|
|
63
|
+
while :; do
|
|
64
|
+
out=$("$@") && { printf '%s' "$out"; return 0; }
|
|
65
|
+
n=$((n+1)); [ "$n" -ge "$max" ] && return 1
|
|
66
|
+
w=$((base * mult * (1 << (n-1)))); [ "$w" -gt $((16*mult)) ] && w=$((16*mult))
|
|
67
|
+
[ "$w" -gt 0 ] && sleep "$w"
|
|
68
|
+
done
|
|
69
|
+
}
|
|
70
|
+
# END gh-retry
|
|
71
|
+
# One review-finding search fetch; fails (so gh_retry retries) on a gh error or non-array output.
|
|
72
|
+
fetch_findings() {
|
|
73
|
+
local raw
|
|
74
|
+
raw=$(gh issue list -R "$GH_REPO" --label "review-finding" --state open --search "PR #${1}" --limit 20 --json number,title,body 2>/dev/null) || return 1
|
|
75
|
+
printf '%s' "$raw" | jq -e 'type == "array"' >/dev/null 2>&1 || return 1
|
|
76
|
+
printf '%s' "$raw"
|
|
77
|
+
}
|
|
53
78
|
ALL_PR_NUMBERS=""
|
|
54
79
|
if [ "$CMD" != bundle ]; then
|
|
55
80
|
[ -n "$PRS_FILE" ] || { echo "review-staging-pre: --prs-file is required" >&2; exit 2; }
|
|
@@ -81,7 +106,7 @@ MERGE_PRS=$(git log origin/$DEFAULT_BRANCH..origin/$STAGING_BRANCH --merges --on
|
|
|
81
106
|
# Merge-subject numbers are PRs by construction; an unclassifiable number is kept (fail open: it may be a PR). forge#4127
|
|
82
107
|
# BEGIN bundle-pr-kind
|
|
83
108
|
# Prints "pr" or "issue" for issue number ${2} in repo ${1}; prints nothing and returns 1 when the read fails or is malformed.
|
|
84
|
-
bundle_pr_kind() { local k; k=$(gh api "repos/${1}/issues/${2}" --jq 'if .pull_request then "pr" else "issue" end' 2>/dev/null) || return 1; case "$k" in pr|issue) printf '%s' "$k" ;; *) return 1 ;; esac; }
|
|
109
|
+
bundle_pr_kind() { local k; k=$(gh_retry gh api "repos/${1}/issues/${2}" --jq 'if .pull_request then "pr" else "issue" end' 2>/dev/null) || return 1; case "$k" in pr|issue) printf '%s' "$k" ;; *) return 1 ;; esac; }
|
|
85
110
|
# END bundle-pr-kind
|
|
86
111
|
SUBJECT_PRS=""; SKIPPED_ISSUES=""
|
|
87
112
|
for n in $BUNDLE_PRS; do
|
|
@@ -115,7 +140,7 @@ DEGRADED_REVIEWS=""
|
|
|
115
140
|
READ_FAILURES=""
|
|
116
141
|
for pr_num in $ALL_PR_NUMBERS; do
|
|
117
142
|
# Capture each gh read and test its own status (as the scope step does): a failed read is never "not degraded" / "no findings".
|
|
118
|
-
IS_REVIEW_DEGRADED=$(gh pr view "$pr_num" -R $GH_REPO --json labels \
|
|
143
|
+
IS_REVIEW_DEGRADED=$(gh_retry gh pr view "$pr_num" -R $GH_REPO --json labels \
|
|
119
144
|
--jq '[.labels[].name] | any(. == "review-degraded")' 2>/dev/null) || IS_REVIEW_DEGRADED=unreadable
|
|
120
145
|
case "$IS_REVIEW_DEGRADED" in true|false) ;; *) READ_FAILURES="$READ_FAILURES #${pr_num}(labels)" ;; esac
|
|
121
146
|
if [ "$IS_REVIEW_DEGRADED" = "true" ]; then
|
|
@@ -128,12 +153,7 @@ for pr_num in $ALL_PR_NUMBERS; do
|
|
|
128
153
|
# narrow it with the anchored FINDING_PR_FILTER: title suffix `PR #N)` or a `**Source**: PR #N` line.
|
|
129
154
|
FINDING_PR_FILTER='[.[] | select((.title | endswith("PR #" + $n + ")")) or ((.body // "") | test("(^|\\n)\\*\\*Source\\*\\*: PR #" + $n + "([^0-9]|$)")))] | .[] | " - #\(.number): \(.title)"' # FINDING_PR_FILTER
|
|
130
155
|
OPEN_FINDINGS=""
|
|
131
|
-
if _RAW=$(
|
|
132
|
-
--label "review-finding" \
|
|
133
|
-
--state open \
|
|
134
|
-
--search "PR #${pr_num}" \
|
|
135
|
-
--limit 20 \
|
|
136
|
-
--json number,title,body 2>/dev/null) && printf '%s' "$_RAW" | jq -e 'type == "array"' >/dev/null 2>&1; then
|
|
156
|
+
if _RAW=$(gh_retry --search fetch_findings "$pr_num"); then
|
|
137
157
|
OPEN_FINDINGS=$(printf '%s' "$_RAW" | jq -r --arg n "${pr_num}" "$FINDING_PR_FILTER" 2>/dev/null) || READ_FAILURES="$READ_FAILURES #${pr_num}(findings)" # uses FINDING_PR_FILTER
|
|
138
158
|
else
|
|
139
159
|
READ_FAILURES="$READ_FAILURES #${pr_num}(findings)"
|
|
@@ -312,7 +332,7 @@ bundle_delta_credit() {
|
|
|
312
332
|
# END bundle-delta-credit
|
|
313
333
|
# BEGIN bundle-pr-kind
|
|
314
334
|
# Prints "pr" or "issue" for issue number ${2} in repo ${1}; prints nothing and returns 1 when the read fails or is malformed.
|
|
315
|
-
bundle_pr_kind() { local k; k=$(gh api "repos/${1}/issues/${2}" --jq 'if .pull_request then "pr" else "issue" end' 2>/dev/null) || return 1; case "$k" in pr|issue) printf '%s' "$k" ;; *) return 1 ;; esac; }
|
|
335
|
+
bundle_pr_kind() { local k; k=$(gh_retry gh api "repos/${1}/issues/${2}" --jq 'if .pull_request then "pr" else "issue" end' 2>/dev/null) || return 1; case "$k" in pr|issue) printf '%s' "$k" ;; *) return 1 ;; esac; }
|
|
316
336
|
# END bundle-pr-kind
|
|
317
337
|
# BEGIN bundle-scope-loop
|
|
318
338
|
_WHY=""
|
|
@@ -13,6 +13,7 @@ bad() { FAILN=$((FAILN+1)); echo "FAIL: $1"; }
|
|
|
13
13
|
command -v jq >/dev/null 2>&1 || { echo "review-staging-pre.test.sh: jq missing, skipped"; exit 0; }
|
|
14
14
|
command -v git >/dev/null 2>&1 || { echo "review-staging-pre.test.sh: git missing, skipped"; exit 0; }
|
|
15
15
|
TMP="$(mktemp -d)"; trap 'rm -rf "$TMP"' EXIT
|
|
16
|
+
export FORGE_GH_RETRY_SLEEP=0 # no real backoff in tests
|
|
16
17
|
export FORGEDOCK_HOME="$ROOT" # resolve helpers from this checkout, not an ambient install
|
|
17
18
|
|
|
18
19
|
# (1) the blocks live only here: present in the script, absent from the spec
|
|
@@ -34,6 +35,9 @@ cat > "$TMP/bin/gh" <<'STUB'
|
|
|
34
35
|
F="$FIX"; a="$*"
|
|
35
36
|
# fail-labels / fail-issues: the degraded-label / review-finding read fails (rate limit, 5xx, expired auth)
|
|
36
37
|
case "$a" in "pr view"*"--json labels"*) [ -f "$F/fail-labels" ] && exit 1 ;; "issue list"*) [ -f "$F/fail-issues" ] && exit 1 ;; esac
|
|
38
|
+
# flaky-<kind>: fail the first N reads of that kind (N = file content), then succeed (transient error)
|
|
39
|
+
flaky() { [ -f "$F/flaky-$1" ] || return 0; local c; c=$(cat "$F/count-$1" 2>/dev/null || echo 0); echo $((c+1)) > "$F/count-$1"; [ "$c" -lt "$(cat "$F/flaky-$1")" ] && exit 1; return 0; }
|
|
40
|
+
case "$a" in "pr view"*"--json labels"*) flaky labels ;; "issue list"*) flaky issues ;; "api repos/o/r/issues/"*) case "$a" in *"/comments"*) ;; *) flaky kind ;; esac ;; esac
|
|
37
41
|
case "$1 $2" in
|
|
38
42
|
"pr comment") while [ $# -gt 0 ]; do [ "$1" = --body ] && { printf '%s\n@@\n' "$2" >> "$F/posted"; }; shift; done; exit 0 ;;
|
|
39
43
|
"pr diff") [ -f "$F/files-$3" ] && { cat "$F/files-$3"; exit 0; }; exit 1 ;;
|
|
@@ -70,6 +74,10 @@ run() { (cd "$TMP/work" && bash "$PRE" "$@"); }
|
|
|
70
74
|
|
|
71
75
|
# (3) bundle: the merge-subject PR is kept, the issue cited in a subject is skipped, the list is persisted
|
|
72
76
|
newfix bundle; : > "$FIX/issue-77"
|
|
77
|
+
newfix bundleflaky; : > "$FIX/issue-77"; echo 2 > "$FIX/flaky-kind"
|
|
78
|
+
OUT=$(run bundle --repo o/r --default main --staging staging --pr 9 --sd "$TMP/sd" 2>&1); RC=$?
|
|
79
|
+
case "$OUT" in *"skipped):"*"77"*) ok ;; *) bad "a transient bundle_pr_kind failure must be retried so issue #77 is still skipped (got '$OUT')" ;; esac
|
|
80
|
+
newfix bundle; : > "$FIX/issue-77"
|
|
73
81
|
OUT=$(run bundle --repo o/r --default main --staging staging --pr 9 --sd "$TMP/sd" 2>&1); RC=$?
|
|
74
82
|
[ "$RC" -eq 0 ] && ok || bad "bundle must exit 0 (got $RC: $OUT)"
|
|
75
83
|
[ "$(cat "$TMP/sd/9_bundle-prs.txt" 2>/dev/null)" = 5 ] && ok || bad "bundle must persist exactly PR 5 (got '$(cat "$TMP/sd/9_bundle-prs.txt" 2>/dev/null)')"
|
|
@@ -106,6 +114,16 @@ OUT=$(run gate --repo o/r --pr 9 --prs-file "$TMP/prs2" 2>&1); RC=$?
|
|
|
106
114
|
case "$OUT" in *"RESULT: BLOCK DEPLOY"*) ok ;; *) bad "gh read failure blocks: must print RESULT: BLOCK DEPLOY (got '$OUT')" ;; esac
|
|
107
115
|
! grep -q 'FORGE:GATE_PASS' "$FIX/posted" 2>/dev/null && ok || bad "gh read failure blocks: must not post GATE_PASS"
|
|
108
116
|
grep -q 'FORGE:GATE_FAILURE:TYPE=gate-read-failure' "$FIX/posted" 2>/dev/null && ok || bad "gh read failure blocks: must post TYPE=gate-read-failure"
|
|
117
|
+
# forge#4572: a transient read failure is retried and does not block; a persistent one (above) still blocks.
|
|
118
|
+
newfix flaky; echo 2 > "$FIX/flaky-labels"; echo 3 > "$FIX/flaky-issues"
|
|
119
|
+
OUT=$(run gate --repo o/r --pr 9 --prs-file "$TMP/prs2" 2>&1); RC=$?
|
|
120
|
+
[ "$RC" -eq 0 ] && grep -q 'FORGE:GATE_PASS' "$FIX/posted" 2>/dev/null && ok || bad "transient label/finding read failures must be retried, not block (rc $RC: $OUT)"
|
|
121
|
+
newfix flakyfew; echo 5 > "$FIX/flaky-issues"
|
|
122
|
+
run gate --repo o/r --pr 9 --prs-file "$TMP/prs2" >/dev/null 2>&1; RC=$?
|
|
123
|
+
[ "$RC" -eq 0 ] && ok || bad "5 transient failures fit inside the default 6 attempts (rc $RC)"
|
|
124
|
+
newfix flakymany; echo 6 > "$FIX/flaky-issues"
|
|
125
|
+
run gate --repo o/r --pr 9 --prs-file "$TMP/prs" >/dev/null 2>&1; RC=$?
|
|
126
|
+
[ "$RC" -eq 1 ] && ! grep -q 'FORGE:GATE_PASS' "$FIX/posted" 2>/dev/null && ok || bad "failures beyond the attempt budget must still block (rc $RC)"
|
|
109
127
|
newfix issuefail; : > "$FIX/fail-issues"
|
|
110
128
|
run gate --repo o/r --pr 9 --prs-file "$TMP/prs" >/dev/null 2>&1; RC=$?
|
|
111
129
|
[ "$RC" -eq 1 ] && ! grep -q 'FORGE:GATE_PASS' "$FIX/posted" 2>/dev/null && ok || bad "a failed review-finding read alone must block without GATE_PASS (rc $RC)"
|