forgedock 1.18.0 → 1.20.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -0
- package/bin/engine/outcome-codes.mjs +13 -0
- package/bin/engine/phases.mjs +54 -4
- package/bin/engine/runlog.mjs +10 -2
- package/bin/engine/telemetry-seam.mjs +3 -0
- package/bin/engine/telemetry.test.mjs +3 -2
- package/bin/engine-cli.mjs +32 -2
- package/bin/engine.mjs +187 -10
- package/bin/runner.mjs +89 -13
- package/bin/stats/aggregate.mjs +7 -3
- package/bin/stats/cache.mjs +7 -3
- package/bin/stats/github-source.mjs +136 -28
- package/bin/stats/index.mjs +46 -9
- package/bin/stats/render.mjs +1 -1
- package/bin/stats/stats.test.mjs +201 -40
- package/commands/orchestrate/phase-4-execution.md +1 -1
- package/commands/review-pr-agents/protocols.md +19 -0
- package/commands/review-pr-staging.md +4 -4
- package/commands/review-pr.md +31 -20
- package/commands/work-on/build/architect.md +54 -1
- package/commands/work-on/build/implement.md +28 -1
- package/commands/work-on/build/validate.md +321 -29
- package/commands/work-on/build.md +184 -17
- package/commands/work-on/close.md +9 -4
- package/commands/work-on/investigate.md +7 -0
- package/commands/work-on/remediate.md +43 -17
- package/commands/work-on/review.md +43 -22
- package/commands/work-on.md +4 -4
- package/docs/CONFIG.md +93 -2
- package/docs/OUTCOME-CODES.md +5 -1
- package/docs/RELEASE-NOTES.md +15 -0
- package/docs/STATS.md +11 -4
- package/docs/TELEMETRY.md +5 -2
- package/docs/WORK-ON-RUNTIME.md +29 -3
- package/package.json +2 -2
- package/packages/protocol/src/types.js +1 -1
- package/scripts/acceptance-red-green.sh +90 -0
- package/scripts/acceptance-red-green.test.sh +74 -0
- package/scripts/architect-resume-match.test.sh +12 -3
- package/scripts/build-knowledge-index.mjs +8 -5
- package/scripts/builder-resume-match.test.sh +12 -3
- package/scripts/calibration.mjs +5 -2
- package/scripts/cap-findings.test.sh +2 -0
- package/scripts/check-contract-scope.test.sh +2 -0
- package/scripts/check-edge-cases.sh +143 -0
- package/scripts/check-edge-cases.test.sh +175 -0
- package/scripts/check-spec-markers.sh +4 -0
- package/scripts/check-test-tmp-cleanup.sh +56 -0
- package/scripts/check-test-tmp-cleanup.test.sh +69 -0
- package/scripts/ci-local/adapters/azure-pipelines.mjs +113 -0
- package/scripts/ci-local/adapters/bitbucket.mjs +64 -0
- package/scripts/ci-local/adapters/buildkite.mjs +58 -0
- package/scripts/ci-local/adapters/circleci.mjs +101 -0
- package/scripts/ci-local/adapters/common.mjs +115 -0
- package/scripts/ci-local/adapters/explicit.mjs +21 -0
- package/scripts/ci-local/adapters/github-actions.mjs +349 -0
- package/scripts/ci-local/adapters/gitlab-ci.mjs +184 -0
- package/scripts/ci-local/adapters/portability.mjs +104 -0
- package/scripts/ci-local/adapters/task-runners.mjs +67 -0
- package/scripts/ci-local/cli.mjs +50 -0
- package/scripts/ci-local/lib.mjs +379 -0
- package/scripts/ci-local/plan.mjs +186 -0
- package/scripts/ci-local/run.mjs +451 -0
- package/scripts/ci-local/scoping.mjs +531 -0
- package/scripts/ci-local.sh +98 -0
- package/scripts/ci-local.test.sh +1108 -0
- package/scripts/classify-finding.sh +33 -7
- package/scripts/classify-finding.test.sh +23 -0
- package/scripts/classify-test-run.sh +69 -0
- package/scripts/classify-test-run.test.sh +90 -0
- package/scripts/close-parent-c3.test.sh +2 -0
- package/scripts/consumer-sweep.sh +569 -0
- package/scripts/consumer-sweep.test.sh +357 -0
- package/scripts/detect-append-registries.sh +106 -0
- package/scripts/detect-append-registries.test.sh +64 -0
- package/scripts/file-deferred-followups.sh +91 -30
- package/scripts/file-deferred-followups.test.sh +101 -8
- package/scripts/fixtures/ci-local/azure/azure-pipelines.yml +11 -0
- package/scripts/fixtures/ci-local/bitbucket/bitbucket-pipelines.yml +12 -0
- package/scripts/fixtures/ci-local/buildkite/.buildkite/pipeline.yml +10 -0
- package/scripts/fixtures/ci-local/circleci/.circleci/config.yml +29 -0
- package/scripts/fixtures/ci-local/gha-basic/.github/workflows/ci.yml +59 -0
- package/scripts/fixtures/ci-local/gha-basic/.github/workflows/docs.yml +10 -0
- package/scripts/fixtures/ci-local/gha-basic/.github/workflows/nightly.yml +10 -0
- package/scripts/fixtures/ci-local/gha-basic/docs/index.md +1 -0
- package/scripts/fixtures/ci-local/gha-basic/src/greet.sh +2 -0
- package/scripts/fixtures/ci-local/gha-basic/tests/greet.test.sh +3 -0
- package/scripts/fixtures/ci-local/gha-dorny/.github/workflows/ci.yml +51 -0
- package/scripts/fixtures/ci-local/gha-dorny/backend/app.py +1 -0
- package/scripts/fixtures/ci-local/gha-dorny/docs/a.md +1 -0
- package/scripts/fixtures/ci-local/gha-dorny/frontend/app.js +1 -0
- package/scripts/fixtures/ci-local/gha-run/.github/workflows/ci.yml +39 -0
- package/scripts/fixtures/ci-local/gha-run/src/x.txt +1 -0
- package/scripts/fixtures/ci-local/gitlab/.gitlab-ci.yml +43 -0
- package/scripts/fixtures/ci-local/gitlab/docs/d.md +1 -0
- package/scripts/fixtures/ci-local/gitlab/src/a.txt +1 -0
- package/scripts/fixtures/ci-local/monorepo/.github/workflows/ci.yml +13 -0
- package/scripts/fixtures/ci-local/monorepo/package-lock.json +1 -0
- package/scripts/fixtures/ci-local/monorepo/packages/a/src/a.sh +2 -0
- package/scripts/fixtures/ci-local/monorepo/packages/a/tests/a.test.sh +3 -0
- package/scripts/fixtures/ci-local/monorepo/packages/b/src/b.sh +2 -0
- package/scripts/fixtures/ci-local/monorepo/packages/b/tests/b.test.sh +3 -0
- package/scripts/fixtures/ci-local/no-ci/Makefile +6 -0
- package/scripts/fixtures/ci-local/no-ci/package.json +1 -0
- package/scripts/fixtures/ci-local/no-ci/src/a.js +1 -0
- package/scripts/fixtures/ci-local/no-ci/src/b.js +1 -0
- package/scripts/fixtures/ci-local/no-ci/tests/a.test.js +2 -0
- package/scripts/fixtures/ci-local/no-ci/tests/b.test.js +2 -0
- package/scripts/forge-root.test.sh +1 -1
- package/scripts/lib/telemetry.sh +3 -0
- package/scripts/remediate-approved-head.test.sh +2 -0
- package/scripts/remediate-basesync.test.sh +1 -1
- package/scripts/review-acceptance-gate.sh +2 -2
- package/scripts/review-acceptance-gate.test.sh +20 -0
- package/scripts/review-delta.sh +8 -1
- package/scripts/review-delta.test.sh +19 -0
- package/scripts/review-finalize.sh +294 -53
- package/scripts/review-finalize.test.sh +347 -9
- package/scripts/review-pr-delta-wiring.test.sh +13 -0
- package/scripts/review-prepare.sh +5 -3
- package/scripts/review-prepare.test.sh +18 -2
- package/scripts/review-roster.sh +18 -5
- package/scripts/review-roster.test.sh +57 -1
- package/scripts/run-shell-tests.sh +5 -0
- package/scripts/selfreview-plan.sh +86 -0
- package/scripts/selfreview-plan.test.sh +90 -0
- package/scripts/spawn-depth-check.sh +5 -4
- package/scripts/spawn-depth-check.test.sh +3 -0
- package/scripts/telemetry.test.sh +10 -0
- package/scripts/transition-label.sh +26 -0
- package/scripts/trusted-comments.test.sh +2 -1
- package/scripts/verify-phase-trail.test.sh +14 -0
- package/scripts/wait-ci-green.sh +61 -9
- package/scripts/wait-ci-green.test.sh +41 -3
- package/scripts/with-run-tmpdir.sh +20 -0
|
@@ -0,0 +1,1108 @@
|
|
|
1
|
+
#!/usr/bin/env bash
|
|
2
|
+
# SPDX-FileCopyrightText: Copyright (c) RapierCraft Studios
|
|
3
|
+
# SPDX-License-Identifier: AGPL-3.0-or-later
|
|
4
|
+
#
|
|
5
|
+
# Fixtures for scripts/ci-local.sh. Run: bash scripts/ci-local.test.sh (bash 3.2 compatible).
|
|
6
|
+
# Every case builds a throwaway git repo from scripts/fixtures/ci-local/<name> (base commit on `main`, change on `feat`)
|
|
7
|
+
# and asserts the plan or the run result. No GitHub access, no real docker (a shim on PATH stands in).
|
|
8
|
+
|
|
9
|
+
set -u
|
|
10
|
+
DIR=$(cd "$(dirname "$0")" && pwd)
|
|
11
|
+
ROOT=$(cd "$DIR/.." && pwd)
|
|
12
|
+
S="$DIR/ci-local.sh"
|
|
13
|
+
FX="$DIR/fixtures/ci-local"
|
|
14
|
+
|
|
15
|
+
for tool in node git jq yq; do
|
|
16
|
+
command -v "$tool" >/dev/null 2>&1 || { echo "SKIP: ci-local.test.sh needs $tool"; exit 0; }
|
|
17
|
+
done
|
|
18
|
+
yq -o=json '.' - <<<'a: 1' 2>/dev/null | grep -q '"a"' || { echo "SKIP: ci-local.test.sh needs mikefarah yq v4"; exit 0; }
|
|
19
|
+
|
|
20
|
+
T=$(mktemp -d "${TMPDIR:-/tmp}/ci-local-test.XXXXXX")
|
|
21
|
+
trap 'rm -rf "$T"' EXIT
|
|
22
|
+
export FORGE_TELEMETRY=off
|
|
23
|
+
export FORGE_CI_LOCAL_HOME="$T/home"
|
|
24
|
+
export FORGE_CI_LOCAL_SLOTS=4
|
|
25
|
+
unset FORGE_CI_LOCAL_SERVICES FORGE_CI_LOCAL_NO_YQ FORGE_CI_LOCAL_BUDGET_SECONDS FORGE_CI_LOCAL_JOB_TIMEOUT FORGE_CI_LOCAL_NO_CACHE
|
|
26
|
+
HOST=$(uname -s)
|
|
27
|
+
|
|
28
|
+
pass=0; fail=0
|
|
29
|
+
ok() { pass=$((pass+1)); }
|
|
30
|
+
bad() { fail=$((fail+1)); echo "FAIL: $1"; [ -n "${2:-}" ] && printf '%s\n' "$2" | sed 's/^/ /' | head -30; }
|
|
31
|
+
eq() { [ "$2" = "$3" ] && ok || bad "$1 (want '$2', got '$3')"; }
|
|
32
|
+
has() { case "$2" in *"$3"*) ok ;; *) bad "$1 (missing '$3')" "$2" ;; esac; }
|
|
33
|
+
hasnt() { case "$2" in *"$3"*) bad "$1 (unexpected '$3')" "$2" ;; *) ok ;; esac; }
|
|
34
|
+
|
|
35
|
+
g() { git -C "$1" -c user.name=t -c user.email=t@example.com -c commit.gpgsign=false "${@:2}" >/dev/null 2>&1; }
|
|
36
|
+
|
|
37
|
+
# mkrepo <fixture> -> prints the repo path; base commit on main, checked out on branch feat
|
|
38
|
+
mkrepo() {
|
|
39
|
+
local r; r=$(mktemp -d "$T/r.XXXXXX")
|
|
40
|
+
cp -R "$FX/$1/." "$r/"
|
|
41
|
+
git -C "$r" init -q -b main >/dev/null 2>&1 || { git -C "$r" init -q; g "$r" checkout -q -b main; }
|
|
42
|
+
g "$r" add -A; g "$r" commit -q -m base
|
|
43
|
+
g "$r" checkout -q -b feat
|
|
44
|
+
printf '%s' "$r"
|
|
45
|
+
}
|
|
46
|
+
# blank <name> -> empty repo (inline cases)
|
|
47
|
+
blank() {
|
|
48
|
+
local r; r=$(mktemp -d "$T/r.XXXXXX")
|
|
49
|
+
git -C "$r" init -q -b main >/dev/null 2>&1 || { git -C "$r" init -q; g "$r" checkout -q -b main; }
|
|
50
|
+
printf '%s' "$r"
|
|
51
|
+
}
|
|
52
|
+
commit_base() { g "$1" add -A; g "$1" commit -q -m base; g "$1" checkout -q -b feat; }
|
|
53
|
+
|
|
54
|
+
plan() { bash "$S" plan --repo "$1" --base main --base-branch main --head-branch feat --json --no-cache "${@:2}" 2>/dev/null; }
|
|
55
|
+
run_() { OUT=$(bash "$S" run --repo "$1" --base main --base-branch main --head-branch feat "${@:2}" 2>&1); RC=$?; }
|
|
56
|
+
state() { printf '%s' "$1" | jq -r --arg id "$2" '[.jobs[] | select(.id == $id)][0] | if . == null then "missing" elif .not_applicable then "not-applicable" elif .deferred_reason then "deferred" else "runnable" end'; }
|
|
57
|
+
reason() { printf '%s' "$1" | jq -r --arg id "$2" '[.jobs[] | select(.id == $id)][0] | (.deferred_reason // .not_applicable // "")'; }
|
|
58
|
+
step_field() { printf '%s' "$1" | jq -r --arg id "$2" --argjson n "$3" "[.jobs[] | select(.id == \$id)][0].steps | map(select(.noop | not))[\$n] | $4"; }
|
|
59
|
+
|
|
60
|
+
# ------------------------------------------------------------------------------------------------ CLI contract
|
|
61
|
+
OUT=$(bash "$S" run 2>&1); RC=$?
|
|
62
|
+
eq "run without --repo is a usage error (exit 2)" 2 "$RC"
|
|
63
|
+
has "usage error prints a DEGRADED result line" "$OUT" "CI_LOCAL_RESULT: DEGRADED"
|
|
64
|
+
OUT=$(bash "$S" bogus 2>&1); RC=$?; eq "unknown subcommand exits 2" 2 "$RC"
|
|
65
|
+
r=$(mkrepo gha-basic); echo x >> "$r/src/greet.sh"
|
|
66
|
+
OUT=$(FORGE_CI_LOCAL_NO_YQ=1 bash "$S" run --repo "$r" --base main 2>&1); RC=$?
|
|
67
|
+
eq "missing yq degrades (exit 2), never passes" 2 "$RC"
|
|
68
|
+
has "missing yq names the cause" "$OUT" "yq not found"
|
|
69
|
+
|
|
70
|
+
# ------------------------------------------------------------------------------------------------ GitHub Actions: classification
|
|
71
|
+
r=$(mkrepo gha-basic); echo 'greet2() { :; }' >> "$r/src/greet.sh"
|
|
72
|
+
P=$(plan "$r")
|
|
73
|
+
eq "unit job is runnable on a src change" runnable "$(state "$P" gha:ci:unit)"
|
|
74
|
+
eq "src change selects the reached test only (selected)" 1 "$(step_field "$P" gha:ci:unit 0 '.scope.selected')"
|
|
75
|
+
eq "the matrix job runs once on the host" runnable "$(state "$P" gha:ci:matrix)"
|
|
76
|
+
case "$HOST" in
|
|
77
|
+
Linux) has "macOS/Windows matrix variants are deferred" "$(reason "$P" gha:ci:matrix:variants)" "macos-14" ;;
|
|
78
|
+
Darwin) has "Windows matrix variant is deferred" "$(reason "$P" gha:ci:matrix:variants)" "windows-latest" ;;
|
|
79
|
+
esac
|
|
80
|
+
eq "service job is runnable when services are enabled" runnable "$(state "$P" gha:ci:withservice)"
|
|
81
|
+
eq "service job lists its container" postgres "$(printf '%s' "$P" | jq -r '.jobs[] | select(.id == "gha:ci:withservice") | .services[0].name')"
|
|
82
|
+
eq "secret-referencing job is deferred" deferred "$(state "$P" gha:ci:needsecret)"
|
|
83
|
+
has "secret job reason" "$(reason "$P" gha:ci:needsecret)" "secrets"
|
|
84
|
+
eq "unknown action job is deferred, never guessed" deferred "$(state "$P" gha:ci:unknownaction)"
|
|
85
|
+
has "unknown action reason names the action" "$(reason "$P" gha:ci:unknownaction)" "some-org/mystery-action"
|
|
86
|
+
eq "environment (deploy) job is deferred" deferred "$(state "$P" gha:ci:deployprod)"
|
|
87
|
+
eq "push-only job (if: false) is not applicable" not-applicable "$(state "$P" gha:ci:conditional)"
|
|
88
|
+
eq "docs workflow path filter excludes a src change" not-applicable "$(state "$P" gha:docs)"
|
|
89
|
+
eq "schedule/dispatch-only workflow is not a PR gate" not-applicable "$(state "$P" gha:nightly)"
|
|
90
|
+
P2=$(FORGE_CI_LOCAL_SERVICES=off plan "$r")
|
|
91
|
+
eq "services: off defers the service job" deferred "$(state "$P2" gha:ci:withservice)"
|
|
92
|
+
|
|
93
|
+
r=$(mkrepo gha-basic); echo more >> "$r/docs/index.md"
|
|
94
|
+
P=$(plan "$r")
|
|
95
|
+
eq "ci workflow path filter excludes a docs-only change" not-applicable "$(state "$P" gha:ci)"
|
|
96
|
+
eq "docs workflow selects a docs change" runnable "$(state "$P" gha:docs:docs)"
|
|
97
|
+
|
|
98
|
+
# trust: workflow definitions come from the base ref, plus the PR's own edits
|
|
99
|
+
r=$(mkrepo gha-basic)
|
|
100
|
+
cat > "$r/.github/workflows/ci.yml" <<'YML'
|
|
101
|
+
name: CI
|
|
102
|
+
on:
|
|
103
|
+
pull_request:
|
|
104
|
+
jobs:
|
|
105
|
+
added:
|
|
106
|
+
runs-on: ubuntu-latest
|
|
107
|
+
steps:
|
|
108
|
+
- run: echo added-by-pr
|
|
109
|
+
YML
|
|
110
|
+
P=$(plan "$r")
|
|
111
|
+
eq "a job the PR adds is included" runnable "$(state "$P" gha:ci:added)"
|
|
112
|
+
eq "a job the PR deletes still runs (base is trusted)" runnable "$(state "$P" gha:ci:unit)"
|
|
113
|
+
|
|
114
|
+
# ------------------------------------------------------------------------------------------------ dorny/paths-filter (AlterLab-shaped)
|
|
115
|
+
r=$(mkrepo gha-dorny); echo '# x' >> "$r/backend/app.py"
|
|
116
|
+
P=$(plan "$r")
|
|
117
|
+
eq "paths-filter: backend change selects the backend job" runnable "$(state "$P" gha:ci:backend)"
|
|
118
|
+
eq "paths-filter: frontend job is not applicable" not-applicable "$(state "$P" gha:ci:frontend)"
|
|
119
|
+
eq "paths-filter: the changes job is resolved locally" not-applicable "$(state "$P" gha:ci:changes)"
|
|
120
|
+
eq "paths-filter: backend service container is planned" redis "$(printf '%s' "$P" | jq -r '.jobs[] | select(.id == "gha:ci:backend") | .services[0].name')"
|
|
121
|
+
eq "paths-filter: secret job is deferred" deferred "$(state "$P" gha:ci:integration)"
|
|
122
|
+
eq "paths-filter: deploy job is deferred" deferred "$(state "$P" gha:ci:deploy-staging)"
|
|
123
|
+
r=$(mkrepo gha-dorny); echo '// x' >> "$r/frontend/app.js"
|
|
124
|
+
P=$(plan "$r")
|
|
125
|
+
eq "paths-filter: frontend change selects frontend" runnable "$(state "$P" gha:ci:frontend)"
|
|
126
|
+
eq "paths-filter: backend job not applicable for a frontend change" not-applicable "$(state "$P" gha:ci:backend)"
|
|
127
|
+
eq "paths-filter: integration not applicable for a frontend change" not-applicable "$(state "$P" gha:ci:integration)"
|
|
128
|
+
r=$(mkrepo gha-dorny); echo '# x' >> "$r/docs/a.md"
|
|
129
|
+
P=$(plan "$r")
|
|
130
|
+
eq "paths-filter: a docs-only change selects neither job" not-applicable "$(state "$P" gha:ci:backend)"
|
|
131
|
+
|
|
132
|
+
# ------------------------------------------------------------------------------------------------ exact scoping: large monorepo
|
|
133
|
+
r=$(blank); mkdir -p "$r/.github/workflows" "$r/.forge-scratch"
|
|
134
|
+
cat > "$r/.github/workflows/ci.yml" <<'YML'
|
|
135
|
+
name: CI
|
|
136
|
+
on:
|
|
137
|
+
pull_request:
|
|
138
|
+
jobs:
|
|
139
|
+
test:
|
|
140
|
+
runs-on: ubuntu-latest
|
|
141
|
+
steps:
|
|
142
|
+
- run: node --test "packages/*/tests/*.test.mjs"
|
|
143
|
+
lint:
|
|
144
|
+
runs-on: ubuntu-latest
|
|
145
|
+
steps:
|
|
146
|
+
- run: shellcheck -S error packages/*/bin/*.sh
|
|
147
|
+
YML
|
|
148
|
+
i=1
|
|
149
|
+
while [ "$i" -le 30 ]; do
|
|
150
|
+
d="$r/packages/pkg$i"; mkdir -p "$d/src" "$d/tests" "$d/bin"
|
|
151
|
+
printf 'export const v%s = %s;\n' "$i" "$i" > "$d/src/mod$i.mjs"
|
|
152
|
+
printf "import test from 'node:test';\nimport { v%s } from '../src/mod%s.mjs';\ntest('t%s', () => { if (v%s !== %s) throw new Error('bad'); });\n" "$i" "$i" "$i" "$i" "$i" > "$d/tests/mod$i.test.mjs"
|
|
153
|
+
printf '#!/usr/bin/env bash\necho %s\n' "$i" > "$d/bin/run$i.sh"
|
|
154
|
+
i=$((i+1))
|
|
155
|
+
done
|
|
156
|
+
commit_base "$r"
|
|
157
|
+
echo '// touched' >> "$r/packages/pkg7/src/mod7.mjs"
|
|
158
|
+
P=$(plan "$r")
|
|
159
|
+
eq "one-file change selects exactly the reaching test (of 30)" "1" "$(step_field "$P" gha:ci:test 0 '.scope.selected')"
|
|
160
|
+
eq "total tests counted" "30" "$(step_field "$P" gha:ci:test 0 '.scope.total')"
|
|
161
|
+
has "selected test is the one importing the file" "$(step_field "$P" gha:ci:test 0 '.command')" "packages/pkg7/tests/mod7.test.mjs"
|
|
162
|
+
hasnt "other packages' tests are not run" "$(step_field "$P" gha:ci:test 0 '.command')" "pkg8"
|
|
163
|
+
eq "shellcheck is skipped when no shell file changed (per-file scope 0 of 30)" "0" "$(step_field "$P" gha:ci:lint 0 '.scope.selected')"
|
|
164
|
+
eq "shellcheck step is skipped for the unchanged inputs" "true" "$(step_field "$P" gha:ci:lint 0 '.skip != null')"
|
|
165
|
+
echo '# touched' >> "$r/packages/pkg9/bin/run9.sh"
|
|
166
|
+
P=$(plan "$r")
|
|
167
|
+
eq "per-file lint runs on the one changed shell file" "1" "$(step_field "$P" gha:ci:lint 0 '.scope.selected')"
|
|
168
|
+
eq "per-file lint total" "30" "$(step_field "$P" gha:ci:lint 0 '.scope.total')"
|
|
169
|
+
has "per-file lint command names only that file" "$(step_field "$P" gha:ci:lint 0 '.command')" "packages/pkg9/bin/run9.sh"
|
|
170
|
+
hasnt "per-file lint command omits siblings" "$(step_field "$P" gha:ci:lint 0 '.command')" "run10.sh"
|
|
171
|
+
# fail-safe widening
|
|
172
|
+
g "$r" add -A; g "$r" commit -q -m wip
|
|
173
|
+
echo '{}' > "$r/package-lock.json"
|
|
174
|
+
P=$(plan "$r")
|
|
175
|
+
has "lockfile change widens and reports why" "$(step_field "$P" gha:ci:test 0 '.scope.widen_reason')" "lockfile"
|
|
176
|
+
eq "widened test step runs the full command (30 of 30)" "30" "$(step_field "$P" gha:ci:test 0 '.scope.selected')"
|
|
177
|
+
rm -f "$r/package-lock.json"
|
|
178
|
+
echo '{}' > "$r/tsconfig.json"
|
|
179
|
+
P=$(plan "$r")
|
|
180
|
+
has "build-config change widens" "$(step_field "$P" gha:ci:lint 0 '.scope.widen_reason')" "build-config"
|
|
181
|
+
rm -f "$r/tsconfig.json"; mkdir -p "$r/.github/workflows"; echo '# c' >> "$r/.github/workflows/ci.yml"
|
|
182
|
+
P=$(plan "$r")
|
|
183
|
+
has "CI definition change widens" "$(step_field "$P" gha:ci:test 0 '.scope.widen_reason')" "ci-config"
|
|
184
|
+
g "$r" checkout -q -- .github/workflows/ci.yml
|
|
185
|
+
mkdir -p "$r/packages/pkg3/data"; echo 'x,y' > "$r/packages/pkg3/data/rows.csv"
|
|
186
|
+
P=$(plan "$r")
|
|
187
|
+
has "unknown file type widens (unknown language)" "$(step_field "$P" gha:ci:test 0 '.scope.widen_reason')" "unknown-language"
|
|
188
|
+
rm -rf "$r/packages/pkg3/data"
|
|
189
|
+
mkdir -p "$r/packages/pkg4/fixtures"; echo '{}' > "$r/packages/pkg4/fixtures/shared.json"
|
|
190
|
+
P=$(plan "$r")
|
|
191
|
+
has "an unreferenced shared fixture widens" "$(step_field "$P" gha:ci:test 0 '.scope.widen_reason')" "unreferenced-fixture"
|
|
192
|
+
rm -rf "$r/packages/pkg4/fixtures"
|
|
193
|
+
|
|
194
|
+
# ------------------------------------------------------------------------------------------------ other CI systems
|
|
195
|
+
r=$(mkrepo gitlab); echo more >> "$r/src/a.txt"
|
|
196
|
+
P=$(plan "$r")
|
|
197
|
+
eq "gitlab: rules:changes + MR source selects lint" runnable "$(state "$P" gitlab:lint)"
|
|
198
|
+
eq "gitlab: extends + only: merge_requests selects unit" runnable "$(state "$P" gitlab:unit)"
|
|
199
|
+
has "gitlab: extended script is inherited" "$(step_field "$P" gitlab:unit 0 '.command')" "echo template"
|
|
200
|
+
eq "gitlab: service containers are deferred" deferred "$(state "$P" gitlab:withdb)"
|
|
201
|
+
eq "gitlab: manual job is deferred" deferred "$(state "$P" gitlab:manualjob)"
|
|
202
|
+
eq "gitlab: environment (deploy) job is deferred" deferred "$(state "$P" gitlab:release)"
|
|
203
|
+
eq "gitlab: docs-only rule is not applicable to a src change" not-applicable "$(state "$P" gitlab:docsonly)"
|
|
204
|
+
eq "gitlab: stage ordering makes unit wait for earlier stages only" "[]" "$(printf '%s' "$P" | jq -c '[.jobs[] | select(.id == "gitlab:lint") | .needs[]]')"
|
|
205
|
+
|
|
206
|
+
r=$(mkrepo no-ci); echo '// t' >> "$r/src/a.js"
|
|
207
|
+
P=$(plan "$r")
|
|
208
|
+
eq "no-CI repo falls back to task runners" task-runners "$(printf '%s' "$P" | jq -r '.sources | index("task-runners") | if . == null then "no" else "task-runners" end')"
|
|
209
|
+
eq "no-CI: npm test job" runnable "$(state "$P" task:npm:test)"
|
|
210
|
+
eq "no-CI: npm lint job" runnable "$(state "$P" task:npm:lint)"
|
|
211
|
+
eq "no-CI: make test job" runnable "$(state "$P" task:make:test)"
|
|
212
|
+
eq "no-CI: make build is never planned" missing "$(state "$P" task:make:build)"
|
|
213
|
+
eq "no-CI: node --test is narrowed to the reaching test" "1" "$(step_field "$P" task:npm:test 1 '.scope.selected')"
|
|
214
|
+
has "no-CI: selected test is a.test.js" "$(step_field "$P" task:npm:test 1 '.command')" "tests/a.test.js"
|
|
215
|
+
|
|
216
|
+
r=$(mkrepo circleci); echo x > "$r/f.txt"
|
|
217
|
+
P=$(plan "$r")
|
|
218
|
+
eq "circleci: plain job runs" runnable "$(state "$P" circleci:main:test)"
|
|
219
|
+
eq "circleci: context (secrets) job deferred" deferred "$(state "$P" circleci:main:secured)"
|
|
220
|
+
eq "circleci: deploy job deferred" deferred "$(state "$P" circleci:main:deploy-prod)"
|
|
221
|
+
|
|
222
|
+
r=$(mkrepo azure); echo x > "$r/f.txt"
|
|
223
|
+
P=$(plan "$r")
|
|
224
|
+
eq "azure: an unknown task defers the job" deferred "$(state "$P" azure:build)"
|
|
225
|
+
has "azure: reason names the task" "$(reason "$P" azure:build)" "SomeUnknownTask@1"
|
|
226
|
+
r=$(blank); cat > "$r/azure-pipelines.yml" <<'YML'
|
|
227
|
+
pool:
|
|
228
|
+
vmImage: ubuntu-latest
|
|
229
|
+
variables:
|
|
230
|
+
greeting: hi
|
|
231
|
+
steps:
|
|
232
|
+
- script: echo "$(greeting) azure"
|
|
233
|
+
YML
|
|
234
|
+
commit_base "$r"; echo x > "$r/f.txt"
|
|
235
|
+
P=$(plan "$r")
|
|
236
|
+
eq "azure: a script job runs" runnable "$(state "$P" azure:build)"
|
|
237
|
+
has "azure: \$(var) macros are expanded from declared variables" "$(step_field "$P" azure:build 0 '.command')" "hi azure"
|
|
238
|
+
|
|
239
|
+
r=$(mkrepo bitbucket); echo x > "$r/f.txt"
|
|
240
|
+
P=$(plan "$r")
|
|
241
|
+
eq "bitbucket: step runs" runnable "$(state "$P" "bitbucket:default:1:Build-and-test")"
|
|
242
|
+
eq "bitbucket: deployment step deferred" deferred "$(state "$P" "bitbucket:default:2:Deploy")"
|
|
243
|
+
|
|
244
|
+
r=$(mkrepo buildkite); echo x > "$r/f.txt"
|
|
245
|
+
P=$(plan "$r")
|
|
246
|
+
eq "buildkite: command step runs" runnable "$(printf '%s' "$P" | jq -r '[.jobs[] | select(.source == "buildkite" and (.name | test("test"))) | if .deferred_reason then "deferred" else "runnable" end][0]')"
|
|
247
|
+
eq "buildkite: plugin step deferred" deferred "$(printf '%s' "$P" | jq -r '[.jobs[] | select(.source == "buildkite" and (.name | test("docker"))) | if .deferred_reason then "deferred" else "runnable" end][0]')"
|
|
248
|
+
eq "buildkite: block step deferred" deferred "$(printf '%s' "$P" | jq -r '[.jobs[] | select(.source == "buildkite" and (.name | test("release"))) | if .deferred_reason then "deferred" else "runnable" end][0]')"
|
|
249
|
+
|
|
250
|
+
# explicit config: verification.ci_local.extra_commands + include/exclude
|
|
251
|
+
r=$(mkrepo gha-basic); echo x >> "$r/src/greet.sh"
|
|
252
|
+
cat > "$r/forge.yaml" <<'YML'
|
|
253
|
+
verification:
|
|
254
|
+
ci_local:
|
|
255
|
+
extra_commands:
|
|
256
|
+
- echo extra-one
|
|
257
|
+
exclude_jobs:
|
|
258
|
+
- gha:ci:matrix
|
|
259
|
+
YML
|
|
260
|
+
P=$(plan "$r")
|
|
261
|
+
eq "extra_commands become explicit jobs" runnable "$(state "$P" explicit:1)"
|
|
262
|
+
eq "exclude_jobs removes a job from the plan" not-applicable "$(state "$P" gha:ci:matrix)"
|
|
263
|
+
|
|
264
|
+
# ------------------------------------------------------------------------------------------------ portability (always runs)
|
|
265
|
+
r=$(blank)
|
|
266
|
+
mkdir -p "$r/.github/workflows" "$r/scripts"
|
|
267
|
+
cat > "$r/.github/workflows/ci.yml" <<'YML'
|
|
268
|
+
on: pull_request
|
|
269
|
+
jobs:
|
|
270
|
+
win:
|
|
271
|
+
runs-on: windows-latest
|
|
272
|
+
steps:
|
|
273
|
+
- run: echo win
|
|
274
|
+
YML
|
|
275
|
+
printf 'seed\n' > "$r/README.md"
|
|
276
|
+
commit_base "$r"
|
|
277
|
+
printf '#!/usr/bin/env bash\necho ok\n' > "$r/scripts/clean.sh"
|
|
278
|
+
P=$(plan "$r")
|
|
279
|
+
eq "portability job is always planned" runnable "$(state "$P" portability:builtin)"
|
|
280
|
+
eq "windows-only CI job is deferred on a non-windows host" deferred "$(state "$P" gha:ci:win)"
|
|
281
|
+
run_ "$r"
|
|
282
|
+
eq "clean change passes the portability checks" 0 "$RC"
|
|
283
|
+
bad_sh="$r/scripts/bad.sh"
|
|
284
|
+
MF="map""file"; RL="read""link"
|
|
285
|
+
{ printf '#!/usr/bin/env bash\n'; printf '%s -t lines < in.txt\n' "$MF"; printf 'p=$(%s -f "$0")\n' "$RL"; printf 'echo hi\r\n'; } > "$bad_sh"
|
|
286
|
+
printf 'x' > "$r/a:b.txt"; printf 'x' > "$r/Readme.md"
|
|
287
|
+
run_ "$r"
|
|
288
|
+
eq "portability violations fail the run" 1 "$RC"
|
|
289
|
+
has "bash 4 construct detected" "$OUT" "bash-3.2"
|
|
290
|
+
has "GNU-only flag detected" "$OUT" "gnu-only"
|
|
291
|
+
has "Windows-invalid filename detected" "$OUT" "windows-filename"
|
|
292
|
+
has "case collision detected" "$OUT" "case-collision"
|
|
293
|
+
has "CRLF detected" "$OUT" "crlf"
|
|
294
|
+
has "failing result line" "$OUT" "CI_LOCAL_RESULT: FAIL"
|
|
295
|
+
# an allowlisted line is not flagged
|
|
296
|
+
rm -f "$bad_sh" "$r/a:b.txt" "$r/Readme.md"
|
|
297
|
+
printf '#!/usr/bin/env bash\nx=$(%s -f "$0") # allowlist: reviewed\n' "$RL" > "$r/scripts/ok.sh"
|
|
298
|
+
run_ "$r"
|
|
299
|
+
eq "an allowlisted line passes" 0 "$RC"
|
|
300
|
+
|
|
301
|
+
# ------------------------------------------------------------------------------------------------ runner
|
|
302
|
+
r=$(mkrepo gha-run); echo x >> "$r/src/x.txt"
|
|
303
|
+
run_ "$r"
|
|
304
|
+
eq "run exits 1 when a runnable job fails" 1 "$RC"
|
|
305
|
+
has "passing job reported" "$OUT" "CI_LOCAL_JOB: pass gha:ci:first"
|
|
306
|
+
has "GITHUB_ENV is carried to later steps" "$OUT" "CI_LOCAL_JOB: pass gha:ci:first"
|
|
307
|
+
has "needs ordering: second runs after first" "$OUT" "CI_LOCAL_JOB: pass gha:ci:second"
|
|
308
|
+
has "failing job is named" "$OUT" "CI_LOCAL_FAIL: gha:ci:failing"
|
|
309
|
+
has "failing job shows its log tail" "$OUT" "about to fail"
|
|
310
|
+
has "a job whose dependency failed is skipped" "$OUT" "CI_LOCAL_JOB: skipped gha:ci:skippedbyneed"
|
|
311
|
+
has "a failed dependency install defers instead of failing" "$OUT" "CI_LOCAL_JOB: deferred gha:ci:broken-install"
|
|
312
|
+
has "deferred jobs are listed" "$OUT" "CI_LOCAL_DEFERRED: gha:ci:broken-install"
|
|
313
|
+
has "result line carries the counts" "$OUT" "jobs_failed=1"
|
|
314
|
+
has "result line is FAIL" "$OUT" "CI_LOCAL_RESULT: FAIL"
|
|
315
|
+
hasnt "no GitHub Actions workflow is triggered" "$OUT" "gh workflow"
|
|
316
|
+
|
|
317
|
+
# passing run: exit 0, deferred jobs listed
|
|
318
|
+
r=$(mkrepo gha-basic); echo x >> "$r/src/greet.sh"
|
|
319
|
+
FORGE_CI_LOCAL_SERVICES=off run_ "$r"
|
|
320
|
+
eq "all runnable jobs pass -> exit 0" 0 "$RC"
|
|
321
|
+
has "PASS result" "$OUT" "CI_LOCAL_RESULT: PASS"
|
|
322
|
+
has "unknown action job is listed as deferred" "$OUT" "CI_LOCAL_DEFERRED: gha:ci:unknownaction"
|
|
323
|
+
has "secret job is listed as deferred" "$OUT" "CI_LOCAL_DEFERRED: gha:ci:needsecret"
|
|
324
|
+
|
|
325
|
+
# result cache: identical inputs are hits on the second iteration and across repos on the same base
|
|
326
|
+
r=$(mkrepo monorepo); echo '# c' >> "$r/packages/a/src/a.sh"
|
|
327
|
+
run_ "$r"; eq "first run passes" 0 "$RC"; has "first run misses the cache" "$OUT" "cache_hits=0"
|
|
328
|
+
run_ "$r"; eq "second run passes" 0 "$RC"
|
|
329
|
+
has "second iteration hits the cache" "$OUT" "CI_LOCAL_JOB: pass gha:ci:lint"
|
|
330
|
+
hits=$(printf '%s\n' "$OUT" | sed -n 's/.*cache_hits=\([0-9]*\).*/\1/p' | tail -n 1)
|
|
331
|
+
[ "${hits:-0}" -ge 1 ] && ok || bad "second iteration should have cache hits (got '${hits:-}')" "$OUT"
|
|
332
|
+
r2=$(mkrepo monorepo); echo '# c' >> "$r2/packages/a/src/a.sh"
|
|
333
|
+
run_ "$r2"
|
|
334
|
+
hits=$(printf '%s\n' "$OUT" | sed -n 's/.*cache_hits=\([0-9]*\).*/\1/p' | tail -n 1)
|
|
335
|
+
[ "${hits:-0}" -ge 1 ] && ok || bad "a second issue with identical per-file inputs should hit the shared cache (got '${hits:-}')" "$OUT"
|
|
336
|
+
echo '# changed' >> "$r/packages/a/src/a.sh"
|
|
337
|
+
run_ "$r"
|
|
338
|
+
has "changed inputs miss the cache" "$OUT" "CI_LOCAL_RESULT: PASS"
|
|
339
|
+
|
|
340
|
+
# parallelism, machine-wide semaphore, stale slots, timeout, budget
|
|
341
|
+
mk_pair() {
|
|
342
|
+
local p; p=$(blank); mkdir -p "$p/.github/workflows"
|
|
343
|
+
cat > "$p/.github/workflows/ci.yml" <<'YML'
|
|
344
|
+
on: pull_request
|
|
345
|
+
jobs:
|
|
346
|
+
ja:
|
|
347
|
+
runs-on: ubuntu-latest
|
|
348
|
+
steps:
|
|
349
|
+
- run: |
|
|
350
|
+
touch a.started
|
|
351
|
+
i=0; while [ "$i" -lt 60 ]; do [ -f b.started ] && exit 0; i=$((i+1)); sleep 0.1; done; exit 1
|
|
352
|
+
jb:
|
|
353
|
+
runs-on: ubuntu-latest
|
|
354
|
+
steps:
|
|
355
|
+
- run: |
|
|
356
|
+
touch b.started
|
|
357
|
+
i=0; while [ "$i" -lt 60 ]; do [ -f a.started ] && exit 0; i=$((i+1)); sleep 0.1; done; exit 1
|
|
358
|
+
YML
|
|
359
|
+
echo seed > "$p/seed.txt"; commit_base "$p"; echo x >> "$p/seed.txt"; printf '%s' "$p"
|
|
360
|
+
}
|
|
361
|
+
export FORGE_CI_LOCAL_NO_CACHE=1
|
|
362
|
+
r=$(mk_pair); run_ "$r" --max-parallel 2
|
|
363
|
+
eq "independent jobs run in parallel (each waits for the other)" 0 "$RC"
|
|
364
|
+
rm -f "$r/a.started" "$r/b.started"
|
|
365
|
+
run_ "$r" --max-parallel 1
|
|
366
|
+
eq "max-parallel 1 serialises jobs (the rendezvous cannot happen)" 1 "$RC"
|
|
367
|
+
rm -f "$r/a.started" "$r/b.started"
|
|
368
|
+
OUT=$(FORGE_CI_LOCAL_SLOTS=1 bash "$S" run --repo "$r" --base main --base-branch main --head-branch feat --max-parallel 2 2>&1); RC=$?
|
|
369
|
+
eq "a one-slot machine-wide semaphore serialises jobs across the pool" 1 "$RC"
|
|
370
|
+
rm -f "$r/a.started" "$r/b.started"
|
|
371
|
+
mkdir -p "$FORGE_CI_LOCAL_HOME/ci-local.lock/slot-0"
|
|
372
|
+
printf '999999 1 stale\n' > "$FORGE_CI_LOCAL_HOME/ci-local.lock/slot-0/owner"
|
|
373
|
+
OUT=$(FORGE_CI_LOCAL_SLOTS=2 bash "$S" run --repo "$r" --base main --base-branch main --head-branch feat --max-parallel 2 2>&1); RC=$?
|
|
374
|
+
eq "a stale slot (dead pid) is reclaimed" 0 "$RC"
|
|
375
|
+
[ ! -d "$FORGE_CI_LOCAL_HOME/ci-local.lock/slot-0" ] && ok || bad "slots are released after the run"
|
|
376
|
+
|
|
377
|
+
r=$(blank); mkdir -p "$r/.github/workflows"
|
|
378
|
+
cat > "$r/.github/workflows/ci.yml" <<'YML'
|
|
379
|
+
on: pull_request
|
|
380
|
+
jobs:
|
|
381
|
+
hang:
|
|
382
|
+
runs-on: ubuntu-latest
|
|
383
|
+
steps:
|
|
384
|
+
- run: sleep 30
|
|
385
|
+
quick:
|
|
386
|
+
runs-on: ubuntu-latest
|
|
387
|
+
steps:
|
|
388
|
+
- run: echo quick
|
|
389
|
+
YML
|
|
390
|
+
echo s > "$r/s.txt"; commit_base "$r"; echo x >> "$r/s.txt"
|
|
391
|
+
OUT=$(FORGE_CI_LOCAL_JOB_TIMEOUT=2 bash "$S" run --repo "$r" --base main --base-branch main --head-branch feat 2>&1); RC=$?
|
|
392
|
+
eq "a job past its timeout is DEGRADED (exit 2), never FAIL (forge#4040)" 2 "$RC"
|
|
393
|
+
has "the timeout is a DEGRADED result line" "$OUT" "CI_LOCAL_RESULT: DEGRADED"
|
|
394
|
+
hasnt "a timeout is never a CI_LOCAL_FAIL" "$OUT" "CI_LOCAL_FAIL:"
|
|
395
|
+
has "timeout is reported" "$OUT" "timed out after 2s"
|
|
396
|
+
has "other jobs still pass" "$OUT" "CI_LOCAL_JOB: pass gha:ci:quick"
|
|
397
|
+
OUT=$(FORGE_CI_LOCAL_BUDGET_SECONDS=2 FORGE_CI_LOCAL_JOB_TIMEOUT=600 bash "$S" run --repo "$r" --base main --base-branch main --head-branch feat 2>&1); RC=$?
|
|
398
|
+
eq "an exhausted budget leaves the run DEGRADED (partial), not a pass and not a fail (forge#4040)" 2 "$RC"
|
|
399
|
+
has "budget exhaustion names the reason" "$OUT" "reason=budget_exhausted"
|
|
400
|
+
has "budget exhaustion is reported" "$OUT" "budget exhausted"
|
|
401
|
+
has "the unfinished job is deferred" "$OUT" "CI_LOCAL_DEFERRED: gha:ci:hang"
|
|
402
|
+
|
|
403
|
+
# ------------------------------------------------------------------------------------------------ services (docker shim)
|
|
404
|
+
r=$(blank); mkdir -p "$r/.github/workflows" "$T/bin"
|
|
405
|
+
cat > "$r/.github/workflows/ci.yml" <<'YML'
|
|
406
|
+
on: pull_request
|
|
407
|
+
jobs:
|
|
408
|
+
db:
|
|
409
|
+
runs-on: ubuntu-latest
|
|
410
|
+
services:
|
|
411
|
+
postgres:
|
|
412
|
+
image: postgres:16
|
|
413
|
+
env:
|
|
414
|
+
POSTGRES_PASSWORD: example
|
|
415
|
+
ports:
|
|
416
|
+
- 54329:5432
|
|
417
|
+
options: --health-cmd pg_isready --health-interval 5s --privileged
|
|
418
|
+
steps:
|
|
419
|
+
- run: echo with-db
|
|
420
|
+
YML
|
|
421
|
+
echo s > "$r/s.txt"; commit_base "$r"; echo x >> "$r/s.txt"
|
|
422
|
+
cat > "$T/bin/docker" <<'SHIM'
|
|
423
|
+
#!/usr/bin/env bash
|
|
424
|
+
echo "$*" >> "$FAKE_DOCKER_LOG"
|
|
425
|
+
case "$1" in
|
|
426
|
+
info) exit "${FAKE_DOCKER_INFO_RC:-0}" ;;
|
|
427
|
+
run) echo fakeid ;;
|
|
428
|
+
inspect) echo "true|healthy" ;;
|
|
429
|
+
*) : ;;
|
|
430
|
+
esac
|
|
431
|
+
SHIM
|
|
432
|
+
chmod +x "$T/bin/docker"
|
|
433
|
+
export FAKE_DOCKER_LOG="$T/docker.log"
|
|
434
|
+
: > "$FAKE_DOCKER_LOG"
|
|
435
|
+
OUT=$(PATH="$T/bin:$PATH" bash "$S" run --repo "$r" --base main --base-branch main --head-branch feat 2>&1); RC=$?
|
|
436
|
+
eq "a service job runs when docker is available" 0 "$RC"
|
|
437
|
+
LOG=$(cat "$FAKE_DOCKER_LOG")
|
|
438
|
+
has "container is started with the same image" "$LOG" "postgres:16"
|
|
439
|
+
has "container publishes the declared port" "$LOG" "-p 127.0.0.1:54329:5432"
|
|
440
|
+
has "container gets the declared env" "$LOG" "-e POSTGRES_PASSWORD=example"
|
|
441
|
+
has "health options are passed through" "$LOG" "--health-cmd"
|
|
442
|
+
hasnt "options off the safe list are dropped" "$LOG" "--privileged"
|
|
443
|
+
has "container is named per run" "$LOG" "forge-ci-"
|
|
444
|
+
has "container is torn down" "$LOG" "rm -f"
|
|
445
|
+
: > "$FAKE_DOCKER_LOG"
|
|
446
|
+
OUT=$(FAKE_DOCKER_INFO_RC=1 PATH="$T/bin:$PATH" bash "$S" run --repo "$r" --base main --base-branch main --head-branch feat 2>&1); RC=$?
|
|
447
|
+
eq "docker absent defers the service job (exit 0)" 0 "$RC"
|
|
448
|
+
has "the deferral says why" "$OUT" "docker is not available"
|
|
449
|
+
hasnt "no container is started without docker" "$(cat "$FAKE_DOCKER_LOG")" "run -d"
|
|
450
|
+
|
|
451
|
+
# ------------------------------------------------------------------------------------------------ telemetry + no remote triggers
|
|
452
|
+
r=$(mkrepo gha-basic); echo x >> "$r/src/greet.sh"
|
|
453
|
+
OUT=$(FORGE_CI_LOCAL_SERVICES=off FORGE_TELEMETRY=on FORGE_REPO=o/r FORGE_ISSUE=1 FORGE_TELEMETRY_GH=/bin/false bash "$S" run --repo "$r" --base main --base-branch main --head-branch feat --iteration 1 2>&1); RC=$?
|
|
454
|
+
eq "telemetry never changes the exit code" 0 "$RC"
|
|
455
|
+
if grep -nE 'gh run|gh workflow|workflow_dispatch' "$S" >/dev/null 2>&1; then bad "ci-local.sh must not trigger or watch GitHub Actions"; else ok; fi
|
|
456
|
+
if grep -rnE "spawn(Sync)?\(['\"]gh['\"]|run\(['\"]gh['\"]|execSync\(['\"]gh " "$DIR/ci-local" >/dev/null 2>&1; then bad "ci-local modules must not invoke the gh CLI"; else ok; fi
|
|
457
|
+
|
|
458
|
+
# ------------------------------------------------------------------------------------------------ environment hygiene, expressions
|
|
459
|
+
r=$(blank); mkdir -p "$r/.github/workflows"
|
|
460
|
+
cat > "$r/.github/workflows/ci.yml" <<'YML'
|
|
461
|
+
on: pull_request
|
|
462
|
+
jobs:
|
|
463
|
+
env:
|
|
464
|
+
runs-on: ubuntu-latest
|
|
465
|
+
env:
|
|
466
|
+
LABELS: ${{ join(github.event.pull_request.labels.*.name, ',') }}
|
|
467
|
+
BASE: ${{ github.base_ref }}
|
|
468
|
+
steps:
|
|
469
|
+
- run: |
|
|
470
|
+
test -z "${LEAKED_SECRET_TOKEN:-}" || { echo "credential leaked"; exit 1; }
|
|
471
|
+
test -z "${FORGEDOCK_HOME:-}" || { echo "ambient install root leaked"; exit 1; }
|
|
472
|
+
test -z "${GIT_INDEX_FILE:-}" || { echo "git hook variable leaked"; exit 1; }
|
|
473
|
+
test "${CUSTOM_FLAG:-}" = 1 || { echo "passthrough missing"; exit 1; }
|
|
474
|
+
test "$CI" = true || { echo "CI not set"; exit 1; }
|
|
475
|
+
test -z "$LABELS" || { echo "labels should be empty locally"; exit 1; }
|
|
476
|
+
test "$BASE" = main || { echo "base_ref not substituted"; exit 1; }
|
|
477
|
+
YML
|
|
478
|
+
echo s > "$r/s.txt"; commit_base "$r"; echo x >> "$r/s.txt"
|
|
479
|
+
OUT=$(LEAKED_SECRET_TOKEN=x FORGEDOCK_HOME=/nonexistent GIT_INDEX_FILE=/nonexistent CUSTOM_FLAG=1 FORGE_CI_LOCAL_PASSTHROUGH=CUSTOM_FLAG bash "$S" run --repo "$r" --base main --base-branch main --head-branch feat 2>&1); RC=$?
|
|
480
|
+
eq "jobs run in a clean runner-like environment (no credentials, no ambient install root, allowlisted passthrough)" 0 "$RC"
|
|
481
|
+
hasnt "no leak reported" "$OUT" "leaked"
|
|
482
|
+
# AUTH-1: credential-bearing names and URL userinfo never reach jobs (checked on scrubbedEnv directly)
|
|
483
|
+
_senv() { env "$@" node -e "import('$ROOT/scripts/ci-local/lib.mjs').then(m=>{const e=m.scrubbedEnv();console.log(['YARN_NPM_AUTH_IDENT','NPM_TOKEN','NODE_AUTH_TOKEN','GH_TOKEN','DEPLOY_PASS','MY_PAT','AWS_ACCESS_KEY','SESSION_COOKIE','HTTPS_PROXY','UV_INDEX_URL','NO_PROXY','LANG','PATH','BENIGN_FLAG','FORGE_CI_LOCAL_PASSTHROUGH'].map(k=>k+'='+(k in e?e[k]:'<unset>')).join('\\n'))})"; }
|
|
484
|
+
SE=$(_senv YARN_NPM_AUTH_IDENT=u:p NPM_TOKEN=t NODE_AUTH_TOKEN=t GH_TOKEN=t DEPLOY_PASS=p MY_PAT=p AWS_ACCESS_KEY=k SESSION_COOKIE=c HTTPS_PROXY=http://u:p@h:3128 UV_INDEX_URL=https://bot:hunter2@pypi.example/simple NO_PROXY=localhost LANG=C.UTF-8 BENIGN_FLAG=1 FORGE_CI_LOCAL_PASSTHROUGH=BENIGN_FLAG)
|
|
485
|
+
for _n in YARN_NPM_AUTH_IDENT NPM_TOKEN NODE_AUTH_TOKEN GH_TOKEN DEPLOY_PASS MY_PAT AWS_ACCESS_KEY SESSION_COOKIE; do has "env: $_n dropped" "$SE" "$_n=<unset>"; done
|
|
486
|
+
has "env: proxy userinfo stripped, host kept" "$SE" "HTTPS_PROXY=http://h:3128"
|
|
487
|
+
has "env: index-url userinfo stripped" "$SE" "UV_INDEX_URL=https://pypi.example/simple"
|
|
488
|
+
hasnt "env: no password text survives" "$SE" "hunter2"
|
|
489
|
+
has "env: NO_PROXY kept" "$SE" "NO_PROXY=localhost"
|
|
490
|
+
has "env: LANG kept" "$SE" "LANG=C.UTF-8"
|
|
491
|
+
has "env: PATH kept" "$SE" "PATH=/"
|
|
492
|
+
has "env: benign passthrough var kept" "$SE" "BENIGN_FLAG=1"
|
|
493
|
+
# native selectors: another language's source does not widen, a data file does
|
|
494
|
+
r=$(blank); mkdir -p "$r/lib" "$r/src"
|
|
495
|
+
printf '{ "name": "js", "version": "1.0.0", "scripts": { "test": "jest" } }\n' > "$r/package.json"
|
|
496
|
+
echo 'module.exports = 1' > "$r/src/a.js"; echo 'x = 1' > "$r/lib/x.py"; echo '{}' > "$r/lib/data.json"
|
|
497
|
+
commit_base "$r"
|
|
498
|
+
echo '# y' >> "$r/lib/x.py"
|
|
499
|
+
P=$(plan "$r")
|
|
500
|
+
eq "jest: a Python-only change selects no JS tests" skip "$(step_field "$P" task:npm:test 1 '.scope.mode')"
|
|
501
|
+
echo '// z' >> "$r/src/a.js"; P=$(plan "$r")
|
|
502
|
+
eq "jest: a JS change uses --findRelatedTests" jest-related "$(step_field "$P" task:npm:test 1 '.scope.selector')"
|
|
503
|
+
echo '{"a":1}' > "$r/lib/data.json"; P=$(plan "$r")
|
|
504
|
+
has "jest: a changed data file widens (not reachable through imports)" "$(step_field "$P" task:npm:test 1 '.scope.widen_reason')" "unknown-language"
|
|
505
|
+
|
|
506
|
+
# ------------------------------------------------------------------------------------------------ ForgeDock's own workflows
|
|
507
|
+
P=$(bash "$S" plan --repo "$ROOT" --base HEAD --base-branch staging --head-branch feat/spec-change --files "commands/work-on/build.md" --json --no-cache 2>/dev/null)
|
|
508
|
+
if [ -z "$P" ]; then bad "plan on ForgeDock's own workflows produced no output"; else
|
|
509
|
+
eq "ForgeDock: a spec change selects a check-commands job" runnable "$(state "$P" gha:check-commands:check-native-conflicts)"
|
|
510
|
+
eq "ForgeDock: a spec change selects the ci unit-tests job" runnable "$(state "$P" gha:ci:unit-tests)"
|
|
511
|
+
eq "ForgeDock: a spec change selects the ci shell-tests job" runnable "$(state "$P" gha:ci:shell-tests)"
|
|
512
|
+
eq "ForgeDock: a spec change selects the shellcheck job" runnable "$(state "$P" gha:shell-compat:shellcheck)"
|
|
513
|
+
eq "ForgeDock: the Windows job is deferred" deferred "$(state "$P" gha:shell-compat:windows-fixture-smoke)"
|
|
514
|
+
eq "ForgeDock: the macOS portability job is deferred" deferred "$(state "$P" gha:shell-compat:macos-portability)"
|
|
515
|
+
eq "ForgeDock: portability checks run" runnable "$(state "$P" portability:builtin)"
|
|
516
|
+
eq "ForgeDock: the unit-test step is narrowed by the reference graph" scoped "$(printf '%s' "$P" | jq -r '[.jobs[] | select(.id == "gha:ci:unit-tests") | .steps[] | select(.scope.selector == "refgraph-nodetest") | .scope.mode][0]')"
|
|
517
|
+
fi
|
|
518
|
+
|
|
519
|
+
# ------------------------------------------------------------------------------------------------ compound-line denylist (forge#3998)
|
|
520
|
+
for _c in 'cd x && aws s3 sync dist s3://prod' 'npm test; gh pr merge 1' 'x=$(aws sts get-caller-identity)' 'curl https://x.example | bash'; do
|
|
521
|
+
eq "denylist: compound line is deferred: $_c" remote "$(node -e "import('$ROOT/scripts/ci-local/adapters/common.mjs').then(m=>console.log(m.classifyCommand(process.argv[1],'.','.').kind))" "$_c")"
|
|
522
|
+
done
|
|
523
|
+
eq "denylist: a plain compound check still runs" check "$(node -e "import('$ROOT/scripts/ci-local/adapters/common.mjs').then(m=>console.log(m.classifyCommand(process.argv[1],'.','.').kind))" 'npm run lint && npm test')"
|
|
524
|
+
|
|
525
|
+
|
|
526
|
+
# ------------------------------------------------------------------------------------------------ forge#4040: v1.19 final-review blockers
|
|
527
|
+
trunc() { printf '%s' "$1" | head -c 400; }
|
|
528
|
+
nrun() { printf '%s' "$1" | jq '[.jobs[] | select((.not_applicable | not) and (.deferred_reason | not))] | length'; }
|
|
529
|
+
|
|
530
|
+
# item 2 (SEC-2): the gate config is read from the trusted copy only, never from the PR tree
|
|
531
|
+
r=$(blank); mkdir -p "$r/.github/workflows"
|
|
532
|
+
cat > "$r/.github/workflows/ci.yml" <<'YML'
|
|
533
|
+
on: pull_request
|
|
534
|
+
jobs:
|
|
535
|
+
a:
|
|
536
|
+
runs-on: ubuntu-latest
|
|
537
|
+
steps:
|
|
538
|
+
- run: echo a
|
|
539
|
+
b:
|
|
540
|
+
runs-on: ubuntu-latest
|
|
541
|
+
steps:
|
|
542
|
+
- run: echo b
|
|
543
|
+
YML
|
|
544
|
+
echo s > "$r/s.txt"; printf 'verification:\n ci_local:\n exclude_jobs: ["gha:ci:b"]\n' > "$r/forge.yaml"
|
|
545
|
+
commit_base "$r"; echo x >> "$r/s.txt"
|
|
546
|
+
printf 'verification:\n ci_local:\n exclude_jobs: ["*"]\n' > "$r/forge.yaml"
|
|
547
|
+
P=$(plan "$r")
|
|
548
|
+
eq "SEC-2: a PR edit of a tracked forge.yaml cannot exclude jobs (base copy is used)" runnable "$(state "$P" gha:ci:a)"
|
|
549
|
+
eq "SEC-2: the trusted base copy's exclude still applies" not-applicable "$(state "$P" gha:ci:b)"
|
|
550
|
+
printf 'verification:\n ci_local:\n mode: blocking\n' > "$r/forge.yaml"; g "$r" checkout -q -- forge.yaml 2>/dev/null
|
|
551
|
+
# worktree: the main checkout's (gitignored) forge.yaml is trusted, the worktree's own copy never is
|
|
552
|
+
m=$(blank); mkdir -p "$m/.github/workflows"; cp "$r/.github/workflows/ci.yml" "$m/.github/workflows/ci.yml"
|
|
553
|
+
echo s > "$m/s.txt"; printf 'forge.yaml\n' > "$m/.gitignore"; printf 'verification:\n ci_local:\n exclude_jobs: ["gha:ci:b"]\n' > "$m/forge.yaml"
|
|
554
|
+
g "$m" add -A; g "$m" commit -q -m base
|
|
555
|
+
git -C "$m" worktree add -q -b feat "$T/wt-sec2" main >/dev/null 2>&1
|
|
556
|
+
echo x >> "$T/wt-sec2/s.txt"; printf 'verification:\n ci_local:\n exclude_jobs: ["*"]\n' > "$T/wt-sec2/forge.yaml"
|
|
557
|
+
P=$(plan "$T/wt-sec2")
|
|
558
|
+
eq "SEC-2: worktree - a forge.yaml dropped in the PR tree cannot exclude jobs" runnable "$(state "$P" gha:ci:a)"
|
|
559
|
+
eq "SEC-2: worktree - the main checkout's config applies" not-applicable "$(state "$P" gha:ci:b)"
|
|
560
|
+
[ "$(nrun "$P")" -ge 1 ] && ok || bad "SEC-2: the gate never ends with zero jobs because of PR config"
|
|
561
|
+
|
|
562
|
+
# item 3 (SEC-3): an empty or unreachable test selection for a changed source file widens (never selects 0 and passes)
|
|
563
|
+
r=$(blank); mkdir -p "$r/app" "$r/tests" "$r/lib"
|
|
564
|
+
echo 'X = 1' > "$r/app/db.py"; echo 'X = 2' > "$r/app/__init__.py"; echo 'Y = 1' > "$r/lib/zz.py"
|
|
565
|
+
printf 'from app import db\n\ndef test_a():\n assert db.X\n' > "$r/tests/test_user.py"
|
|
566
|
+
printf 'import app\n\ndef test_pkg():\n assert app.X\n' > "$r/tests/test_pkg.py"
|
|
567
|
+
printf 'def test_other():\n assert True\n' > "$r/tests/test_other.py"; echo '# doc' > "$r/README.md"
|
|
568
|
+
commit_base "$r"
|
|
569
|
+
_imp() { node -e "
|
|
570
|
+
import('$ROOT/scripts/ci-local/scoping.mjs').then(m=>{const c=process.argv.slice(1);const s=m.makeScopeCtx({repo:'$r',baseRef:'main',from:'main',changed:c});
|
|
571
|
+
const t=['tests/test_user.py','tests/test_pkg.py','tests/test_other.py'];const r=m.impactedTests(s,t);console.log(JSON.stringify({sel:r.selected,widen:r.widen}))})" "$@"; }
|
|
572
|
+
echo '# c' >> "$r/app/db.py"
|
|
573
|
+
R=$(_imp app/db.py)
|
|
574
|
+
eq "SEC-3: a module name under 4 chars (app/db.py) selects its importer" '["tests/test_user.py"]' "$(printf '%s' "$R" | jq -c '.sel')"
|
|
575
|
+
eq "SEC-3: ...without widening" "" "$(printf '%s' "$R" | jq -r '.widen')"
|
|
576
|
+
R=$(_imp app/__init__.py)
|
|
577
|
+
has "SEC-3: __init__.py selects the importer of the package" "$(printf '%s' "$R" | jq -c '.sel')" "tests/test_pkg.py"
|
|
578
|
+
R=$(_imp lib/zz.py)
|
|
579
|
+
has "SEC-3: an empty selection for a changed source file widens" "$(printf '%s' "$R" | jq -r '.widen')" "empty-selection"
|
|
580
|
+
eq "SEC-3: the widened selection is the full set" 3 "$(printf '%s' "$R" | jq '.sel | length')"
|
|
581
|
+
R=$(_imp README.md)
|
|
582
|
+
eq "SEC-3: a docs-only change still selects nothing (no widening)" '[]' "$(printf '%s' "$R" | jq -c '.sel')"
|
|
583
|
+
|
|
584
|
+
# item 4 (REG-3): the built-in bash-3.2/GNU portability job runs only when CI targets macOS/Windows
|
|
585
|
+
r=$(blank); mkdir -p "$r/.github/workflows" "$r/scripts"
|
|
586
|
+
cat > "$r/.github/workflows/ci.yml" <<'YML'
|
|
587
|
+
on: pull_request
|
|
588
|
+
jobs:
|
|
589
|
+
lint:
|
|
590
|
+
runs-on: ubuntu-latest
|
|
591
|
+
steps:
|
|
592
|
+
- run: echo lint
|
|
593
|
+
YML
|
|
594
|
+
printf '#!/usr/bin/env bash\necho hi\n' > "$r/scripts/x.sh"; commit_base "$r"
|
|
595
|
+
printf 'mapfile -t lines < /dev/null\n' >> "$r/scripts/x.sh"
|
|
596
|
+
P=$(plan "$r")
|
|
597
|
+
eq "REG-3: Linux-only CI: the portability job is skipped (a mapfile repo is not hard-blocked)" not-applicable "$(state "$P" portability:builtin)"
|
|
598
|
+
has "REG-3: the skip carries a note" "$(reason "$P" portability:builtin)" "Linux only"
|
|
599
|
+
run_ "$r"; eq "REG-3: Linux-only repo with mapfile passes the gate" 0 "$RC"
|
|
600
|
+
sed -i.bak 's/runs-on: ubuntu-latest/runs-on: ${{ matrix.os }}\n strategy:\n matrix:\n os: [ubuntu-latest, macos-14]/' "$r/.github/workflows/ci.yml"; rm -f "$r/.github/workflows/ci.yml.bak"
|
|
601
|
+
P=$(plan "$r")
|
|
602
|
+
eq "REG-3: CI that targets macOS keeps the portability job" runnable "$(state "$P" portability:builtin)"
|
|
603
|
+
r=$(mkrepo no-ci); echo x >> "$r/$(git -C "$r" ls-files | head -1)"
|
|
604
|
+
P=$(plan "$r")
|
|
605
|
+
eq "REG-3: a repo with no CI definitions skips the portability job" not-applicable "$(state "$P" portability:builtin)"
|
|
606
|
+
|
|
607
|
+
# item 5 (REG-4): a failure under a major tool-version drift is DEFERRED, never FAIL
|
|
608
|
+
r=$(blank); mkdir -p "$r/.github/workflows"
|
|
609
|
+
cat > "$r/.github/workflows/ci.yml" <<'YML'
|
|
610
|
+
on: pull_request
|
|
611
|
+
jobs:
|
|
612
|
+
drift:
|
|
613
|
+
runs-on: ubuntu-latest
|
|
614
|
+
steps:
|
|
615
|
+
- uses: actions/setup-node@v4
|
|
616
|
+
with:
|
|
617
|
+
node-version: 6
|
|
618
|
+
- run: exit 1
|
|
619
|
+
YML
|
|
620
|
+
echo s > "$r/s.txt"; commit_base "$r"; echo x >> "$r/s.txt"
|
|
621
|
+
run_ "$r"
|
|
622
|
+
eq "REG-4: setup-node 6 vs the host: the failing job does not fail the gate (exit 0)" 0 "$RC"
|
|
623
|
+
has "REG-4: it is DEFERRED with the version-drift reason" "$OUT" "CI_LOCAL_DEFERRED: gha:ci:drift"
|
|
624
|
+
has "REG-4: reason names the drift" "$OUT" "version drift"
|
|
625
|
+
hasnt "REG-4: never a CI_LOCAL_FAIL" "$OUT" "CI_LOCAL_FAIL:"
|
|
626
|
+
HM=$(node -p 'process.versions.node.split(".")[0]')
|
|
627
|
+
sed -i.bak "s/node-version: 6/node-version: $HM/" "$r/.github/workflows/ci.yml"; rm -f "$r/.github/workflows/ci.yml.bak"; g "$r" add -A; g "$r" commit -q -m wf; echo y >> "$r/s.txt"
|
|
628
|
+
run_ "$r"; eq "REG-4: with no drift the same failing step is a real FAIL (exit 1)" 1 "$RC"
|
|
629
|
+
|
|
630
|
+
# item 6 (REG-5): slot-wait time does not count toward the job timeout; a real timeout is DEGRADED
|
|
631
|
+
r=$(blank); mkdir -p "$r/.github/workflows"
|
|
632
|
+
cat > "$r/.github/workflows/ci.yml" <<'YML'
|
|
633
|
+
on: pull_request
|
|
634
|
+
jobs:
|
|
635
|
+
one:
|
|
636
|
+
runs-on: ubuntu-latest
|
|
637
|
+
steps:
|
|
638
|
+
- run: sleep 2
|
|
639
|
+
two:
|
|
640
|
+
runs-on: ubuntu-latest
|
|
641
|
+
steps:
|
|
642
|
+
- run: sleep 2
|
|
643
|
+
YML
|
|
644
|
+
echo s > "$r/s.txt"; commit_base "$r"; echo x >> "$r/s.txt"
|
|
645
|
+
OUT=$(FORGE_CI_LOCAL_SLOTS=1 FORGE_CI_LOCAL_JOB_TIMEOUT=3 bash "$S" run --repo "$r" --base main --base-branch main --head-branch feat --max-parallel 2 2>&1); RC=$?
|
|
646
|
+
eq "REG-5: the job that waited ~2s for a slot is not timed out by its wait (jobs take 2s, timeout 3s)" 0 "$RC"
|
|
647
|
+
hasnt "REG-5: no timeout reported" "$OUT" "timed out"
|
|
648
|
+
|
|
649
|
+
# item 7 + 8 + validate.md fences (V2 skip removed, V4.8 mode, DEGRADED_CHECKS name)
|
|
650
|
+
VAL="$ROOT/commands/work-on/build/validate.md"
|
|
651
|
+
STUB="$T/stubroot"; mkdir -p "$STUB/scripts/lib"; cp "$ROOT/scripts/lib/forge-resolve.sh" "$STUB/scripts/lib/"
|
|
652
|
+
cat > "$STUB/scripts/ci-local.sh" <<'STB'
|
|
653
|
+
#!/usr/bin/env bash
|
|
654
|
+
case "${STUB_CL:-fail}" in
|
|
655
|
+
fail) echo "CI_LOCAL_FAIL: gha:ci:unit | step: unit | exit 1"; echo "CI_LOCAL_RESULT: FAIL jobs_run=1 jobs_failed=1"; exit 1 ;;
|
|
656
|
+
degraded) echo "CI_LOCAL_RESULT: DEGRADED jobs_run=1 reason=job_timeout"; exit 2 ;;
|
|
657
|
+
pass) echo "CI_LOCAL_DEFERRED: gha:ci:e2e | e2e | needs secrets"; echo "CI_LOCAL_RESULT: PASS jobs_run=1"; exit 0 ;;
|
|
658
|
+
esac
|
|
659
|
+
STB
|
|
660
|
+
chmod +x "$STUB/scripts/ci-local.sh"
|
|
661
|
+
_fence() { awk -v h="$1" '$0 ~ h {p=1} p && /^```bash$/ {f=1; next} f && /^```$/ {exit} f {print}' "$VAL"; }
|
|
662
|
+
_fence '^## Phase V4.8' > "$T/v48.sh"
|
|
663
|
+
mm=$(blank); echo s > "$mm/s.txt"; printf 'forge.yaml\n' > "$mm/.gitignore"; g "$mm" add -A; g "$mm" commit -q -m base
|
|
664
|
+
git -C "$mm" worktree add -q -b feat48 "$T/wt48" main >/dev/null 2>&1
|
|
665
|
+
_v48() { # _v48 <iter> -> runs the real V4.8 fence against the stub gate; output in $OUT
|
|
666
|
+
sed -e "s|{WORKTREE_PATH}|$T/wt48|g" -e 's|{GH_REPO}|o/r|g' -e 's|{NUMBER}|1|g' -e 's|{PR_BASE}|main|g' -e 's|{BRANCH}|feat48|g' -e "s|^CL_ITER=0 |CL_ITER=$1 |" "$T/v48.sh" > "$T/v48.run.sh"
|
|
667
|
+
OUT=$(FORGEDOCK_HOME="$STUB" FORGE_ROOT="$STUB" bash "$T/v48.run.sh" 2>&1); }
|
|
668
|
+
[ -s "$T/v48.sh" ] && ok || bad "V4.8 fence extracted"
|
|
669
|
+
STUB_CL=fail; export STUB_CL
|
|
670
|
+
_v48 0
|
|
671
|
+
has "mode: iteration 0 FAIL stays a FAIL so the fix loop runs" "$OUT" "CI_LOCAL_STATE: FAIL"
|
|
672
|
+
has "mode: iteration 0 FAIL sets GATE_PASSED false" "$OUT" "GATE_PASSED: false"
|
|
673
|
+
_v48 2
|
|
674
|
+
has "mode: default advisory - unfixable after 2 iterations is DEGRADED" "$OUT" "CI_LOCAL_STATE: DEGRADED"
|
|
675
|
+
has "mode: DEGRADED_CHECKS names ci-local and the failing job" "$OUT" "DEGRADED_CHECKS+=ci-local — unfixable after 2 fix iterations: gha:ci:unit"
|
|
676
|
+
hasnt "mode: advisory never sets GATE_PASSED false (no BUILD_BLOCKED)" "$OUT" "GATE_PASSED: false"
|
|
677
|
+
has "mode: the unresolved failure is surfaced for FORGE:BUILDER" "$OUT" "CI_LOCAL_UNRESOLVED: gha:ci:unit"
|
|
678
|
+
hasnt "REG-2: DEGRADED is never printed as SKIPPED_CHECKS" "$OUT" "SKIPPED_CHECKS+="
|
|
679
|
+
printf 'verification:\n ci_local:\n mode: blocking\n' > "$mm/forge.yaml"
|
|
680
|
+
_v48 2
|
|
681
|
+
has "mode: blocking (main checkout config) keeps FAIL after the fix iterations" "$OUT" "CI_LOCAL_STATE: FAIL"
|
|
682
|
+
has "mode: blocking sets GATE_PASSED false" "$OUT" "GATE_PASSED: false"
|
|
683
|
+
rm -f "$mm/forge.yaml"; printf 'verification:\n ci_local:\n mode: blocking\n' > "$T/wt48/forge.yaml"
|
|
684
|
+
_v48 2
|
|
685
|
+
has "mode: a PR-tree forge.yaml cannot flip the mode" "$OUT" "CI_LOCAL_STATE: DEGRADED"
|
|
686
|
+
rm -f "$T/wt48/forge.yaml"
|
|
687
|
+
STUB_CL=degraded; _v48 0
|
|
688
|
+
has "REG-2: a DEGRADED gate prints DEGRADED_CHECKS+=ci-local" "$OUT" "DEGRADED_CHECKS+=ci-local — job_timeout"
|
|
689
|
+
hasnt "REG-2: ...and not SKIPPED_CHECKS" "$OUT" "SKIPPED_CHECKS+="
|
|
690
|
+
STUB_CL=pass; _v48 0
|
|
691
|
+
has "a passing gate prints PASS and its deferred lines" "$OUT" "CI_LOCAL_DEFERRED: gha:ci:e2e"
|
|
692
|
+
STUB_CL=fail; unset STUB_CL
|
|
693
|
+
|
|
694
|
+
# item 1: V2 runs the configured test command even when ci-local.sh resolves (a failing suite fails V2)
|
|
695
|
+
{ for fn in degrade_check run_verification_command apply_known_slow_filter run_classified_test; do sed -n "/^${fn}() {/,/^}/p" "$VAL"; done; } > "$T/v2fns.sh"
|
|
696
|
+
_fence '^### Test commands: classified runs' | sed '/^cd {WORKTREE_PATH}$/d' > "$T/v2.sh"
|
|
697
|
+
mkdir -p "$T/wt-v2"; printf 'verification:\n commands:\n python:\n test: "echo suite-ran; exit 3"\n' > "$T/wt-v2/forge.yaml"
|
|
698
|
+
git -C "$T/wt-v2" init -q >/dev/null 2>&1
|
|
699
|
+
OUT=$(cd "$T/wt-v2" && DEGRADED_CHECKS="" FORGEDOCK_HOME="$STUB" bash -c ". '$T/v2fns.sh'; . '$T/v2.sh'" 2>&1)
|
|
700
|
+
has "item 1: the configured test command runs although ci-local.sh resolves" "$OUT" "suite-ran"
|
|
701
|
+
has "item 1: its failure fails V2 (TEST_FAILED=1)" "$OUT" "TEST_FAILED=1"
|
|
702
|
+
hasnt "item 1: no 'skipped: change-scoped local CI gate' shortcut" "$OUT" "covers it"
|
|
703
|
+
|
|
704
|
+
# item 9 (REG-6): one invocation is capped below the Bash tool limit; beyond it the run is a partial DEGRADED
|
|
705
|
+
r=$(blank); mkdir -p "$r/.github/workflows"
|
|
706
|
+
cat > "$r/.github/workflows/ci.yml" <<'YML'
|
|
707
|
+
on: pull_request
|
|
708
|
+
jobs:
|
|
709
|
+
hang:
|
|
710
|
+
runs-on: ubuntu-latest
|
|
711
|
+
steps:
|
|
712
|
+
- run: sleep 30
|
|
713
|
+
YML
|
|
714
|
+
echo s > "$r/s.txt"; commit_base "$r"; echo x >> "$r/s.txt"
|
|
715
|
+
OUT=$(FORGE_CI_LOCAL_MAX_INVOCATION_SECONDS=2 FORGE_CI_LOCAL_JOB_TIMEOUT=600 bash "$S" run --repo "$r" --base main --base-branch main --head-branch feat --budget-minutes 30 2>&1); RC=$?
|
|
716
|
+
eq "REG-6: a 30 minute budget is clamped to the invocation cap: partial run is DEGRADED (exit 2)" 2 "$RC"
|
|
717
|
+
has "REG-6: the cap is named" "$OUT" "reason=budget_capped_at_invocation_limit"
|
|
718
|
+
_cap=$(grep -o 'MAX_INVOCATION_SECONDS) || [0-9]*' "$DIR/ci-local/run.mjs" | grep -o '[0-9]*$')
|
|
719
|
+
[ "${_cap:-9999}" -le 540 ] && ok || bad "REG-6: default invocation cap must be <= 540 s (got '$_cap')"
|
|
720
|
+
has "REG-6: the V4.8 block passes a <= 9 minute budget" "$(cat "$T/v48.sh")" "--budget-minutes 9"
|
|
721
|
+
|
|
722
|
+
# item 11 (SEC-4): the working-directory guard is a path-segment check
|
|
723
|
+
_in() { node -e "import('$ROOT/scripts/ci-local/run.mjs').then(m=>console.log(m.insideRepo(process.argv[1],process.argv[2])))" "$1" "$2"; }
|
|
724
|
+
eq "SEC-4: ../r6-sibling is outside r6" false "$(_in /tmp/x/r6 /tmp/x/r6/../r6-sibling)"
|
|
725
|
+
eq "SEC-4: a plain sibling prefix is outside" false "$(_in /tmp/x/r6 /tmp/x/r6-sibling/sub)"
|
|
726
|
+
eq "SEC-4: a nested directory is inside" true "$(_in /tmp/x/r6 /tmp/x/r6/pkg/a)"
|
|
727
|
+
eq "SEC-4: the root itself is inside" true "$(_in /tmp/x/r6 /tmp/x/r6)"
|
|
728
|
+
eq "SEC-4: parent traversal is outside" false "$(_in /tmp/x/r6 /tmp/x/r6/../other)"
|
|
729
|
+
|
|
730
|
+
# item 12 (SEC-5): github.* expressions go through the environment, never into the script text
|
|
731
|
+
r=$(blank); mkdir -p "$r/.github/workflows"
|
|
732
|
+
cat > "$r/.github/workflows/ci.yml" <<'YML'
|
|
733
|
+
on: pull_request
|
|
734
|
+
jobs:
|
|
735
|
+
inj:
|
|
736
|
+
runs-on: ubuntu-latest
|
|
737
|
+
steps:
|
|
738
|
+
- run: |
|
|
739
|
+
echo "branch=${{ github.head_ref }}" > "$RUNNER_TEMP/out.txt"
|
|
740
|
+
test "$(cat "$RUNNER_TEMP/out.txt")" = 'branch=feat/x";touch PWNED;"' || { echo "value mangled"; exit 1; }
|
|
741
|
+
YML
|
|
742
|
+
echo s > "$r/s.txt"; commit_base "$r"; g "$r" checkout -q -b 'feat/x";touch PWNED;"' 2>/dev/null; echo x >> "$r/s.txt"
|
|
743
|
+
P=$(bash "$S" plan --repo "$r" --base main --base-branch main --head-branch 'feat/x";touch PWNED;"' --json --no-cache 2>/dev/null)
|
|
744
|
+
CMD=$(step_field "$P" gha:ci:inj 0 '.command')
|
|
745
|
+
hasnt "SEC-5: the branch name is not substituted into the script text" "$CMD" "PWNED"
|
|
746
|
+
has "SEC-5: the script reads it from the environment" "$CMD" 'FORGE_CI_EXPR_1'
|
|
747
|
+
OUT=$(bash "$S" run --repo "$r" --base main --base-branch main --head-branch 'feat/x";touch PWNED;"' 2>&1); RC=$?
|
|
748
|
+
eq "SEC-5: a hostile head ref reaches the script as data (value intact, exit 0)" 0 "$RC"
|
|
749
|
+
[ ! -e "$r/PWNED" ] && ok || bad "SEC-5: injected command executed"
|
|
750
|
+
|
|
751
|
+
# item 13 (SEC-6): CircleCI secondary containers never use host networking
|
|
752
|
+
r=$(blank); mkdir -p "$r/.circleci"
|
|
753
|
+
cat > "$r/.circleci/config.yml" <<'YML'
|
|
754
|
+
version: 2.1
|
|
755
|
+
jobs:
|
|
756
|
+
test:
|
|
757
|
+
docker:
|
|
758
|
+
- image: cimg/base:stable
|
|
759
|
+
- image: cimg/postgres:16.1
|
|
760
|
+
ports: ["54331:5432"]
|
|
761
|
+
environment:
|
|
762
|
+
POSTGRES_PASSWORD: example
|
|
763
|
+
steps:
|
|
764
|
+
- run: echo with-db
|
|
765
|
+
workflows:
|
|
766
|
+
main:
|
|
767
|
+
jobs:
|
|
768
|
+
- test
|
|
769
|
+
YML
|
|
770
|
+
echo s > "$r/s.txt"; commit_base "$r"; echo x >> "$r/s.txt"
|
|
771
|
+
: > "$FAKE_DOCKER_LOG"
|
|
772
|
+
OUT=$(PATH="$T/bin:$PATH" bash "$S" run --repo "$r" --base main --base-branch main --head-branch feat 2>&1); RC=$?
|
|
773
|
+
LOG=$(cat "$FAKE_DOCKER_LOG")
|
|
774
|
+
eq "SEC-6: the circleci service job runs" 0 "$RC"
|
|
775
|
+
hasnt "SEC-6: no --network host" "$LOG" "--network host"
|
|
776
|
+
has "SEC-6: a per-job docker network is created" "$LOG" "network create forge-ci-net-"
|
|
777
|
+
has "SEC-6: the container joins it" "$LOG" "--network forge-ci-net-"
|
|
778
|
+
has "SEC-6: only the declared/default port is published, on 127.0.0.1" "$LOG" "-p 127.0.0.1:54331:5432"
|
|
779
|
+
has "SEC-6: the network is removed afterwards" "$LOG" "network rm forge-ci-net-"
|
|
780
|
+
|
|
781
|
+
# item 14: task-runner checks are a fallback ONLY for repos with no CI definitions
|
|
782
|
+
r=$(blank); mkdir -p "$r/.github/workflows" "$r/docs"
|
|
783
|
+
cat > "$r/.github/workflows/ci.yml" <<'YML'
|
|
784
|
+
on:
|
|
785
|
+
pull_request:
|
|
786
|
+
paths: ["src/**"]
|
|
787
|
+
jobs:
|
|
788
|
+
build:
|
|
789
|
+
runs-on: ubuntu-latest
|
|
790
|
+
steps:
|
|
791
|
+
- run: echo build
|
|
792
|
+
YML
|
|
793
|
+
printf 'test:\n\techo full-suite\n' > "$r/Makefile"; printf '{ "name": "x", "scripts": { "test": "echo full" } }\n' > "$r/package.json"
|
|
794
|
+
echo doc > "$r/docs/a.md"; mkdir -p "$r/src"; echo 1 > "$r/src/a.txt"; commit_base "$r"; echo more >> "$r/docs/a.md"
|
|
795
|
+
P=$(plan "$r")
|
|
796
|
+
eq "item 14: CI exists but no job applies: no task-runner fallback" 0 "$(printf '%s' "$P" | jq '[.jobs[] | select(.source == "task-runners")] | length')"
|
|
797
|
+
eq "item 14: no_applicable_jobs is recorded" true "$(printf '%s' "$P" | jq '.no_applicable_jobs')"
|
|
798
|
+
run_ "$r"
|
|
799
|
+
eq "item 14: the run executes nothing and passes" 0 "$RC"
|
|
800
|
+
has "item 14: the run records no_applicable_jobs" "$OUT" "no_applicable_jobs"
|
|
801
|
+
hasnt "item 14: no suite ran" "$OUT" "full-suite"
|
|
802
|
+
r=$(mkrepo no-ci); f=$(git -C "$r" ls-files | head -1); echo x >> "$r/$f"
|
|
803
|
+
P=$(plan "$r"); eq "item 14: a repo with no CI definitions keeps the task-runner fallback (unchanged)" false "$(printf '%s' "$P" | jq '.no_applicable_jobs')"
|
|
804
|
+
|
|
805
|
+
# item 15: matrix include semantics (extend matching combos, add non-matching ones)
|
|
806
|
+
r=$(blank); mkdir -p "$r/.github/workflows"
|
|
807
|
+
cat > "$r/.github/workflows/ci.yml" <<'YML'
|
|
808
|
+
on: pull_request
|
|
809
|
+
jobs:
|
|
810
|
+
m:
|
|
811
|
+
runs-on: ${{ matrix.os }}
|
|
812
|
+
strategy:
|
|
813
|
+
matrix:
|
|
814
|
+
os: [ubuntu-latest, macos-14]
|
|
815
|
+
node: [18, 20]
|
|
816
|
+
include:
|
|
817
|
+
- os: ubuntu-latest
|
|
818
|
+
node: 20
|
|
819
|
+
extra: yes
|
|
820
|
+
- os: ubuntu-latest
|
|
821
|
+
node: 22
|
|
822
|
+
steps:
|
|
823
|
+
- run: echo ${{ matrix.node }}-${{ matrix.extra }}
|
|
824
|
+
YML
|
|
825
|
+
echo s > "$r/s.txt"; commit_base "$r"; echo x >> "$r/s.txt"
|
|
826
|
+
_mx() { node -e "
|
|
827
|
+
import('$ROOT/scripts/ci-local/adapters/github-actions.mjs').then(m=>{console.log(JSON.stringify(m.expandMatrix(JSON.parse(process.argv[1]),{})))})" "$1"; }
|
|
828
|
+
MX=$(_mx '{"os":["a","b"],"node":[18,20],"include":[{"os":"a","node":20,"extra":"y"},{"os":"a","node":22}]}')
|
|
829
|
+
eq "item 15: 4 base combos + 1 non-matching include = 5, matched include extends (no duplicate)" 5 "$(printf '%s' "$MX" | jq 'length')"
|
|
830
|
+
eq "item 15: the matching include extended exactly one combo" 1 "$(printf '%s' "$MX" | jq '[.[] | select(.extra == "y")] | length')"
|
|
831
|
+
eq "item 15: the non-matching include added a new combo" 1 "$(printf '%s' "$MX" | jq '[.[] | select(.node == 22)] | length')"
|
|
832
|
+
MX=$(_mx '{"os":["a","b"],"include":[{"color":"green"}]}')
|
|
833
|
+
eq "item 15: an include naming no axis extends every combo" 2 "$(printf '%s' "$MX" | jq '[.[] | select(.color == "green")] | length')"
|
|
834
|
+
eq "item 15: ...without adding combos" 2 "$(printf '%s' "$MX" | jq 'length')"
|
|
835
|
+
MX=$(_mx '{"include":[{"a":1},{"a":2}]}')
|
|
836
|
+
eq "item 15: an include-only matrix yields one combo per entry" 2 "$(printf '%s' "$MX" | jq 'length')"
|
|
837
|
+
|
|
838
|
+
|
|
839
|
+
# ------------------------------------------------------------------------------------------------ forge#4040 review fixes
|
|
840
|
+
# FIX-1: widening is per changed source file (a changed test file or another selecting file must not hide an unreached one)
|
|
841
|
+
r=$(blank); mkdir -p "$r/app" "$r/tests" "$r/lib"
|
|
842
|
+
echo 'X = 1' > "$r/app/db.py"; echo 'Y = 1' > "$r/lib/zz.py"
|
|
843
|
+
printf 'from app import db\n\ndef test_a():\n assert db.X\n' > "$r/tests/test_user.py"
|
|
844
|
+
printf 'from lib import zz\n\ndef test_b():\n assert zz.Y\n' > "$r/tests/test_numbers.py"
|
|
845
|
+
printf 'def test_other():\n assert True\n' > "$r/tests/test_other.py"
|
|
846
|
+
commit_base "$r"; echo '# c' >> "$r/app/db.py"; echo '# c' >> "$r/lib/zz.py"; echo '# t' >> "$r/tests/test_other.py"
|
|
847
|
+
_imp2() { node -e "
|
|
848
|
+
import('$ROOT/scripts/ci-local/scoping.mjs').then(m=>{const c=process.argv.slice(1);const s=m.makeScopeCtx({repo:'$r',baseRef:'main',from:'main',changed:c});
|
|
849
|
+
const t=['tests/test_user.py','tests/test_numbers.py','tests/test_other.py'];const r=m.impactedTests(s,t);console.log(JSON.stringify({sel:r.selected,widen:r.widen}))})" "$@"; }
|
|
850
|
+
R=$(_imp2 tests/test_other.py lib/zz.py)
|
|
851
|
+
has "FIX-1: changed test file + lib/zz.py imported by a differently named test: that test runs" "$(printf '%s' "$R" | jq -c '.sel')" "tests/test_numbers.py"
|
|
852
|
+
R=$(_imp2 tests/test_other.py app/db.py lib/zz.py)
|
|
853
|
+
eq "FIX-1: every changed source file reaches a test: narrowed, no widening" "" "$(printf '%s' "$R" | jq -r '.widen')"
|
|
854
|
+
echo 'W = 1' > "$r/lib/unref.py"
|
|
855
|
+
R=$(_imp2 tests/test_other.py app/db.py lib/unref.py)
|
|
856
|
+
has "FIX-1: one unreached source file widens the job even though others select tests" "$(printf '%s' "$R" | jq -r '.widen')" "lib/unref.py"
|
|
857
|
+
eq "FIX-1: ...to the full test set" 3 "$(printf '%s' "$R" | jq '.sel | length')"
|
|
858
|
+
|
|
859
|
+
# FIX-2: no expression is substituted into script text, direct or through env
|
|
860
|
+
r=$(blank); mkdir -p "$r/.github/workflows"
|
|
861
|
+
cat > "$r/.github/workflows/ci.yml" <<'YML'
|
|
862
|
+
on: pull_request
|
|
863
|
+
env:
|
|
864
|
+
WF_REF: ${{ github.head_ref }}
|
|
865
|
+
jobs:
|
|
866
|
+
inj:
|
|
867
|
+
runs-on: ubuntu-latest
|
|
868
|
+
env:
|
|
869
|
+
JOB_REF: ${{ github.head_ref }}
|
|
870
|
+
steps:
|
|
871
|
+
- env:
|
|
872
|
+
STEP_REF: ${{ github.head_ref }}
|
|
873
|
+
run: |
|
|
874
|
+
echo "direct=${{ github.head_ref }}" > "$RUNNER_TEMP/d.txt"
|
|
875
|
+
echo "wf=${{ env.WF_REF }}" > "$RUNNER_TEMP/w.txt"
|
|
876
|
+
echo "job=${{ env.JOB_REF }}" > "$RUNNER_TEMP/j.txt"
|
|
877
|
+
echo 'single='${{ env.STEP_REF }}'' > "$RUNNER_TEMP/s.txt"
|
|
878
|
+
echo ${{ env.STEP_REF }} > "$RUNNER_TEMP/b.txt"
|
|
879
|
+
for f in d w j s b; do grep -q 'PWNED' "$RUNNER_TEMP/$f.txt" || { echo "value lost in $f"; exit 1; }; done
|
|
880
|
+
YML
|
|
881
|
+
echo s > "$r/s.txt"; commit_base "$r"
|
|
882
|
+
HB='x$(touch${IFS}PWNED)'
|
|
883
|
+
git -C "$r" checkout -q -b 'hostile' 2>/dev/null; echo x >> "$r/s.txt"
|
|
884
|
+
P=$(bash "$S" plan --repo "$r" --base main --base-branch main --head-branch "$HB" --json --no-cache 2>/dev/null)
|
|
885
|
+
CMD=$(step_field "$P" gha:ci:inj 0 '.command')
|
|
886
|
+
hasnt "FIX-2: no payload text in the script (direct or via env.X)" "$CMD" 'touch'
|
|
887
|
+
has "FIX-2: indirect env.X is a quoted variable reference" "$CMD" '"${FORGE_CI_EXPR_'
|
|
888
|
+
OUT=$(cd "$r" && bash "$S" run --repo "$r" --base main --base-branch main --head-branch "$HB" 2>&1); RC=$?
|
|
889
|
+
eq "FIX-2: direct and indirect hostile head ref reach the script as data (exit 0)" 0 "$RC" || echo "$OUT" | head -20
|
|
890
|
+
[ ! -e "$r/PWNED" ] && ok || bad "FIX-2: injected command executed"
|
|
891
|
+
[ ! -e "$(pwd)/PWNED" ] && ok || { bad "FIX-2: injected command executed in cwd"; rm -f "$(pwd)/PWNED"; }
|
|
892
|
+
|
|
893
|
+
# FIX-3: an unknown-port secondary container is DEFERRED, not run unreachable
|
|
894
|
+
r=$(blank); mkdir -p "$r/.circleci"
|
|
895
|
+
cat > "$r/.circleci/config.yml" <<'YML'
|
|
896
|
+
version: 2.1
|
|
897
|
+
jobs:
|
|
898
|
+
test:
|
|
899
|
+
docker:
|
|
900
|
+
- image: cimg/base:stable
|
|
901
|
+
- image: example/custom-service:1.2
|
|
902
|
+
steps:
|
|
903
|
+
- run: echo with-custom
|
|
904
|
+
workflows:
|
|
905
|
+
main:
|
|
906
|
+
jobs:
|
|
907
|
+
- test
|
|
908
|
+
YML
|
|
909
|
+
echo s > "$r/s.txt"; commit_base "$r"; echo x >> "$r/s.txt"
|
|
910
|
+
P=$(plan "$r")
|
|
911
|
+
eq "FIX-3: unknown service ports: job is deferred" deferred "$(state "$P" circleci:main:test)"
|
|
912
|
+
has "FIX-3: the reason names the cause" "$(reason "$P" circleci:main:test)" "unknown service ports"
|
|
913
|
+
: > "$FAKE_DOCKER_LOG"
|
|
914
|
+
OUT=$(PATH="$T/bin:$PATH" bash "$S" run --repo "$r" --base main --base-branch main --head-branch feat 2>&1); RC=$?
|
|
915
|
+
eq "FIX-3: the run passes with the job deferred" 0 "$RC"
|
|
916
|
+
hasnt "FIX-3: no container was started" "$(cat "$FAKE_DOCKER_LOG")" "run -d"
|
|
917
|
+
|
|
918
|
+
# ------------------------------------------------------------------------------------------------ forge#4057: concurrency
|
|
919
|
+
# SEC-3 universe: a single-test step is judged against the repo-wide tests, not the one test it runs
|
|
920
|
+
r=$(blank); mkdir -p "$r/scripts"
|
|
921
|
+
echo 'echo a' > "$r/scripts/a.sh"; echo 'bash scripts/a.sh' > "$r/scripts/a.test.sh"
|
|
922
|
+
echo 'echo x' > "$r/scripts/x.test.sh"; echo 'echo z' > "$r/scripts/zz.sh"
|
|
923
|
+
commit_base "$r"
|
|
924
|
+
_scope() { # _scope <command> <changed files...>
|
|
925
|
+
node -e "
|
|
926
|
+
import('$ROOT/scripts/ci-local/scoping.mjs').then(m=>{const [cmd,...c]=process.argv.slice(1);const s=m.makeScopeCtx({repo:'$r',baseRef:'main',from:'main',changed:c});
|
|
927
|
+
const r=m.scopeCommand(cmd,'',s);console.log(JSON.stringify({action:r.action,widen:r.widen_reason||''}))})" "$@"; }
|
|
928
|
+
echo '# c' >> "$r/scripts/a.sh"
|
|
929
|
+
R=$(_scope 'bash scripts/x.test.sh' scripts/a.sh)
|
|
930
|
+
eq "4057: an unrelated x.test.sh is skipped when a.sh has a sibling a.test.sh" skip "$(printf '%s' "$R" | jq -r '.action')" # edge_e2_unrelated_test_skipped
|
|
931
|
+
R=$(_scope 'bash scripts/a.test.sh' scripts/a.sh)
|
|
932
|
+
eq "4057: the sibling a.test.sh still runs" run "$(printf '%s' "$R" | jq -r '.action')"
|
|
933
|
+
echo '# c' >> "$r/scripts/zz.sh"
|
|
934
|
+
R=$(_scope 'bash scripts/x.test.sh' scripts/zz.sh)
|
|
935
|
+
has "4057: SEC-3 widen still fires when a changed source reaches no test anywhere" "$(printf '%s' "$R" | jq -r '.widen')" "empty-selection" # edge_e1_sec3_widen_still_fires
|
|
936
|
+
eq "4057: ...and the step runs" run "$(printf '%s' "$R" | jq -r '.action')"
|
|
937
|
+
R=$(_scope 'bash scripts/x.test.sh' scripts/a.sh scripts/zz.sh)
|
|
938
|
+
has "4057: one unreached source file among several still widens a single-test step" "$(printf '%s' "$R" | jq -r '.widen')" "scripts/zz.sh"
|
|
939
|
+
# SEC-3 universe is per ecosystem: a test in another language never covers a changed source (polyglot under-block)
|
|
940
|
+
r=$(blank); mkdir -p "$r/pkg" "$r/tests"
|
|
941
|
+
echo 'def f(): return 1' > "$r/pkg/utils.py"; echo 'def test_core(): assert True' > "$r/tests/test_core.py"
|
|
942
|
+
echo 'test("u", () => {})' > "$r/tests/utils.test.ts"
|
|
943
|
+
commit_base "$r"
|
|
944
|
+
echo '# c' >> "$r/pkg/utils.py"
|
|
945
|
+
R=$(_scope 'pytest tests/test_core.py' pkg/utils.py)
|
|
946
|
+
eq "SEC-3 ecosystem: a pytest step is not skipped because an unrelated .ts test matches the changed .py" run "$(printf '%s' "$R" | jq -r '.action')"
|
|
947
|
+
has "SEC-3 ecosystem: ...it widens and names the unreached source" "$(printf '%s' "$R" | jq -r '.widen')" "pkg/utils.py"
|
|
948
|
+
R=$(_scope 'pytest' pkg/utils.py)
|
|
949
|
+
eq "SEC-3 ecosystem: a bare pytest runner step widens too" run "$(printf '%s' "$R" | jq -r '.action')"
|
|
950
|
+
R=$(_scope 'python3 tests/test_core.py' pkg/utils.py)
|
|
951
|
+
eq "SEC-3 ecosystem: a single python test step widens (no py test reaches the change)" run "$(printf '%s' "$R" | jq -r '.action')"
|
|
952
|
+
# DRR-1: a single-language change never widens another ecosystem's runner step (it skips/narrows as at a0d45ae)
|
|
953
|
+
r=$(blank); mkdir -p "$r/pkg" "$r/tests"
|
|
954
|
+
echo 'def f(): return 1' > "$r/pkg/utils.py"; printf 'from pkg import utils\ndef test_core(): assert utils.f()\n' > "$r/tests/test_core.py"
|
|
955
|
+
echo 'import test from "node:test"; test("a", () => {})' > "$r/tests/a.test.mjs"
|
|
956
|
+
commit_base "$r"
|
|
957
|
+
echo '# c' >> "$r/pkg/utils.py"
|
|
958
|
+
R=$(_scope 'node --test tests/a.test.mjs' pkg/utils.py)
|
|
959
|
+
eq "DRR-1: a node step is not widened by a Python-only change" skip "$(printf '%s' "$R" | jq -r '.action')"
|
|
960
|
+
eq "DRR-1: ...and carries no widen reason" "" "$(printf '%s' "$R" | jq -r '.widen')"
|
|
961
|
+
R=$(_scope 'pytest tests/test_core.py' pkg/utils.py)
|
|
962
|
+
eq "DRR-1: the pytest step still runs the reaching test" run "$(printf '%s' "$R" | jq -r '.action')"
|
|
963
|
+
eq "DRR-1: ...scoped, not widened" "" "$(printf '%s' "$R" | jq -r '.widen')"
|
|
964
|
+
|
|
965
|
+
# FIFO slots (tickets), dead-owner reaping, deadline while queued
|
|
966
|
+
_slot() { FORGE_CI_LOCAL_HOME="$SLOT_HOME" FORGE_CI_LOCAL_SLOTS=1 node --input-type=module -e "
|
|
967
|
+
import('$ROOT/scripts/ci-local/run.mjs').then(async (m) => {
|
|
968
|
+
const [id, hold, budget] = process.argv.slice(1).map(Number);
|
|
969
|
+
const t0 = Date.now();
|
|
970
|
+
const rel = await m.acquireSlot('t' + id, t0 + budget);
|
|
971
|
+
if (!rel) { console.log('timeout ' + id); return; }
|
|
972
|
+
console.log('acq ' + id + (process.env.SLOT_SHOW_WAIT === '1' ? ' wait=' + (Date.now() - t0) : '')); await new Promise((r) => setTimeout(r, hold)); rel();
|
|
973
|
+
})" "$@"; }
|
|
974
|
+
SLOT_HOME="$T/fifo-home"; mkdir -p "$SLOT_HOME"; LOG="$T/fifo.log"; : > "$LOG"
|
|
975
|
+
# hold (1500ms) exceeds the cumulative spawn spacing (3 x 0.3s), so invocations 2-4 all queue behind invocation 1
|
|
976
|
+
for i in 1 2 3 4; do SLOT_SHOW_WAIT=1 _slot "$i" 1500 30000 >> "$LOG" 2>&1 & sleep 0.3; done; wait
|
|
977
|
+
eq "4057: 4 invocations, 1 slot: acquired strictly in ticket order, none starved" "acq 1 acq 2 acq 3 acq 4" "$(sed 's/ wait=.*//' "$LOG" | tr '\n' ' ' | sed 's/ $//')" # edge_e4_fifo_order
|
|
978
|
+
W1=$(sed -n 's/^acq 1 wait=//p' "$LOG")
|
|
979
|
+
case "$W1" in ''|*[!0-9]*) bad "4059: invocation 1 wait not recorded ('$W1')" ;; *) [ "$W1" -lt 500 ] && ok || bad "4059: invocation 1 acquired immediately (wait ${W1}ms, want < 500)" ;; esac
|
|
980
|
+
for i in 2 3 4; do # edge_e2_queued_waits
|
|
981
|
+
Wi=$(sed -n "s/^acq $i wait=//p" "$LOG")
|
|
982
|
+
case "$Wi" in ''|*[!0-9]*) bad "4059: invocation $i wait not recorded ('$Wi')" ;; *) [ "$Wi" -ge 500 ] && ok || bad "4059: invocation $i queued behind the holder (wait ${Wi}ms, want >= 500)" ;; esac
|
|
983
|
+
done
|
|
984
|
+
eq "4057: no ticket is left behind" 0 "$(ls "$SLOT_HOME/ci-local.lock/queue" | wc -l | tr -d ' ')"
|
|
985
|
+
SLOT_HOME="$T/fifo-home2"; mkdir -p "$SLOT_HOME/ci-local.lock/slot-0" "$SLOT_HOME/ci-local.lock/queue"
|
|
986
|
+
( : ) & DEAD=$!; wait "$DEAD" 2>/dev/null
|
|
987
|
+
printf '%s 1 ghost\n' "$DEAD" > "$SLOT_HOME/ci-local.lock/slot-0/owner"
|
|
988
|
+
: > "$SLOT_HOME/ci-local.lock/queue/00000000000001-000001-$DEAD-abcdef"
|
|
989
|
+
eq "4057: a dead slot owner and a dead earlier ticket are reaped, the queue is not blocked" "acq 9" "$(_slot 9 0 5000)" # edge_e3_dead_owner_reaped
|
|
990
|
+
eq "4057: no ticket left after a reaped acquire" 0 "$(ls "$SLOT_HOME/ci-local.lock/queue" | wc -l | tr -d ' ')"
|
|
991
|
+
mkdir -p "$SLOT_HOME/ci-local.lock/slot-0"; printf '%s %s000 live\n' "$$" "$(date +%s)" > "$SLOT_HOME/ci-local.lock/slot-0/owner"
|
|
992
|
+
eq "4057: the deadline while queued returns no slot" "timeout 8" "$(_slot 8 0 700)" # edge_e6_deadline_ticket_removed
|
|
993
|
+
eq "4057: ...and removes its ticket (no leak)" 0 "$(ls "$SLOT_HOME/ci-local.lock/queue" | wc -l | tr -d ' ')"
|
|
994
|
+
|
|
995
|
+
# single-flight dedup: identical check in two jobs runs once; a different env must not dedup
|
|
996
|
+
r=$(blank); mkdir -p "$r/.github/workflows"
|
|
997
|
+
cat > "$r/.github/workflows/ci.yml" <<'YML'
|
|
998
|
+
on: pull_request
|
|
999
|
+
jobs:
|
|
1000
|
+
a:
|
|
1001
|
+
runs-on: ubuntu-latest
|
|
1002
|
+
steps:
|
|
1003
|
+
- run: echo x >> counter.txt; sleep 1
|
|
1004
|
+
b:
|
|
1005
|
+
runs-on: ubuntu-latest
|
|
1006
|
+
steps:
|
|
1007
|
+
- run: echo x >> counter.txt; sleep 1
|
|
1008
|
+
c:
|
|
1009
|
+
runs-on: ubuntu-latest
|
|
1010
|
+
env:
|
|
1011
|
+
VARIANT: "1"
|
|
1012
|
+
steps:
|
|
1013
|
+
- run: echo x >> counter.txt; sleep 1
|
|
1014
|
+
YML
|
|
1015
|
+
echo s > "$r/s.txt"; commit_base "$r"; echo x >> "$r/s.txt"
|
|
1016
|
+
OUT=$(FORGE_CI_LOCAL_NO_CACHE=1 bash "$S" run --repo "$r" --base main --base-branch main --head-branch feat 2>&1); RC=$?
|
|
1017
|
+
eq "4057: dedup run passes" 0 "$RC"
|
|
1018
|
+
eq "4057: identical step ran once, the different-env step ran too (2 executions of 3 jobs)" 2 "$(wc -l < "$r/counter.txt" | tr -d ' ')" # edge_e8_different_env_not_deduped
|
|
1019
|
+
has "4057: the duplicate job reports dedup" "$OUT" "dedup=1"
|
|
1020
|
+
has "4057: the result line carries the dedup count and wait_ms" "$OUT" "wait_ms="
|
|
1021
|
+
r=$(blank); mkdir -p "$r/.github/workflows"
|
|
1022
|
+
cat > "$r/.github/workflows/ci.yml" <<'YML'
|
|
1023
|
+
on: pull_request
|
|
1024
|
+
jobs:
|
|
1025
|
+
a:
|
|
1026
|
+
runs-on: ubuntu-latest
|
|
1027
|
+
steps:
|
|
1028
|
+
- run: echo y >> fail.txt; sleep 1; exit 3
|
|
1029
|
+
b:
|
|
1030
|
+
runs-on: ubuntu-latest
|
|
1031
|
+
steps:
|
|
1032
|
+
- run: echo y >> fail.txt; sleep 1; exit 3
|
|
1033
|
+
YML
|
|
1034
|
+
echo s > "$r/s.txt"; commit_base "$r"; echo x >> "$r/s.txt"
|
|
1035
|
+
OUT=$(FORGE_CI_LOCAL_NO_CACHE=1 bash "$S" run --repo "$r" --base main --base-branch main --head-branch feat 2>&1); RC=$?
|
|
1036
|
+
eq "4057: a failing check shared by two jobs fails the gate" 1 "$RC"
|
|
1037
|
+
eq "4057: ...and was executed once, not re-run by the duplicate" 1 "$(wc -l < "$r/fail.txt" | tr -d ' ')" # edge_e5_failure_shared_once
|
|
1038
|
+
eq "4057: ...both jobs report the failure" 2 "$(printf '%s\n' "$OUT" | grep -c '^CI_LOCAL_JOB: fail ')"
|
|
1039
|
+
|
|
1040
|
+
# forge#4060: jobs with the same command but different service containers must not share a dedup result or a cached pass
|
|
1041
|
+
mksvc() { # $1 = repo, $2 = image of job b's postgres, $3 = job c's second-service ordering flag
|
|
1042
|
+
mkdir -p "$1/.github/workflows"
|
|
1043
|
+
cat > "$1/.github/workflows/ci.yml" <<YML
|
|
1044
|
+
on: pull_request
|
|
1045
|
+
jobs:
|
|
1046
|
+
a:
|
|
1047
|
+
runs-on: ubuntu-latest
|
|
1048
|
+
services:
|
|
1049
|
+
postgres:
|
|
1050
|
+
image: postgres:14
|
|
1051
|
+
steps:
|
|
1052
|
+
- run: echo x >> counter.txt; sleep 1
|
|
1053
|
+
b:
|
|
1054
|
+
runs-on: ubuntu-latest
|
|
1055
|
+
services:
|
|
1056
|
+
postgres:
|
|
1057
|
+
image: $2
|
|
1058
|
+
steps:
|
|
1059
|
+
- run: echo x >> counter.txt; sleep 1
|
|
1060
|
+
YML
|
|
1061
|
+
}
|
|
1062
|
+
r=$(blank); mksvc "$r" postgres:16
|
|
1063
|
+
echo s > "$r/s.txt"; commit_base "$r"; echo x >> "$r/s.txt"
|
|
1064
|
+
: > "$FAKE_DOCKER_LOG"
|
|
1065
|
+
OUT=$(PATH="$T/bin:$PATH" FORGE_CI_LOCAL_NO_CACHE=1 bash "$S" run --repo "$r" --base main --base-branch main --head-branch feat 2>&1); RC=$?
|
|
1066
|
+
eq "4060: different-services run passes" 0 "$RC"
|
|
1067
|
+
eq "4060: same command with different services ran in both jobs, no shared flight result" 2 "$(wc -l < "$r/counter.txt" | tr -d ' ')" # edge_e4_different_services_not_deduped
|
|
1068
|
+
hasnt "4060: no job reports dedup" "$OUT" "dedup=1"
|
|
1069
|
+
r=$(blank); mksvc "$r" postgres:14
|
|
1070
|
+
echo s > "$r/s.txt"; commit_base "$r"; echo x >> "$r/s.txt"
|
|
1071
|
+
OUT=$(PATH="$T/bin:$PATH" FORGE_CI_LOCAL_NO_CACHE=1 bash "$S" run --repo "$r" --base main --base-branch main --head-branch feat 2>&1); RC=$?
|
|
1072
|
+
eq "4060: identical services still dedup (one execution)" 1 "$(wc -l < "$r/counter.txt" | tr -d ' ')"
|
|
1073
|
+
has "4060: the identical-services duplicate reports dedup" "$OUT" "dedup=1"
|
|
1074
|
+
# persisted cache: a pass for services X is not a hit for services Y
|
|
1075
|
+
r=$(blank); mksvc "$r" postgres:16
|
|
1076
|
+
echo s > "$r/s.txt"; commit_base "$r"; echo x >> "$r/s.txt"
|
|
1077
|
+
OUT=$(PATH="$T/bin:$PATH" FORGE_CI_LOCAL_NO_CACHE=0 FORGE_CI_LOCAL_SLOTS=2 bash "$S" run --repo "$r" --base main --base-branch main --head-branch feat --max-parallel 1 2>&1)
|
|
1078
|
+
eq "4060: persisted cache is not shared across different service sets" 2 "$(wc -l < "$r/counter.txt" | tr -d ' ')" # edge_e5_cache_not_shared
|
|
1079
|
+
|
|
1080
|
+
# wait_ms is reported apart from duration; cached jobs wait for nothing; cheap jobs go first once durations are known
|
|
1081
|
+
r=$(blank); mkdir -p "$r/.github/workflows"
|
|
1082
|
+
cat > "$r/.github/workflows/ci.yml" <<'YML'
|
|
1083
|
+
on: pull_request
|
|
1084
|
+
jobs:
|
|
1085
|
+
big:
|
|
1086
|
+
runs-on: ubuntu-latest
|
|
1087
|
+
steps:
|
|
1088
|
+
- run: sleep 1.5 # ord-big
|
|
1089
|
+
small:
|
|
1090
|
+
runs-on: ubuntu-latest
|
|
1091
|
+
steps:
|
|
1092
|
+
- run: echo ord-small
|
|
1093
|
+
YML
|
|
1094
|
+
echo s > "$r/s.txt"; commit_base "$r"; echo x >> "$r/s.txt"
|
|
1095
|
+
OUT=$(FORGE_CI_LOCAL_NO_CACHE=0 FORGE_CI_LOCAL_SLOTS=1 bash "$S" run --repo "$r" --base main --base-branch main --head-branch feat --max-parallel 2 2>&1); RC=$?
|
|
1096
|
+
eq "4057: wait run passes" 0 "$RC"
|
|
1097
|
+
WM=$(printf '%s\n' "$OUT" | sed -n 's/^CI_LOCAL_RESULT:.* wait_ms=\([0-9]*\).*/\1/p')
|
|
1098
|
+
[ "${WM:-0}" -ge 500 ] && ok || bad "4057: one job queued for the single slot, so wait_ms is reported (got '${WM:-}')" "$OUT"
|
|
1099
|
+
has "4057: job lines carry wait=" "$OUT" " wait="
|
|
1100
|
+
OUT=$(FORGE_CI_LOCAL_NO_CACHE=0 FORGE_CI_LOCAL_SLOTS=1 bash "$S" run --repo "$r" --base main --base-branch main --head-branch feat --max-parallel 2 2>&1)
|
|
1101
|
+
has "4057: a fully cached job waits for no slot (wait=0.0s)" "$OUT" "cached=1 wait=0.0s" # edge_e7_cached_job_no_wait
|
|
1102
|
+
OUT=$(FORGE_CI_LOCAL_NO_CACHE=1 FORGE_CI_LOCAL_SLOTS=1 bash "$S" run --repo "$r" --base main --base-branch main --head-branch feat --max-parallel 1 2>&1)
|
|
1103
|
+
FIRST=$(printf '%s\n' "$OUT" | grep '^CI_LOCAL_JOB: pass' | head -n 1)
|
|
1104
|
+
has "4057: the cheapest known job runs first (small before big)" "$FIRST" "gha:ci:small"
|
|
1105
|
+
FORGE_TELEMETRY=off bash -c 'grep -q "wait_ms" "$0"' "$S" && ok || bad "4057: ci-local.sh forwards wait_ms to telemetry"
|
|
1106
|
+
|
|
1107
|
+
echo "ci-local.test.sh: $pass passed, $fail failed"
|
|
1108
|
+
[ "$fail" -eq 0 ]
|