forgedock 1.16.2 → 1.18.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/engine/outcome-codes.mjs +114 -0
- package/bin/engine/phases.mjs +211 -100
- package/bin/engine/projector.mjs +48 -1
- package/bin/engine/runlog.mjs +20 -5
- package/bin/engine/telemetry-seam.mjs +62 -0
- package/bin/engine/telemetry-seam.test.mjs +60 -0
- package/bin/engine/telemetry.mjs +148 -0
- package/bin/engine/telemetry.test.mjs +120 -0
- package/bin/engine-cli.mjs +17 -1
- package/bin/engine.mjs +227 -48
- package/bin/forgedock.mjs +8 -2
- package/bin/hooks/interactive-engine.mjs +10 -0
- package/bin/hooks/pre-tool-use.mjs +220 -0
- package/bin/runner.mjs +70 -4
- package/bin/stats/aggregate.mjs +231 -0
- package/bin/stats/cache.mjs +40 -0
- package/bin/stats/github-source.mjs +211 -0
- package/bin/stats/index.mjs +154 -0
- package/bin/stats/local-source.mjs +79 -0
- package/bin/stats/package.json +6 -0
- package/bin/stats/render.mjs +82 -0
- package/bin/stats/stats.test.mjs +312 -0
- package/commands/cleanup.md +90 -2
- package/commands/deploy-info.md +8 -4
- package/commands/deploy-pr.md +64 -0
- package/commands/fix-ci.md +20 -6
- package/commands/milestone.md +34 -17
- package/commands/orchestrate/phase-1-resolve.md +4 -1
- package/commands/orchestrate/phase-4-execution.md +28 -6
- package/commands/orchestrate/phase-5-cleanup.md +18 -2
- package/commands/pipeline-health/phase-1-context.md +2 -0
- package/commands/pipeline-health/phase-2-metrics.md +12 -5
- package/commands/pipeline-status.md +21 -1
- package/commands/review-pr-agents/protocols.md +1 -0
- package/commands/review-pr-agents.md +10 -2
- package/commands/review-pr-staging.md +196 -496
- package/commands/review-pr.md +512 -3134
- package/commands/test-gate.md +25 -10
- package/commands/work-on/build/architect.md +13 -1
- package/commands/work-on/build/implement.md +28 -6
- package/commands/work-on/build/validate.md +5 -2
- package/commands/work-on/build.md +197 -145
- package/commands/work-on/close.md +66 -20
- package/commands/work-on/decompose.md +113 -34
- package/commands/work-on/investigate.md +15 -14
- package/commands/work-on/remediate.md +49 -19
- package/commands/work-on/review.md +7 -5
- package/commands/work-on.md +1 -2
- package/docs/CONFIG.md +76 -24
- package/docs/OUTCOME-CODES.md +53 -0
- package/docs/RELEASE-NOTES.md +24 -0
- package/docs/STATS.md +63 -0
- package/docs/TELEMETRY.md +65 -0
- package/docs/WORK-ON-RUNTIME.md +19 -0
- package/docs/site/command-reference.md +1 -1
- package/docs/spec/forge-protocol-v1.md +2 -2
- package/docs/spec/label-state-machine.md +1 -1
- package/package.json +3 -2
- package/packages/protocol/src/phases.js +5 -5
- package/scripts/builder-resume-match.test.sh +58 -0
- package/scripts/cap-findings.sh +110 -0
- package/scripts/cap-findings.test.sh +130 -0
- package/scripts/check-spec-bash.sh +5 -1
- package/scripts/check-spec-bash.test.sh +20 -0
- package/scripts/check-spec-markers.sh +2 -0
- package/scripts/classify-finding.sh +65 -16
- package/scripts/classify-finding.test.sh +43 -1
- package/scripts/classify-grep-exit.sh +20 -0
- package/scripts/classify-grep-exit.test.sh +46 -0
- package/scripts/close-parent-c3.test.sh +123 -0
- package/scripts/cost-report.sh +146 -0
- package/scripts/cost-report.test.sh +80 -0
- package/scripts/diff-size.sh +97 -3
- package/scripts/diff-size.test.sh +59 -2
- package/scripts/doctor-pipeline-state.sh +43 -0
- package/scripts/file-deferred-followups.sh +159 -0
- package/scripts/file-deferred-followups.test.sh +106 -0
- package/scripts/fixtures/cost-report/a-merged1.jsonl +6 -0
- package/scripts/fixtures/cost-report/a-merged2-remediated.jsonl +7 -0
- package/scripts/fixtures/cost-report/a-needs-human.jsonl +4 -0
- package/scripts/fixtures/cost-report/b-merged.jsonl +3 -0
- package/scripts/fixtures/cost-report/c-merged-attempt-usage.jsonl +8 -0
- package/scripts/fixtures/cost-report/hydrated-no-usage.jsonl +3 -0
- package/scripts/fixtures/cost-report/old-log-no-version.jsonl +3 -0
- package/scripts/fixtures/cost-report/unfinished-no-terminal.jsonl +2 -0
- package/scripts/fixtures/cost-report/unfinished-reopened.jsonl +5 -0
- package/scripts/forge-root.test.sh +18 -11
- package/scripts/lib/fence-bootstrap.sh +4 -3
- package/scripts/lib/pr-issue-number.sh +18 -0
- package/scripts/lib/review-template-resolve.sh +41 -0
- package/scripts/lib/telemetry.sh +168 -0
- package/scripts/publish-staging-sync.test.sh +48 -0
- package/scripts/remediate-liveness.sh +42 -6
- package/scripts/resolve-deploy-gate.sh +32 -6
- package/scripts/resolve-deploy-gate.test.sh +42 -1
- package/scripts/review-acceptance-gate.sh +16 -7
- package/scripts/review-acceptance-gate.test.sh +13 -0
- package/scripts/review-finalize.sh +1427 -0
- package/scripts/review-finalize.test.sh +833 -0
- package/scripts/review-pr-delta-wiring.test.sh +19 -72
- package/scripts/review-pr-domain-map.test.sh +23 -13
- package/scripts/review-pr-gate2.test.sh +24 -65
- package/scripts/review-pr-staging-finding-filter.test.sh +44 -0
- package/scripts/review-pr-staging-scope.test.sh +56 -80
- package/scripts/review-prepare.sh +553 -0
- package/scripts/review-prepare.test.sh +288 -0
- package/scripts/review-roster.sh +186 -0
- package/scripts/review-roster.test.sh +185 -0
- package/scripts/run-shell-tests.sh +3 -0
- package/scripts/telemetry.test.sh +106 -0
- package/scripts/trusted-comments.test.sh +9 -2
- package/scripts/wait-ci-green.sh +8 -1
- package/scripts/wait-ci-green.test.sh +3 -3
- package/templates/forge.yaml.minimal +9 -0
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
# Gate Telemetry
|
|
2
|
+
|
|
3
|
+
Gate scripts record their decisions as structured events so gate block rates, finding funnels and malformed-check
|
|
4
|
+
counts can be measured without re-reading comments with a model. GitHub is the source of truth: records are written
|
|
5
|
+
by code (never by a model) into a `FORGE:TELEMETRY` comment, and everything can be rebuilt from the GitHub API alone.
|
|
6
|
+
Local files are only a cache.
|
|
7
|
+
|
|
8
|
+
## Where records live
|
|
9
|
+
|
|
10
|
+
- One `<!-- FORGE:TELEMETRY v1 -->` comment per issue, or per PR for review-side events (`review_panel`,
|
|
11
|
+
`finding_disposition`, `ci_gate` go to the PR when one is known, everything else to the issue).
|
|
12
|
+
- The comment is edited in place. The records are JSONL inside a fenced `jsonl` block.
|
|
13
|
+
- Near 60,000 characters (GitHub's limit is 65,536) a new `<!-- FORGE:TELEMETRY v1 part=N -->` comment is started.
|
|
14
|
+
- Written with the pipeline's own `gh` token. Readers must take only comments that pass `scripts/trusted-comments.sh`
|
|
15
|
+
(OWNER/MEMBER/COLLABORATOR, or a Bot author); the emitter itself only edits trusted telemetry comments.
|
|
16
|
+
- If a write fails the record is appended to `~/.forge/telemetry-buffer/<owner>_<repo>__<number>.jsonl` and flushed
|
|
17
|
+
with the next emit for the same target, from any machine that touches the issue. Lines already present in the
|
|
18
|
+
comment are not written twice. Deleting the buffer loses only records that never reached GitHub.
|
|
19
|
+
|
|
20
|
+
## Record shape
|
|
21
|
+
|
|
22
|
+
```json
|
|
23
|
+
{"v":1,"ts":"2026-10-11T19:38:54Z","forge_version":"1.16.1","repo":"o/r","issue":5,"pr":null,"run":null,"phase":null,"event":"diff_size","lines":10}
|
|
24
|
+
```
|
|
25
|
+
|
|
26
|
+
`issue`, `pr`, `run` and `phase` come from `FORGE_ISSUE`, `FORGE_PR`, `FORGE_RUN` and `FORGE_PHASE` (null when unset);
|
|
27
|
+
`repo` from `FORGE_REPO`, else the `origin` remote. A record with no repo or no numeric issue/PR has no destination and
|
|
28
|
+
is dropped. Payload keys are `[A-Za-z0-9_]+` and cannot shadow the shared keys; integers and `true`/`false` are typed.
|
|
29
|
+
|
|
30
|
+
## Event catalogue
|
|
31
|
+
|
|
32
|
+
| Event | Emitted by | Payload |
|
|
33
|
+
|-------|-----------|---------|
|
|
34
|
+
| `diff_size` | `scripts/diff-size.sh`; `work-on/build.md` B5.5 (final result) | `lines`, `excluded`, `ws_lines`, `review_weight`, `modules`, `result`; the script emits the legacy `threshold` and `OK` \| `OVER` \| `COHESIVE`, the B5.5 call site emits the final `result` = `OK` \| `EXTENDED_REVIEW` \| `SCOPE_FLAGGED` \| `OVERRIDDEN` plus `estimate` (plan), `actual` (whitespace-insensitive lines), `scope` = `ok` \| `out_of_plan` \| `no_baseline` and `review_depth` = `STANDARD` \| `EXTENDED` (calibration inputs, forge#3862) |
|
|
35
|
+
| `acceptance_gate` | `work-on/build.md` B6.5 (`phase=build`); `scripts/review-acceptance-gate.sh` (`phase=review`) | `phase`, `verified`, `failed`, `malformed`, `deferred` |
|
|
36
|
+
| `finding_disposition` | `scripts/classify-finding.sh`, one per finding | `severity`, `confidence`, `agent`, `disposition` = `ISSUE` \| `NOTE` \| `INPR_FIX`, `reason` |
|
|
37
|
+
| `review_panel` | `review-pr.md` Phase 4 | `agents_selected`, `agents_completed`, `mode` = `FULL` \| `DELTA` \| `BASE_SYNC_ONLY` |
|
|
38
|
+
| `ci_gate` | `scripts/wait-ci-green.sh` | `result` = `PASS` \| `FAIL` \| `ERROR` \| `TIMEOUT`, `rc`, `checks` |
|
|
39
|
+
| `liveness` | `scripts/remediate-liveness.sh` | `decision` = `PROCEED` \| `DEFER` \| `EXHAUSTED`, `why` |
|
|
40
|
+
| `phase_attempt` | the engine (`bin/engine/telemetry-seam.mjs`, one per phase attempt; `phase` = the phase id) | `attempt`, `model`, `turns`, `duration_ms`, `cost_usd`, `input_tokens`, `output_tokens`, `cache_read_tokens`, `cache_creation_tokens` (any value the runner did not report is omitted) |
|
|
41
|
+
| `run_terminal` | the engine (`terminate()`, one per run segment) | `reason` (terminal reason), `code` (see `docs/OUTCOME-CODES.md`), `wall_ms` (this engine process, start to terminal) |
|
|
42
|
+
|
|
43
|
+
The engine records carry `run` = `r_<issue>_<start epoch seconds>` (one per `forgedock run-issue` process) so `forgedock stats`
|
|
44
|
+
(`docs/STATS.md`) can group the attempts of a run and tell a finished run from an unfinished one. They are the GitHub-side
|
|
45
|
+
source for Cost, Time and outcome codes (`run` is per process, but stats count one run per issue, last wins): without them a fresh machine can rebuild only the gate and review sections.
|
|
46
|
+
|
|
47
|
+
The classifier and the scripts emit only when a destination is known: callers export `FORGE_ISSUE` and/or `FORGE_PR`
|
|
48
|
+
(`review-pr.md` exports them before classifying; the other scripts default `FORGE_REPO`/`FORGE_ISSUE`/`FORGE_PR` from
|
|
49
|
+
their own arguments).
|
|
50
|
+
|
|
51
|
+
## Guarantees
|
|
52
|
+
|
|
53
|
+
- Never fails or alters the caller: `forge_emit` always returns 0, runs in a subshell with all output discarded, bounds
|
|
54
|
+
each `gh` call to 30 seconds, and gate stdout and exit codes are identical with it on or off.
|
|
55
|
+
- `FORGE_TELEMETRY=off` makes every emit a no-op (the test runner sets it so suites never touch `~/.forge`).
|
|
56
|
+
- Emitters: `scripts/lib/telemetry.sh` (`forge_emit <event> key=value...`, also runnable as
|
|
57
|
+
`bash scripts/lib/telemetry.sh <event> key=value...`) and `bin/engine/telemetry.mjs` (`emit(event, payload)`),
|
|
58
|
+
with the same record shape.
|
|
59
|
+
|
|
60
|
+
## Privacy note
|
|
61
|
+
|
|
62
|
+
Records contain ids, counts and codes only: never comment bodies, issue text, file contents, diffs or secrets. Values
|
|
63
|
+
are restricted to `A-Za-z0-9 _.:/#@+=,-` (anything else becomes `_`) and truncated to 200 characters, so a value
|
|
64
|
+
cannot carry markup, backticks or newlines into the comment. Treat the comment as public to anyone who can read the
|
|
65
|
+
issue or PR.
|
package/docs/WORK-ON-RUNTIME.md
CHANGED
|
@@ -51,6 +51,12 @@ Only the router may invoke them, so they always run at router layer + 1 (L2 unde
|
|
|
51
51
|
L1 solo). When another phase needs one, it returns a handoff (`status: NEXT`, `next: <phase>`) and
|
|
52
52
|
the router invokes it. Concretely: review returns `next: remediate` instead of calling remediation.
|
|
53
53
|
|
|
54
|
+
The same rule gives the `PHASE_TRAIL_FAILED` re-dispatch a single owner: `work-on:review` Phase R4 alone
|
|
55
|
+
re-runs the missing phases. `/review-pr` (Phase 8, via `scripts/review-finalize.sh merge-gate`) and
|
|
56
|
+
`work-on:remediate` (Phase M6) only report the status with its `MISSING:` lines and never re-dispatch. When
|
|
57
|
+
`/review-pr` finds neither `Task` nor `Agent` ("neither tool available"), the usual cause is this depth limit
|
|
58
|
+
(F1/F5): the review was invoked from inside a fork instead of by the router, so `Agent` was withheld.
|
|
59
|
+
|
|
54
60
|
**R2 — Non-dispatching forks may nest.** Phases and children that never spawn sub-agents may invoke
|
|
55
61
|
further forks at any depth (F5): build → context/architect/implement/validate → quality-gate,
|
|
56
62
|
review → quality-gate / missing-phase re-dispatch, remediate → close. Forking them keeps each
|
|
@@ -117,3 +123,16 @@ remediate L2, reviewers L3, same as the table.
|
|
|
117
123
|
- `/orchestrate` still dispatches one worker `Agent` per issue (L1).
|
|
118
124
|
- Phases keep their specs, `*_RESULT` contracts and GitHub markers; phase-trail verification is unchanged.
|
|
119
125
|
- Solo `/work-on` gets one extra layer of headroom.
|
|
126
|
+
|
|
127
|
+
## 7. Pipeline cost report
|
|
128
|
+
|
|
129
|
+
`bash scripts/cost-report.sh [--runs-dir DIR] [--last N] [--versions a,b] [--min-n N]` reads the engine run logs (`~/.forge/runs/*.jsonl`) and prints a markdown table per ForgeDock version and repo: finished-run count `n`, mean and median cache-read tokens per issue, mean per phase, remediated %, merged %, and cache-read tokens per merged issue. Groups with `n` below `--min-n` (default 10) are flagged `low-n`. The staging review (`commands/review-pr-staging.md` Phase 8) embeds it for the current version and the previous two (`--last 3`).
|
|
130
|
+
|
|
131
|
+
- Only finished runs count (a `RUN_TERMINAL` after any `RUN_REOPEN` / `RUN_NEXT_CYCLE`). Unfinished runs and hydrated runs (reconstructed from `FORGE:STATE`, no usage data) are excluded and counted in the footer.
|
|
132
|
+
- Attribution comes from `RUN_START`, which now records `version` (from `package.json`) and `repo` (`owner/repo`) on all three writers: the engine's fresh start, the hydrate path (`hydrated: true`), and the interactive hook. Both are best-effort and null when unresolved; `deriveState` ignores them.
|
|
133
|
+
- Older logs have neither: the repo shows `unknown` and the version is estimated from the log's mtime against release times (git tags `v*`, or `--release-times FILE` with `<version> <epoch>` lines), marked `~`.
|
|
134
|
+
- Fixture test: `bash scripts/cost-report.test.sh` (fixtures in `scripts/fixtures/cost-report/`).
|
|
135
|
+
|
|
136
|
+
## 8. Outcome codes
|
|
137
|
+
|
|
138
|
+
Every `PHASE_FAILED`, blocked `PHASE_COMMIT` and `RUN_TERMINAL` run-log event carries a machine-readable `code` from a closed, versioned enum (`bin/engine/outcome-codes.mjs`), and `FORGE:STATE` records the causing code as `outcomeCode`. Codes come from structured evidence only, never from the prose `detail`; an unmapped outcome is `unclassified` with the raw reason kept. See `docs/OUTCOME-CODES.md`.
|
|
@@ -419,7 +419,7 @@ Cleans up stale labels, missing workflow state, project board gaps, dangling wor
|
|
|
419
419
|
/cleanup branches # Prune merged branches
|
|
420
420
|
/cleanup milestones # Archive shipped milestones
|
|
421
421
|
/cleanup board # Fix project board gaps
|
|
422
|
-
/cleanup orphans # Remove dangling worktrees
|
|
422
|
+
/cleanup orphans # Remove dangling worktrees; close dead orchestrate claims boards
|
|
423
423
|
/cleanup all # Everything
|
|
424
424
|
```
|
|
425
425
|
|
|
@@ -535,8 +535,8 @@ needed.
|
|
|
535
535
|
|
|
536
536
|
| Marker | Category | Meaning |
|
|
537
537
|
|--------|----------|---------|
|
|
538
|
-
| `<!-- FORGE:DIFF_SIZE -->` | Lifecycle (no completion sentinel) |
|
|
539
|
-
| `<!-- FORGE:SIZE_OVERRIDE -->` | Control | Posted by a human to
|
|
538
|
+
| `<!-- FORGE:DIFF_SIZE -->` | Lifecycle (no completion sentinel) | Record of the build's post-build scope check (B5.5): `diff_lines`, `excluded_lines`, `ws_lines`, `review_weight`, `modules`, `estimated_diff_lines`, `scope: ok \| out_of_plan \| no_baseline`, `review_depth: STANDARD \| EXTENDED`, `result: OK \| EXTENDED_REVIEW \| SCOPE_FLAGGED \| OVERRIDDEN` plus `out_of_plan:` / `overrun:` / `trimmed:` notes. Never a size gate and never a decompose signal (a finished build is not routed to decompose). Refreshed in place; readers take the latest trusted comment. Consumed by review-pr (review depth) and close (records `diff_lines` in `FORGE:TRAJECTORY`). Older records carrying `threshold` / `result: OVER \| COHESIVE` still parse |
|
|
539
|
+
| `<!-- FORGE:SIZE_OVERRIDE -->` | Control | Posted by a human to suppress the scope-check flag (out-of-plan files or estimate overrun) and the decompose loop guard. Valid only from a trusted author and with a non-empty justification on the line after the marker |
|
|
540
540
|
|
|
541
541
|
Both lookups are anchored at the start of the comment body and trust-filtered
|
|
542
542
|
(`scripts/trusted-comments.sh`); an unreadable lookup fails closed.
|
|
@@ -20,7 +20,7 @@ See also: [FORGE Annotation Protocol §6](forge-protocol-v1.md#6-label-state-mac
|
|
|
20
20
|
| `workflow:invalid` | Issue closed as invalid | Investigator agent |
|
|
21
21
|
| `workflow:decomposed` | Issue decomposed into sub-issues | Decomposer agent |
|
|
22
22
|
| `needs-human` | Pipeline blocked, human intervention required | Any agent on error |
|
|
23
|
-
| `workflow:engine-error` | Pipeline blocked by the engine/tool itself breaking (CLI crash, missing SDK/API key, exhausted retries with no attempt ever reaching a real outcome) — not a genuine content-level judgment call | `terminate()` in `bin/engine.mjs` |
|
|
23
|
+
| `workflow:engine-error` | Pipeline blocked by the engine/tool itself breaking (CLI crash, missing SDK/API key, exhausted retries with no attempt ever reaching a real outcome) — not a genuine content-level judgment call | `terminate()` in `bin/engine.mjs` (also removes a stacked `workflow:remediating`, and adds `needs-human` when the run was remediating) |
|
|
24
24
|
|
|
25
25
|
`workflow:remediating` vs `needs-human` (forge#3541): `needs-human` is reserved for genuine human decisions and external actions (an exhausted remediation bound, an unresolvable conflict, a policy gate, spend, real-environment validation). Work the pipeline fixes by itself (a pure base-sync, a red CI gate, an in-PR finding round) is marked `workflow:remediating` instead, so operators are not alerted and orchestrate does not hold DAG dependents as GATED. `work-on:remediate` accepts either label (a bare `needs-human` from an older run still works), swaps it for `workflow:in-review` on a FIXABLE run, and on every BLOCKED/UNFIXABLE exit clears `workflow:remediating` and asserts `needs-human`.
|
|
26
26
|
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "forgedock",
|
|
3
|
-
"version": "1.
|
|
3
|
+
"version": "1.18.0",
|
|
4
4
|
"description": "Autonomous development pipeline for coding agents. GitHub as institutional memory, with native Claude Code and OpenCode entrypoints.",
|
|
5
5
|
"bin": {
|
|
6
6
|
"forgedock": "bin/forgedock.mjs"
|
|
@@ -35,6 +35,7 @@
|
|
|
35
35
|
"files": [
|
|
36
36
|
"bin/",
|
|
37
37
|
"!bin/tests/",
|
|
38
|
+
"!bin/stats/fixtures/",
|
|
38
39
|
"commands/",
|
|
39
40
|
"docs/",
|
|
40
41
|
"!docs/site/.vitepress/",
|
|
@@ -47,7 +48,7 @@
|
|
|
47
48
|
"README.md"
|
|
48
49
|
],
|
|
49
50
|
"scripts": {
|
|
50
|
-
"test": "node --test \"bin/tests/**/*.test.mjs\" \"bin/engine/**/*.test.mjs\" \"packages/protocol/test/**/*.test.mjs\"",
|
|
51
|
+
"test": "node --test \"bin/tests/**/*.test.mjs\" \"bin/engine/**/*.test.mjs\" \"bin/stats/**/*.test.mjs\" \"packages/protocol/test/**/*.test.mjs\"",
|
|
51
52
|
"test:sh": "bash scripts/run-shell-tests.sh",
|
|
52
53
|
"docs:dev": "vitepress dev docs/site",
|
|
53
54
|
"docs:build": "vitepress build docs/site",
|
|
@@ -57,8 +57,8 @@ export const PHASE_IDS = ['investigate', 'decompose', 'build', 'review', 'remedi
|
|
|
57
57
|
* investigate.md emits it as the HTML comment `<!-- DECOMPOSE:YES -->`.
|
|
58
58
|
* @property {string} [notDecomposedMarker] - (investigate only) the explicit not-decomposed
|
|
59
59
|
* verdict, emitted as `<!-- DECOMPOSE:NO -->`.
|
|
60
|
-
* @property {string} [sizeGateMarker] - (build only) bare annotation-opener of the diff
|
|
61
|
-
* @property {string} [sizeOverrideMarker] - (build only) bare annotation-opener of the
|
|
60
|
+
* @property {string} [sizeGateMarker] - (build only) bare annotation-opener of the scope-check record (diff size, review weight, scope outcome).
|
|
61
|
+
* @property {string} [sizeOverrideMarker] - (build only) bare annotation-opener of the scope-check override.
|
|
62
62
|
* @property {string} [investigationMarker] - (build only) bare marker of the durable
|
|
63
63
|
* investigation-deliverables comment, posted as `<!-- FORGE:INVESTIGATION:DELIVERABLES -->`.
|
|
64
64
|
* @property {string} [partialMarker] - sentinel for an interrupted/partial annotation.
|
|
@@ -108,9 +108,9 @@ export const PHASE_MARKERS = {
|
|
|
108
108
|
build: {
|
|
109
109
|
completionMarker: RESERVED_TYPES.BUILDER.completionSentinel, // 'FORGE:BUILDER:COMPLETE'
|
|
110
110
|
header: 'FORGE:BUILDER',
|
|
111
|
-
//
|
|
112
|
-
//
|
|
113
|
-
//
|
|
111
|
+
// Scope-check (B5.5) records. Neither is a completion sentinel and neither routes anywhere
|
|
112
|
+
// (forge#3862): a finished build is never sent to decompose. DIFF_SIZE is the record of the
|
|
113
|
+
// scope-check outcome; SIZE_OVERRIDE suppresses the scope flag and the decompose loop guard.
|
|
114
114
|
sizeGateMarker: `FORGE:${RESERVED_TYPES.DIFF_SIZE.type}`, // 'FORGE:DIFF_SIZE'
|
|
115
115
|
sizeOverrideMarker: `FORGE:${RESERVED_TYPES.SIZE_OVERRIDE.type}`, // 'FORGE:SIZE_OVERRIDE'
|
|
116
116
|
// Investigation exit (implement.md I2): no code, no PR, so no FORGE:BUILDER:COMPLETE.
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
#!/usr/bin/env bash
|
|
2
|
+
# builder-resume-match.test.sh — the anchored, trusted FORGE:BUILDER resume selector (forge#3668, mirrors forge#3648)
|
|
3
|
+
# Exercises the exact jq used by commands/work-on/build/implement.md Resume Check. No network.
|
|
4
|
+
# Usage: bash scripts/builder-resume-match.test.sh
|
|
5
|
+
# SPDX-FileCopyrightText: Copyright (c) RapierCraft Studios
|
|
6
|
+
# SPDX-License-Identifier: AGPL-3.0-or-later
|
|
7
|
+
|
|
8
|
+
set -uo pipefail
|
|
9
|
+
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
10
|
+
ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
|
|
11
|
+
|
|
12
|
+
PASS=0; FAILN=0
|
|
13
|
+
ok() { PASS=$((PASS+1)); echo "ok - $1"; }
|
|
14
|
+
bad() { FAILN=$((FAILN+1)); echo "FAIL - $1"; }
|
|
15
|
+
eq() { if [ "$2" = "$3" ]; then ok "$1"; else bad "$1 (got '$2', want '$3')"; fi; }
|
|
16
|
+
|
|
17
|
+
SELECT_JQ='[.[][] | select((.body // "") | startswith("<!-- FORGE:BUILDER -->")) | select((.user.type // "") == "Bot" or ((.author_association // "") | IN("OWNER","MEMBER","COLLABORATOR"))) | {id: .id, complete: (.body | contains("<!-- FORGE:BUILDER:COMPLETE -->")), other: (.body | test("<!-- FORGE:(INVESTIGATOR|CONTEXT|CONTRACT|ARCHITECT)"))}]'
|
|
18
|
+
DELETE_JQ='[.[] | select(.complete | not)] | if length == 1 and (.[0].other | not) then .[0].id else empty end'
|
|
19
|
+
|
|
20
|
+
# c <id> <assoc> <type> <body>
|
|
21
|
+
c() { jq -nc --argjson i "$1" --arg a "$2" --arg t "$3" --arg b "$4" '{id:$i,author_association:$a,user:{type:$t},body:$b}'; }
|
|
22
|
+
arch() { printf '%s' "$1" | jq -sc . | jq -s "$SELECT_JQ"; }
|
|
23
|
+
n_complete() { arch "$1" | jq '[.[] | select(.complete)] | length'; }
|
|
24
|
+
del_id() { arch "$1" | jq -r "$DELETE_JQ"; }
|
|
25
|
+
|
|
26
|
+
PLAN=$'<!-- FORGE:BUILDER -->\n## Implementation\n\n<!-- FORGE:BUILDER:COMPLETE -->'
|
|
27
|
+
PARTIAL=$'<!-- FORGE:BUILDER -->\n## Implementation (in progress)'
|
|
28
|
+
INVEST=$'<!-- FORGE:INVESTIGATOR -->\nThe build phase later posts FORGE:BUILDER and FORGE:BUILDER:COMPLETE.'
|
|
29
|
+
CTX=$'<!-- FORGE:CONTEXT -->\nmentions <!-- FORGE:BUILDER -->'
|
|
30
|
+
MIXED=$'<!-- FORGE:BUILDER -->\nquoted <!-- FORGE:INVESTIGATOR -->'
|
|
31
|
+
|
|
32
|
+
eq "INVESTIGATOR mentioning FORGE:BUILDER is not selected" "$(arch "$(c 1 NONE Bot "$INVEST")" | jq length)" 0
|
|
33
|
+
eq "INVESTIGATOR mentioning FORGE:BUILDER is never deleted" "$(del_id "$(c 1 NONE Bot "$INVEST")")" ""
|
|
34
|
+
eq "CONTEXT mentioning the marker mid-body is not selected" "$(arch "$(c 2 NONE Bot "$CTX")" | jq length)" 0
|
|
35
|
+
eq "complete comment by bot is complete" "$(n_complete "$(c 3 NONE Bot "$PLAN")")" 1
|
|
36
|
+
eq "complete comment by MEMBER is complete" "$(n_complete "$(c 3 MEMBER User "$PLAN")")" 1
|
|
37
|
+
eq "single partial comment -> delete its id" "$(del_id "$(c 4 NONE Bot "$PARTIAL")")" 4
|
|
38
|
+
eq "two partial comments -> delete nothing" "$(del_id "$(c 4 NONE Bot "$PARTIAL")"$'\n'"$(c 5 NONE Bot "$PARTIAL")")" ""
|
|
39
|
+
eq "untrusted author partial ignored" "$(del_id "$(c 6 NONE User "$PARTIAL")")" ""
|
|
40
|
+
eq "untrusted author complete ignored" "$(n_complete "$(c 6 NONE User "$PLAN")")" 0
|
|
41
|
+
eq "partial carrying another FORGE marker is not deleted" "$(del_id "$(c 7 NONE Bot "$MIXED")")" ""
|
|
42
|
+
eq "investigator + partial comment -> only the comment id" "$(del_id "$(c 1 NONE Bot "$INVEST")"$'\n'"$(c 8 NONE Bot "$PARTIAL")")" 8
|
|
43
|
+
eq "paginated concatenated arrays" "$(printf '[%s]\n[%s]' "$(c 1 NONE Bot "$INVEST")" "$(c 9 NONE Bot "$PLAN")" | jq -s "$SELECT_JQ" | jq '[.[] | select(.complete)] | length')" 1
|
|
44
|
+
|
|
45
|
+
ARCHC=$'<!-- FORGE:ARCHITECT -->\nplan; the builder will post <!-- FORGE:BUILDER -->'
|
|
46
|
+
eq "ARCHITECT mentioning the marker is not selected" "$(arch "$(c 10 NONE Bot "$ARCHC")" | jq length)" 0
|
|
47
|
+
eq "unparsable input fails closed" "$(printf 'not json' | jq -s "$SELECT_JQ" 2>/dev/null | jq -r "$DELETE_JQ" 2>/dev/null)" ""
|
|
48
|
+
|
|
49
|
+
# Spec drift guard: every mutating site carries this exact predicate and no unanchored selector.
|
|
50
|
+
for f in commands/work-on/build/implement.md commands/work-on/build.md commands/work-on/build/validate.md; do
|
|
51
|
+
SPEC="$ROOT/$f"
|
|
52
|
+
! grep -qE 'contains\(.FORGE:BUILDER.\) and' "$SPEC" && ok "$f has no unanchored selector" || bad "$f still uses unanchored selector"
|
|
53
|
+
grep -qF "$SELECT_JQ" "$SPEC" && ok "$f uses the tested select jq" || bad "$f select jq drifted from this test"
|
|
54
|
+
grep -qF "$DELETE_JQ" "$SPEC" && ok "$f uses the tested pick jq" || bad "$f pick jq drifted from this test"
|
|
55
|
+
done
|
|
56
|
+
|
|
57
|
+
echo "passed=$PASS failed=$FAILN"
|
|
58
|
+
[ "$FAILN" -eq 0 ]
|
|
@@ -0,0 +1,110 @@
|
|
|
1
|
+
#!/usr/bin/env bash
|
|
2
|
+
# SPDX-FileCopyrightText: Copyright (c) RapierCraft Studios
|
|
3
|
+
# SPDX-License-Identifier: AGPL-3.0-or-later
|
|
4
|
+
#
|
|
5
|
+
# cap-findings.sh — PR-level cap on filed review findings (review-pr §6B.5, review-pr-staging §7B).
|
|
6
|
+
# classify-finding.sh decides one finding at a time; this is the aggregation
|
|
7
|
+
# step after it: at most N ISSUE dispositions per review run are kept, the rest
|
|
8
|
+
# become `NOTE capped` (listed in FORGE:NOTE_DISPOSITION, never dropped).
|
|
9
|
+
#
|
|
10
|
+
# Usage:
|
|
11
|
+
# cap-findings.sh [--cap N] [--lineage none|review-finding] [--input FILE]
|
|
12
|
+
#
|
|
13
|
+
# Input (stdin, or --input FILE): TSV rows
|
|
14
|
+
# ID<TAB>SEVERITY<TAB>CONFIDENCE<TAB>DISPOSITION[<TAB>AGENT]
|
|
15
|
+
# DISPOSITION is the classifier output (`ISSUE <reason>`, `NOTE <reason>`, `INPR_FIX <reason>`).
|
|
16
|
+
#
|
|
17
|
+
# Output (stdout): one `ID<TAB>DISPOSITION` row per input row, in input order, then a final
|
|
18
|
+
# `CAPPED=<N>` line (no tab) counting the rows rewritten to `NOTE capped`. Callers read the rows
|
|
19
|
+
# with `awk -F'\t' 'NF >= 2'`.
|
|
20
|
+
#
|
|
21
|
+
# Rules:
|
|
22
|
+
# - Only ISSUE rows are considered. NOTE and INPR_FIX rows pass through untouched and never
|
|
23
|
+
# consume the cap.
|
|
24
|
+
# - HIGH/CRITICAL ISSUE rows are NEVER capped (they always file and still block via the review's
|
|
25
|
+
# severity gate).
|
|
26
|
+
# - `ISSUE fix-efficacy-*` rows (the classifier's never-damped "the linked issue is not fixed / the
|
|
27
|
+
# PR re-introduces its defect" exemption, #3857) are NEVER capped either: capping one would merge
|
|
28
|
+
# a known-broken fix and close its issue as fixed. They are treated like HIGH rows (under
|
|
29
|
+
# --lineage review-finding they consume the budget).
|
|
30
|
+
# - Remaining ISSUE rows are ranked by severity (MEDIUM > LOW), then confidence (CONFIRMED >
|
|
31
|
+
# LIKELY > POSSIBLE), then safety domain (Security/Auth/Billing/Concurrency/Database agent
|
|
32
|
+
# first), then input order. Unparseable severity or confidence ranks as MEDIUM / LIKELY, so the
|
|
33
|
+
# cap still applies to it (it is not an automatic ISSUE).
|
|
34
|
+
# - Cap default: 2. Under --lineage review-finding the default is 1 and every filed HIGH/CRITICAL
|
|
35
|
+
# consumes that budget (a fix for a finding must not mint a new generation of findings).
|
|
36
|
+
# An explicit --cap overrides the default in both lineages. 0 is valid (cap everything
|
|
37
|
+
# cappable). A non-numeric or negative --cap falls back to the default with a warning on stderr.
|
|
38
|
+
#
|
|
39
|
+
# Exit codes: 0 ok, 2 usage error. Bash 3.2 compatible (no mapfile, no associative arrays).
|
|
40
|
+
|
|
41
|
+
set -u
|
|
42
|
+
|
|
43
|
+
CAP=""; LINEAGE="none"; INPUT=""
|
|
44
|
+
while [ $# -gt 0 ]; do
|
|
45
|
+
case "$1" in
|
|
46
|
+
--cap) [ $# -ge 2 ] || { echo "cap-findings.sh: --cap needs a value" >&2; exit 2; }; CAP="$2"; shift 2 ;;
|
|
47
|
+
--lineage) [ $# -ge 2 ] || { echo "cap-findings.sh: --lineage needs a value" >&2; exit 2; }; LINEAGE="$2"; shift 2 ;;
|
|
48
|
+
--input) [ $# -ge 2 ] || { echo "cap-findings.sh: --input needs a value" >&2; exit 2; }; INPUT="$2"; shift 2 ;;
|
|
49
|
+
*) echo "cap-findings.sh: unknown argument: $1" >&2; exit 2 ;;
|
|
50
|
+
esac
|
|
51
|
+
done
|
|
52
|
+
|
|
53
|
+
case "$LINEAGE" in
|
|
54
|
+
none|review-finding) ;;
|
|
55
|
+
*) echo "cap-findings.sh: --lineage must be none or review-finding (got '$LINEAGE')" >&2; exit 2 ;;
|
|
56
|
+
esac
|
|
57
|
+
|
|
58
|
+
if [ "$LINEAGE" = "review-finding" ]; then DEFAULT_CAP=1; else DEFAULT_CAP=2; fi
|
|
59
|
+
case "$CAP" in
|
|
60
|
+
"") CAP="$DEFAULT_CAP" ;;
|
|
61
|
+
*[!0-9]*) echo "cap-findings.sh: invalid cap '$CAP', using default $DEFAULT_CAP" >&2; CAP="$DEFAULT_CAP" ;;
|
|
62
|
+
esac
|
|
63
|
+
|
|
64
|
+
if [ -n "$INPUT" ] && [ ! -r "$INPUT" ]; then
|
|
65
|
+
echo "cap-findings.sh: cannot read --input $INPUT" >&2; exit 2
|
|
66
|
+
fi
|
|
67
|
+
|
|
68
|
+
awk -F'\t' -v cap="$CAP" -v lineage="$LINEAGE" '
|
|
69
|
+
function up(s) { return toupper(s) }
|
|
70
|
+
function sevclass(s) { s = up(s); gsub(/[^A-Z]/, "", s); if (s == "CRITICAL" || s == "HIGH") return 3; if (s == "LOW") return 1; return 2 }
|
|
71
|
+
function confrank(c) { c = up(c); if (c == "CONFIRMED") return 3; if (c == "POSSIBLE") return 1; return 2 }
|
|
72
|
+
function domain(a) { a = tolower(a); return (a ~ /(security|auth|billing|concurrency|database)/) ? 1 : 0 }
|
|
73
|
+
/^[ \t\r]*$/ { next }
|
|
74
|
+
{
|
|
75
|
+
n++
|
|
76
|
+
sub(/\r$/, "")
|
|
77
|
+
id[n] = $1; disp[n] = $4; agent[n] = $5
|
|
78
|
+
if (NF < 4 || $4 == "") disp[n] = "ISSUE malformed-row"
|
|
79
|
+
split(disp[n], w, " ")
|
|
80
|
+
isissue[n] = (w[1] == "ISSUE")
|
|
81
|
+
sc = sevclass($2)
|
|
82
|
+
# the classifier already decided HIGH/CRITICAL: never cap those, whatever the raw severity text looked like
|
|
83
|
+
if (w[2] ~ /^severity-(HIGH|CRITICAL)$/) sc = 3
|
|
84
|
+
# fix-efficacy is never damped (classify-finding.sh): the per-review cap must not re-damp it
|
|
85
|
+
if (w[2] ~ /^fix-efficacy-/) sc = 3
|
|
86
|
+
if (isissue[n] && sc == 3) { high++ ; capable[n] = 0 }
|
|
87
|
+
else if (isissue[n]) { capable[n] = 1; score[n] = sc * 1000 + confrank($3) * 100 + domain($5) * 10 }
|
|
88
|
+
else capable[n] = 0
|
|
89
|
+
}
|
|
90
|
+
END {
|
|
91
|
+
budget = cap + 0
|
|
92
|
+
if (lineage == "review-finding") budget -= high
|
|
93
|
+
if (budget < 0) budget = 0
|
|
94
|
+
kept = 0
|
|
95
|
+
while (kept < budget) {
|
|
96
|
+
best = 0
|
|
97
|
+
for (i = 1; i <= n; i++) {
|
|
98
|
+
if (!capable[i] || keep[i]) continue
|
|
99
|
+
if (best == 0 || score[i] > score[best]) best = i
|
|
100
|
+
}
|
|
101
|
+
if (best == 0) break
|
|
102
|
+
keep[best] = 1; kept++
|
|
103
|
+
}
|
|
104
|
+
capped = 0
|
|
105
|
+
for (i = 1; i <= n; i++) {
|
|
106
|
+
if (capable[i] && !keep[i]) { printf "%s\tNOTE capped\n", id[i]; capped++ }
|
|
107
|
+
else printf "%s\t%s\n", id[i], disp[i]
|
|
108
|
+
}
|
|
109
|
+
printf "CAPPED=%d\n", capped
|
|
110
|
+
}' ${INPUT:+"$INPUT"}
|
|
@@ -0,0 +1,130 @@
|
|
|
1
|
+
#!/usr/bin/env bash
|
|
2
|
+
# SPDX-FileCopyrightText: Copyright (c) RapierCraft Studios
|
|
3
|
+
# SPDX-License-Identifier: AGPL-3.0-or-later
|
|
4
|
+
#
|
|
5
|
+
# Fixtures for scripts/cap-findings.sh (review-pr §6B.5 / review-pr-staging §7B per-review cap).
|
|
6
|
+
# Run: bash scripts/cap-findings.test.sh (bash 3.2 compatible)
|
|
7
|
+
|
|
8
|
+
set -u
|
|
9
|
+
DIR=$(cd "$(dirname "$0")" && pwd)
|
|
10
|
+
S="$DIR/cap-findings.sh"
|
|
11
|
+
pass=0; fail=0
|
|
12
|
+
T=$(printf '\t')
|
|
13
|
+
|
|
14
|
+
# rows <id:sev:conf:disp[:agent]>... -> TSV on stdout (disp may use ~ for space)
|
|
15
|
+
rows() {
|
|
16
|
+
local spec id sev conf disp agent
|
|
17
|
+
for spec in "$@"; do
|
|
18
|
+
IFS=: read -r id sev conf disp agent <<EOF2
|
|
19
|
+
$spec
|
|
20
|
+
EOF2
|
|
21
|
+
disp=$(printf '%s' "$disp" | tr '~' ' ')
|
|
22
|
+
printf '%s\t%s\t%s\t%s\t%s\n' "$id" "$sev" "$conf" "$disp" "$agent"
|
|
23
|
+
done
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
# expect <description> <expected "id=CLASS ..."> <script args> -- <row specs>
|
|
27
|
+
# Compares the first word of each output row's disposition, in input order, plus the CAPPED trailer.
|
|
28
|
+
expect() {
|
|
29
|
+
local desc="$1" want="$2"; shift 2
|
|
30
|
+
local args=()
|
|
31
|
+
while [ "$1" != "--" ]; do args+=("$1"); shift; done
|
|
32
|
+
shift
|
|
33
|
+
local got
|
|
34
|
+
got=$(rows "$@" | bash "$S" ${args[@]+"${args[@]}"} 2>/dev/null | awk -F'\t' 'NF >= 2 { split($2, w, " "); printf "%s=%s%s ", $1, w[1], (w[2] == "capped" ? "(capped)" : "") } /^CAPPED=/ { printf "%s", $0 }')
|
|
35
|
+
if [ "$got" = "$want" ]; then pass=$((pass+1)); else fail=$((fail+1)); echo "FAIL: $desc"; echo " want: $want"; echo " got: $got"; fi
|
|
36
|
+
}
|
|
37
|
+
expect_exit() { # expect_exit <code> <description> -- args...
|
|
38
|
+
local want="$1" desc="$2"; shift 3
|
|
39
|
+
bash "$S" "$@" </dev/null >/dev/null 2>&1; local got=$?
|
|
40
|
+
if [ "$got" = "$want" ]; then pass=$((pass+1)); else fail=$((fail+1)); echo "FAIL: $desc (want exit $want, got $got)"; fi
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
# --- Default cap is 2; ranking by severity, then confidence ---
|
|
44
|
+
expect "under the cap: nothing capped" "a=ISSUE b=ISSUE CAPPED=0" -- \
|
|
45
|
+
a:MEDIUM:LIKELY:ISSUE~x b:MEDIUM:CONFIRMED:ISSUE~x
|
|
46
|
+
expect "overflow becomes NOTE capped, lowest rank loses" "a=NOTE(capped) b=ISSUE c=ISSUE CAPPED=1" -- \
|
|
47
|
+
a:MEDIUM:POSSIBLE:ISSUE~x b:MEDIUM:CONFIRMED:ISSUE~x c:MEDIUM:LIKELY:ISSUE~x
|
|
48
|
+
expect "ranks by confidence CONFIRMED > LIKELY > POSSIBLE" "a=NOTE(capped) b=ISSUE c=ISSUE d=NOTE(capped) CAPPED=2" -- \
|
|
49
|
+
a:MEDIUM:POSSIBLE:ISSUE~x b:MEDIUM:CONFIRMED:ISSUE~x c:MEDIUM:LIKELY:ISSUE~x d:MEDIUM:POSSIBLE:ISSUE~x
|
|
50
|
+
expect "MEDIUM outranks LOW even at lower confidence" "a=NOTE(capped) b=ISSUE c=ISSUE CAPPED=1" -- \
|
|
51
|
+
a:LOW:CONFIRMED:ISSUE~x b:MEDIUM:POSSIBLE:ISSUE~x c:MEDIUM:LIKELY:ISSUE~x
|
|
52
|
+
expect "ties broken by safety-domain agent" "a=NOTE(capped) b=NOTE(capped) c=ISSUE d=ISSUE CAPPED=2" -- \
|
|
53
|
+
a:MEDIUM:LIKELY:ISSUE~x:General b:MEDIUM:LIKELY:ISSUE~x:General c:MEDIUM:LIKELY:ISSUE~x:Auth d:MEDIUM:LIKELY:ISSUE~x:Concurrency_reviewer
|
|
54
|
+
expect "remaining ties broken by input order (stable)" "a=ISSUE b=ISSUE c=NOTE(capped) CAPPED=1" -- \
|
|
55
|
+
a:MEDIUM:LIKELY:ISSUE~x b:MEDIUM:LIKELY:ISSUE~x c:MEDIUM:LIKELY:ISSUE~x
|
|
56
|
+
|
|
57
|
+
# --- HIGH/CRITICAL are never capped and do not consume the default budget ---
|
|
58
|
+
expect "HIGH and CRITICAL never capped past the cap" "a=ISSUE b=ISSUE c=ISSUE d=ISSUE e=ISSUE CAPPED=0" -- \
|
|
59
|
+
a:HIGH:POSSIBLE:ISSUE~x b:CRITICAL:LIKELY:ISSUE~x c:HIGH:CONFIRMED:ISSUE~x d:MEDIUM:CONFIRMED:ISSUE~x e:MEDIUM:LIKELY:ISSUE~x
|
|
60
|
+
expect "HIGH does not consume the default budget" "a=ISSUE b=ISSUE c=ISSUE d=NOTE(capped) CAPPED=1" -- \
|
|
61
|
+
a:HIGH:LIKELY:ISSUE~x b:MEDIUM:CONFIRMED:ISSUE~x c:MEDIUM:LIKELY:ISSUE~x d:MEDIUM:POSSIBLE:ISSUE~x
|
|
62
|
+
expect "lowercase high is never capped" "a=ISSUE b=NOTE(capped) CAPPED=1" --cap 0 -- \
|
|
63
|
+
a:high:POSSIBLE:ISSUE~x b:medium:LIKELY:ISSUE~x
|
|
64
|
+
|
|
65
|
+
# --- Unparseable severity/confidence ranks as MEDIUM/LIKELY and IS capped ---
|
|
66
|
+
expect "unparseable severity is capped like MEDIUM" "a=NOTE(capped) b=ISSUE c=ISSUE CAPPED=1" -- \
|
|
67
|
+
a:INFO:POSSIBLE:ISSUE~x b:MEDIUM:CONFIRMED:ISSUE~x c:MEDIUM:LIKELY:ISSUE~x
|
|
68
|
+
expect "missing severity and confidence rank MEDIUM/LIKELY (above POSSIBLE, below CONFIRMED)" "a=ISSUE b=NOTE(capped) c=ISSUE CAPPED=1" -- \
|
|
69
|
+
a:MEDIUM:CONFIRMED:ISSUE~x b:MEDIUM:POSSIBLE:ISSUE~x c::ISSUE~x
|
|
70
|
+
expect "unparseable severity cannot displace a ranked MEDIUM CONFIRMED when cap is 1" "a=NOTE(capped) b=ISSUE CAPPED=1" --cap 1 -- \
|
|
71
|
+
a::ISSUE~x b:MEDIUM:CONFIRMED:ISSUE~x
|
|
72
|
+
|
|
73
|
+
# --- Lineage review-finding: default 1, filed HIGH+ consumes the budget ---
|
|
74
|
+
expect "lineage default 1 keeps only the best" "a=NOTE(capped) b=ISSUE c=NOTE(capped) CAPPED=2" --lineage review-finding -- \
|
|
75
|
+
a:MEDIUM:LIKELY:ISSUE~x b:MEDIUM:CONFIRMED:ISSUE~x c:MEDIUM:LIKELY:ISSUE~x
|
|
76
|
+
expect "lineage: a filed HIGH consumes the budget, all MEDIUM capped" "a=ISSUE b=NOTE(capped) c=NOTE(capped) CAPPED=2" --lineage review-finding -- \
|
|
77
|
+
a:HIGH:LIKELY:ISSUE~x b:MEDIUM:CONFIRMED:ISSUE~x c:MEDIUM:LIKELY:ISSUE~x
|
|
78
|
+
expect "lineage: CRITICAL still never capped" "a=ISSUE b=ISSUE CAPPED=0" --lineage review-finding -- \
|
|
79
|
+
a:CRITICAL:CONFIRMED:ISSUE~x b:HIGH:CONFIRMED:ISSUE~x
|
|
80
|
+
expect "lineage: explicit cap 3 with one HIGH keeps two MEDIUM" "a=ISSUE b=ISSUE c=ISSUE d=NOTE(capped) CAPPED=1" --lineage review-finding --cap 3 -- \
|
|
81
|
+
a:HIGH:LIKELY:ISSUE~x b:MEDIUM:CONFIRMED:ISSUE~x c:MEDIUM:LIKELY:ISSUE~x d:MEDIUM:POSSIBLE:ISSUE~x
|
|
82
|
+
|
|
83
|
+
# --- Explicit cap, 0, invalid cap ---
|
|
84
|
+
expect "explicit cap 3 overrides default 2" "a=ISSUE b=ISSUE c=ISSUE CAPPED=0" --cap 3 -- \
|
|
85
|
+
a:MEDIUM:LIKELY:ISSUE~x b:MEDIUM:LIKELY:ISSUE~x c:MEDIUM:LIKELY:ISSUE~x
|
|
86
|
+
expect "cap 0 caps every cappable row, never HIGH" "a=NOTE(capped) b=ISSUE c=NOTE(capped) CAPPED=2" --cap 0 -- \
|
|
87
|
+
a:MEDIUM:CONFIRMED:ISSUE~x b:HIGH:LIKELY:ISSUE~x c:LOW:CONFIRMED:ISSUE~x
|
|
88
|
+
expect "invalid cap falls back to default 2" "a=ISSUE b=ISSUE c=NOTE(capped) CAPPED=1" --cap abc -- \
|
|
89
|
+
a:MEDIUM:CONFIRMED:ISSUE~x b:MEDIUM:LIKELY:ISSUE~x c:MEDIUM:POSSIBLE:ISSUE~x
|
|
90
|
+
expect "negative cap falls back to default 2" "a=ISSUE b=ISSUE c=NOTE(capped) CAPPED=1" --cap -1 -- \
|
|
91
|
+
a:MEDIUM:CONFIRMED:ISSUE~x b:MEDIUM:LIKELY:ISSUE~x c:MEDIUM:POSSIBLE:ISSUE~x
|
|
92
|
+
expect "invalid cap under lineage falls back to 1" "a=ISSUE b=NOTE(capped) CAPPED=1" --lineage review-finding --cap x -- \
|
|
93
|
+
a:MEDIUM:CONFIRMED:ISSUE~x b:MEDIUM:LIKELY:ISSUE~x
|
|
94
|
+
|
|
95
|
+
# --- Pass-through rows never consume the cap ---
|
|
96
|
+
expect "NOTE and INPR_FIX rows pass through" "a=NOTE b=INPR_FIX c=ISSUE d=ISSUE e=NOTE(capped) CAPPED=1" -- \
|
|
97
|
+
a:MEDIUM:LIKELY:NOTE~reflag-note b:MEDIUM:CONFIRMED:INPR_FIX~in-pr c:MEDIUM:CONFIRMED:ISSUE~x d:MEDIUM:LIKELY:ISSUE~x e:MEDIUM:POSSIBLE:ISSUE~x
|
|
98
|
+
expect "empty input prints only the trailer" "CAPPED=0" --
|
|
99
|
+
|
|
100
|
+
# --- Reason text and original rows survive ---
|
|
101
|
+
out=$(rows "a:MEDIUM:CONFIRMED:ISSUE~medium~confirmed" | bash "$S" 2>/dev/null | head -n 1)
|
|
102
|
+
if [ "$out" = "a${T}ISSUE medium confirmed" ]; then pass=$((pass+1)); else fail=$((fail+1)); echo "FAIL: kept row keeps its classifier reason (got '$out')"; fi
|
|
103
|
+
out=$(rows "a:MEDIUM:CONFIRMED:NOTE~stale" | bash "$S" 2>/dev/null | head -n 1)
|
|
104
|
+
if [ "$out" = "a${T}NOTE stale" ]; then pass=$((pass+1)); else fail=$((fail+1)); echo "FAIL: NOTE row passes through verbatim (got '$out')"; fi
|
|
105
|
+
|
|
106
|
+
# --- Severity text noise never demotes HIGH/CRITICAL (classifier strips whitespace and uppercases) ---
|
|
107
|
+
expect "whitespace/markdown HIGH still uncapped at cap 0" "a=ISSUE b=ISSUE c=ISSUE d=NOTE(capped) CAPPED=1" --cap 0 -- \
|
|
108
|
+
"a:HIGH :CONFIRMED:ISSUE~severity-HIGH" 'b:**HIGH**:CONFIRMED:ISSUE~severity-HIGH' c:High:LIKELY:ISSUE~severity-HIGH d:MEDIUM:LIKELY:ISSUE~x
|
|
109
|
+
expect "fix-efficacy ISSUE rows are never capped (#3857 exemption survives the cap)" "a=ISSUE b=ISSUE c=NOTE(capped) CAPPED=1" --cap 1 -- \
|
|
110
|
+
a:MEDIUM:LIKELY:ISSUE~fix-efficacy-fails b:MEDIUM:CONFIRMED:ISSUE~x c:MEDIUM:LIKELY:ISSUE~x
|
|
111
|
+
expect "lineage: fix-efficacy beats an unrelated CONFIRMED MEDIUM and consumes the budget" "a=NOTE(capped) b=ISSUE CAPPED=1" --lineage review-finding -- \
|
|
112
|
+
a:MEDIUM:CONFIRMED:ISSUE~lineage-medium-CONFIRMED b:MEDIUM:LIKELY:ISSUE~fix-efficacy-regresses
|
|
113
|
+
expect "classifier severity-HIGH/CRITICAL reason is never capped" "a=ISSUE b=ISSUE CAPPED=0" --cap 0 -- \
|
|
114
|
+
"a:weird:CONFIRMED:ISSUE~severity-HIGH" "b:weird:CONFIRMED:ISSUE~severity-CRITICAL"
|
|
115
|
+
expect "lineage budget counts noisy HIGH" "a=ISSUE b=NOTE(capped) CAPPED=1" --cap 1 --lineage review-finding -- \
|
|
116
|
+
"a:HIGH :CONFIRMED:ISSUE~severity-HIGH" b:MEDIUM:CONFIRMED:ISSUE~x
|
|
117
|
+
|
|
118
|
+
# --- --input file and usage errors ---
|
|
119
|
+
tmp=$(mktemp)
|
|
120
|
+
rows a:MEDIUM:LIKELY:ISSUE~x b:MEDIUM:LIKELY:ISSUE~x c:MEDIUM:LIKELY:ISSUE~x > "$tmp"
|
|
121
|
+
got=$(bash "$S" --input "$tmp" 2>/dev/null | tail -n 1)
|
|
122
|
+
rm -f "$tmp"
|
|
123
|
+
if [ "$got" = "CAPPED=1" ]; then pass=$((pass+1)); else fail=$((fail+1)); echo "FAIL: --input file (got '$got')"; fi
|
|
124
|
+
expect_exit 2 "unknown arg is a usage error" -- --bogus
|
|
125
|
+
expect_exit 2 "bad lineage is a usage error" -- --lineage sideways
|
|
126
|
+
expect_exit 2 "missing --input file is a usage error" -- --input /nonexistent/cap-findings-rows
|
|
127
|
+
expect_exit 0 "empty stdin exits 0" --
|
|
128
|
+
|
|
129
|
+
echo "cap-findings.test.sh: $pass passed, $fail failed"
|
|
130
|
+
[ "$fail" -eq 0 ]
|
|
@@ -33,6 +33,8 @@
|
|
|
33
33
|
# `export VAR=`, `read VAR`, `for VAR in`) at or before its first read. Comment-only lines are ignored; `$REPO`
|
|
34
34
|
# does not match `$REPO_FLAG`. Suppress a deliberate read by putting
|
|
35
35
|
# `allowlist:fence-state` on the same line. Pseudo-tool-call fences are scanned too.
|
|
36
|
+
# A line that sources scripts/lib/fence-bootstrap.sh and ends in `# FENCE_BOOTSTRAP` binds the variables that
|
|
37
|
+
# library derives (PR_NUMBER ALL_PR_NUMBERS GH_REPO DEFAULT_BRANCH STAGING_BRANCH CONFIG_FILE _SD FORGE_RUN_KEY).
|
|
36
38
|
# Output: `FAIL <file>:<line> fence-state ...` then `SPEC-FENCE-STATE: ...`.
|
|
37
39
|
# --fence-funcs Different check, opt-in: shell functions are per-shell too, so a function defined
|
|
38
40
|
# (`name() {`, `function name`) in one ```bash/sh fence is undefined in every other
|
|
@@ -113,15 +115,17 @@ if [ "$FENCESTATE" = 1 ]; then
|
|
|
113
115
|
}
|
|
114
116
|
}
|
|
115
117
|
function reset( i) { for (i = 1; i <= nv; i++) { asg[vn[i]] = 0; first[vn[i]] = ""; firsttxt[vn[i]] = "" } }
|
|
116
|
-
BEGIN { nv = split(vars, vn, " ") }
|
|
118
|
+
BEGIN { nv = split(vars, vn, " "); nb = split("PR_NUMBER ALL_PR_NUMBERS GH_REPO DEFAULT_BRANCH STAGING_BRANCH CONFIG_FILE _SD FORGE_RUN_KEY", bv, " ") }
|
|
117
119
|
!inb && /^[ \t]*```(bash|sh)[ \t]*$/ { inb = 1; reset(); next }
|
|
118
120
|
!inb && /^[ \t]*```/ { other = !other; next }
|
|
119
121
|
other { next }
|
|
120
122
|
inb && /^[ \t]*```[ \t]*$/ { flush(); inb = 0; next }
|
|
121
123
|
inb {
|
|
122
124
|
if ($0 ~ /^[ \t]*#/) next
|
|
125
|
+
boot = ($0 ~ /fence-bootstrap\.sh/ && $0 ~ /# FENCE_BOOTSTRAP[ \t]*$/)
|
|
123
126
|
for (i = 1; i <= nv; i++) {
|
|
124
127
|
v = vn[i]
|
|
128
|
+
if (boot && first[v] == "") for (j = 1; j <= nb; j++) if (bv[j] == v) asg[v] = 1
|
|
125
129
|
if (first[v] == "" && ($0 ~ ("(^|[;&|( \t])(export[ \t]+|local[ \t]+|declare[ \t]+(-[A-Za-z]+[ \t]+)?|readonly[ \t]+)?" v "=") ||
|
|
126
130
|
$0 ~ ("(^|[;&|( \t])read[ \t]+(-[A-Za-z]+[ \t]+)*([A-Za-z0-9_]+[ \t]+)*" v "([ \t;]|$)") ||
|
|
127
131
|
$0 ~ ("(^|[;&|( \t])for[ \t]+" v "[ \t]+in[ \t]"))) asg[v] = 1
|
|
@@ -165,6 +165,26 @@ check 0 "violations=0" "--fence-state passes declared, loop, e
|
|
|
165
165
|
check 2 "Usage" "--fence-state without -- is a usage error" -- --fence-state PR_NUMBER fs-good.md
|
|
166
166
|
check 2 "Usage" "--fence-state without vars is a usage error" -- --fence-state -- fs-good.md
|
|
167
167
|
|
|
168
|
+
# --fence-state: a FENCE_BOOTSTRAP source line binds the variables scripts/lib/fence-bootstrap.sh derives (and only those, and only on a marked line)
|
|
169
|
+
cat > "$T/fs-boot.md" <<EOF
|
|
170
|
+
${FENCE}bash
|
|
171
|
+
. "\$(forge_resolve lib/fence-bootstrap.sh)" || exit 1 # FENCE_BOOTSTRAP
|
|
172
|
+
gh pr view "\$PR_NUMBER" -R "\$GH_REPO" --json state
|
|
173
|
+
${FENCE}
|
|
174
|
+
EOF
|
|
175
|
+
cat > "$T/fs-boot-bad.md" <<EOF
|
|
176
|
+
${FENCE}bash
|
|
177
|
+
. "\$(forge_resolve lib/fence-bootstrap.sh)" || exit 1
|
|
178
|
+
gh pr view "\$PR_NUMBER" -R "\$GH_REPO" --json state
|
|
179
|
+
${FENCE}
|
|
180
|
+
${FENCE}bash
|
|
181
|
+
. "\$(forge_resolve lib/fence-bootstrap.sh)" || exit 1 # FENCE_BOOTSTRAP
|
|
182
|
+
echo "\$MERGE_ISSUE"
|
|
183
|
+
${FENCE}
|
|
184
|
+
EOF
|
|
185
|
+
check 0 "violations=0" "--fence-state treats a FENCE_BOOTSTRAP source line as binding PR_NUMBER and GH_REPO" -- --fence-state PR_NUMBER GH_REPO -- fs-boot.md
|
|
186
|
+
check 1 "violations=3" "--fence-state ignores an unmarked source line and a var the bootstrap does not derive" -- --fence-state PR_NUMBER GH_REPO MERGE_ISSUE -- fs-boot-bad.md
|
|
187
|
+
|
|
168
188
|
# --fence-funcs: a function defined only in another fence, then called, is flagged.
|
|
169
189
|
cat > "$T/ff-bad.md" <<EOF
|
|
170
190
|
${FENCE}bash
|
|
@@ -165,12 +165,14 @@ MARKER_REGISTRY="
|
|
|
165
165
|
REMEDIATION
|
|
166
166
|
REMEDIATION_HELD
|
|
167
167
|
REMEDIATION_DEFERRED
|
|
168
|
+
REMEDIATION_DISPATCH
|
|
168
169
|
REREVIEW_DISPATCHED
|
|
169
170
|
REREVIEW_RELEASED
|
|
170
171
|
REREVIEW_SKIPPED
|
|
171
172
|
REVIEW
|
|
172
173
|
REVIEWER
|
|
173
174
|
REVIEW_ROUTE
|
|
175
|
+
REVIEW_TIER
|
|
174
176
|
REVIEW_STARTED
|
|
175
177
|
SECURITY_AUDIT
|
|
176
178
|
SIGNAL_RESOLVED
|