forgedock 1.16.2 → 1.17.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/engine/outcome-codes.mjs +92 -0
- package/bin/engine/phases.mjs +86 -26
- package/bin/engine/projector.mjs +48 -1
- package/bin/engine/runlog.mjs +13 -5
- package/bin/engine/telemetry-seam.mjs +62 -0
- package/bin/engine/telemetry-seam.test.mjs +60 -0
- package/bin/engine/telemetry.mjs +148 -0
- package/bin/engine/telemetry.test.mjs +120 -0
- package/bin/engine-cli.mjs +12 -1
- package/bin/engine.mjs +145 -40
- package/bin/forgedock.mjs +7 -1
- package/bin/hooks/interactive-engine.mjs +10 -0
- package/bin/hooks/pre-tool-use.mjs +187 -0
- package/bin/runner.mjs +7 -0
- package/bin/stats/aggregate.mjs +231 -0
- package/bin/stats/cache.mjs +40 -0
- package/bin/stats/github-source.mjs +140 -0
- package/bin/stats/index.mjs +154 -0
- package/bin/stats/local-source.mjs +79 -0
- package/bin/stats/package.json +6 -0
- package/bin/stats/render.mjs +82 -0
- package/bin/stats/stats.test.mjs +224 -0
- package/commands/cleanup.md +83 -2
- package/commands/deploy-info.md +8 -4
- package/commands/orchestrate/phase-1-resolve.md +4 -1
- package/commands/orchestrate/phase-4-execution.md +17 -2
- package/commands/orchestrate/phase-5-cleanup.md +18 -2
- package/commands/pipeline-health/phase-1-context.md +2 -0
- package/commands/pipeline-health/phase-2-metrics.md +12 -5
- package/commands/pipeline-status.md +21 -1
- package/commands/review-pr-agents/protocols.md +1 -0
- package/commands/review-pr-agents.md +10 -2
- package/commands/review-pr-staging.md +102 -25
- package/commands/review-pr.md +119 -103
- package/commands/work-on/build/implement.md +26 -5
- package/commands/work-on/build/validate.md +3 -2
- package/commands/work-on/build.md +95 -64
- package/commands/work-on/close.md +17 -6
- package/commands/work-on/decompose.md +45 -4
- package/commands/work-on/investigate.md +6 -2
- package/commands/work-on/remediate.md +22 -6
- package/commands/work-on/review.md +2 -2
- package/docs/CONFIG.md +33 -0
- package/docs/OUTCOME-CODES.md +48 -0
- package/docs/RELEASE-NOTES.md +12 -0
- package/docs/STATS.md +62 -0
- package/docs/TELEMETRY.md +65 -0
- package/docs/WORK-ON-RUNTIME.md +13 -0
- package/docs/site/command-reference.md +1 -1
- package/docs/spec/label-state-machine.md +1 -1
- package/package.json +3 -2
- package/scripts/builder-resume-match.test.sh +58 -0
- package/scripts/cap-findings.sh +110 -0
- package/scripts/cap-findings.test.sh +130 -0
- package/scripts/check-spec-markers.sh +2 -0
- package/scripts/classify-finding.sh +65 -16
- package/scripts/classify-finding.test.sh +43 -1
- package/scripts/classify-grep-exit.sh +20 -0
- package/scripts/classify-grep-exit.test.sh +46 -0
- package/scripts/close-parent-c3.test.sh +54 -0
- package/scripts/cost-report.sh +146 -0
- package/scripts/cost-report.test.sh +80 -0
- package/scripts/diff-size.sh +5 -0
- package/scripts/doctor-pipeline-state.sh +43 -0
- package/scripts/file-deferred-followups.sh +159 -0
- package/scripts/file-deferred-followups.test.sh +106 -0
- package/scripts/fixtures/cost-report/a-merged1.jsonl +6 -0
- package/scripts/fixtures/cost-report/a-merged2-remediated.jsonl +7 -0
- package/scripts/fixtures/cost-report/a-needs-human.jsonl +4 -0
- package/scripts/fixtures/cost-report/b-merged.jsonl +3 -0
- package/scripts/fixtures/cost-report/c-merged-attempt-usage.jsonl +8 -0
- package/scripts/fixtures/cost-report/hydrated-no-usage.jsonl +3 -0
- package/scripts/fixtures/cost-report/old-log-no-version.jsonl +3 -0
- package/scripts/fixtures/cost-report/unfinished-no-terminal.jsonl +2 -0
- package/scripts/fixtures/cost-report/unfinished-reopened.jsonl +5 -0
- package/scripts/forge-root.test.sh +8 -3
- package/scripts/lib/fence-bootstrap.sh +4 -3
- package/scripts/lib/pr-issue-number.sh +18 -0
- package/scripts/lib/review-template-resolve.sh +41 -0
- package/scripts/lib/telemetry.sh +168 -0
- package/scripts/remediate-liveness.sh +42 -6
- package/scripts/review-acceptance-gate.sh +16 -7
- package/scripts/review-acceptance-gate.test.sh +13 -0
- package/scripts/review-pr-domain-map.test.sh +8 -3
- package/scripts/review-pr-gate2.test.sh +15 -0
- package/scripts/review-pr-staging-finding-filter.test.sh +44 -0
- package/scripts/review-pr-staging-scope.test.sh +65 -5
- package/scripts/review-prepare.sh +517 -0
- package/scripts/review-prepare.test.sh +270 -0
- package/scripts/review-roster.sh +172 -0
- package/scripts/review-roster.test.sh +163 -0
- package/scripts/run-shell-tests.sh +3 -0
- package/scripts/telemetry.test.sh +106 -0
- package/scripts/wait-ci-green.sh +8 -1
- package/templates/forge.yaml.minimal +9 -0
|
@@ -562,9 +562,9 @@ gh issue view {NUMBER} {GH_FLAG} --json state --jq '.state'
|
|
|
562
562
|
|
|
563
563
|
- `REVIEW_RESULT: status: BLOCKED` from /review-pr whose blocker contains "ci gate" (Phase 8 refused to merge because checks failed, were cancelled, or stayed pending past the gate retries): do NOT attempt the manual merge below — that would bypass the CI gate. Fixing red CI is pipeline work, not a human decision: hand remediation to the router **once**. Bound: count `<!-- FORGE:CI_REMEDIATION: pr={PR_NUMBER} -->` comments on the issue first; if ≥ 1, do not remediate again — remove `workflow:remediating`, label the issue `needs-human` (the bound is exhausted, so a human is now genuinely needed) and return `REVIEW_RESULT: status: BLOCKED` with blocker "ci gate not green after remediation". Otherwise run the remediation-pending handoff block above (sets `workflow:remediating`, removes the `needs-human` that Phase 8 set) and return `REVIEW_RESULT: status: NEXT` with `next: remediate` and `remediation: ci-gate` (see Output). Do not post the marker here: the router posts it immediately before it invokes remediation (Phase 4R), so a handoff interrupted before remediation starts is retried instead of being counted as used. **Never invoke `work-on:remediate` from this phase** (forge#3398): remediation re-reviews through `/review-pr`, which spawns domain-reviewer sub-agents, so it must run one level below the router, not nested inside this fork. Nested here, it lands at the sub-agent depth limit under `/orchestrate` with no `Agent` tool and strands the PR at `REREVIEW-REQUIRED` (forge#3391). The router runs `work-on:remediate`, which classifies a CI-gate refusal as FIXABLE, swaps `workflow:remediating` (or a legacy `needs-human`) for `workflow:in-review`, fixes the failing jobs on the PR branch, re-runs the quality gate and re-reviews the new head (scoped by /review-pr: full, delta or carried forward), and auto-lands through the same CI gate. <!-- forge#3191 -->
|
|
564
564
|
|
|
565
|
-
- `REVIEW_RESULT: status: BLOCKED` from /review-pr whose blocker contains "in-pr fix required" (review-pr §6B.6: CONFIRMED MEDIUM findings in files this PR changed, requested on the PR in a `FORGE:INPR_FIX` comment at the current head): do NOT merge and do NOT leave the issue at `needs-human` because of this gate. Run **one** fix round, bounded by `<!-- FORGE:INPR_REMEDIATION: pr={PR_NUMBER} -->` on the issue (`{BOUND}` = `INPR_REMEDIATION` below).
|
|
565
|
+
- `REVIEW_RESULT: status: BLOCKED` from /review-pr whose blocker contains "in-pr fix required" (review-pr §6B.6: CONFIRMED MEDIUM/HIGH/CRITICAL findings in files this PR changed, requested on the PR in a `FORGE:INPR_FIX` comment at the current head): do NOT merge and do NOT leave the issue at `needs-human` because of this gate. Run **one** fix round, bounded by `<!-- FORGE:INPR_REMEDIATION: pr={PR_NUMBER} -->` on the issue (`{BOUND}` = `INPR_REMEDIATION` below).
|
|
566
566
|
- **Bound unused**: run the remediation-pending handoff block above (set `workflow:remediating`, no `needs-human`; remediation accepts that state and its Phase M1 clears it as FIXABLE). Then return `REVIEW_RESULT: status: NEXT` with `next: remediate` and `remediation: inpr-fix` (the router posts the bound marker, as in the ci-gate case) — never invoke `work-on:remediate` from this phase (same reason as the ci-gate case). The router runs remediation, which reads the `FORGE:INPR_FIX` work order, fixes exactly those findings, re-reviews the new head and auto-lands. On the re-review, any finding still present is filed as an issue (the round is used). If remediation does not land, the router re-invokes this phase, which then finds the bound used and takes the branch below.
|
|
567
|
-
- **Bound already used** (including the re-invocation after a remediation that did not land): waive the gate for the current head. Remove `needs-human` and `workflow:remediating` (restoring `workflow:in-review`), post `<!-- FORGE:INPR_FIX_WAIVED: head=<current head SHA> -->` on the **PR**, then re-invoke Phase R3 once under the `STALE_REREVIEW` bound. That review files the remaining findings as issues and merges as it would have before this gate. Only if `STALE_REREVIEW` is also exhausted, label the issue `needs-human`.
|
|
567
|
+
- **Bound already used** (including the re-invocation after a remediation that did not land): first read `max_severity=` from the latest trusted `FORGE:INPR_FIX` marker on the PR (`gh pr view {PR_NUMBER} {GH_FLAG} --json comments`, author-trusted as elsewhere in this phase; absent means MEDIUM). **HIGH/CRITICAL are never waived**: remove `workflow:remediating`, label the issue `needs-human`, leave the PR unmerged and return `REVIEW_RESULT: status: BLOCKED` with blocker "in-pr fix not landed (HIGH/CRITICAL, no waiver)". The `FORGE:INPR_FIX` work order stays on the PR for the human. For MEDIUM, waive the gate for the current head. Remove `needs-human` and `workflow:remediating` (restoring `workflow:in-review`), post `<!-- FORGE:INPR_FIX_WAIVED: head=<current head SHA> -->` on the **PR**, then re-invoke Phase R3 once under the `STALE_REREVIEW` bound. That review files the remaining findings as issues and merges as it would have before this gate. Only if `STALE_REREVIEW` is also exhausted, label the issue `needs-human`.
|
|
568
568
|
```bash
|
|
569
569
|
INPR_WAIVE_HEAD=$(gh pr view {PR_NUMBER} {GH_FLAG} --json headRefOid --jq .headRefOid)
|
|
570
570
|
gh issue edit {NUMBER} {GH_FLAG} --remove-label "needs-human,workflow:remediating" --add-label "workflow:in-review" 2>/dev/null || true # allowlist:check-command-side-effects
|
package/docs/CONFIG.md
CHANGED
|
@@ -308,6 +308,11 @@ orchestration:
|
|
|
308
308
|
# shrink cascade queue growth.
|
|
309
309
|
max_concurrent: 12
|
|
310
310
|
|
|
311
|
+
# Age (hours since the issue last updated) after which a predecessor still at
|
|
312
|
+
# `workflow:remediating` or `workflow:engine-error` is classified GATED
|
|
313
|
+
# (stalled, needs a human) instead of IN_PROGRESS. Default: 48.
|
|
314
|
+
in_progress_max_hours: 48
|
|
315
|
+
|
|
311
316
|
cascade:
|
|
312
317
|
# Named preset expanding to the granular keys below. Any granular key set
|
|
313
318
|
# explicitly overrides just that field — the preset supplies defaults, not
|
|
@@ -531,6 +536,30 @@ build:
|
|
|
531
536
|
|
|
532
537
|
---
|
|
533
538
|
|
|
539
|
+
## `decomposition` (OPTIONAL)
|
|
540
|
+
|
|
541
|
+
Controls what happens to a decomposed parent issue. A parent is only a tracker for its sub-issues, but while it is open it inflates GitHub's open-issue count and every backlog view.
|
|
542
|
+
|
|
543
|
+
```yaml
|
|
544
|
+
decomposition:
|
|
545
|
+
# Close the parent at decomposition. Default: false (parent stays open until its last child closes).
|
|
546
|
+
close_parent: true
|
|
547
|
+
```
|
|
548
|
+
|
|
549
|
+
| Field | Type | Required | Description |
|
|
550
|
+
|-------|------|----------|-------------|
|
|
551
|
+
| `close_parent` | boolean | No | When `true`, `work-on:decompose` (Phase D6.5) closes the parent after the sub-issues exist, `FORGE:DECOMPOSED:COMPLETE` is posted and `workflow:decomposed` is set, with the comment "Decomposed into #a #b #c; tracked here until all complete". The label is kept. Every child body is also checked for the `**Parent**: #N` reference. Default: `false`. Any value other than `true`, or an unreadable config, leaves the parent open |
|
|
552
|
+
|
|
553
|
+
**With `close_parent: true`**: `work-on:close` Phase C3 keeps checking children off the closed parent's checklist. When the last child closes, the parent gets an "All sub-issues complete" comment and `workflow:merged`; it is not closed a second time and never reopened. `/pipeline-status` and `/pipeline-health` exclude `workflow:decomposed` from work counts and report them separately as `trackers (N)`. `/orchestrate` expands an explicitly requested closed decomposed parent to its open children.
|
|
554
|
+
|
|
555
|
+
**With `close_parent: false`** (default): unchanged. The parent stays open and is closed by C3 when the last child closes.
|
|
556
|
+
|
|
557
|
+
**Limitation**: `--retry` and engine resume refuse closed issues, so a closed decomposed parent cannot be re-run. It has nothing left to run; work on its open children instead.
|
|
558
|
+
|
|
559
|
+
**Commands that use this section**: `work-on` (`decompose` D6.5, `close` C3), `orchestrate`
|
|
560
|
+
|
|
561
|
+
---
|
|
562
|
+
|
|
534
563
|
## `services` (OPTIONAL)
|
|
535
564
|
|
|
536
565
|
External service endpoints and identifiers. Used by analytics and monitoring commands.
|
|
@@ -595,6 +624,8 @@ review:
|
|
|
595
624
|
auth: "JWT + refresh token rotation; sessions stored in Redis"
|
|
596
625
|
database: "Always use docker exec <container> for psql/migrations"
|
|
597
626
|
|
|
627
|
+
# max_filed_findings: 2 # per-review cap on filed findings (default 2; 1 on review-finding lineage)
|
|
628
|
+
|
|
598
629
|
# key_paths:
|
|
599
630
|
# auth: ["services/api/auth/**", "web/src/lib/auth/**"]
|
|
600
631
|
# billing: ["services/api/billing/**", "web/src/app/billing/**"]
|
|
@@ -606,6 +637,7 @@ review:
|
|
|
606
637
|
| `tech_stack` | string | No | One-line tech stack summary. Injected into all review agent prompts |
|
|
607
638
|
| `context` | string (multiline) | No | Freeform context about architecture, deploy model, and known pitfalls |
|
|
608
639
|
| `domains.*` | string | No | Domain-specific notes keyed by review agent name (`billing`, `auth`, `database`, `frontend`, `api`, `infra`, `security`) |
|
|
640
|
+
| `max_filed_findings` | integer >= 0 | No | Per-review cap on ISSUE dispositions (`scripts/cap-findings.sh`, applied by `review-pr` §6B.5 and `review-pr-staging` §7B.5 after classification). Default `2`; `1` when the reviewed PR is itself a fix for a `review-finding` issue, where every filed HIGH/CRITICAL consumes that budget. ISSUE findings are ranked by severity, then confidence, then safety-domain agent (security, auth, billing, concurrency, database), then input order; an unparseable severity or confidence ranks as MEDIUM/LIKELY. The overflow becomes `NOTE capped`: listed in the PR body and counted as `notes_capped` in `FORGE:NOTE_DISPOSITION`, never filed and never dropped. **HIGH/CRITICAL findings are never capped** and still block; `ISSUE fix-efficacy-*` findings (the linked issue is not fixed, or the PR re-introduces its defect) are never capped either and count against the review-finding budget like HIGH. `0` is valid (cap every MEDIUM/LOW finding); a non-numeric or negative value falls back to the default with a warning. An explicit value overrides both defaults. |
|
|
609
641
|
| `key_paths` | map(string→list of strings) | No | Domain-to-file mapping used by `/work-on` investigation and `/review-pr` agents to quickly locate relevant files. Keys are domain names matching issue labels; values are lists of file path patterns (glob-style, relative to repo root). When present, agents use this table instead of inferring files from labels and issue body. |
|
|
610
642
|
|
|
611
643
|
**Commands that use this section**: `review-pr`, `review-pr-agents` (all domain agents), `work-on` (Phase 1B investigation)
|
|
@@ -939,6 +971,7 @@ These variables are read from the operator's runner environment, not from `forge
|
|
|
939
971
|
| `FORGE_TRAIL_OVERRIDE_ASSOCIATIONS` | the resolved `FORGE_TRAIL_TRUSTED_ASSOCIATIONS` value | Comma-separated `author_association` values prefiltering break-glass override candidates (the permission API stays authoritative). Never affects which FORGE markers count toward the gate. An approver with concealed org membership can show `CONTRIBUTOR`/`NONE`, so widen this variable (not the trail list) to accept them. An explicitly empty value means no override candidates. |
|
|
940
972
|
| `FORGE_TRAIL_PIPELINE_LOGINS` | empty | Comma-separated GitHub logins (case-insensitive) that are the pipeline's own identity and can never approve a phase-trail break-glass override. The authors of trusted FORGE marker comments on the issue are added automatically. |
|
|
941
973
|
| `FORGE_TRAIL_QG_SINCE` | `2026-10-07T03:40:12Z` | ISO-8601 UTC cutoff for the legacy `FORGE:QUALITY_GATE` grace. An issue whose trusted `FORGE:BUILDER:COMPLETE` comment was last updated before this time has `QUALITY_GATE` waived. Set to an empty string to disable the grace. A malformed or future value (including a runner clock earlier than this default) fails closed: the verifier exits 2, an infrastructure block rather than `MISSING`. |
|
|
974
|
+
| `FORGE_TELEMETRY` | `on` | Set to `off` to disable gate telemetry. Gate scripts append one structured record per decision to a trusted `FORGE:TELEMETRY v1` comment; emission is best-effort and never changes a gate's output or exit code. See [TELEMETRY.md](TELEMETRY.md). |
|
|
942
975
|
|
|
943
976
|
### Install root resolution
|
|
944
977
|
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
# Engine outcome codes
|
|
2
|
+
|
|
3
|
+
Version: 1 (`OUTCOME_CODES_VERSION` in `bin/engine/outcome-codes.mjs`, the single source of truth).
|
|
4
|
+
|
|
5
|
+
Every `PHASE_FAILED`, blocked `PHASE_COMMIT` (a handoff or block) and `RUN_TERMINAL` run-log event carries a
|
|
6
|
+
`code` from this closed enum, so stall causes can be counted and grouped without reading prose. The code is
|
|
7
|
+
derived in code from structured evidence only (`REVIEW_RESULT` fields, FORGE markers and sentinels,
|
|
8
|
+
`**Re-gate outcome**`, `err.code`), never from the free-text `detail`/`reason`, which stays alongside for humans.
|
|
9
|
+
`terminate()` also records the code in the issue's `FORGE:STATE` block (`outcomeCode`), so it survives a rebuild
|
|
10
|
+
or resume and is recoverable from GitHub alone. Adding a code is additive; renaming or removing one bumps the
|
|
11
|
+
version.
|
|
12
|
+
|
|
13
|
+
An absent or unknown code is recorded as `unclassified`, with the raw reason kept (`PHASE_FAILED.reason`,
|
|
14
|
+
`RUN_TERMINAL.detail`), so gaps show up in a report instead of being hidden.
|
|
15
|
+
|
|
16
|
+
| Code | Meaning | Emitted by | Source evidence |
|
|
17
|
+
|------|---------|------------|-----------------|
|
|
18
|
+
| `review.changes_requested` | Review asked for an in-PR fix and handed off to remediate | review | `REVIEW_RESULT` `status: NEXT`, `next: remediate` |
|
|
19
|
+
| `review.escalated` | Review escalated the PR to a human or remediation | review | PR/issue carries `needs-human` or `workflow:remediating` |
|
|
20
|
+
| `review.rereview_skipped` | Re-review was skipped | reserved | No structured evidence source yet |
|
|
21
|
+
| `build.acceptance_failed` | Newest acceptance gate marker is blocked | build | `FORGE:ACCEPTANCE_GATE:FAILED`/`:BLOCKED` or `FORGE:BUILD_BLOCKED` |
|
|
22
|
+
| `build.acceptance_malformed` | Acceptance spec was malformed | reserved | Indistinguishable from `build.acceptance_failed` by sentinel alone |
|
|
23
|
+
| `build.size_over_gate` | Size gate routed the build to decompose | build | `FORGE:DIFF_SIZE` `result: OVER` with `### Split Proposal` |
|
|
24
|
+
| `build.cohesive` | Size gate judged an oversized diff cohesive | reserved | `result: COHESIVE` is a pass, so no failure to code |
|
|
25
|
+
| `remediate.no_marker` | Remediation ended without its completion marker | remediate | No trusted `FORGE:REMEDIATION:COMPLETE` |
|
|
26
|
+
| `remediate.deferred_live_run` | Remediation deferred to a live earlier run | remediate, engine | Trusted `FORGE:REMEDIATION_DEFERRED` record |
|
|
27
|
+
| `remediate.re_escalated` | Remediation re-escalated to a human | remediate | `**Re-gate outcome**: RE-ESCALATED` |
|
|
28
|
+
| `remediate.unfixable` | Remediation judged the findings unfixable | remediate | `**Re-gate outcome**: UNFIXABLE` |
|
|
29
|
+
| `ci.red` | CI failed | reserved | `REVIEW_RESULT` carries no CI field yet |
|
|
30
|
+
| `ci.timeout` | CI timed out | reserved | `REVIEW_RESULT` carries no CI field yet |
|
|
31
|
+
| `base.conflict` | Base-branch conflict blocked the PR | reserved | `REVIEW_RESULT` carries no conflict field yet |
|
|
32
|
+
| `rate_limited` | Session or usage limit outlasted the pause budget | engine | `err.usageLimit` / `err.resetAtEpochMs` |
|
|
33
|
+
| `engine.error` | Engine or tool failure (missing key/SDK, CLI crash, max turns, unresolved args, runner threw every attempt) | engine | `err.code` |
|
|
34
|
+
| `lease.held` | Another live agent holds the issue lease | engine | Foreign live lease on acquire |
|
|
35
|
+
| `invalid` | Issue is invalid or in a divergent closed state | investigate, engine | `INVESTIGATION:INVALID` marker, or live snapshot |
|
|
36
|
+
| `decomposed` | Issue was decomposed into sub-issues | investigate, decompose | `FORGE:DECOMPOSED` or the decomposition assessment |
|
|
37
|
+
| `completed` | Non-failure terminal (merged, phase-complete, investigation) | engine | Terminal reason |
|
|
38
|
+
| `awaiting_merge` | Clean result held for a merge decision | remediate | Terminal reason `awaiting-merge` |
|
|
39
|
+
| `unclassified` | Fallback for an absent or unknown code | engine | The raw reason is kept alongside |
|
|
40
|
+
|
|
41
|
+
## Known limits
|
|
42
|
+
|
|
43
|
+
- `build.acceptance_failed` covers malformed specs too until the gate sentinel distinguishes them.
|
|
44
|
+
- `ci.red`, `ci.timeout` and `base.conflict` stay reserved until `REVIEW_RESULT` carries structured fields for
|
|
45
|
+
them; the engine will not infer them from prose. Such outcomes are `unclassified` today.
|
|
46
|
+
- `needs-human` has many causes; a pause on a pre-existing `needs-human` label is `unclassified`.
|
|
47
|
+
- The `FORGE:TELEMETRY` comment write is owned by #3834. The engine forwards coded deferrals through the
|
|
48
|
+
existing `emitTelemetry` hook and exposes the code on the run log and `FORGE:STATE`.
|
package/docs/RELEASE-NOTES.md
CHANGED
|
@@ -2,6 +2,18 @@
|
|
|
2
2
|
|
|
3
3
|
Operator-facing notes for behavior changes that need action or awareness. Newest first.
|
|
4
4
|
|
|
5
|
+
## Gate scripts emit structured telemetry (#3834)
|
|
6
|
+
|
|
7
|
+
`diff-size.sh`, `review-acceptance-gate.sh`, `classify-finding.sh`, `wait-ci-green.sh`, `remediate-liveness.sh` and the build/review spec call sites (B5.5, B6.5, review panel) now append one JSONL record per decision to a trusted `<!-- FORGE:TELEMETRY v1 -->` comment on the issue or PR (shell emitter `scripts/lib/telemetry.sh`, Node twin `bin/engine/telemetry.mjs`). A failed write is buffered under `~/.forge/telemetry-buffer/` and flushed on the next emit. Catalogue and privacy note: `docs/TELEMETRY.md`.
|
|
8
|
+
|
|
9
|
+
Action: none. Set `FORGE_TELEMETRY=off` to disable. Gate stdout and exit codes are unchanged.
|
|
10
|
+
|
|
11
|
+
## Deferred follow-ups are folded or batched into one issue (#3843)
|
|
12
|
+
|
|
13
|
+
Build B2.0 no longer files one issue per `deferred → #N` item. A same-mechanical-change sibling that fits the remaining `0.8 × build.diff_size.threshold` budget (1000 when the threshold is 0 or unset) is folded into the contract's Deliverables. The rest go through `scripts/file-deferred-followups.sh`: one follow-up per parent with a checklist (known-flaw items flagged `known-flaw`), or an append to the open trusted follow-up for the same parent (exact path match). The script prints a `DEFERRED_TELEMETRY deferred_items= folded= batched_into= filed=` line and calls the #3834 emitter only if present. The `FORGE:DEFERRED_FROM` marker and every path stay in the body, so review-pr deferred verification is unchanged.
|
|
14
|
+
|
|
15
|
+
Action: none. Expect fewer, larger follow-up issues titled `follow-up: N deferred item(s) (deferred from #P)`.
|
|
16
|
+
|
|
5
17
|
## remediate.md uses the shared resolver stub (#3779)
|
|
6
18
|
|
|
7
19
|
`commands/work-on/remediate.md` no longer inlines the `FORGE_ROOT` bootstrap or the `TRUSTED_SCRIPT` / `CI_GATE_SCRIPT` resolvers; each fence uses the pinned `forge_resolve` locator from `scripts/lib/forge-resolve.sh` (unresolved script => empty => the existing strict branch). The spec shrinks from 95,283 to 80,957 bytes (-15.0%).
|
package/docs/STATS.md
ADDED
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
<!-- SPDX-FileCopyrightText: Copyright (c) RapierCraft Studios -->
|
|
2
|
+
<!-- SPDX-License-Identifier: AGPL-3.0-or-later -->
|
|
3
|
+
|
|
4
|
+
# forgedock stats
|
|
5
|
+
|
|
6
|
+
A deterministic, version-over-version report. No model is involved: code reads records and counts them.
|
|
7
|
+
|
|
8
|
+
```bash
|
|
9
|
+
npx forgedock stats [--since <version|date>] [--repo <owner/repo>] [--json] [--refresh]
|
|
10
|
+
[--compact] [--no-local] [--no-cache] [--min-n N] [--release-times FILE]
|
|
11
|
+
```
|
|
12
|
+
|
|
13
|
+
`--repo` defaults to `project.owner`/`project.repo` in `forge.yaml`, else the `origin` remote. `--since 1.16.0` keeps
|
|
14
|
+
versions at or above it; `--since 2026-10-01` keeps activity on or after the date. `--compact` prints one row per
|
|
15
|
+
version (the table embedded in the staging review summary). `--json` is machine-readable.
|
|
16
|
+
|
|
17
|
+
## Sources
|
|
18
|
+
|
|
19
|
+
1. **GitHub (primary).** Issues, PRs and labels of the target repo, plus trusted `<!-- FORGE:TELEMETRY v1 -->` comments
|
|
20
|
+
(`docs/TELEMETRY.md`). Only comments from OWNER/MEMBER/COLLABORATOR or a Bot are read (same predicate as
|
|
21
|
+
`scripts/trusted-comments.sh`); `part=N` comments are concatenated and records are deduped on `eid`. All `gh` calls
|
|
22
|
+
are argv-based. Needs an authenticated `gh`.
|
|
23
|
+
2. **Local cache (optional).** `~/.forge/stats-cache/<owner>_<repo>/cache.json` (`FORGE_STATS_CACHE_DIR`) holds what was
|
|
24
|
+
fetched plus an `updated_at` watermark, so the next run only re-reads items updated since. It is safe to delete (the
|
|
25
|
+
cost is one full re-fetch); a corrupt cache is treated as empty. `--refresh` ignores it, `--no-cache` never touches it.
|
|
26
|
+
3. **Local run logs (optional).** `~/.forge/runs/*.jsonl` (`FORGE_RUNS_DIR`) for runs that predate the engine's
|
|
27
|
+
`phase_attempt` / `run_terminal` records. A run already present on GitHub is not counted twice. Only runs whose
|
|
28
|
+
`RUN_START.repo` equals the target repo are used. `--no-local` skips them.
|
|
29
|
+
|
|
30
|
+
With no local state at all, `forgedock stats --repo <r>` reproduces the report from GitHub alone.
|
|
31
|
+
|
|
32
|
+
## Sections (per version and repo)
|
|
33
|
+
|
|
34
|
+
- **Cost**: cache-read tokens per issue (mean, median), per phase (mean over all runs, so phases add up to the per-issue
|
|
35
|
+
mean), per merged issue, output tokens, USD where recorded.
|
|
36
|
+
- **Time**: p50 and p90 duration per phase and wall time per issue.
|
|
37
|
+
- **Flow**: merged %, needs-human %, remediation rate, outcome-code distribution (`docs/OUTCOME-CODES.md`).
|
|
38
|
+
- **Review funnel**: findings per review by severity, then ISSUE / NOTE / in-PR fix, plus `validated` and `false-positive` issues.
|
|
39
|
+
- **Gates**: block rate per gate (acceptance, diff size OVER, CI, liveness), malformed-check rate, COHESIVE and OVER counts.
|
|
40
|
+
- **Issue flow**: issues created by source (review-finding, pipeline bot, human), closed by kind, net flow, findings per merged issue.
|
|
41
|
+
|
|
42
|
+
Each group shows a delta against the previous version of the same repo and is flagged `low-n` below `--min-n`
|
|
43
|
+
(default 10 finished runs).
|
|
44
|
+
|
|
45
|
+
## Counting rules
|
|
46
|
+
|
|
47
|
+
They match `scripts/cost-report.sh`, so the two reports agree on the same logs: only finished runs count; hydrated runs
|
|
48
|
+
and runs with no usage events are excluded; tokens are cache-read tokens; `PHASE_ATTEMPT_USAGE` supersedes the cumulative
|
|
49
|
+
`PHASE_COMMIT` usage of the same phase. On GitHub a run is the set of `phase_attempt` records sharing a `run` id and it is
|
|
50
|
+
finished once a `run_terminal` record exists. Wall time covers one engine process; a resumed run is a separate run.
|
|
51
|
+
|
|
52
|
+
## Versions and n/a
|
|
53
|
+
|
|
54
|
+
Records carry `forge_version`. Records without one (older runs, issues with no telemetry) are placed by release time:
|
|
55
|
+
`--release-times FILE` (`<version> <epoch-seconds>` lines), else the git tags of the ForgeDock checkout, shown with a `~`
|
|
56
|
+
suffix; otherwise `unknown`. Anything that cannot be computed (no timestamps, no outcome codes, no records) prints `n/a`
|
|
57
|
+
instead of failing.
|
|
58
|
+
|
|
59
|
+
## Limits
|
|
60
|
+
|
|
61
|
+
Issues and PRs are listed by `updated_at`, so the first run on a large repo makes one comment request per item that has
|
|
62
|
+
comments. Telemetry comments are edited in place; if a cached item looks stale, run with `--refresh`.
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
# Gate Telemetry
|
|
2
|
+
|
|
3
|
+
Gate scripts record their decisions as structured events so gate block rates, finding funnels and malformed-check
|
|
4
|
+
counts can be measured without re-reading comments with a model. GitHub is the source of truth: records are written
|
|
5
|
+
by code (never by a model) into a `FORGE:TELEMETRY` comment, and everything can be rebuilt from the GitHub API alone.
|
|
6
|
+
Local files are only a cache.
|
|
7
|
+
|
|
8
|
+
## Where records live
|
|
9
|
+
|
|
10
|
+
- One `<!-- FORGE:TELEMETRY v1 -->` comment per issue, or per PR for review-side events (`review_panel`,
|
|
11
|
+
`finding_disposition`, `ci_gate` go to the PR when one is known, everything else to the issue).
|
|
12
|
+
- The comment is edited in place. The records are JSONL inside a fenced `jsonl` block.
|
|
13
|
+
- Near 60,000 characters (GitHub's limit is 65,536) a new `<!-- FORGE:TELEMETRY v1 part=N -->` comment is started.
|
|
14
|
+
- Written with the pipeline's own `gh` token. Readers must take only comments that pass `scripts/trusted-comments.sh`
|
|
15
|
+
(OWNER/MEMBER/COLLABORATOR, or a Bot author); the emitter itself only edits trusted telemetry comments.
|
|
16
|
+
- If a write fails the record is appended to `~/.forge/telemetry-buffer/<owner>_<repo>__<number>.jsonl` and flushed
|
|
17
|
+
with the next emit for the same target, from any machine that touches the issue. Lines already present in the
|
|
18
|
+
comment are not written twice. Deleting the buffer loses only records that never reached GitHub.
|
|
19
|
+
|
|
20
|
+
## Record shape
|
|
21
|
+
|
|
22
|
+
```json
|
|
23
|
+
{"v":1,"ts":"2026-10-11T19:38:54Z","forge_version":"1.16.1","repo":"o/r","issue":5,"pr":null,"run":null,"phase":null,"event":"diff_size","lines":10}
|
|
24
|
+
```
|
|
25
|
+
|
|
26
|
+
`issue`, `pr`, `run` and `phase` come from `FORGE_ISSUE`, `FORGE_PR`, `FORGE_RUN` and `FORGE_PHASE` (null when unset);
|
|
27
|
+
`repo` from `FORGE_REPO`, else the `origin` remote. A record with no repo or no numeric issue/PR has no destination and
|
|
28
|
+
is dropped. Payload keys are `[A-Za-z0-9_]+` and cannot shadow the shared keys; integers and `true`/`false` are typed.
|
|
29
|
+
|
|
30
|
+
## Event catalogue
|
|
31
|
+
|
|
32
|
+
| Event | Emitted by | Payload |
|
|
33
|
+
|-------|-----------|---------|
|
|
34
|
+
| `diff_size` | `scripts/diff-size.sh`; `work-on/build.md` B5.5 (final result) | `lines`, `excluded`, `threshold`, `result` = `OK` \| `OVER` \| `COHESIVE` (`OVERRIDDEN` from the B5.5 call site) |
|
|
35
|
+
| `acceptance_gate` | `work-on/build.md` B6.5 (`phase=build`); `scripts/review-acceptance-gate.sh` (`phase=review`) | `phase`, `verified`, `failed`, `malformed`, `deferred` |
|
|
36
|
+
| `finding_disposition` | `scripts/classify-finding.sh`, one per finding | `severity`, `confidence`, `agent`, `disposition` = `ISSUE` \| `NOTE` \| `INPR_FIX`, `reason` |
|
|
37
|
+
| `review_panel` | `review-pr.md` Phase 4 | `agents_selected`, `agents_completed`, `mode` = `FULL` \| `DELTA` \| `BASE_SYNC_ONLY` |
|
|
38
|
+
| `ci_gate` | `scripts/wait-ci-green.sh` | `result` = `PASS` \| `FAIL` \| `ERROR` \| `TIMEOUT`, `rc`, `checks` |
|
|
39
|
+
| `liveness` | `scripts/remediate-liveness.sh` | `decision` = `PROCEED` \| `DEFER` \| `EXHAUSTED`, `why` |
|
|
40
|
+
| `phase_attempt` | the engine (`bin/engine/telemetry-seam.mjs`, one per phase attempt; `phase` = the phase id) | `attempt`, `model`, `turns`, `duration_ms`, `cost_usd`, `input_tokens`, `output_tokens`, `cache_read_tokens`, `cache_creation_tokens` (any value the runner did not report is omitted) |
|
|
41
|
+
| `run_terminal` | the engine (`terminate()`, one per run segment) | `reason` (terminal reason), `code` (see `docs/OUTCOME-CODES.md`), `wall_ms` (this engine process, start to terminal) |
|
|
42
|
+
|
|
43
|
+
The engine records carry `run` = `r_<issue>_<start epoch seconds>` (one per `forgedock run-issue` process) so `forgedock stats`
|
|
44
|
+
(`docs/STATS.md`) can group the attempts of a run and tell a finished run from an unfinished one. They are the GitHub-side
|
|
45
|
+
source for Cost, Time and outcome codes: without them a fresh machine can rebuild only the gate and review sections.
|
|
46
|
+
|
|
47
|
+
The classifier and the scripts emit only when a destination is known: callers export `FORGE_ISSUE` and/or `FORGE_PR`
|
|
48
|
+
(`review-pr.md` exports them before classifying; the other scripts default `FORGE_REPO`/`FORGE_ISSUE`/`FORGE_PR` from
|
|
49
|
+
their own arguments).
|
|
50
|
+
|
|
51
|
+
## Guarantees
|
|
52
|
+
|
|
53
|
+
- Never fails or alters the caller: `forge_emit` always returns 0, runs in a subshell with all output discarded, bounds
|
|
54
|
+
each `gh` call to 30 seconds, and gate stdout and exit codes are identical with it on or off.
|
|
55
|
+
- `FORGE_TELEMETRY=off` makes every emit a no-op (the test runner sets it so suites never touch `~/.forge`).
|
|
56
|
+
- Emitters: `scripts/lib/telemetry.sh` (`forge_emit <event> key=value...`, also runnable as
|
|
57
|
+
`bash scripts/lib/telemetry.sh <event> key=value...`) and `bin/engine/telemetry.mjs` (`emit(event, payload)`),
|
|
58
|
+
with the same record shape.
|
|
59
|
+
|
|
60
|
+
## Privacy note
|
|
61
|
+
|
|
62
|
+
Records contain ids, counts and codes only: never comment bodies, issue text, file contents, diffs or secrets. Values
|
|
63
|
+
are restricted to `A-Za-z0-9 _.:/#@+=,-` (anything else becomes `_`) and truncated to 200 characters, so a value
|
|
64
|
+
cannot carry markup, backticks or newlines into the comment. Treat the comment as public to anyone who can read the
|
|
65
|
+
issue or PR.
|
package/docs/WORK-ON-RUNTIME.md
CHANGED
|
@@ -117,3 +117,16 @@ remediate L2, reviewers L3, same as the table.
|
|
|
117
117
|
- `/orchestrate` still dispatches one worker `Agent` per issue (L1).
|
|
118
118
|
- Phases keep their specs, `*_RESULT` contracts and GitHub markers; phase-trail verification is unchanged.
|
|
119
119
|
- Solo `/work-on` gets one extra layer of headroom.
|
|
120
|
+
|
|
121
|
+
## 7. Pipeline cost report
|
|
122
|
+
|
|
123
|
+
`bash scripts/cost-report.sh [--runs-dir DIR] [--last N] [--versions a,b] [--min-n N]` reads the engine run logs (`~/.forge/runs/*.jsonl`) and prints a markdown table per ForgeDock version and repo: finished-run count `n`, mean and median cache-read tokens per issue, mean per phase, remediated %, merged %, and cache-read tokens per merged issue. Groups with `n` below `--min-n` (default 10) are flagged `low-n`. The staging review (`commands/review-pr-staging.md` Phase 8) embeds it for the current version and the previous two (`--last 3`).
|
|
124
|
+
|
|
125
|
+
- Only finished runs count (a `RUN_TERMINAL` after any `RUN_REOPEN` / `RUN_NEXT_CYCLE`). Unfinished runs and hydrated runs (reconstructed from `FORGE:STATE`, no usage data) are excluded and counted in the footer.
|
|
126
|
+
- Attribution comes from `RUN_START`, which now records `version` (from `package.json`) and `repo` (`owner/repo`) on all three writers: the engine's fresh start, the hydrate path (`hydrated: true`), and the interactive hook. Both are best-effort and null when unresolved; `deriveState` ignores them.
|
|
127
|
+
- Older logs have neither: the repo shows `unknown` and the version is estimated from the log's mtime against release times (git tags `v*`, or `--release-times FILE` with `<version> <epoch>` lines), marked `~`.
|
|
128
|
+
- Fixture test: `bash scripts/cost-report.test.sh` (fixtures in `scripts/fixtures/cost-report/`).
|
|
129
|
+
|
|
130
|
+
## 8. Outcome codes
|
|
131
|
+
|
|
132
|
+
Every `PHASE_FAILED`, blocked `PHASE_COMMIT` and `RUN_TERMINAL` run-log event carries a machine-readable `code` from a closed, versioned enum (`bin/engine/outcome-codes.mjs`), and `FORGE:STATE` records the causing code as `outcomeCode`. Codes come from structured evidence only, never from the prose `detail`; an unmapped outcome is `unclassified` with the raw reason kept. See `docs/OUTCOME-CODES.md`.
|
|
@@ -419,7 +419,7 @@ Cleans up stale labels, missing workflow state, project board gaps, dangling wor
|
|
|
419
419
|
/cleanup branches # Prune merged branches
|
|
420
420
|
/cleanup milestones # Archive shipped milestones
|
|
421
421
|
/cleanup board # Fix project board gaps
|
|
422
|
-
/cleanup orphans # Remove dangling worktrees
|
|
422
|
+
/cleanup orphans # Remove dangling worktrees; close dead orchestrate claims boards
|
|
423
423
|
/cleanup all # Everything
|
|
424
424
|
```
|
|
425
425
|
|
|
@@ -20,7 +20,7 @@ See also: [FORGE Annotation Protocol §6](forge-protocol-v1.md#6-label-state-mac
|
|
|
20
20
|
| `workflow:invalid` | Issue closed as invalid | Investigator agent |
|
|
21
21
|
| `workflow:decomposed` | Issue decomposed into sub-issues | Decomposer agent |
|
|
22
22
|
| `needs-human` | Pipeline blocked, human intervention required | Any agent on error |
|
|
23
|
-
| `workflow:engine-error` | Pipeline blocked by the engine/tool itself breaking (CLI crash, missing SDK/API key, exhausted retries with no attempt ever reaching a real outcome) — not a genuine content-level judgment call | `terminate()` in `bin/engine.mjs` |
|
|
23
|
+
| `workflow:engine-error` | Pipeline blocked by the engine/tool itself breaking (CLI crash, missing SDK/API key, exhausted retries with no attempt ever reaching a real outcome) — not a genuine content-level judgment call | `terminate()` in `bin/engine.mjs` (also removes a stacked `workflow:remediating`, and adds `needs-human` when the run was remediating) |
|
|
24
24
|
|
|
25
25
|
`workflow:remediating` vs `needs-human` (forge#3541): `needs-human` is reserved for genuine human decisions and external actions (an exhausted remediation bound, an unresolvable conflict, a policy gate, spend, real-environment validation). Work the pipeline fixes by itself (a pure base-sync, a red CI gate, an in-PR finding round) is marked `workflow:remediating` instead, so operators are not alerted and orchestrate does not hold DAG dependents as GATED. `work-on:remediate` accepts either label (a bare `needs-human` from an older run still works), swaps it for `workflow:in-review` on a FIXABLE run, and on every BLOCKED/UNFIXABLE exit clears `workflow:remediating` and asserts `needs-human`.
|
|
26
26
|
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "forgedock",
|
|
3
|
-
"version": "1.
|
|
3
|
+
"version": "1.17.0",
|
|
4
4
|
"description": "Autonomous development pipeline for coding agents. GitHub as institutional memory, with native Claude Code and OpenCode entrypoints.",
|
|
5
5
|
"bin": {
|
|
6
6
|
"forgedock": "bin/forgedock.mjs"
|
|
@@ -35,6 +35,7 @@
|
|
|
35
35
|
"files": [
|
|
36
36
|
"bin/",
|
|
37
37
|
"!bin/tests/",
|
|
38
|
+
"!bin/stats/fixtures/",
|
|
38
39
|
"commands/",
|
|
39
40
|
"docs/",
|
|
40
41
|
"!docs/site/.vitepress/",
|
|
@@ -47,7 +48,7 @@
|
|
|
47
48
|
"README.md"
|
|
48
49
|
],
|
|
49
50
|
"scripts": {
|
|
50
|
-
"test": "node --test \"bin/tests/**/*.test.mjs\" \"bin/engine/**/*.test.mjs\" \"packages/protocol/test/**/*.test.mjs\"",
|
|
51
|
+
"test": "node --test \"bin/tests/**/*.test.mjs\" \"bin/engine/**/*.test.mjs\" \"bin/stats/**/*.test.mjs\" \"packages/protocol/test/**/*.test.mjs\"",
|
|
51
52
|
"test:sh": "bash scripts/run-shell-tests.sh",
|
|
52
53
|
"docs:dev": "vitepress dev docs/site",
|
|
53
54
|
"docs:build": "vitepress build docs/site",
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
#!/usr/bin/env bash
|
|
2
|
+
# builder-resume-match.test.sh — the anchored, trusted FORGE:BUILDER resume selector (forge#3668, mirrors forge#3648)
|
|
3
|
+
# Exercises the exact jq used by commands/work-on/build/implement.md Resume Check. No network.
|
|
4
|
+
# Usage: bash scripts/builder-resume-match.test.sh
|
|
5
|
+
# SPDX-FileCopyrightText: Copyright (c) RapierCraft Studios
|
|
6
|
+
# SPDX-License-Identifier: AGPL-3.0-or-later
|
|
7
|
+
|
|
8
|
+
set -uo pipefail
|
|
9
|
+
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
10
|
+
ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
|
|
11
|
+
|
|
12
|
+
PASS=0; FAILN=0
|
|
13
|
+
ok() { PASS=$((PASS+1)); echo "ok - $1"; }
|
|
14
|
+
bad() { FAILN=$((FAILN+1)); echo "FAIL - $1"; }
|
|
15
|
+
eq() { if [ "$2" = "$3" ]; then ok "$1"; else bad "$1 (got '$2', want '$3')"; fi; }
|
|
16
|
+
|
|
17
|
+
SELECT_JQ='[.[][] | select((.body // "") | startswith("<!-- FORGE:BUILDER -->")) | select((.user.type // "") == "Bot" or ((.author_association // "") | IN("OWNER","MEMBER","COLLABORATOR"))) | {id: .id, complete: (.body | contains("<!-- FORGE:BUILDER:COMPLETE -->")), other: (.body | test("<!-- FORGE:(INVESTIGATOR|CONTEXT|CONTRACT|ARCHITECT)"))}]'
|
|
18
|
+
DELETE_JQ='[.[] | select(.complete | not)] | if length == 1 and (.[0].other | not) then .[0].id else empty end'
|
|
19
|
+
|
|
20
|
+
# c <id> <assoc> <type> <body>
|
|
21
|
+
c() { jq -nc --argjson i "$1" --arg a "$2" --arg t "$3" --arg b "$4" '{id:$i,author_association:$a,user:{type:$t},body:$b}'; }
|
|
22
|
+
arch() { printf '%s' "$1" | jq -sc . | jq -s "$SELECT_JQ"; }
|
|
23
|
+
n_complete() { arch "$1" | jq '[.[] | select(.complete)] | length'; }
|
|
24
|
+
del_id() { arch "$1" | jq -r "$DELETE_JQ"; }
|
|
25
|
+
|
|
26
|
+
PLAN=$'<!-- FORGE:BUILDER -->\n## Implementation\n\n<!-- FORGE:BUILDER:COMPLETE -->'
|
|
27
|
+
PARTIAL=$'<!-- FORGE:BUILDER -->\n## Implementation (in progress)'
|
|
28
|
+
INVEST=$'<!-- FORGE:INVESTIGATOR -->\nThe build phase later posts FORGE:BUILDER and FORGE:BUILDER:COMPLETE.'
|
|
29
|
+
CTX=$'<!-- FORGE:CONTEXT -->\nmentions <!-- FORGE:BUILDER -->'
|
|
30
|
+
MIXED=$'<!-- FORGE:BUILDER -->\nquoted <!-- FORGE:INVESTIGATOR -->'
|
|
31
|
+
|
|
32
|
+
eq "INVESTIGATOR mentioning FORGE:BUILDER is not selected" "$(arch "$(c 1 NONE Bot "$INVEST")" | jq length)" 0
|
|
33
|
+
eq "INVESTIGATOR mentioning FORGE:BUILDER is never deleted" "$(del_id "$(c 1 NONE Bot "$INVEST")")" ""
|
|
34
|
+
eq "CONTEXT mentioning the marker mid-body is not selected" "$(arch "$(c 2 NONE Bot "$CTX")" | jq length)" 0
|
|
35
|
+
eq "complete comment by bot is complete" "$(n_complete "$(c 3 NONE Bot "$PLAN")")" 1
|
|
36
|
+
eq "complete comment by MEMBER is complete" "$(n_complete "$(c 3 MEMBER User "$PLAN")")" 1
|
|
37
|
+
eq "single partial comment -> delete its id" "$(del_id "$(c 4 NONE Bot "$PARTIAL")")" 4
|
|
38
|
+
eq "two partial comments -> delete nothing" "$(del_id "$(c 4 NONE Bot "$PARTIAL")"$'\n'"$(c 5 NONE Bot "$PARTIAL")")" ""
|
|
39
|
+
eq "untrusted author partial ignored" "$(del_id "$(c 6 NONE User "$PARTIAL")")" ""
|
|
40
|
+
eq "untrusted author complete ignored" "$(n_complete "$(c 6 NONE User "$PLAN")")" 0
|
|
41
|
+
eq "partial carrying another FORGE marker is not deleted" "$(del_id "$(c 7 NONE Bot "$MIXED")")" ""
|
|
42
|
+
eq "investigator + partial comment -> only the comment id" "$(del_id "$(c 1 NONE Bot "$INVEST")"$'\n'"$(c 8 NONE Bot "$PARTIAL")")" 8
|
|
43
|
+
eq "paginated concatenated arrays" "$(printf '[%s]\n[%s]' "$(c 1 NONE Bot "$INVEST")" "$(c 9 NONE Bot "$PLAN")" | jq -s "$SELECT_JQ" | jq '[.[] | select(.complete)] | length')" 1
|
|
44
|
+
|
|
45
|
+
ARCHC=$'<!-- FORGE:ARCHITECT -->\nplan; the builder will post <!-- FORGE:BUILDER -->'
|
|
46
|
+
eq "ARCHITECT mentioning the marker is not selected" "$(arch "$(c 10 NONE Bot "$ARCHC")" | jq length)" 0
|
|
47
|
+
eq "unparsable input fails closed" "$(printf 'not json' | jq -s "$SELECT_JQ" 2>/dev/null | jq -r "$DELETE_JQ" 2>/dev/null)" ""
|
|
48
|
+
|
|
49
|
+
# Spec drift guard: every mutating site carries this exact predicate and no unanchored selector.
|
|
50
|
+
for f in commands/work-on/build/implement.md commands/work-on/build.md commands/work-on/build/validate.md; do
|
|
51
|
+
SPEC="$ROOT/$f"
|
|
52
|
+
! grep -qE 'contains\(.FORGE:BUILDER.\) and' "$SPEC" && ok "$f has no unanchored selector" || bad "$f still uses unanchored selector"
|
|
53
|
+
grep -qF "$SELECT_JQ" "$SPEC" && ok "$f uses the tested select jq" || bad "$f select jq drifted from this test"
|
|
54
|
+
grep -qF "$DELETE_JQ" "$SPEC" && ok "$f uses the tested pick jq" || bad "$f pick jq drifted from this test"
|
|
55
|
+
done
|
|
56
|
+
|
|
57
|
+
echo "passed=$PASS failed=$FAILN"
|
|
58
|
+
[ "$FAILN" -eq 0 ]
|
|
@@ -0,0 +1,110 @@
|
|
|
1
|
+
#!/usr/bin/env bash
|
|
2
|
+
# SPDX-FileCopyrightText: Copyright (c) RapierCraft Studios
|
|
3
|
+
# SPDX-License-Identifier: AGPL-3.0-or-later
|
|
4
|
+
#
|
|
5
|
+
# cap-findings.sh — PR-level cap on filed review findings (review-pr §6B.5, review-pr-staging §7B.5).
|
|
6
|
+
# classify-finding.sh decides one finding at a time; this is the aggregation
|
|
7
|
+
# step after it: at most N ISSUE dispositions per review run are kept, the rest
|
|
8
|
+
# become `NOTE capped` (listed in FORGE:NOTE_DISPOSITION, never dropped).
|
|
9
|
+
#
|
|
10
|
+
# Usage:
|
|
11
|
+
# cap-findings.sh [--cap N] [--lineage none|review-finding] [--input FILE]
|
|
12
|
+
#
|
|
13
|
+
# Input (stdin, or --input FILE): TSV rows
|
|
14
|
+
# ID<TAB>SEVERITY<TAB>CONFIDENCE<TAB>DISPOSITION[<TAB>AGENT]
|
|
15
|
+
# DISPOSITION is the classifier output (`ISSUE <reason>`, `NOTE <reason>`, `INPR_FIX <reason>`).
|
|
16
|
+
#
|
|
17
|
+
# Output (stdout): one `ID<TAB>DISPOSITION` row per input row, in input order, then a final
|
|
18
|
+
# `CAPPED=<N>` line (no tab) counting the rows rewritten to `NOTE capped`. Callers read the rows
|
|
19
|
+
# with `awk -F'\t' 'NF >= 2'`.
|
|
20
|
+
#
|
|
21
|
+
# Rules:
|
|
22
|
+
# - Only ISSUE rows are considered. NOTE and INPR_FIX rows pass through untouched and never
|
|
23
|
+
# consume the cap.
|
|
24
|
+
# - HIGH/CRITICAL ISSUE rows are NEVER capped (they always file and still block via the review's
|
|
25
|
+
# severity gate).
|
|
26
|
+
# - `ISSUE fix-efficacy-*` rows (the classifier's never-damped "the linked issue is not fixed / the
|
|
27
|
+
# PR re-introduces its defect" exemption, #3857) are NEVER capped either: capping one would merge
|
|
28
|
+
# a known-broken fix and close its issue as fixed. They are treated like HIGH rows (under
|
|
29
|
+
# --lineage review-finding they consume the budget).
|
|
30
|
+
# - Remaining ISSUE rows are ranked by severity (MEDIUM > LOW), then confidence (CONFIRMED >
|
|
31
|
+
# LIKELY > POSSIBLE), then safety domain (Security/Auth/Billing/Concurrency/Database agent
|
|
32
|
+
# first), then input order. Unparseable severity or confidence ranks as MEDIUM / LIKELY, so the
|
|
33
|
+
# cap still applies to it (it is not an automatic ISSUE).
|
|
34
|
+
# - Cap default: 2. Under --lineage review-finding the default is 1 and every filed HIGH/CRITICAL
|
|
35
|
+
# consumes that budget (a fix for a finding must not mint a new generation of findings).
|
|
36
|
+
# An explicit --cap overrides the default in both lineages. 0 is valid (cap everything
|
|
37
|
+
# cappable). A non-numeric or negative --cap falls back to the default with a warning on stderr.
|
|
38
|
+
#
|
|
39
|
+
# Exit codes: 0 ok, 2 usage error. Bash 3.2 compatible (no mapfile, no associative arrays).
|
|
40
|
+
|
|
41
|
+
set -u
|
|
42
|
+
|
|
43
|
+
CAP=""; LINEAGE="none"; INPUT=""
|
|
44
|
+
while [ $# -gt 0 ]; do
|
|
45
|
+
case "$1" in
|
|
46
|
+
--cap) [ $# -ge 2 ] || { echo "cap-findings.sh: --cap needs a value" >&2; exit 2; }; CAP="$2"; shift 2 ;;
|
|
47
|
+
--lineage) [ $# -ge 2 ] || { echo "cap-findings.sh: --lineage needs a value" >&2; exit 2; }; LINEAGE="$2"; shift 2 ;;
|
|
48
|
+
--input) [ $# -ge 2 ] || { echo "cap-findings.sh: --input needs a value" >&2; exit 2; }; INPUT="$2"; shift 2 ;;
|
|
49
|
+
*) echo "cap-findings.sh: unknown argument: $1" >&2; exit 2 ;;
|
|
50
|
+
esac
|
|
51
|
+
done
|
|
52
|
+
|
|
53
|
+
case "$LINEAGE" in
|
|
54
|
+
none|review-finding) ;;
|
|
55
|
+
*) echo "cap-findings.sh: --lineage must be none or review-finding (got '$LINEAGE')" >&2; exit 2 ;;
|
|
56
|
+
esac
|
|
57
|
+
|
|
58
|
+
if [ "$LINEAGE" = "review-finding" ]; then DEFAULT_CAP=1; else DEFAULT_CAP=2; fi
|
|
59
|
+
case "$CAP" in
|
|
60
|
+
"") CAP="$DEFAULT_CAP" ;;
|
|
61
|
+
*[!0-9]*) echo "cap-findings.sh: invalid cap '$CAP', using default $DEFAULT_CAP" >&2; CAP="$DEFAULT_CAP" ;;
|
|
62
|
+
esac
|
|
63
|
+
|
|
64
|
+
if [ -n "$INPUT" ] && [ ! -r "$INPUT" ]; then
|
|
65
|
+
echo "cap-findings.sh: cannot read --input $INPUT" >&2; exit 2
|
|
66
|
+
fi
|
|
67
|
+
|
|
68
|
+
awk -F'\t' -v cap="$CAP" -v lineage="$LINEAGE" '
|
|
69
|
+
function up(s) { return toupper(s) }
|
|
70
|
+
function sevclass(s) { s = up(s); gsub(/[^A-Z]/, "", s); if (s == "CRITICAL" || s == "HIGH") return 3; if (s == "LOW") return 1; return 2 }
|
|
71
|
+
function confrank(c) { c = up(c); if (c == "CONFIRMED") return 3; if (c == "POSSIBLE") return 1; return 2 }
|
|
72
|
+
function domain(a) { a = tolower(a); return (a ~ /(security|auth|billing|concurrency|database)/) ? 1 : 0 }
|
|
73
|
+
/^[ \t\r]*$/ { next }
|
|
74
|
+
{
|
|
75
|
+
n++
|
|
76
|
+
sub(/\r$/, "")
|
|
77
|
+
id[n] = $1; disp[n] = $4; agent[n] = $5
|
|
78
|
+
if (NF < 4 || $4 == "") disp[n] = "ISSUE malformed-row"
|
|
79
|
+
split(disp[n], w, " ")
|
|
80
|
+
isissue[n] = (w[1] == "ISSUE")
|
|
81
|
+
sc = sevclass($2)
|
|
82
|
+
# the classifier already decided HIGH/CRITICAL: never cap those, whatever the raw severity text looked like
|
|
83
|
+
if (w[2] ~ /^severity-(HIGH|CRITICAL)$/) sc = 3
|
|
84
|
+
# fix-efficacy is never damped (classify-finding.sh): the per-review cap must not re-damp it
|
|
85
|
+
if (w[2] ~ /^fix-efficacy-/) sc = 3
|
|
86
|
+
if (isissue[n] && sc == 3) { high++ ; capable[n] = 0 }
|
|
87
|
+
else if (isissue[n]) { capable[n] = 1; score[n] = sc * 1000 + confrank($3) * 100 + domain($5) * 10 }
|
|
88
|
+
else capable[n] = 0
|
|
89
|
+
}
|
|
90
|
+
END {
|
|
91
|
+
budget = cap + 0
|
|
92
|
+
if (lineage == "review-finding") budget -= high
|
|
93
|
+
if (budget < 0) budget = 0
|
|
94
|
+
kept = 0
|
|
95
|
+
while (kept < budget) {
|
|
96
|
+
best = 0
|
|
97
|
+
for (i = 1; i <= n; i++) {
|
|
98
|
+
if (!capable[i] || keep[i]) continue
|
|
99
|
+
if (best == 0 || score[i] > score[best]) best = i
|
|
100
|
+
}
|
|
101
|
+
if (best == 0) break
|
|
102
|
+
keep[best] = 1; kept++
|
|
103
|
+
}
|
|
104
|
+
capped = 0
|
|
105
|
+
for (i = 1; i <= n; i++) {
|
|
106
|
+
if (capable[i] && !keep[i]) { printf "%s\tNOTE capped\n", id[i]; capped++ }
|
|
107
|
+
else printf "%s\t%s\n", id[i], disp[i]
|
|
108
|
+
}
|
|
109
|
+
printf "CAPPED=%d\n", capped
|
|
110
|
+
}' ${INPUT:+"$INPUT"}
|