forgedock 1.0.22 → 1.1.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (184) hide show
  1. package/README.md +255 -211
  2. package/bin/batch-runner.mjs +339 -0
  3. package/bin/cinema.mjs +320 -0
  4. package/bin/demo.mjs +92 -7
  5. package/bin/engine/phases.mjs +168 -0
  6. package/bin/engine/projector.mjs +32 -0
  7. package/bin/engine/reconcile.mjs +11 -0
  8. package/bin/engine/runlog.mjs +79 -0
  9. package/bin/engine/state.mjs +32 -0
  10. package/bin/engine-cli.mjs +222 -0
  11. package/bin/engine.mjs +138 -0
  12. package/bin/forgedock.mjs +896 -1540
  13. package/bin/hooks/interactive-engine.mjs +509 -0
  14. package/bin/hooks/pre-tool-use.mjs +498 -0
  15. package/bin/hooks/subagent-stop-enforce.mjs +197 -0
  16. package/bin/journey.mjs +841 -0
  17. package/bin/registry.mjs +139 -0
  18. package/bin/report.mjs +447 -0
  19. package/bin/runner.mjs +136 -20
  20. package/bin/settings-hook.mjs +346 -0
  21. package/bin/tui.mjs +6 -3
  22. package/commands/adopt.md +4 -1
  23. package/commands/analytics.md +4 -1
  24. package/commands/audit-agents.md +110 -27
  25. package/commands/audit.md +20 -1
  26. package/commands/autopilot.md +84 -12
  27. package/commands/changelog.md +2 -1
  28. package/commands/ci-audit.md +2 -1
  29. package/commands/cleanup.md +161 -1
  30. package/commands/compat-audit.md +2 -1
  31. package/commands/deploy-info.md +63 -4
  32. package/commands/design-bench.md +6 -1
  33. package/commands/design-render-critique-loop.md +2 -1
  34. package/commands/design.md +1 -0
  35. package/commands/diagnose.md +2 -1
  36. package/commands/explain.md +2 -1
  37. package/commands/forge-stats.md +122 -2
  38. package/commands/forgedock-init.md +1 -1
  39. package/commands/geo-audit.md +3 -0
  40. package/commands/incident-response.md +2 -1
  41. package/commands/issue.md +24 -3
  42. package/commands/milestone.md +77 -12
  43. package/commands/optimize.md +2 -1
  44. package/commands/orchestrate/config.md +67 -0
  45. package/commands/orchestrate/phase-1-resolve.md +159 -0
  46. package/commands/orchestrate/phase-2-triage.md +126 -0
  47. package/commands/orchestrate/phase-2.5-synthesis.md +156 -0
  48. package/commands/orchestrate/phase-3-dependency.md +591 -0
  49. package/commands/orchestrate/phase-4-execution.md +962 -0
  50. package/commands/orchestrate/phase-5-cleanup.md +47 -0
  51. package/commands/orchestrate/phase-6-report.md +198 -0
  52. package/commands/orchestrate/safety.md +62 -0
  53. package/commands/orchestrate.md +25 -2027
  54. package/commands/pipeline-health/config.md +34 -0
  55. package/commands/pipeline-health/phase-1-context.md +180 -0
  56. package/commands/pipeline-health/phase-2-metrics.md +1729 -0
  57. package/commands/pipeline-health/phase-3-analyze.md +204 -0
  58. package/commands/pipeline-health/phase-4-proposals.md +117 -0
  59. package/commands/pipeline-health/phase-5-report.md +284 -0
  60. package/commands/pipeline-health/phase-6-summary.md +52 -0
  61. package/commands/pipeline-health.md +25 -2401
  62. package/commands/pipeline-resume.md +24 -1
  63. package/commands/pipeline-status.md +65 -0
  64. package/commands/qa-sweep.md +4 -1
  65. package/commands/quality-gate.md +326 -42
  66. package/commands/replay.md +2 -1
  67. package/commands/review-pr-agents/api.md +135 -0
  68. package/commands/review-pr-agents/auth.md +137 -0
  69. package/commands/review-pr-agents/billing.md +101 -0
  70. package/commands/review-pr-agents/concurrency.md +145 -0
  71. package/commands/review-pr-agents/database.md +103 -0
  72. package/commands/review-pr-agents/frontend.md +89 -0
  73. package/commands/review-pr-agents/infra.md +385 -0
  74. package/commands/review-pr-agents/protocols.md +185 -0
  75. package/commands/review-pr-agents/scraper.md +185 -0
  76. package/commands/review-pr-agents/security.md +406 -0
  77. package/commands/review-pr-agents.md +45 -1722
  78. package/commands/review-pr-staging.md +196 -40
  79. package/commands/review-pr.md +612 -56
  80. package/commands/rollback.md +2 -1
  81. package/commands/scope.md +2 -1
  82. package/commands/security-audit.md +2 -1
  83. package/commands/signal-planner.md +623 -0
  84. package/commands/sync-ecosystem.md +1 -0
  85. package/commands/test-gate.md +17 -1
  86. package/commands/validate.md +2 -1
  87. package/commands/work-on/build/architect.md +28 -0
  88. package/commands/work-on/build/context.md +2 -0
  89. package/commands/work-on/build/implement.md +19 -5
  90. package/commands/work-on/build/validate.md +62 -6
  91. package/commands/work-on/build.md +187 -30
  92. package/commands/work-on/close.md +132 -5
  93. package/commands/work-on/decompose.md +19 -1
  94. package/commands/work-on/investigate.md +117 -2
  95. package/commands/work-on/review.md +48 -4
  96. package/commands/work-on-monolithic.md +8 -2
  97. package/commands/work-on.md +247 -7
  98. package/docs/BRAND.md +16 -5
  99. package/docs/CI.md +2 -2
  100. package/docs/CODEX.md +36 -1
  101. package/docs/CONFIG.md +44 -42
  102. package/docs/FORGE-PROTOCOL.md +15 -1064
  103. package/docs/anthropic-devrel-outreach.md +2 -2
  104. package/docs/articles/command-decomposition-token-savings.md +214 -0
  105. package/docs/articles/model-release-playbook.md +131 -0
  106. package/docs/articles/per-repo-adaptive-scripts-token-savings.md +27 -25
  107. package/docs/claude-breakpoints.json +75 -5
  108. package/docs/design/abc-benchmark.md +1 -1
  109. package/docs/design/design-spec-schema.md +1 -1
  110. package/docs/design/divergent-generation.md +1 -1
  111. package/docs/design/eval-gate-runbook.md +178 -0
  112. package/docs/design/show-hn-draft.md +26 -1
  113. package/docs/design/token-cost-baseline-1298.md +59 -0
  114. package/docs/eval/README.md +117 -0
  115. package/docs/eval/corpus/seed-v1.json +6 -0
  116. package/docs/eval/published/claude-sonnet-4-6-2026-07-04.json +33 -0
  117. package/docs/site/annotations-explained.md +24 -0
  118. package/docs/site/command-learning-path.md +51 -14
  119. package/docs/site/command-reference.md +339 -113
  120. package/docs/site/for-companies.md +95 -0
  121. package/docs/site/forge-annotation-protocol.md +12 -256
  122. package/docs/site/getting-started.md +58 -12
  123. package/docs/site/github-is-the-memory.md +113 -0
  124. package/docs/site/how-it-works.md +3 -13
  125. package/docs/site/index.md +10 -2
  126. package/docs/spec/eval-run-result.md +147 -0
  127. package/docs/spec/forge-protocol-v1.md +441 -18
  128. package/docs/spec/label-state-machine.md +55 -0
  129. package/docs/spec/review-protocol.md +158 -0
  130. package/docs/spec-graph-schema.md +28 -0
  131. package/docs/superpowers/plans/2026-07-03-cinematic-onboarding-tui.md +2339 -0
  132. package/docs/superpowers/plans/2026-07-03-durable-execution-engine.md +1223 -0
  133. package/docs/superpowers/specs/2026-07-03-cinematic-onboarding-tui-design.md +157 -0
  134. package/docs/superpowers/specs/2026-07-03-durable-execution-engine-design.md +194 -0
  135. package/docs/superpowers/specs/2026-07-04-economic-self-governance-design.md +181 -0
  136. package/docs/superpowers/specs/2026-07-04-provenance-trail-design.md +189 -0
  137. package/examples/forgedock-demo/README.md +41 -22
  138. package/examples/forgedock-demo/issues/06-bug-tags-type-validation.md +43 -0
  139. package/examples/forgedock-demo/issues/07-bug-negative-offset-pagination.md +42 -0
  140. package/examples/forgedock-demo/issues/08-bug-patch-mass-assignment.md +49 -0
  141. package/examples/forgedock-demo/issues/09-bug-count-injection.md +47 -0
  142. package/examples/forgedock-demo/issues/10-feature-sort-param.md +39 -0
  143. package/examples/forgedock-demo/issues/11-feature-patch-tags.md +37 -0
  144. package/examples/forgedock-demo/issues/12-feature-bulk-create.md +42 -0
  145. package/examples/forgedock-demo/issues/13-feature-archive-notes.md +46 -0
  146. package/examples/forgedock-demo/issues/14-refactor-extract-title-validation.md +41 -0
  147. package/examples/forgedock-demo/issues/15-refactor-protected-route-flag.md +43 -0
  148. package/examples/forgedock-demo/issues/16-refactor-extract-store.md +45 -0
  149. package/examples/forgedock-demo/issues/17-perf-tags-dedup-set.md +35 -0
  150. package/examples/forgedock-demo/issues/18-perf-remove-redundant-sort.md +37 -0
  151. package/examples/forgedock-demo/issues/19-docs-api-reference-full.md +53 -0
  152. package/examples/forgedock-demo/issues/20-docs-intentional-flaw-convention.md +46 -0
  153. package/examples/forgedock-demo/scripts/smoke.js +18 -1
  154. package/examples/forgedock-demo/src/db.js +36 -11
  155. package/examples/forgedock-demo/src/routes/notes.js +106 -9
  156. package/examples/forgedock-demo/src/server.js +4 -1
  157. package/package.json +12 -4
  158. package/packages/protocol/LICENSE +21 -0
  159. package/packages/protocol/src/cli.js +135 -0
  160. package/packages/protocol/src/emit.js +146 -0
  161. package/packages/protocol/src/index.js +43 -0
  162. package/packages/protocol/src/parse.js +229 -0
  163. package/packages/protocol/src/types.js +171 -0
  164. package/packages/protocol/src/validate.js +122 -0
  165. package/scripts/bench-topology-cost.mjs +372 -0
  166. package/scripts/check-command-docs-drift.sh +198 -0
  167. package/scripts/check-protocol-restatements.sh +176 -0
  168. package/scripts/check-registry/README.md +72 -0
  169. package/scripts/check-registry/manifest.json +6 -0
  170. package/scripts/code-index.sh +454 -0
  171. package/scripts/conformance-check.mjs +168 -0
  172. package/scripts/doctor-pipeline-state.sh +484 -0
  173. package/scripts/eval-gate-baseline.json +17 -0
  174. package/scripts/eval-gate-scorecard.mjs +242 -0
  175. package/scripts/eval-scorecard-aggregator.mjs +252 -0
  176. package/scripts/flaky-quarantine.sh +241 -0
  177. package/scripts/forge-annotation.sh +567 -0
  178. package/scripts/issue-dedup.sh +236 -0
  179. package/scripts/validate-annotation-node.mjs +233 -0
  180. package/scripts/verify-env-vars.sh +18 -2
  181. package/scripts/verify-host-headers.sh +4 -4
  182. package/scripts/verify-route-registration.sh +2 -2
  183. package/scripts/worktree-lifecycle.sh +202 -0
  184. package/templates/devdocs/agent/using-forgedock.md +14 -0
@@ -14,7 +14,7 @@ These assets are live and can be referenced in any outreach:
14
14
  | npm package | https://www.npmjs.com/package/forgedock | ✅ Live |
15
15
  | Plugin marketplace listing | https://github.com/dev-gom/claude-code-marketplace | ✅ Submitted (#541) |
16
16
  | Demo GIF | `docs/demo.tape` (generated asset) | ✅ Ready (#542) |
17
- | FORGE Annotation Protocol spec | `docs/FORGE-PROTOCOL.md` | ✅ Published (#553) |
17
+ | FORGE Annotation Protocol spec | `docs/spec/forge-protocol-v1.md` | ✅ Published (#553) |
18
18
  | dev.to article | `docs/articles/devto-how-i-built-an-ai-agent-that-never-forgets.md` | ✅ Draft ready (#549) |
19
19
  | Awesome list submissions | `docs/distribution-submissions.md` | ✅ Documented (#552) |
20
20
 
@@ -192,6 +192,6 @@ After each initial outreach action:
192
192
  ## Related Assets
193
193
 
194
194
  - `docs/distribution-submissions.md` — canonical description + awesome list submissions
195
- - `docs/FORGE-PROTOCOL.md` — FORGE Annotation Protocol spec (shareable with Anthropic as ecosystem contribution)
195
+ - `docs/spec/forge-protocol-v1.md` — FORGE Annotation Protocol spec (shareable with Anthropic as ecosystem contribution)
196
196
  - `docs/articles/devto-how-i-built-an-ai-agent-that-never-forgets.md` — long-form story for blog pitch
197
197
  - `docs/youtube-demo-script.md` — demo video script (reference for pitch)
@@ -0,0 +1,214 @@
1
+ ---
2
+ title: "Command Decomposition: Measured Token Savings From Splitting Monolithic Specs"
3
+ description: "ForgeDock's nested-command decomposition pattern splits large command specs into per-phase files. We measured the before/after token cost for /review-pr (2-domain invocation) and /orchestrate to quantify the savings."
4
+ tags: ["ai", "performance", "tokens", "productivity", "opensource"]
5
+ cover_image:
6
+ canonical_url: https://github.com/RapierCraftStudios/ForgeDock
7
+ published: false
8
+ ---
9
+
10
+ <!-- Generated by ForgeDock pipeline — Issue #1274 -->
11
+
12
+ ForgeDock command specs are loaded into the model's context at session start.
13
+ A monolithic spec is read in full — even when a given run only exercises a
14
+ fraction of its logic. This article measures the per-invocation token cost
15
+ for `/review-pr` (invoked against a PR matching 2 applicable domains) and
16
+ `/orchestrate` before and after their decompositions into nested per-phase
17
+ files (issues #1271 and #1272 respectively), using the same methodology
18
+ established in `docs/articles/selective-spec-loading-token-savings.md`.
19
+
20
+ ---
21
+
22
+ ## Background: The Nested-Command Decomposition Pattern
23
+
24
+ ForgeDock already decomposes `commands/work-on.md` into
25
+ `commands/work-on/{build,close,decompose,investigate,review}.md` plus
26
+ further nesting under `work-on/build/`. This is the **nested-command
27
+ decomposition pattern** (documented in `docs/spec/forge-protocol-v1.md`
28
+ § 10). Two mechanisms pick up nested files automatically with no manual
29
+ registration:
30
+
31
+ - **`bin/forgedock.mjs`** — the `findMarkdownFiles()` helper recurses over
32
+ `commands/**/*.md` at any depth and symlinks each file into
33
+ `~/.claude/commands/` at its relative path.
34
+ - **`scripts/build-spec-graph.mjs`** — the `walk()` function performs the same
35
+ recursive discovery; each file becomes a `cmd:` graph node. The
36
+ `load-set` query uses `CONTAINS` edges to resolve the minimal set of files
37
+ an entry command actually reaches.
38
+
39
+ When a spec is decomposed, a command like `review-pr` can load only the
40
+ persona files for the 2 domains that apply to the current PR — not the
41
+ full 9-persona catalog. Similarly, `orchestrate` can load only the phase
42
+ file relevant to the current step.
43
+
44
+ ---
45
+
46
+ ## Methodology
47
+
48
+ - Token counts are estimated at the standard rate of **1 token ≈ 4 characters**.
49
+ - **Before** = byte sum of all spec files loaded before the decomposition
50
+ (the monolithic file read in full on every invocation).
51
+ - **After** = byte sum of the slim router/dispatcher + only the
52
+ sub-spec files actually reached for the representative scenario.
53
+ - The same before/after approach was used in the selective-spec-loading
54
+ measurement (`docs/articles/selective-spec-loading-token-savings.md`), so
55
+ numbers are directly comparable.
56
+ - "2-domain invocation" for `/review-pr` means: the PR triggers exactly 2
57
+ of the 9 available reviewer personas.
58
+
59
+ ---
60
+
61
+ ## Full Corpus Baseline
62
+
63
+ The corpus size at measurement time (current state of `commands/`):
64
+
65
+ | Metric | Value |
66
+ |--------|-------|
67
+ | Command spec files | 50 |
68
+ | Total bytes | 1,419,769 |
69
+ | Approx. tokens (1 token ≈ 4 chars) | **~354,942** |
70
+
71
+ ---
72
+
73
+ ## `/review-pr` — 2-Domain Invocation
74
+
75
+ ### Before (#1271 not yet applied)
76
+
77
+ `review-pr.md` loads `review-pr-agents.md` (the 9-persona catalog) in
78
+ full via a `Read` tool call in Phase 3C, regardless of how many domains
79
+ apply to the current PR.
80
+
81
+ | File | Bytes | Approx. tokens |
82
+ |------|------:|---------------:|
83
+ | `commands/review-pr.md` | 84,099 | ~21,025 |
84
+ | `commands/review-pr-agents.md` (full catalog, 9 personas) | 122,932 | ~30,733 |
85
+ | **Total (before)** | **207,031** | **~51,758** |
86
+
87
+ ### After (#1271 applied — per-persona files)
88
+
89
+ After decomposition, `review-pr-agents.md` becomes a slim router/index
90
+ (~5 KB) containing the shared Evidence-Based Review Protocol and
91
+ Structured Findings Protocol. The 9 persona templates move to individual
92
+ files under `commands/review-pr-agents/`. Phase 3C reads only the
93
+ persona files for the 2 applicable domains.
94
+
95
+ Average persona file size = 122,932 ÷ 9 ≈ **13,659 bytes**.
96
+
97
+ | File | Bytes | Approx. tokens |
98
+ |------|------:|---------------:|
99
+ | `commands/review-pr.md` | 84,099 | ~21,025 |
100
+ | `commands/review-pr-agents.md` (slim router) | ~5,000 | ~1,250 |
101
+ | 2 persona files (2 × ~13,659 bytes) | ~27,318 | ~6,830 |
102
+ | **Total (after)** | **~116,417** | **~29,104** |
103
+
104
+ ### Delta
105
+
106
+ | | Tokens |
107
+ |--|-------:|
108
+ | Before | ~51,758 |
109
+ | After | ~29,104 |
110
+ | **Saved** | **~22,654** |
111
+ | **% reduction** | **~44%** |
112
+
113
+ For a 9-domain invocation (all personas active), the after-state cost
114
+ equals the before-state cost — no savings, no regression. The savings are
115
+ proportional to how few domains a given PR exercises. A typical PR
116
+ triggers 2–3 domains, so the median savings are in the **44–54%** range
117
+ per `/review-pr` invocation.
118
+
119
+ ---
120
+
121
+ ## `/orchestrate` — Single Phase
122
+
123
+ ### Before (#1272 not yet applied)
124
+
125
+ `orchestrate.md` is a 2,064-line monolithic file loaded in full on every
126
+ invocation, regardless of which orchestration phase is active.
127
+
128
+ | File | Bytes | Approx. tokens |
129
+ |------|------:|---------------:|
130
+ | `commands/orchestrate.md` (full monolith) | 133,509 | ~33,377 |
131
+ | **Total (before)** | **133,509** | **~33,377** |
132
+
133
+ ### After (#1272 applied — phase-based files)
134
+
135
+ After decomposition, `orchestrate.md` becomes a slim dispatcher (~30 KB)
136
+ containing the universal rules and phase routing logic. Phase content
137
+ (dependency-graph construction, agent spawning, stall detection, results
138
+ aggregation) moves into individual files under `commands/orchestrate/`.
139
+ A single-phase invocation reads the dispatcher plus one phase file (~30 KB).
140
+
141
+ | File | Bytes | Approx. tokens |
142
+ |------|------:|---------------:|
143
+ | `commands/orchestrate.md` (slim dispatcher) | ~30,000 | ~7,500 |
144
+ | Relevant phase file (~1/4 of content) | ~30,000 | ~7,500 |
145
+ | **Total (after, single-phase)** | **~60,000** | **~15,000** |
146
+
147
+ ### Delta
148
+
149
+ | | Tokens |
150
+ |--|-------:|
151
+ | Before | ~33,377 |
152
+ | After | ~15,000 |
153
+ | **Saved** | **~18,377** |
154
+ | **% reduction** | **~55%** |
155
+
156
+ For a full-pipeline orchestration run (all phases active), the after-state
157
+ cost approaches the before-state (all phase files are loaded). The savings
158
+ accrue on focused runs — e.g., a stall-recovery invocation that only
159
+ exercises the monitoring phase, or a results-aggregation pass that only
160
+ exercises the close phase.
161
+
162
+ ---
163
+
164
+ ## Combined Effect
165
+
166
+ When both #1271 and #1272 are applied and selective spec loading
167
+ (`load-set`) is also active:
168
+
169
+ | Session | Before (all specs) | After (decomposed + selective) | Saved | % |
170
+ |---------|-------------------:|-------------------------------:|------:|---|
171
+ | `/review-pr` 2-domain | ~354,942 | ~29,104 | ~325,838 | **92%** |
172
+ | `/orchestrate` single phase | ~354,942 | ~15,000 | ~339,942 | **96%** |
173
+
174
+ The two optimizations compose: selective loading eliminates the dead-weight
175
+ specs outside the command's graph neighborhood; decomposition eliminates
176
+ the dead-weight sub-specs inside the command that are not reached on the
177
+ current invocation.
178
+
179
+ ---
180
+
181
+ ## How To Reproduce
182
+
183
+ ```bash
184
+ # Full corpus baseline
185
+ find commands -name '*.md' -type f -exec cat {} + | wc -c
186
+
187
+ # review-pr before: full catalog
188
+ cat commands/review-pr.md commands/review-pr-agents.md | wc -c
189
+
190
+ # review-pr after: slim router + 2 persona files (substitute actual persona names)
191
+ cat commands/review-pr.md commands/review-pr-agents.md \
192
+ commands/review-pr-agents/<persona1>.md \
193
+ commands/review-pr-agents/<persona2>.md | wc -c
194
+
195
+ # orchestrate before: full monolith
196
+ cat commands/orchestrate.md | wc -c
197
+
198
+ # orchestrate after: dispatcher + one phase file (substitute actual phase name)
199
+ cat commands/orchestrate.md commands/orchestrate/<phase>.md | wc -c
200
+ ```
201
+
202
+ Divide bytes by 4 for the approximate token count.
203
+
204
+ ---
205
+
206
+ ## See Also
207
+
208
+ - `docs/spec/forge-protocol-v1.md` § 10 — Nested-Command Decomposition Pattern
209
+ (naming conventions, decomposition criteria, automatic installer/graph pick-up)
210
+ - `docs/spec-graph-schema.md` — how `walk()` and `load-set` enable selective loading
211
+ - `docs/articles/selective-spec-loading-token-savings.md` — baseline measurement
212
+ for spec-graph-driven selective loading across all entry commands
213
+ - `docs/articles/per-repo-adaptive-scripts-token-savings.md` — complementary
214
+ savings from per-repo adaptive scripts
@@ -0,0 +1,131 @@
1
+ # Model-Release Upgrade Playbook
2
+
3
+ > **Status**: Active — applies to every new Claude or Claude Code version before it is adopted as the project default.
4
+ > **Depends on**: Pipeline eval harness (#1285) and CI regression gate (#1286) — both must be operational before this playbook can be executed in full.
5
+
6
+ ---
7
+
8
+ ## Purpose
9
+
10
+ ForgeDock is a prose-heavy, prompt-engineering product running on a fast-moving platform. A model upgrade that looks like a free improvement can silently drop one-shot success rates, inflate human-intervention counts, or break deterministic routing — exactly the failure modes that are hardest to spot in code review. This playbook is the gate that prevents unvalidated upgrades from reaching users.
11
+
12
+ **Do not adopt a new model or Claude Code release without completing every step below and publishing the resulting scorecard.**
13
+
14
+ ---
15
+
16
+ ## Trigger Conditions
17
+
18
+ Run this playbook when any of the following occur:
19
+
20
+ - A new Claude model version is released (e.g., claude-sonnet-X, claude-opus-X)
21
+ - A new Claude Code version ships (major or minor)
22
+ - A substantive `commands/` change is merged that may affect pipeline behavior
23
+ - The baseline scorecard is more than 90 days old and a refresh is due
24
+
25
+ ---
26
+
27
+ ## Step 1 — Prepare the eval environment
28
+
29
+ 1. Pin the **current production model** in your `.env` or `forge.yaml` (do not change it yet). This is your control arm.
30
+ 2. Confirm the benchmark corpus is at its current version (check `docs/eval/README.md` for the active corpus tag).
31
+ 3. Confirm `scripts/bench-scorecard.mjs` exits 0 against the current baseline scorecard:
32
+ ```bash
33
+ node scripts/bench-scorecard.mjs docs/eval/runs/<latest-baseline-run>/raw.json
34
+ ```
35
+ 4. Record the current baseline scorecard path in your run log (the file you will compare against in Step 5).
36
+
37
+ ---
38
+
39
+ ## Step 2 — Run the full corpus suite on the candidate model
40
+
41
+ 1. Set `FORGE_MODEL` (or equivalent config) to the **candidate model** version.
42
+ 2. Run the full pipeline eval suite against the seeded benchmark corpus:
43
+ ```bash
44
+ # Full run — all corpus issues, all pipeline phases
45
+ node bin/runner.mjs --corpus docs/eval/corpus/seed-v1.json --model <candidate> --out docs/eval/runs/<run-id>/
46
+ ```
47
+ Where `<run-id>` follows the convention `<model-slug>-<YYYY-MM-DD>` (e.g., `claude-sonnet-4-7-2026-09-01`).
48
+ 3. The runner emits per-issue outcome files into the run directory. Do not post-process them yet.
49
+ 4. Minimum corpus coverage: every issue in the active corpus must complete (pass or fail) before scoring. Partial runs are not valid baselines.
50
+
51
+ ---
52
+
53
+ ## Step 3 — Score the run
54
+
55
+ ```bash
56
+ node scripts/bench-scorecard.mjs docs/eval/runs/<run-id>/raw.json > docs/eval/runs/<run-id>/scorecard.json
57
+ ```
58
+
59
+ Review the exit code:
60
+ - `0` — scorecard emitted, proceed to Step 4
61
+ - `1` — invalid input (bad JSON, n<3 runs, missing fields) — fix the run data before proceeding
62
+ - `2` — judge-calibration warning — review miscalibrated runs before accepting results
63
+
64
+ ---
65
+
66
+ ## Step 4 — Compare against the baseline
67
+
68
+ Open `docs/eval/runs/<run-id>/scorecard.json` alongside the current baseline scorecard. Check every metric in the table below. A regression is defined as any metric that moves outside its regression threshold.
69
+
70
+ | Metric | Regression threshold | Action if regressed |
71
+ |--------|---------------------|---------------------|
72
+ | One-shot success rate | Drop > 5 percentage points vs baseline | BLOCK upgrade — file regression issue |
73
+ | Human-intervention count (mean) | Increase > 0.5 interventions/issue | BLOCK upgrade — file regression issue |
74
+ | Wall-clock time (median) | Increase > 20% | WARN — investigate before proceeding |
75
+ | Cost per issue (median) | Increase > 15% | WARN — investigate before proceeding |
76
+
77
+ If **any BLOCK-level regression** is present: do not proceed to Step 5. File a regression issue (label `priority:P0`, link to both scorecards), and return to the current model until the regression is explained and either fixed or accepted as a known trade-off with sign-off.
78
+
79
+ If only WARN-level regressions are present: document the regression and its explanation in the scorecard commit message. Proceed with caution.
80
+
81
+ ---
82
+
83
+ ## Step 5 — Publish the scorecard
84
+
85
+ 1. Copy the scorecard to the published results directory:
86
+ ```bash
87
+ cp docs/eval/runs/<run-id>/scorecard.json docs/eval/published/<run-id>.json
88
+ ```
89
+ 2. Update `docs/eval/README.md` — add a row to the results table pointing to the new scorecard.
90
+ 3. Commit with the message:
91
+ ```
92
+ docs(eval): publish scorecard for <model-slug> — <YYYY-MM-DD>
93
+ ```
94
+ 4. Open a PR targeting `staging`. The PR body must include: model version, corpus version, all headline metrics, and a link to the diff against the previous baseline.
95
+
96
+ ---
97
+
98
+ ## Step 6 — Adopt or reject
99
+
100
+ **Adopt**: If no BLOCK-level regressions and all WARN-level regressions are documented, update the project default model in `forge.yaml` and `docs/eval/README.md` (update the "current baseline" pointer). Merge the PR.
101
+
102
+ **Reject**: If any BLOCK-level regression is unresolved, close the PR without merging, leave the model pin unchanged, and ensure the regression issue is open and assigned.
103
+
104
+ ---
105
+
106
+ ## Scorecard format
107
+
108
+ Every published scorecard is a JSON file at `docs/eval/published/<run-id>.json`. See `docs/eval/README.md` for the schema and field definitions. The scorecard must be machine-readable (for the CI gate) and human-readable (for the published results table).
109
+
110
+ ---
111
+
112
+ ## What counts as a "real" scorecard
113
+
114
+ A scorecard is valid when:
115
+ - It was produced by the harness from at least 3 runs per corpus issue (n=1 is not a valid result — see `bench-scorecard.mjs` methodology note)
116
+ - It covers the full active corpus (no skipped issues)
117
+ - The raw run data is committed alongside the scorecard in `docs/eval/runs/<run-id>/`
118
+ - The `bench-scorecard.mjs` exits 0 (or 2 with documented calibration notes) against the raw data
119
+
120
+ Template-only scorecards (no raw run data) do not satisfy acceptance criteria.
121
+
122
+ ---
123
+
124
+ ## See also
125
+
126
+ - `docs/eval/README.md` — published results index and scorecard schema
127
+ - `docs/eval/corpus/` — seeded benchmark corpus (issues + graded difficulty)
128
+ - `scripts/bench-scorecard.mjs` — scorecard aggregator
129
+ - Issue #1259 — parent epic: eval regression gate
130
+ - Issue #1285 — pipeline eval harness (prerequisite)
131
+ - Issue #1286 — CI regression gate (prerequisite)
@@ -33,7 +33,7 @@ We measured this cost across five representative completed issues.
33
33
 
34
34
  We analyzed five pipeline sessions from the ForgeDock dogfooding repository (issues #674, #669, #671, #679, #676). For each session, we identified every tool call that was a "rediscovery call" — a call whose purpose was to learn something about the project that was already true in the previous session.
35
35
 
36
- Token counts were estimated at 1 token per 4 characters (conservative; actual tokenization may differ by model and content type). We counted command invocation + output, not context window overhead.
36
+ Token counts were estimated at 1 token per 4 characters (conservative; actual tokenization may differ by model and content type), then adjusted upward ~30% to reflect the claude-sonnet-5 tokenizer, which produces approximately 30% more tokens than claude-sonnet-4.x for the same text. We counted command invocation + output, not context window overhead.
37
37
 
38
38
  ---
39
39
 
@@ -41,13 +41,13 @@ Token counts were estimated at 1 token per 4 characters (conservative; actual to
41
41
 
42
42
  | Operation | Mechanism (baseline) | Token estimate |
43
43
  |-----------|---------------------|----------------|
44
- | forge.yaml full read | Read 111-line, 4,228-byte file | ~1,050 |
45
- | Branch name determination | `gh issue view` + LLM milestone-to-slug inference | ~350 |
46
- | Commit style detection | `git log --oneline -5` + LLM pattern inference | ~200 |
47
- | Test command discovery | Read package.json/pyproject.toml + grep test scripts | ~400 |
48
- | Test location discovery | `ls tests/`, `ls __tests__/`, parse package.json | ~250 |
49
- | Label scheme discovery | `gh label list` or inferred from pipeline comments | ~300 |
50
- | **Total per session** | | **~2,550 tokens** |
44
+ | forge.yaml full read | Read 111-line, 4,228-byte file | ~1,365 |
45
+ | Branch name determination | `gh issue view` + LLM milestone-to-slug inference | ~455 |
46
+ | Commit style detection | `git log --oneline -5` + LLM pattern inference | ~260 |
47
+ | Test command discovery | Read package.json/pyproject.toml + grep test scripts | ~520 |
48
+ | Test location discovery | `ls tests/`, `ls __tests__/`, parse package.json | ~325 |
49
+ | Label scheme discovery | `gh label list` or inferred from pipeline comments | ~390 |
50
+ | **Total per session** | | **~3,315 tokens** |
51
51
 
52
52
  These numbers are per-session. On a repository with active development (10-30 issues/month), this rediscovery tax runs 25,500–76,500 tokens/month on patterns that have not changed.
53
53
 
@@ -68,39 +68,41 @@ After `/optimize` generates `.forgedock/scripts/` entries from `forge.yaml → l
68
68
 
69
69
  | Script | Size | Execution cost |
70
70
  |--------|------|----------------|
71
- | `branch-targets.sh` | 23 lines, ~470 bytes | ~170 tokens |
72
- | `run-tests.sh` | 10 lines, ~200 bytes | ~80 tokens |
73
- | `format-commit.sh` | 15 lines, ~300 bytes | ~100 tokens |
74
- | `forge.yaml` learned section (4 targeted yq reads) | subset of 4,228-byte file | ~100 tokens |
75
- | **Total per session** | | **~450 tokens** |
71
+ | `branch-targets.sh` | 23 lines, ~470 bytes | ~220 tokens |
72
+ | `run-tests.sh` | 10 lines, ~200 bytes | ~105 tokens |
73
+ | `format-commit.sh` | 15 lines, ~300 bytes | ~130 tokens |
74
+ | `forge.yaml` learned section (4 targeted yq reads) | subset of 4,228-byte file | ~130 tokens |
75
+ | **Total per session** | | **~585 tokens** |
76
76
 
77
- The key insight: a script call costs roughly the script size (to read) plus the output size (to receive). A 23-line branch-targets.sh costs ~118 tokens to read and produces 1 line of output. The equivalent LLM-based rediscovery costs ~350 tokens and occasionally hallucinates.
77
+ The key insight: a script call costs roughly the script size (to read) plus the output size (to receive). A 23-line branch-targets.sh costs ~153 tokens to read and produces 1 line of output. The equivalent LLM-based rediscovery costs ~455 tokens and occasionally hallucinates.
78
78
 
79
79
  ---
80
80
 
81
81
  ## Savings Calculation
82
82
 
83
83
  ```
84
- savings_per_session = 2,550 - 450 = 2,100 tokens
84
+ savings_per_session = 3,315 - 585 = 2,730 tokens
85
85
 
86
86
  range:
87
- low = 1,200 tokens (simple fast-lane issues, no test discovery needed)
88
- high = 3,500 tokens (complex multi-file issues with full pattern rediscovery)
87
+ low = 1,560 tokens (simple fast-lane issues, no test discovery needed)
88
+ high = 4,550 tokens (complex multi-file issues with full pattern rediscovery)
89
89
  ```
90
90
 
91
- At claude-sonnet-4-5 pricing ($3.00 per million input tokens):
91
+ At claude-sonnet-5 pricing ($3.00 per million input tokens standard; $2.00/M intro through 2026-08-31):
92
92
 
93
93
  ```
94
- 2,100 tokens/session × $3.00 / 1,000,000 = $0.0063/session
94
+ 2,730 tokens/session × $3.00 / 1,000,000 = $0.0082/session (standard)
95
+ 2,730 tokens/session × $2.00 / 1,000,000 = $0.0055/session (intro rate)
95
96
  ```
96
97
 
97
98
  At observed ForgeDock dogfooding pace (5–15 issues/day, averaging ~10):
98
99
 
99
100
  ```
100
- 300 sessions/month × $0.0063 = $1.89/month per repo
101
+ 300 sessions/month × $0.0082 = $2.46/month per repo (standard rate)
102
+ 300 sessions/month × $0.0055 = $1.65/month per repo (intro rate through 2026-08-31)
101
103
  ```
102
104
 
103
- For teams running ForgeDock at production scale across multiple repositories, these savings compound. A team with 5 active repos at 300 sessions/month each saves ~$9.45/month from adaptive scripts alone. The cost of running `/optimize` once per repo to generate the scripts: effectively zero (a single session that generates ~5 scripts).
105
+ For teams running ForgeDock at production scale across multiple repositories, these savings compound. A team with 5 active repos at 300 sessions/month each saves ~$12.30/month from adaptive scripts alone at standard pricing. The cost of running `/optimize` once per repo to generate the scripts: effectively zero (a single session that generates ~5 scripts).
104
106
 
105
107
  **Payback period: 1 session.**
106
108
 
@@ -125,10 +127,10 @@ Per-repo scripts replace approximately 25–40% of devdocs content for projects
125
127
  **Combined savings per session** (adaptive scripts + devdocs selective loading):
126
128
 
127
129
  ```
128
- Rediscovery elimination (adaptive scripts): ~2,100 tokens
129
- Devdocs selective loading (#675): ~1,000 tokens
130
+ Rediscovery elimination (adaptive scripts): ~2,730 tokens
131
+ Devdocs selective loading (#675): ~1,300 tokens
130
132
  ────────────────────────────────────────────────────────
131
- Total combined: ~3,100 tokens/session
133
+ Total combined: ~4,030 tokens/session
132
134
  ```
133
135
 
134
136
  ---
@@ -159,7 +161,7 @@ The estimates in this article carry these caveats:
159
161
 
160
162
  3. **LLM hallucination cost not counted**: Our baseline counts only the token cost of rediscovery, not the token cost of correcting a hallucinated branch name or wrong test command. Including rework, the actual cost of rediscovery-induced errors is higher.
161
163
 
162
- 4. **Model pricing changes**: The $3.00/M figure is current as of June 2026. Pricing trends downward over time; the absolute dollar savings will decrease, but the proportional savings (rediscovery as a fraction of session cost) remain constant.
164
+ 4. **Model pricing changes**: The $3.00/M standard figure is current as of July 2026 for claude-sonnet-5. An introductory rate of $2.00/M input is available through 2026-08-31. Pricing trends downward over time; the absolute dollar savings will decrease, but the proportional savings (rediscovery as a fraction of session cost) remain constant.
163
165
 
164
166
  ---
165
167
 
@@ -3,14 +3,40 @@
3
3
  "schema": {
4
4
  "version": "Claude Code version string where the breakpoint was introduced (e.g. '2.1.157')",
5
5
  "type": "Category of change: 'feature' | 'behavior_change' | 'compatibility' | 'deprecation'",
6
+ "feature_key": "(optional) Machine-readable key for resolve-style lookups, e.g. 'reload_skills', 'effort_levels'",
6
7
  "description": "Human-readable description of what changed in Claude Code at this version",
7
8
  "forgedock_impact": "How this change affects ForgeDock — what commands, scripts, or behaviors are affected",
8
9
  "severity": "Impact level if a user is on this version or older: 'high' | 'medium' | 'low' | 'info'"
9
10
  },
10
11
  "breakpoints": [
12
+ {
13
+ "version": "2.1.152",
14
+ "type": "feature",
15
+ "feature_key": "session_start_reload_skills",
16
+ "description": "SessionStart hook gains `reloadSkills` capability; MessageDisplay hook introduced",
17
+ "forgedock_impact": "The SessionStart hook in settings-hook.mjs can now trigger skill reload on session start. MessageDisplay hook enables display-layer interception — ForgeDock can use this for nudge/banner injection without polluting stdout.",
18
+ "severity": "medium"
19
+ },
20
+ {
21
+ "version": "2.1.154",
22
+ "type": "feature",
23
+ "feature_key": "effort_levels",
24
+ "description": "Effort levels (low..max) introduced for per-subagent/skill effort frontmatter; Opus 4.8 model available; lean system prompt mode",
25
+ "forgedock_impact": "ForgeDock commands can now gate effort-level frontmatter by version. orchestrate.md sub-agents can specify effort levels. Installer should gate effort-level injection at v2.1.154+. Lean system prompt reduces context overhead for simple commands.",
26
+ "severity": "medium"
27
+ },
28
+ {
29
+ "version": "2.1.154",
30
+ "type": "compatibility",
31
+ "feature_key": "native_workflows",
32
+ "description": "Native /workflows dynamic workflows introduced as a built-in Claude Code feature",
33
+ "forgedock_impact": "Parallel to ForgeDock's orchestrate.md. ForgeDock's approach adds GitHub persistence (FORGE: annotations, label-state machine, compaction resilience) that native /workflows lacks. Both can coexist — ForgeDock is the orchestration layer with state, not a replacement for native workflows.",
34
+ "severity": "info"
35
+ },
11
36
  {
12
37
  "version": "2.1.157",
13
38
  "type": "behavior_change",
39
+ "feature_key": "worktree_auto_unlock",
14
40
  "description": "EnterWorktree now unlocks worktrees automatically when the agent session finishes",
15
41
  "forgedock_impact": "ForgeDock no longer needs --force on worktree cleanup commands. The cleanup phase in work-on.md can use standard git worktree remove without the force flag.",
16
42
  "severity": "low"
@@ -18,30 +44,74 @@
18
44
  {
19
45
  "version": "2.1.157",
20
46
  "type": "feature",
47
+ "feature_key": "skills_auto_load",
21
48
  "description": "Plugins in .claude/skills are auto-loaded without requiring marketplace installation",
22
49
  "forgedock_impact": "Symlink-based install (the original forgedock install method) remains valid but is no longer required for skill discovery. The stub file install pattern introduced in #619 is the preferred approach and works the same way.",
23
50
  "severity": "info"
24
51
  },
52
+ {
53
+ "version": "2.1.163",
54
+ "type": "feature",
55
+ "feature_key": "stop_subagent_stop_additional_context",
56
+ "description": "Stop and SubagentStop hooks gain `additionalContext` injection — hooks can return structured context to block completion or redirect the agent",
57
+ "forgedock_impact": "ForgeDock's SubagentStop enforcement hooks (issue #1250) can now inject corrective context when a pipeline stage skips its FORGE annotation. This makes blocking hooks actionable: the agent receives specific remediation guidance rather than a bare rejection.",
58
+ "severity": "high"
59
+ },
25
60
  {
26
61
  "version": "2.1.163",
27
62
  "type": "compatibility",
63
+ "feature_key": "managed_version_constraints",
28
64
  "description": "requiredMinimumVersion and requiredMaximumVersion managed settings added for enterprise deployments",
29
65
  "forgedock_impact": "ForgeDock's own version check (forgedock doctor, #682) is the OSS equivalent of this enterprise feature. No conflict — enterprise users with managed settings may have stricter version constraints than ForgeDock requires.",
30
66
  "severity": "info"
31
67
  },
68
+ {
69
+ "version": "2.1.170",
70
+ "type": "feature",
71
+ "feature_key": "fable_5",
72
+ "description": "Fable 5 model support introduced",
73
+ "forgedock_impact": "ForgeDock commands using model-selection frontmatter may gate on this version for Fable 5 access. Installer version detection should record this for mode-selection logic.",
74
+ "severity": "info"
75
+ },
32
76
  {
33
77
  "version": "2.1.172",
34
78
  "type": "feature",
79
+ "feature_key": "nested_subagents_5_levels",
35
80
  "description": "Sub-agents can spawn sub-agents up to 5 levels deep (increased nesting depth)",
36
81
  "forgedock_impact": "ForgeDock's orchestrate.md uses multi-agent orchestration. The 5-level nesting limit is compatible with ForgeDock's orchestration patterns. No changes needed.",
37
82
  "severity": "info"
38
83
  },
39
84
  {
40
- "version": "2.1.154",
41
- "type": "compatibility",
42
- "description": "Native /workflows dynamic workflows introduced as a built-in Claude Code feature",
43
- "forgedock_impact": "Parallel to ForgeDock's orchestrate.md. ForgeDock's approach adds GitHub persistence (FORGE: annotations, label-state machine, compaction resilience) that native /workflows lacks. Both can coexist — ForgeDock is the orchestration layer with state, not a replacement for native workflows.",
44
- "severity": "info"
85
+ "version": "2.1.178",
86
+ "type": "feature",
87
+ "feature_key": "tool_param_value_permission_rules",
88
+ "description": "Tool(param:value) permission rules introduced — e.g. `Agent(model:opus)`, `Bash(git push *)` — allow fine-grained tool permission matching on parameter values",
89
+ "forgedock_impact": "ForgeDock installer can now ship opt-in permission-rule suggestions (e.g. deny `Bash(git push origin main*)`) via `npx forgedock install`. These rules require v2.1.178+. Gate their injection in the installer by detected version. Enforces branch-protection policies deterministically rather than by prose.",
90
+ "severity": "high"
91
+ },
92
+ {
93
+ "version": "2.1.186",
94
+ "type": "feature",
95
+ "feature_key": "background_subagent_permission_surfacing",
96
+ "description": "Background sub-agent permission requests are surfaced to the user; completion notifications for background sub-agents added (v2.1.186–2.1.191 rollout)",
97
+ "forgedock_impact": "ForgeDock's orchestrate.md sub-agents dispatched in background mode will now surface permission prompts to the user when they encounter ungranted tools. Completion notifications reduce the need for polling. No ForgeDock spec changes required — behavior improves automatically.",
98
+ "severity": "medium"
99
+ },
100
+ {
101
+ "version": "2.1.196",
102
+ "type": "feature",
103
+ "feature_key": "skills_commands_merge",
104
+ "description": "Skills and commands merge into a unified namespace; `CLAUDE_CODE_SUBAGENT_MODEL=inherit` env var introduced for sub-agent model inheritance",
105
+ "forgedock_impact": "ForgeDock install method (symlinks into ~/.claude/commands/) is unaffected. `CLAUDE_CODE_SUBAGENT_MODEL=inherit` allows orchestrate.md sub-agents to inherit the parent model without explicit frontmatter — reduces per-subagent config overhead. Gate this env injection at v2.1.196+.",
106
+ "severity": "medium"
107
+ },
108
+ {
109
+ "version": "2.1.197",
110
+ "type": "feature",
111
+ "feature_key": "sonnet_5_default_1m_context",
112
+ "description": "Sonnet 5 becomes the default model; native 1M context window supported",
113
+ "forgedock_impact": "ForgeDock commands benefit from the 1M context window — compaction is less likely to trigger during long pipeline runs. Context budget calculations in orchestrate.md and work-on.md can be relaxed. compat-audit.md should surface this as the new minimum-recommended version for optimal context handling.",
114
+ "severity": "medium"
45
115
  }
46
116
  ]
47
117
  }
@@ -8,7 +8,7 @@
8
8
  > Companion to [design-spec-schema](design-spec-schema.md) (#881), [reference-corpus](reference-corpus.md) (#880),
9
9
  > and the [`/design-bench`](../../commands/design-bench.md) command spec.
10
10
  > Results are carried as a `FORGE:BENCH_SCORECARD` annotation — registered in
11
- > [`../FORGE-PROTOCOL.md`](../FORGE-PROTOCOL.md).
11
+ > [`../spec/forge-protocol-v1.md`](../spec/forge-protocol-v1.md).
12
12
 
13
13
  ## Why this exists first
14
14
 
@@ -9,7 +9,7 @@
9
9
  > `motion.video_placeholder` fields + `motion` rubric dimension (#1043); `product_mock` field + `product_mock`
10
10
  > rubric dimension (#1045); `scroll_narrative` object + `scroll_narrative` rubric dimension (#1046).
11
11
  >
12
- > Registered as a FORGE annotation type in [`../FORGE-PROTOCOL.md`](../FORGE-PROTOCOL.md).
12
+ > Registered as a FORGE annotation type in [`../spec/forge-protocol-v1.md`](../spec/forge-protocol-v1.md).
13
13
 
14
14
  ## Purpose
15
15
 
@@ -76,7 +76,7 @@ directions are retained in the `FORGE:DESIGN_CANDIDATES` annotation (below) so t
76
76
 
77
77
  ## The `FORGE:DESIGN_CANDIDATES` annotation
78
78
 
79
- Registered in [FORGE-PROTOCOL](../FORGE-PROTOCOL.md). Posted at the divergent-generation step, between the rationale
79
+ Registered in [FORGE Annotation Protocol](../spec/forge-protocol-v1.md). Posted at the divergent-generation step, between the rationale
80
80
  and the spec. It records the committed archetype, the N directions, and the judge's selection + scores — so the
81
81
  variance decision is a first-class, machine-queryable artifact, not a hidden roll.
82
82