forgedock 1.1.1 → 1.1.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (171) hide show
  1. package/README.md +53 -8
  2. package/bin/batch-runner.mjs +339 -0
  3. package/bin/demo.mjs +92 -7
  4. package/bin/engine/phases.mjs +20 -0
  5. package/bin/engine/state.mjs +8 -3
  6. package/bin/engine-cli.mjs +177 -5
  7. package/bin/forgedock.mjs +409 -26
  8. package/bin/hooks/interactive-engine.mjs +509 -0
  9. package/bin/hooks/pre-tool-use.mjs +498 -0
  10. package/bin/hooks/subagent-stop-enforce.mjs +197 -0
  11. package/bin/journey.mjs +72 -14
  12. package/bin/registry.mjs +139 -0
  13. package/bin/report.mjs +447 -0
  14. package/bin/runner.mjs +68 -19
  15. package/bin/settings-hook.mjs +235 -1
  16. package/commands/adopt.md +4 -1
  17. package/commands/analytics.md +4 -1
  18. package/commands/audit-agents.md +110 -27
  19. package/commands/audit.md +20 -1
  20. package/commands/autopilot.md +80 -8
  21. package/commands/changelog.md +2 -1
  22. package/commands/ci-audit.md +2 -1
  23. package/commands/cleanup.md +161 -1
  24. package/commands/compat-audit.md +2 -1
  25. package/commands/deploy-info.md +58 -1
  26. package/commands/design-bench.md +5 -1
  27. package/commands/design-render-critique-loop.md +1 -1
  28. package/commands/diagnose.md +2 -1
  29. package/commands/explain.md +2 -1
  30. package/commands/forge-stats.md +121 -2
  31. package/commands/forgedock-init.md +1 -1
  32. package/commands/geo-audit.md +3 -0
  33. package/commands/incident-response.md +2 -1
  34. package/commands/issue.md +23 -2
  35. package/commands/milestone.md +77 -12
  36. package/commands/optimize.md +2 -1
  37. package/commands/orchestrate/config.md +67 -0
  38. package/commands/orchestrate/phase-1-resolve.md +159 -0
  39. package/commands/orchestrate/phase-2-triage.md +126 -0
  40. package/commands/orchestrate/phase-2.5-synthesis.md +156 -0
  41. package/commands/orchestrate/phase-3-dependency.md +591 -0
  42. package/commands/orchestrate/phase-4-execution.md +962 -0
  43. package/commands/orchestrate/phase-5-cleanup.md +47 -0
  44. package/commands/orchestrate/phase-6-report.md +198 -0
  45. package/commands/orchestrate/safety.md +62 -0
  46. package/commands/orchestrate.md +25 -2045
  47. package/commands/pipeline-health/config.md +34 -0
  48. package/commands/pipeline-health/phase-1-context.md +180 -0
  49. package/commands/pipeline-health/phase-2-metrics.md +1729 -0
  50. package/commands/pipeline-health/phase-3-analyze.md +204 -0
  51. package/commands/pipeline-health/phase-4-proposals.md +117 -0
  52. package/commands/pipeline-health/phase-5-report.md +284 -0
  53. package/commands/pipeline-health/phase-6-summary.md +52 -0
  54. package/commands/pipeline-health.md +25 -2401
  55. package/commands/pipeline-resume.md +24 -1
  56. package/commands/pipeline-status.md +65 -0
  57. package/commands/qa-sweep.md +4 -1
  58. package/commands/quality-gate.md +248 -10
  59. package/commands/replay.md +2 -1
  60. package/commands/review-pr-agents/api.md +135 -0
  61. package/commands/review-pr-agents/auth.md +137 -0
  62. package/commands/review-pr-agents/billing.md +101 -0
  63. package/commands/review-pr-agents/concurrency.md +145 -0
  64. package/commands/review-pr-agents/database.md +103 -0
  65. package/commands/review-pr-agents/frontend.md +89 -0
  66. package/commands/review-pr-agents/infra.md +385 -0
  67. package/commands/review-pr-agents/protocols.md +185 -0
  68. package/commands/review-pr-agents/scraper.md +185 -0
  69. package/commands/review-pr-agents/security.md +406 -0
  70. package/commands/review-pr-agents.md +45 -1722
  71. package/commands/review-pr-staging.md +196 -40
  72. package/commands/review-pr.md +591 -52
  73. package/commands/rollback.md +2 -1
  74. package/commands/scope.md +2 -1
  75. package/commands/security-audit.md +2 -1
  76. package/commands/signal-planner.md +623 -0
  77. package/commands/sync-ecosystem.md +1 -0
  78. package/commands/test-gate.md +17 -1
  79. package/commands/validate.md +2 -1
  80. package/commands/work-on/build/architect.md +28 -0
  81. package/commands/work-on/build/context.md +2 -0
  82. package/commands/work-on/build/implement.md +3 -1
  83. package/commands/work-on/build/validate.md +26 -7
  84. package/commands/work-on/build.md +163 -24
  85. package/commands/work-on/close.md +130 -3
  86. package/commands/work-on/decompose.md +19 -1
  87. package/commands/work-on/investigate.md +117 -2
  88. package/commands/work-on/review.md +24 -4
  89. package/commands/work-on-monolithic.md +7 -2
  90. package/commands/work-on.md +202 -4
  91. package/docs/BRAND.md +14 -7
  92. package/docs/CI.md +2 -2
  93. package/docs/CODEX.md +36 -1
  94. package/docs/CONFIG.md +5 -8
  95. package/docs/FORGE-PROTOCOL.md +15 -1064
  96. package/docs/anthropic-devrel-outreach.md +2 -2
  97. package/docs/articles/command-decomposition-token-savings.md +214 -0
  98. package/docs/articles/model-release-playbook.md +131 -0
  99. package/docs/articles/per-repo-adaptive-scripts-token-savings.md +27 -25
  100. package/docs/claude-breakpoints.json +75 -5
  101. package/docs/design/abc-benchmark.md +1 -1
  102. package/docs/design/design-spec-schema.md +1 -1
  103. package/docs/design/divergent-generation.md +1 -1
  104. package/docs/design/eval-gate-runbook.md +178 -0
  105. package/docs/design/show-hn-draft.md +26 -1
  106. package/docs/design/token-cost-baseline-1298.md +59 -0
  107. package/docs/eval/README.md +117 -0
  108. package/docs/eval/corpus/seed-v1.json +6 -0
  109. package/docs/eval/published/claude-sonnet-4-6-2026-07-04.json +33 -0
  110. package/docs/site/annotations-explained.md +24 -0
  111. package/docs/site/command-learning-path.md +51 -14
  112. package/docs/site/command-reference.md +339 -113
  113. package/docs/site/for-companies.md +95 -0
  114. package/docs/site/forge-annotation-protocol.md +12 -256
  115. package/docs/site/getting-started.md +43 -4
  116. package/docs/site/github-is-the-memory.md +113 -0
  117. package/docs/site/how-it-works.md +3 -13
  118. package/docs/site/index.md +9 -1
  119. package/docs/spec/eval-run-result.md +147 -0
  120. package/docs/spec/forge-protocol-v1.md +441 -18
  121. package/docs/spec/label-state-machine.md +55 -0
  122. package/docs/spec/review-protocol.md +158 -0
  123. package/docs/spec-graph-schema.md +28 -0
  124. package/docs/superpowers/specs/2026-07-04-economic-self-governance-design.md +181 -0
  125. package/docs/superpowers/specs/2026-07-04-provenance-trail-design.md +189 -0
  126. package/examples/forgedock-demo/README.md +41 -22
  127. package/examples/forgedock-demo/issues/06-bug-tags-type-validation.md +43 -0
  128. package/examples/forgedock-demo/issues/07-bug-negative-offset-pagination.md +42 -0
  129. package/examples/forgedock-demo/issues/08-bug-patch-mass-assignment.md +49 -0
  130. package/examples/forgedock-demo/issues/09-bug-count-injection.md +47 -0
  131. package/examples/forgedock-demo/issues/10-feature-sort-param.md +39 -0
  132. package/examples/forgedock-demo/issues/11-feature-patch-tags.md +37 -0
  133. package/examples/forgedock-demo/issues/12-feature-bulk-create.md +42 -0
  134. package/examples/forgedock-demo/issues/13-feature-archive-notes.md +46 -0
  135. package/examples/forgedock-demo/issues/14-refactor-extract-title-validation.md +41 -0
  136. package/examples/forgedock-demo/issues/15-refactor-protected-route-flag.md +43 -0
  137. package/examples/forgedock-demo/issues/16-refactor-extract-store.md +45 -0
  138. package/examples/forgedock-demo/issues/17-perf-tags-dedup-set.md +35 -0
  139. package/examples/forgedock-demo/issues/18-perf-remove-redundant-sort.md +37 -0
  140. package/examples/forgedock-demo/issues/19-docs-api-reference-full.md +53 -0
  141. package/examples/forgedock-demo/issues/20-docs-intentional-flaw-convention.md +46 -0
  142. package/examples/forgedock-demo/scripts/smoke.js +18 -1
  143. package/examples/forgedock-demo/src/db.js +36 -11
  144. package/examples/forgedock-demo/src/routes/notes.js +106 -9
  145. package/examples/forgedock-demo/src/server.js +4 -1
  146. package/package.json +3 -1
  147. package/packages/protocol/LICENSE +21 -0
  148. package/packages/protocol/src/cli.js +135 -0
  149. package/packages/protocol/src/emit.js +146 -0
  150. package/packages/protocol/src/index.js +43 -0
  151. package/packages/protocol/src/parse.js +229 -0
  152. package/packages/protocol/src/types.js +171 -0
  153. package/packages/protocol/src/validate.js +122 -0
  154. package/scripts/bench-topology-cost.mjs +372 -0
  155. package/scripts/check-command-docs-drift.sh +198 -0
  156. package/scripts/check-protocol-restatements.sh +176 -0
  157. package/scripts/check-registry/README.md +72 -0
  158. package/scripts/check-registry/manifest.json +6 -0
  159. package/scripts/code-index.sh +454 -0
  160. package/scripts/conformance-check.mjs +168 -0
  161. package/scripts/doctor-pipeline-state.sh +484 -0
  162. package/scripts/eval-gate-baseline.json +17 -0
  163. package/scripts/eval-gate-scorecard.mjs +242 -0
  164. package/scripts/eval-scorecard-aggregator.mjs +252 -0
  165. package/scripts/flaky-quarantine.sh +241 -0
  166. package/scripts/forge-annotation.sh +567 -0
  167. package/scripts/issue-dedup.sh +236 -0
  168. package/scripts/validate-annotation-node.mjs +233 -0
  169. package/scripts/worktree-lifecycle.sh +3 -3
  170. package/templates/devdocs/agent/using-forgedock.md +14 -0
  171. package/docs/site/.vitepress/config.mjs +0 -85
package/README.md CHANGED
@@ -150,7 +150,7 @@ Issue → Investigate → Context → Architect → Build → Quality Gate → R
150
150
  | **Review** | PR diff, contract, gate results | `FORGE:REVIEW_STARTED` on the issue; per-agent findings as structured `FINDING` blocks on the PR |
151
151
  | **Close** | All of the above | `FORGE:TRAJECTORY` — the full audit trail of the run |
152
152
 
153
- **GitHub as the database.** Every annotation is wrapped in an HTML comment (`<!-- FORGE:INVESTIGATOR -->`) that makes it machine-parseable. When an agent starts — even in a brand-new conversation after compaction — it queries the issue via `gh` and reconstructs full context from these tags. Workflow labels (`workflow:investigating`, `workflow:in-review`, `workflow:merged`…) track state, and the pipeline resumes from whatever state GitHub reports. The annotation format is an open standard — see the [FORGE Annotation Protocol](docs/FORGE-PROTOCOL.md).
153
+ **GitHub as the database.** Every annotation is wrapped in an HTML comment (`<!-- FORGE:INVESTIGATOR -->`) that makes it machine-parseable. When an agent starts — even in a brand-new conversation after compaction — it queries the issue via `gh` and reconstructs full context from these tags. Workflow labels (`workflow:investigating`, `workflow:in-review`, `workflow:merged`…) track state, and the pipeline resumes from whatever state GitHub reports. The annotation format is an open standard — see the [FORGE Annotation Protocol](docs/spec/forge-protocol-v1.md).
154
154
 
155
155
  **Durable by design.** Headless runs are backed by a real execution engine, not prompt-hope: every phase transition is appended to an event-sourced, crash-safe run log, mirrored to the issue as a compact `FORGE:STATE` index, and guarded by leases so two agents can never own the same issue. Kill the process mid-run and restart it — the engine reconciles local state against GitHub (GitHub wins), adopts branches and PRs that already exist instead of re-running the LLM, and escalates to `needs-human` after bounded retries instead of looping. Phase selection is a pure rule-based state machine: **the engine, not the model, decides what happens next.** The headless core shipped in [PR #1326](https://github.com/RapierCraftStudios/ForgeDock/pull/1326); wiring the interactive path onto the same engine is in progress ([#1323](https://github.com/RapierCraftStudios/ForgeDock/issues/1323)–[#1325](https://github.com/RapierCraftStudios/ForgeDock/issues/1325)).
156
156
 
@@ -221,9 +221,12 @@ More ship today (web-property analytics, browser QA sweeps, self-benchmarking)
221
221
  **Requirements:** [Claude Code](https://docs.anthropic.com/en/docs/claude-code) · [GitHub CLI](https://cli.github.com/) (authenticated) · Node.js ≥ 18.
222
222
 
223
223
  ```bash
224
- npx forgedock # checks your environment, installs the commands, detects your repo, and hands you a reviewed forge.yaml
224
+ npx forgedock # project-scoped install: checks your environment, installs commands for this repo, detects your repo, and hands you a reviewed forge.yaml
225
+ npx forgedock --global # global install: installs into ~/.claude/commands/ — available in every Claude Code session on this machine
225
226
  ```
226
227
 
228
+ **Project-scoped is the default.** `npx forgedock` installs commands scoped to the current project directory. Use `--global` only when you want the commands available across all your projects on this machine.
229
+
227
230
  One command does everything: it checks your environment, installs the slash commands into Claude Code, detects your repo (owner, branches, paths), and hands you a single annotated `forge.yaml` to review — press Enter to accept. Run `npx forgedock init` any time afterward to re-generate the config only.
228
231
 
229
232
  Installing also registers a SessionStart hook, so every Claude Code session
@@ -248,6 +251,15 @@ Commands then appear as `/forgedock:work-on`, etc. You still run `npx forgedock
248
251
 
249
252
  **Headless / CI:** the pipeline also runs outside Claude Code. `npx forgedock run work-on <issue> --dry-run` previews the assembled prompt and tool plan; with an `ANTHROPIC_API_KEY`, `npx forgedock run` drives the same command specs through a hardened tool-use loop, and `npx forgedock run-issue <issue>` executes them on the durable engine (event-sourced run log, leases, crash-safe resume).
250
253
 
254
+ **Install modes:**
255
+
256
+ ```bash
257
+ npx forgedock install # project-scoped: <cwd>/.claude/commands/ (default)
258
+ npx forgedock install --global # global: ~/.claude/commands/ (opt-in)
259
+ ```
260
+
261
+ `uninstall`, `update`, and `doctor` auto-detect which mode is installed — no need to re-specify `--global` after a global install. To explicitly target global: `npx forgedock doctor --global`.
262
+
251
263
  **Maintenance:**
252
264
 
253
265
  ```bash
@@ -255,8 +267,9 @@ npx forgedock update # relink commands + refresh the SessionStart hook
255
267
  npx forgedock enable # turn ForgeDock on for this directory
256
268
  npx forgedock disable # turn ForgeDock off for this directory
257
269
  npx forgedock status # show ForgeDock's state for this directory
258
- npx forgedock doctor # installation health check with fix hints
259
- npx forgedock uninstall # remove commands, the hook, and tracked copies
270
+ npx forgedock doctor # installation health check with fix hints (auto-detects mode)
271
+ npx forgedock report # 30-day pipeline impact receipts (--md for Markdown, --json for scripting)
272
+ npx forgedock uninstall # remove commands, the hook, and tracked copies (auto-detects mode)
260
273
  npx forgedock help # show everything
261
274
  ```
262
275
 
@@ -264,6 +277,35 @@ npx forgedock help # show everything
264
277
 
265
278
  </details>
266
279
 
280
+ <details>
281
+ <summary><strong>Migrating from a global install</strong></summary>
282
+
283
+ If you installed ForgeDock before project-scoped became the default, your commands live in `~/.claude/commands/`. You have two paths forward:
284
+
285
+ **Keep the global install** — add `--global` going forward and nothing changes:
286
+
287
+ ```bash
288
+ npx forgedock --global # install / update globally as before
289
+ ```
290
+
291
+ **Switch to project-scoped** (recommended for multi-repo setups):
292
+
293
+ ```bash
294
+ # 1. Check your current install mode
295
+ npx forgedock status
296
+
297
+ # 2. Remove the global install
298
+ npx forgedock uninstall --global
299
+
300
+ # 3. Install project-scoped in each repo you use ForgeDock in
301
+ cd /path/to/your/project
302
+ npx forgedock
303
+ ```
304
+
305
+ Run `npx forgedock doctor` after switching to confirm everything is healthy.
306
+
307
+ </details>
308
+
267
309
  ---
268
310
 
269
311
  ## For companies
@@ -272,8 +314,8 @@ The core is AGPL-3.0 and stays that way: engineers run the full pipeline on thei
272
314
 
273
315
  Two things are for sale:
274
316
 
275
- - **A [commercial license](COMMERCIAL-LICENSE.md)** — for organizations that need ForgeDock inside proprietary workflows or products without AGPL copyleft obligations. Contact [licensing@rapiercraft.studio](mailto:licensing@rapiercraft.studio).
276
- - **The fleet layer** *(in development)* — org-wide observability over every pipeline run: the receipts on this page, live, across all your repos, plus policy controls and audit-grade provenance for autonomous merges. We're onboarding a small group of design partners — [open a Discussion](https://github.com/RapierCraftStudios/ForgeDock/discussions) or email the address above to get in early.
317
+ - **A [commercial license](COMMERCIAL-LICENSE.md)** — for organizations that need ForgeDock inside proprietary workflows or products without AGPL copyleft obligations. Contact [support@rapiercraftstudios.com](mailto:support@rapiercraftstudios.com).
318
+ - **The fleet layer** *(in development)* — org-wide observability over every pipeline run: the receipts on this page, live, across all your repos, plus policy controls and audit-grade provenance for autonomous merges. We're onboarding a small group of design partners — see [ForgeDock for Companies](docs/site/for-companies.md) for details and intake.
277
319
 
278
320
  ---
279
321
 
@@ -282,7 +324,7 @@ Two things are for sale:
282
324
  Month one built the execution layer. The open roadmap — tracked in the [five-foundations epic (#1320)](https://github.com/RapierCraftStudios/ForgeDock/issues/1320) — is about earning trust while unattended:
283
325
 
284
326
  1. **Durability** — engine-owned state instead of prose-owned state. Headless core shipped ([PR #1326](https://github.com/RapierCraftStudios/ForgeDock/pull/1326)); interactive wiring in progress.
285
- 2. **Verification** — an outcome-based acceptance gate and a graded eval corpus, so "done" is machine-checkable before anything claims success.
327
+ 2. **Verification** — an outcome-based acceptance gate and a graded eval corpus, so "done" is machine-checkable before anything claims success. Per-release pipeline scorecards are published in [`docs/eval/`](docs/eval/README.md); model upgrades follow the [model-release playbook](docs/articles/model-release-playbook.md).
286
328
  3. **Learning** — per-codebase memory that compounds across runs.
287
329
  4. **Economics** — per-run cost accounting and risk×cost dispatch decisions.
288
330
  5. **Provenance** — signed, replayable records of every autonomous change.
@@ -320,15 +362,18 @@ Using ForgeDock in your pipeline? Add the badge — each one is a backlink and a
320
362
 
321
363
  ## Docs & community
322
364
 
365
+ - [GitHub Is Already Your Agents' Memory](docs/site/github-is-the-memory.md) — the canonical argument: why GitHub is the right place for agent memory, how FORGE annotations make it machine-readable, and how to adopt the protocol without ForgeDock
323
366
  - [Getting Started in 5 Minutes](docs/site/getting-started.md)
324
367
  - [How the Knowledge Graph Works](docs/site/how-it-works.md)
325
368
  - [What Are Those FORGE Comments?](docs/site/annotations-explained.md) — 2-minute explainer for annotations you meet in the wild
326
- - [FORGE Annotation Protocol](docs/FORGE-PROTOCOL.md) — the open standard for AI context passing ([formal v1.0 spec](docs/spec/forge-protocol-v1.md))
369
+ - [FORGE Annotation Protocol](docs/spec/forge-protocol-v1.md) — the open standard for AI context passing (CC-BY-4.0)
327
370
  - [ForgeDock vs. Manual Claude Code Workflows](docs/site/vs-manual-workflows.md)
328
371
  - [ForgeDock vs. DeepWiki, AGENTS.md, and Cursor Memories](docs/comparison.md)
329
372
  - [Command Learning Path](docs/site/command-learning-path.md) — which commands to learn first
330
373
  - [Complete Command Reference](docs/site/command-reference.md)
331
374
  - [Troubleshooting & Recovery](docs/site/troubleshooting.md)
375
+ - [Pipeline Eval Scorecards](docs/eval/README.md) — per-release published results for every model/Claude Code upgrade
376
+ - [Model-Release Upgrade Playbook](docs/articles/model-release-playbook.md) — how to validate a new model before adopting it
332
377
 
333
378
  **Contributing:** PRs welcome — every change goes through a PR, tested against 3+ scenarios, using conventional commits (`fix(command):`, `feat(command):`). **License:** [AGPL-3.0](LICENSE) — free to use, modify, and distribute; network use of modifications must be open-sourced under the same license. Commercial licenses are available for proprietary use — see [COMMERCIAL-LICENSE.md](COMMERCIAL-LICENSE.md).
334
379
 
@@ -0,0 +1,339 @@
1
+ #!/usr/bin/env node
2
+ /**
3
+ * bin/batch-runner.mjs — Batch driver for the ForgeDock pipeline eval harness (#1285)
4
+ *
5
+ * Iterates a corpus of GitHub issue numbers, invokes runCommand() (from
6
+ * bin/runner.mjs) for each issue against the /work-on spec, and writes a
7
+ * structured per-run results file consumable by scripts/eval-gate-scorecard.mjs.
8
+ *
9
+ * This is the batch/corpus increment of the standalone runtime — the natural
10
+ * next step after bin/runner.mjs's single-command foundational increment (#1151).
11
+ *
12
+ * Exports (all pure, no SDK, no network):
13
+ * loadCorpus(path) → {corpus_version, issues}
14
+ * makeRunResult(issue, status, opts) → RunResult object
15
+ * classifyRunnerResult(result) → "success" | "failure" | "incomplete"
16
+ * writeResults(outputPath, results, meta) → void
17
+ * runCorpus(opts) → Promise<RunResult[]>
18
+ *
19
+ * CLI usage:
20
+ * node bin/batch-runner.mjs <corpus.json> [output.json]
21
+ * node bin/batch-runner.mjs <corpus.json> [output.json] --dry-run
22
+ *
23
+ * ANTHROPIC_API_KEY must be set for live runs.
24
+ * FORGEDOCK_MODEL overrides the model (default: runner.mjs default).
25
+ *
26
+ * Corpus file format (docs/spec/eval-run-result.md):
27
+ * { "corpus_version": "v1", "issues": [1001, 1002, 1003] }
28
+ *
29
+ * Output file format:
30
+ * { "corpus_version": "v1", "runs": [ ...RunResult ] }
31
+ *
32
+ * See docs/spec/eval-run-result.md for the full JSON schema.
33
+ */
34
+
35
+ import { readFileSync, writeFileSync, mkdirSync } from "node:fs";
36
+ import { dirname, resolve } from "node:path";
37
+
38
+ // ---------------------------------------------------------------------------
39
+ // Corpus loading
40
+ // ---------------------------------------------------------------------------
41
+
42
+ /**
43
+ * Load and validate a corpus JSON file.
44
+ *
45
+ * Throws a descriptive Error if the file is missing, invalid JSON, or does not
46
+ * contain a non-empty integer array at the "issues" key.
47
+ *
48
+ * @param {string} corpusPath - Absolute or relative path to the corpus JSON.
49
+ * @returns {{ corpus_version: string|null, issues: number[] }}
50
+ */
51
+ export function loadCorpus(corpusPath) {
52
+ let raw;
53
+ try {
54
+ raw = readFileSync(corpusPath, "utf-8");
55
+ } catch (e) {
56
+ throw new Error(`Cannot read corpus file "${corpusPath}": ${e.message}`);
57
+ }
58
+
59
+ let data;
60
+ try {
61
+ data = JSON.parse(raw);
62
+ } catch (e) {
63
+ throw new Error(`Corpus file "${corpusPath}" contains invalid JSON: ${e.message}`);
64
+ }
65
+
66
+ if (!data || typeof data !== "object") {
67
+ throw new Error(`Corpus file must be a JSON object; got ${typeof data}`);
68
+ }
69
+ if (!Array.isArray(data.issues) || data.issues.length === 0) {
70
+ throw new Error(`Corpus file must contain a non-empty "issues" array`);
71
+ }
72
+ for (let i = 0; i < data.issues.length; i++) {
73
+ const n = data.issues[i];
74
+ if (typeof n !== "number" || !Number.isInteger(n) || n <= 0) {
75
+ throw new Error(
76
+ `issues[${i}]: must be a positive integer (got ${JSON.stringify(n)})`,
77
+ );
78
+ }
79
+ }
80
+
81
+ return {
82
+ corpus_version: data.corpus_version ?? null,
83
+ issues: data.issues,
84
+ };
85
+ }
86
+
87
+ // ---------------------------------------------------------------------------
88
+ // Run-result construction helpers
89
+ // ---------------------------------------------------------------------------
90
+
91
+ /**
92
+ * Map a runCommand() result status + stop reason to a scorecard status.
93
+ *
94
+ * runCommand() returns:
95
+ * status: "complete" | "incomplete" | "dry-run" | "max-iterations"
96
+ *
97
+ * We map these to the eval-result schema's status:
98
+ * "success" — run completed cleanly (status "complete")
99
+ * "incomplete" — run was cut short (status "incomplete" or "max-iterations" or "dry-run")
100
+ *
101
+ * The caller is responsible for "failure" — that requires inspecting GitHub
102
+ * state (whether the issue reached workflow:merged). For the purposes of the
103
+ * batch driver, a clean runner completion is classified as "success"; the
104
+ * scorecard consumer or a post-run hook can downgrade to "failure" by
105
+ * inspecting the issue label state.
106
+ *
107
+ * @param {{ status: string, stopReason?: string }} result
108
+ * @returns {"success" | "incomplete"}
109
+ */
110
+ export function classifyRunnerResult(result) {
111
+ if (result.status === "complete") return "success";
112
+ return "incomplete";
113
+ }
114
+
115
+ /**
116
+ * Construct a per-run result object conforming to docs/spec/eval-run-result.md.
117
+ *
118
+ * @param {number} issue - GitHub issue number.
119
+ * @param {"success"|"failure"|"incomplete"|"error"} status
120
+ * @param {object} [opts]
121
+ * @param {number} [opts.wallClockMs] - Wall-clock duration (ms).
122
+ * @param {number} [opts.interventionCount] - Human intervention count.
123
+ * @param {number|null} [opts.cost] - Token cost (null until #1255).
124
+ * @param {number} [opts.iterations] - Tool-loop iteration count.
125
+ * @param {string} [opts.stopReason] - Anthropic stop_reason or "max_iterations".
126
+ * @param {string|null} [opts.error] - Error message when status === "error".
127
+ * @param {string} [opts.specVersion] - Package version at run time.
128
+ * @param {string} [opts.model] - Model used.
129
+ * @param {string} [opts.runAt] - ISO-8601 timestamp.
130
+ * @returns {object}
131
+ */
132
+ export function makeRunResult(issue, status, opts = {}) {
133
+ return {
134
+ issue,
135
+ status,
136
+ wallClockMs: opts.wallClockMs ?? 0,
137
+ interventionCount: opts.interventionCount ?? 0,
138
+ cost: opts.cost !== undefined ? opts.cost : null,
139
+ iterations: opts.iterations ?? null,
140
+ stopReason: opts.stopReason ?? null,
141
+ error: opts.error ?? null,
142
+ specVersion: opts.specVersion ?? null,
143
+ model: opts.model ?? null,
144
+ runAt: opts.runAt ?? new Date().toISOString(),
145
+ };
146
+ }
147
+
148
+ // ---------------------------------------------------------------------------
149
+ // Output writing
150
+ // ---------------------------------------------------------------------------
151
+
152
+ /**
153
+ * Serialize run results to a JSON file.
154
+ * Creates parent directories as needed.
155
+ *
156
+ * @param {string} outputPath - Absolute or relative path to write.
157
+ * @param {object[]} results - Array of per-run result objects.
158
+ * @param {object} [meta] - Top-level metadata (corpus_version, spec_version, model, corpus_size).
159
+ * @returns {void}
160
+ */
161
+ export function writeResults(outputPath, results, meta = {}) {
162
+ const resolved = resolve(outputPath);
163
+ mkdirSync(dirname(resolved), { recursive: true });
164
+ const payload = {
165
+ corpus_version: meta.corpus_version ?? null,
166
+ corpus_size: meta.corpus_size ?? null,
167
+ spec_version: meta.spec_version ?? null,
168
+ model: meta.model ?? null,
169
+ generated_at: new Date().toISOString(),
170
+ runs: results,
171
+ };
172
+ writeFileSync(resolved, JSON.stringify(payload, null, 2) + "\n", "utf-8");
173
+ }
174
+
175
+ // ---------------------------------------------------------------------------
176
+ // Batch orchestrator
177
+ // ---------------------------------------------------------------------------
178
+
179
+ /**
180
+ * Run the full corpus headlessly.
181
+ *
182
+ * For each issue number in `corpus.issues`, invokes runCommand() against the
183
+ * /work-on spec in a shared working directory, records a per-run result, and
184
+ * returns the full array of results.
185
+ *
186
+ * Does NOT write output itself — callers should call writeResults() on the
187
+ * returned array. The CLI entry point below does this automatically.
188
+ *
189
+ * @param {object} opts
190
+ * @param {string} opts.commandsDir - Absolute path to commands/.
191
+ * @param {{ corpus_version: string|null, issues: number[] }} opts.corpus
192
+ * @param {string} [opts.cwd] - Working directory for runner (default process.cwd()).
193
+ * @param {string} [opts.apiKey] - Anthropic API key.
194
+ * @param {string} [opts.model] - Model id.
195
+ * @param {number} [opts.maxIterations] - Per-issue tool-loop bound.
196
+ * @param {boolean} [opts.dryRun] - If true, no API calls are made.
197
+ * @param {{log: Function, error?: Function}} [opts.logger]
198
+ * @returns {Promise<object[]>} - Array of per-run result objects.
199
+ */
200
+ export async function runCorpus(opts = {}) {
201
+ const {
202
+ commandsDir,
203
+ corpus,
204
+ cwd = process.cwd(),
205
+ apiKey = process.env.ANTHROPIC_API_KEY,
206
+ model,
207
+ maxIterations,
208
+ dryRun = false,
209
+ logger = console,
210
+ } = opts;
211
+
212
+ // Lazy import of runCommand so this module stays import-safe without the SDK.
213
+ const { runCommand } = await import("./runner.mjs");
214
+
215
+ const results = [];
216
+
217
+ for (const issue of corpus.issues) {
218
+ logger.log(`[batch-runner] Starting issue #${issue}…`);
219
+ const startMs = Date.now();
220
+ let result;
221
+ try {
222
+ result = await runCommand({
223
+ commandsDir,
224
+ commandName: "work-on",
225
+ args: [String(issue)],
226
+ cwd,
227
+ apiKey,
228
+ ...(model ? { model } : {}),
229
+ ...(maxIterations !== undefined ? { maxIterations } : {}),
230
+ dryRun,
231
+ logger,
232
+ });
233
+ const wallClockMs = Date.now() - startMs;
234
+ const status = classifyRunnerResult(result);
235
+ results.push(
236
+ makeRunResult(issue, status, {
237
+ wallClockMs,
238
+ interventionCount: 0,
239
+ cost: null, // wired in once #1255 lands
240
+ iterations: result.iterations ?? null,
241
+ stopReason: result.stopReason ?? null,
242
+ model: model ?? result.model ?? null,
243
+ runAt: new Date().toISOString(),
244
+ }),
245
+ );
246
+ logger.log(`[batch-runner] Issue #${issue} → ${status} (${wallClockMs}ms)`);
247
+ } catch (e) {
248
+ const wallClockMs = Date.now() - startMs;
249
+ results.push(
250
+ makeRunResult(issue, "error", {
251
+ wallClockMs,
252
+ interventionCount: 0,
253
+ cost: null,
254
+ error: e.message,
255
+ model: model ?? null,
256
+ runAt: new Date().toISOString(),
257
+ }),
258
+ );
259
+ logger.error
260
+ ? logger.error(`[batch-runner] Issue #${issue} → error: ${e.message}`)
261
+ : logger.log(`[batch-runner] Issue #${issue} → error: ${e.message}`);
262
+ }
263
+ }
264
+
265
+ return results;
266
+ }
267
+
268
+ // ---------------------------------------------------------------------------
269
+ // CLI entry point
270
+ // ---------------------------------------------------------------------------
271
+
272
+ async function main() {
273
+ const [, , corpusArg, outputArg, ...flags] = process.argv;
274
+ const dryRun = flags.includes("--dry-run");
275
+
276
+ if (!corpusArg) {
277
+ process.stderr.write(
278
+ "Usage: node bin/batch-runner.mjs <corpus.json> [output.json] [--dry-run]\n",
279
+ );
280
+ process.exit(1);
281
+ }
282
+
283
+ // Resolve paths relative to cwd of the calling shell, not __dirname.
284
+ const corpusPath = resolve(corpusArg);
285
+ const outputPath = outputArg ? resolve(outputArg) : resolve("eval-results.json");
286
+
287
+ let corpus;
288
+ try {
289
+ corpus = loadCorpus(corpusPath);
290
+ } catch (e) {
291
+ process.stderr.write(`ERROR: ${e.message}\n`);
292
+ process.exit(1);
293
+ }
294
+
295
+ // commandsDir: sibling of bin/ at the repo root.
296
+ const { fileURLToPath } = await import("node:url");
297
+ const { join } = await import("node:path");
298
+ const binDir = dirname(fileURLToPath(import.meta.url));
299
+ const commandsDir = join(binDir, "..", "commands");
300
+
301
+ // Read package version for specVersion field.
302
+ let specVersion = null;
303
+ try {
304
+ const pkgPath = join(binDir, "..", "package.json");
305
+ specVersion = JSON.parse(readFileSync(pkgPath, "utf-8")).version ?? null;
306
+ } catch {
307
+ // non-fatal — specVersion stays null
308
+ }
309
+
310
+ console.log(
311
+ `[batch-runner] Running ${corpus.issues.length} issue(s) from corpus "${corpus.corpus_version ?? "(no version)"}"${dryRun ? " (dry-run)" : ""}`,
312
+ );
313
+
314
+ const results = await runCorpus({
315
+ commandsDir,
316
+ corpus,
317
+ dryRun,
318
+ logger: console,
319
+ });
320
+
321
+ const meta = {
322
+ corpus_version: corpus.corpus_version,
323
+ corpus_size: corpus.issues.length,
324
+ spec_version: specVersion,
325
+ model: results.find((r) => r.model)?.model ?? null,
326
+ };
327
+
328
+ writeResults(outputPath, results, meta);
329
+ console.log(`[batch-runner] Results written to ${outputPath}`);
330
+
331
+ const nSuccess = results.filter((r) => r.status === "success").length;
332
+ const nTotal = results.length;
333
+ console.log(`[batch-runner] ${nSuccess}/${nTotal} runs succeeded`);
334
+ }
335
+
336
+ import { fileURLToPath } from "node:url";
337
+ if (process.argv[1] && fileURLToPath(import.meta.url) === process.argv[1]) {
338
+ main();
339
+ }
package/bin/demo.mjs CHANGED
@@ -61,6 +61,9 @@ export const DEFAULT_DEMO_DIRNAME = "forgedock-demo";
61
61
  * --repo <slug> | --repo=<slug> Override the demo repo (owner/name).
62
62
  * --open Launch Claude Code in the demo dir when found.
63
63
  * --no-open Explicitly disable launching (the default).
64
+ * --run Run the seeded issue through /work-on automatically
65
+ * (requires ANTHROPIC_API_KEY; falls back to print-only
66
+ * instructions when the key is absent).
64
67
  * --help | -h Show usage.
65
68
  *
66
69
  * The first non-flag argument is treated as the target directory (positional
@@ -68,8 +71,8 @@ export const DEFAULT_DEMO_DIRNAME = "forgedock-demo";
68
71
  *
69
72
  * @param {string[]} args
70
73
  * @returns {{positional: string|undefined, dir: string|undefined,
71
- * repo: string|undefined, open: boolean, help: boolean,
72
- * error: string|null}}
74
+ * repo: string|undefined, open: boolean, run: boolean,
75
+ * help: boolean, error: string|null}}
73
76
  */
74
77
  export function parseDemoArgs(args = []) {
75
78
  const out = {
@@ -77,6 +80,7 @@ export function parseDemoArgs(args = []) {
77
80
  dir: undefined,
78
81
  repo: undefined,
79
82
  open: false,
83
+ run: false,
80
84
  help: false,
81
85
  error: null,
82
86
  };
@@ -88,6 +92,8 @@ export function parseDemoArgs(args = []) {
88
92
  out.open = true;
89
93
  } else if (a === "--no-open") {
90
94
  out.open = false;
95
+ } else if (a === "--run") {
96
+ out.run = true;
91
97
  } else if (a === "--dir") {
92
98
  out.dir = args[++i];
93
99
  if (out.dir === undefined) out.error = "--dir requires a path argument";
@@ -145,7 +151,7 @@ function isEmptyDir(dir, fsx) {
145
151
 
146
152
  export function demoUsage() {
147
153
  return [
148
- "Usage: forgedock demo [dir] [--dir <path>] [--repo <owner/name>] [--open]",
154
+ "Usage: forgedock demo [dir] [--dir <path>] [--repo <owner/name>] [--open] [--run]",
149
155
  "",
150
156
  "Stand up a runnable ForgeDock demo at a predictable location (default",
151
157
  "~/forgedock-demo) and print the exact next steps. No configuration needed.",
@@ -155,6 +161,9 @@ export function demoUsage() {
155
161
  " --repo <slug> Demo repo to clone (default: RapierCraftStudios/forgedock-demo).",
156
162
  " --open Launch Claude Code in the demo directory if available.",
157
163
  " --no-open Do not launch Claude Code (default).",
164
+ " --run Automatically run the seeded issue through /work-on to a",
165
+ " merged PR (requires ANTHROPIC_API_KEY; falls back to",
166
+ " print-only instructions when the key is absent).",
158
167
  " -h, --help Show this help.",
159
168
  ].join("\n");
160
169
  }
@@ -186,10 +195,10 @@ export function demoForgeYaml(repoSlug) {
186
195
 
187
196
  /**
188
197
  * Render the success / next-steps card.
189
- * @param {{target: string, source: string, claudeAvailable: boolean}} ctx
198
+ * @param {{target: string, source: string, claudeAvailable: boolean, apiKeyAvailable?: boolean}} ctx
190
199
  * @returns {string}
191
200
  */
192
- export function renderNextSteps({ target, source, claudeAvailable }) {
201
+ export function renderNextSteps({ target, source, claudeAvailable, apiKeyAvailable = false }) {
193
202
  const sourceNote = {
194
203
  cloned: "Cloned the live demo repo.",
195
204
  pulled: "Demo already present — pulled the latest.",
@@ -214,6 +223,15 @@ export function renderNextSteps({ target, source, claudeAvailable }) {
214
223
  ? " claude # open Claude Code here (or re-run with --open)"
215
224
  : " # then open Claude Code in this directory",
216
225
  ];
226
+
227
+ if (!apiKeyAvailable) {
228
+ lines.push(
229
+ "",
230
+ "Tip: set ANTHROPIC_API_KEY and re-run with --run to execute the pipeline",
231
+ "automatically and watch a PR merge without opening Claude Code.",
232
+ );
233
+ }
234
+
217
235
  return lines.join("\n");
218
236
  }
219
237
 
@@ -261,11 +279,12 @@ const defaultLogger = {
261
279
  * Stand up the demo.
262
280
  *
263
281
  * @param {object} opts
264
- * @param {string} opts.forgeHome - Package root (for bundled scaffold).
282
+ * @param {string} opts.forgeHome - Package root (for bundled scaffold and commands/).
265
283
  * @param {string[]} [opts.args] - CLI args after `demo`.
266
284
  * @param {string} [opts.home] - Home dir (default os.homedir()).
267
285
  * @param {string} [opts.cwd] - Working dir (default process.cwd()).
268
286
  * @param {string} [opts.demoRepo] - Override demo repo slug.
287
+ * @param {string} [opts.apiKey] - Anthropic API key (default ANTHROPIC_API_KEY env).
269
288
  * @param {Function} [opts.exec] - (cmd, args, {cwd}) => {status,...}.
270
289
  * @param {object} [opts.fsx] - Filesystem facade.
271
290
  * @param {Function} [opts.commandExists] - (bin) => boolean.
@@ -279,6 +298,7 @@ export async function runDemo(opts = {}) {
279
298
  args = [],
280
299
  home = homedir(),
281
300
  cwd = process.cwd(),
301
+ apiKey = process.env.ANTHROPIC_API_KEY,
282
302
  exec = defaultExec,
283
303
  fsx = defaultFsx,
284
304
  commandExists = defaultCommandExists,
@@ -371,7 +391,72 @@ export async function runDemo(opts = {}) {
371
391
  }
372
392
 
373
393
  const claudeAvailable = commandExists("claude");
374
- logger.log(renderNextSteps({ target, source, claudeAvailable }));
394
+ const apiKeyAvailable = Boolean(apiKey);
395
+
396
+ // --run: drive the seeded issue through /work-on automatically.
397
+ // Requires ANTHROPIC_API_KEY; falls back to print-only instructions gracefully.
398
+ if (parsed.run) {
399
+ if (!apiKeyAvailable) {
400
+ logger.log(
401
+ "\nANTHROPIC_API_KEY is not set — cannot run the pipeline automatically.\n" +
402
+ "Export your key and re-run with --run, or open Claude Code manually:\n",
403
+ );
404
+ logger.log(renderNextSteps({ target, source, claudeAvailable, apiKeyAvailable: false }));
405
+ return { status: "ok", target, source };
406
+ }
407
+
408
+ const commandsDir = forgeHome ? join(forgeHome, "commands") : null;
409
+ if (!commandsDir || !fsx.existsSync(commandsDir)) {
410
+ logger.error(
411
+ "Could not locate the ForgeDock commands/ directory. " +
412
+ "Falling back to manual instructions.",
413
+ );
414
+ logger.log(renderNextSteps({ target, source, claudeAvailable, apiKeyAvailable }));
415
+ return { status: "ok", target, source };
416
+ }
417
+
418
+ logger.log(
419
+ "\nRunning /work-on 1 in the demo repo — watch the full pipeline execute…\n",
420
+ );
421
+
422
+ let runCommand;
423
+ try {
424
+ ({ runCommand } = await import("./runner.mjs"));
425
+ } catch {
426
+ logger.error(
427
+ "Could not load bin/runner.mjs. Falling back to manual instructions.",
428
+ );
429
+ logger.log(renderNextSteps({ target, source, claudeAvailable, apiKeyAvailable }));
430
+ return { status: "ok", target, source };
431
+ }
432
+
433
+ try {
434
+ const result = await runCommand({
435
+ commandsDir,
436
+ commandName: "work-on",
437
+ args: ["1"],
438
+ cwd: target,
439
+ apiKey,
440
+ logger: { log: logger.log, error: logger.error },
441
+ });
442
+ logger.log(`\nDemo pipeline finished with status: ${result.status}`);
443
+ return { status: result.status === "ok" ? "ok" : result.status, target, source };
444
+ } catch (err) {
445
+ if (err.code === "NO_SDK") {
446
+ logger.error(
447
+ "\n@anthropic-ai/sdk is not installed. Install it with:\n" +
448
+ " npm install @anthropic-ai/sdk\n" +
449
+ "Then re-run with --run, or open Claude Code manually:\n",
450
+ );
451
+ } else {
452
+ logger.error(`\nPipeline error: ${err.message}\nFalling back to manual instructions.\n`);
453
+ }
454
+ logger.log(renderNextSteps({ target, source, claudeAvailable, apiKeyAvailable }));
455
+ return { status: "ok", target, source };
456
+ }
457
+ }
458
+
459
+ logger.log(renderNextSteps({ target, source, claudeAvailable, apiKeyAvailable }));
375
460
 
376
461
  if (parsed.open) {
377
462
  if (claudeAvailable) {