forgedock 1.27.0 → 1.29.0

package/README.md

@@ -268,7 +268,9 @@ Then just open Claude Code and run `/work-on <issue>`.
 
 Commands then appear as `/forgedock:work-on`, etc. You still run `npx forgedock init` to generate `forge.yaml`.
 
-**Headless / CI:** the pipeline also runs outside Claude Code. `npx forgedock run work-on <issue> --dry-run` previews the assembled prompt and tool plan. `npx forgedock run` picks an execution backend automatically (`--backend auto`, the default): if the Claude Code CLI (`claude`) is installed and already authenticated (Pro/Max subscription or a CLI-managed key), it drives the command through that — **no separate `ANTHROPIC_API_KEY` required**. Otherwise it falls back to the Anthropic API directly (`ANTHROPIC_API_KEY` required). Force either path explicitly with `--backend cli` / `--backend api` (or `FORGEDOCK_BACKEND=cli|api`) — the API backend is what CI environments without an interactive `claude` login should use. `npx forgedock run-issue <issue>` executes the same command specs on the durable engine (event-sourced run log, leases, crash-safe resume).
+**Headless / CI:** the pipeline also runs outside Claude Code. `npx forgedock run work-on <issue> --dry-run` previews the assembled prompt and tool plan. `npx forgedock run` picks an execution backend automatically (`--backend auto`, the default): if the Claude Code CLI (`claude`) is installed and already authenticated (Pro/Max subscription or a CLI-managed key), it drives the command through that — **no separate `ANTHROPIC_API_KEY` required**. It never falls back to the paid Anthropic API implicitly: without a CLI it fails with `CLI_BACKEND_UNAVAILABLE` unless the owner opted in (forge.yaml `execution.agent_backend: api` plus `FORGEDOCK_ALLOW_API_BACKEND=1`; see [CONFIG.md](docs/CONFIG.md#execution-optional)). Force either path explicitly with `--backend cli` / `--backend api` (or `FORGEDOCK_BACKEND=cli|api`) — the API backend is what CI environments without an interactive `claude` login should use. `npx forgedock run-issue <issue>` executes the same command specs on the durable engine (event-sourced run log, leases, crash-safe resume).
+
+**Brand-new project?** `/bootstrap "<idea>"` (or `npx forgedock bootstrap "<idea>" --dry-run` to preview) turns an idea into a validated, milestone-ordered GitHub backlog that `/orchestrate` can build. See [Start a New Project with ForgeDock](docs/site/start-a-new-project.md).
 
 **Explicit install command:**
 
@@ -363,6 +365,7 @@ Using ForgeDock in your pipeline? Add the badge — each one is a backlink and a
 
 - [GitHub Is Already Your Agents' Memory](docs/site/github-is-the-memory.md) — the canonical argument: why GitHub is the right place for agent memory, how FORGE annotations make it machine-readable, and how to adopt the protocol without ForgeDock
 - [Getting Started in 5 Minutes](docs/site/getting-started.md)
+- [Start a New Project with ForgeDock](docs/site/start-a-new-project.md) — idea → `/bootstrap` → backlog → `/orchestrate`, with a worked example
 - [How the Knowledge Graph Works](docs/site/how-it-works.md)
 - [What Are Those FORGE Comments?](docs/site/annotations-explained.md) — 2-minute explainer for annotations you meet in the wild
 - [FORGE Annotation Protocol](docs/spec/forge-protocol-v1.md) — the open standard for AI context passing (CC-BY-4.0)

package/bin/bootstrap.mjs

@@ -0,0 +1,55 @@
+// SPDX-FileCopyrightText: Copyright (c) RapierCraft Studios
+// SPDX-License-Identifier: AGPL-3.0-or-later
+/**
+ * `forgedock bootstrap` (#4177, part of #4155): the headless `/bootstrap`.
+ *   - Plan mode, `--plan <plan.json> [--dry-run] [--repo] [--config] [--base] [--json] [--cost-per-issue <usd>]`:
+ *     scripts/bootstrap-create.mjs validates and previews (dry run, no GitHub access) or creates the backlog. No LLM
+ *     call. Re-running the same command resumes from GitHub state: existing spike, milestones, branches, issues and
+ *     marker comments are reused, so no local state is needed.
+ *   - Idea mode, `<free text | docs path> [--dry-run] [--spike <path>] [--model <id>] [--backend <cli|api|auto>]`:
+ *     runs commands/bootstrap.md headlessly through runner.mjs runCommand. `--dry-run` and `--spike` are spec
+ *     arguments; `--dry-run` is never the runner's own prompt-preview dryRun.
+ * Exit: 0 ok; 1 failed run (or invalid plan); 2 usage; 3 GitHub error (plan mode, re-run to resume).
+ */
+
+const USAGE = "usage: forgedock bootstrap <idea | docs path> [--dry-run] [--spike <path>] [--model <id>] [--backend <cli|api|auto>]\n" +
+  "       forgedock bootstrap --plan <plan.json> [--dry-run] [--repo OWNER/NAME] [--config forge.yaml] [--base <branch>] [--json] [--cost-per-issue <usd>]\n";
+
+/**
+ * @param {string[]} argv  arguments after `bootstrap`
+ * @param {{runCommand?: Function, createMain?: Function, commandsDir?: string, cwd?: string, out?: {write}, err?: {write}}} deps
+ * @returns {Promise<number>} exit code
+ */
+export async function runBootstrap(argv = [], deps = {}) {
+  const { commandsDir, cwd = process.cwd(), out = process.stdout, err = process.stderr } = deps;
+  const needValue = (i) => i + 1 < argv.length && !argv[i + 1].startsWith("--");
+  const planIdx = argv.indexOf("--plan");
+  if (planIdx !== -1) {
+    if (!needValue(planIdx)) { err.write(`forgedock bootstrap: --plan requires a value\n${USAGE}`); return 2; }
+    const createMain = deps.createMain ?? (await import("../scripts/bootstrap-create.mjs")).main;
+    const rest = [...argv.slice(0, planIdx), ...argv.slice(planIdx + 2)];
+    return createMain([argv[planIdx + 1], ...rest], { out, err });
+  }
+
+  const args = [];
+  let model, backend;
+  for (let i = 0; i < argv.length; i++) {
+    const a = argv[i];
+    if (a === "--dry-run") args.push(a);
+    else if (a === "--spike" || a === "--model" || a === "--backend") {
+      if (!needValue(i)) { err.write(`forgedock bootstrap: ${a} requires a value\n${USAGE}`); return 2; }
+      const v = argv[++i];
+      if (a === "--spike") args.push(a, v);
+      else if (a === "--model") model = v;
+      else backend = v;
+    } else if (a.startsWith("--")) { err.write(`forgedock bootstrap: unknown argument ${a}\n${USAGE}`); return 2; }
+    else args.push(a);
+  }
+  if (!args.some((a, i) => !a.startsWith("--") && args[i - 1] !== "--spike")) { err.write(USAGE); return 2; }
+  const runCommand = deps.runCommand ?? (await import("./runner.mjs")).runCommand;
+  const result = await runCommand({
+    commandsDir, commandName: "bootstrap", args, cwd, dryRun: false,
+    ...(model ? { model } : {}), ...(backend ? { backend } : {}),
+  });
+  return result && (result.status === "max-iterations" || result.status === "incomplete") ? 1 : 0;
+}

package/bin/engine/outcome-codes.mjs

@@ -47,6 +47,9 @@ export const OUTCOME_CODES = Object.freeze({
   INFRA_CLI_SPAWN_TRANSIENT: "infra.cli_spawn_transient",
   CLI_TIMEOUT: "cli.timeout",
   CLI_SPAWN_FAILED: "cli.spawn_failed", // forge#4129: deterministic spawn failure (E2BIG, ENOENT, EACCES, ...)
+  ENV_SETUP_FAILED: "env.setup_failed", // forge#4163: forge.yaml env.setup failed before the builder; an environment error, not a build failure
+  SANDBOX_START_FAILED: "sandbox.start_failed", // forge#4254: a lane sandbox could not start; needs-human, not relaunched
+  SANDBOX_CAPACITY: "sandbox.capacity", // forge#4254: the sandbox provider had no capacity; deferred with backoff
   LEASE_HELD: "lease.held",
   DEPENDENCY_OPEN: "dependency.open",
   DEPENDENCY_UNREADABLE: "dependency.unreadable",
@@ -77,8 +80,14 @@ export const OUTCOME_CODES = Object.freeze({
   CONFORMANCE_PRESERVE_VIOLATED: "conformance.preserve_violated",
   CONFORMANCE_REGRESSION: "conformance.regression",
   STRUCTURE_UNDECLARED_EDGE: "structure.undeclared_edge", // forge#4159: a workspace package.json edge the contract does not name
+  DOCS_STALE: "docs.stale", // the target repo's devdocs freshness rule still fails after one builder resume
+  MAINTENANCE_HOTSPOT_GROWTH: "maintenance.hotspot_growth", // forge#4195: an over-threshold file grew past growth_lines without hotspot-exempt
   FOLLOWUP_UNAVAILABLE: "followup.unavailable",
   MERGE_HEAD_MISMATCH: "merge.head_mismatch",
+  // refactor:split lane (forge#4211): deterministic refusals and the move proof; every one leaves no edit behind.
+  SPLIT_UNSUPPORTED: "split.unsupported", // no codemod or index tool for the target language, or libcst or ts-morph missing; refused before any edit
+  SPLIT_PLAN_INVALID: "split.plan_invalid", // missing target/test command, unparseable grouping reply, or a plan symbol-move refuses
+  SPLIT_VERIFY_FAILED: "split.verify_failed", // refactor-verify found a logic diff, missing symbol, changed counts or new cycle; branch reset
   // Non-failure terminals, so a RUN_TERMINAL for a healthy run is not counted as a gap.
   COMPLETED: "completed",
   AWAITING_MERGE: "awaiting_merge",
@@ -133,6 +142,7 @@ export function codeForRunnerError(err) {
       return OUTCOME_CODES.CLI_SPAWN_FAILED;
     case "NO_API_KEY":
     case "NO_SDK":
+    case "CLI_BACKEND_UNAVAILABLE":
     case "CLI_BACKEND_FAILED":
     case "PHASE_ARGS_INVALID":
     case "CLI_MAX_TURNS":
@@ -165,3 +175,32 @@ export function codeForTerminalReason(reason) {
     default: return UNCLASSIFIED;
   }
 }
+
+/**
+ * Batch recovery classes of a lane result (forge#4267). The batch runner recovers `recoverable` rows once
+ * (`orchestration.recovery.max_attempts`) and re-queues them; only `human` rows are escalations.
+ */
+export const RECOVERABLE_CODES = Object.freeze([OUTCOME_CODES.CI_RED, OUTCOME_CODES.BASE_CONFLICT, OUTCOME_CODES.BUILDER_BLOCK_UNTYPED, OUTCOME_CODES.CONTRACT_INVALID]);
+/** `builder.blocked` is only reached by a typed human block: credentials, spend, promotion, customer, product-decision. */
+export const HUMAN_CODES = Object.freeze([OUTCOME_CODES.BUILDER_BLOCKED]);
+
+/**
+ * `recoverable` | `human` | `terminal` for a lane result row ({outcome|terminalReason, code, issueEdited?, conflicting?}).
+ * Recoverable: needs-human `ci.red`, `base.conflict`, `builder.block_untyped`, `contract.invalid` after a mid-run issue
+ * edit (`issueEdited`), and `ci-pending` on a conflicting PR (`conflicting`). Human: `builder.blocked`. Anything else
+ * (another code, or not a needs-human/ci-pending row) is terminal. Pure, never throws.
+ */
+export function classifyNeedsHuman(row) {
+  if (!row || typeof row !== "object") return "terminal";
+  const outcome = row.outcome ?? row.terminalReason;
+  if (outcome === "ci-pending") return row.conflicting === true ? "recoverable" : "terminal";
+  if (outcome !== "needs-human") return "terminal";
+  if (HUMAN_CODES.includes(row.code)) return "human";
+  if (row.code === OUTCOME_CODES.CONTRACT_INVALID) return row.issueEdited === true ? "recoverable" : "terminal";
+  return RECOVERABLE_CODES.includes(row.code) ? "recoverable" : "terminal";
+}
+
+/** The code a recovery lane works on: a conflicting `ci-pending` is recovered as a base conflict. */
+export function recoveryCodeOf(row) {
+  return (row?.outcome ?? row?.terminalReason) === "ci-pending" ? OUTCOME_CODES.BASE_CONFLICT : normalizeCode(row?.code);
+}

package/bin/engine/telemetry-seam.mjs

@@ -3,9 +3,13 @@
 /**
  * forge#3835: the engine's `emitTelemetry` seam writer. Maps the run-log shaped records the engine hands to the
  * seam onto FORGE:TELEMETRY events so `forgedock stats` can rebuild Cost, Time and outcome codes from GitHub alone:
- *   PHASE_ATTEMPT_USAGE -> `phase_attempt` (attempt, model, turns, duration_ms, cost_usd, *_tokens)
- *   RUN_TERMINAL        -> `run_terminal`  (reason, code, wall_ms)
+ *   PHASE_ATTEMPT_USAGE -> `phase_attempt` (attempt, model, backend, turns, duration_ms, cost_usd, *_tokens)
+ *   RUN_TERMINAL        -> `run_terminal`  (reason, code, wall_ms; lean adds milestone, conformance_mode, shadow_would and the pr)
+ *   SANDBOX_USAGE       -> `sandbox_usage` (provider, sandbox_id, minutes, cost_usd, batch; one per settled remote lane, forge#4255)
  *   STRUCTURE_DELTA     -> `structure_delta` (edges_added/removed, import_files_added/removed, boundary_changed/loosened 0|1; forge#4159)
+ *                          lean's merge-time record (forge#4165) adds milestone, the pr and string lists: deps_added and
+ *                          new_edges / removed_edges as "from:to" comma-joined, top_growth "file+N", decisions "K1:lint",
+ *                          plus decisions_doc/lint/precedent/spec counts. Values are capped at 200 chars by the emitter.
  * Other records are ignored. The returned function NEVER throws and honours FORGE_TELEMETRY=off. Telemetry values
  * are scalars (the emitter stringifies objects), so the `usage` object is flattened here; absent values are omitted.
  */
@@ -24,24 +28,48 @@ const scalar = (v) => (typeof v === "number" ? Number.isFinite(v) : typeof v ===
 const compact = (o) => Object.fromEntries(Object.entries(o).filter(([, v]) => scalar(v)));
 const tok = (u, k) => (u && typeof u === "object" && Number.isFinite(u[k]) ? u[k] : undefined);
 
-/** Pure mapping, exported for tests: returns { event, payload, phase } or null when the record is not mapped. */
+const SOURCES = ["doc", "lint", "precedent", "spec"];
+const prOf = (rec) => (Number.isSafeInteger(rec.pr) && rec.pr > 0 ? { pr: rec.pr } : {});
+/** "a:b,c:d" from [{from,to}] (forge#4165); undefined for a non-array so compact() drops it. An empty list is "". */
+const pairs = (v) => (Array.isArray(v) ? v.map((e) => `${e?.from}:${e?.to}`).join(",") : undefined);
+const nonEmpty = (v) => (v === "" ? "none" : v);
+/** Lean's forge#4165 extras of a STRUCTURE_DELTA record; {} for the implement-time counts-only shape. */
+function structureLists(rec) {
+  if (!Array.isArray(rec.decisions) && !Array.isArray(rec.new_edges) && !Array.isArray(rec.deps_added)) return {};
+  const dec = Array.isArray(rec.decisions) ? rec.decisions : [];
+  const out = {
+    milestone: rec.milestone, deps_added: nonEmpty(pairs(rec.deps_added)), new_edges: nonEmpty(pairs(rec.new_edges)), removed_edges: nonEmpty(pairs(rec.removed_edges)),
+    top_growth: Array.isArray(rec.top_growth) ? nonEmpty(rec.top_growth.map((g) => `${g?.file}+${g?.lines}`).join(",")) : undefined,
+    decisions: nonEmpty(dec.map((k) => `${k?.id}:${k?.source ?? "none"}`).join(",")),
+  };
+  for (const src of SOURCES) out[`decisions_${src}`] = dec.filter((k) => k?.source === src).length;
+  return out;
+}
+
+/** Pure mapping, exported for tests: returns { event, payload, phase, pr? } or null when the record is not mapped. */
 export function mapRecord(rec, wallMs) {
   if (!rec || typeof rec !== "object") return null;
   if (rec.event === "PHASE_ATTEMPT_USAGE") {
     return { event: "phase_attempt", phase: rec.phase, payload: compact({
-      attempt: rec.attempt, model: rec.model, stages: rec.stages, turns: rec.turns, duration_ms: rec.duration_ms,
+      attempt: rec.attempt, model: rec.model, backend: rec.backend ?? undefined, stages: rec.stages, turns: rec.turns, duration_ms: rec.duration_ms,
       cost_usd: Number.isFinite(rec.cost_usd) ? Math.round(rec.cost_usd * 1e6) / 1e6 : undefined,
       input_tokens: tok(rec.usage, "input_tokens"), output_tokens: tok(rec.usage, "output_tokens"),
       cache_read_tokens: tok(rec.usage, "cache_read_input_tokens"), cache_creation_tokens: tok(rec.usage, "cache_creation_input_tokens"),
     }) };
   }
   if (rec.event === "RUN_TERMINAL") {
-    return { event: "run_terminal", phase: null, payload: compact({ reason: rec.reason, code: rec.code, wall_ms: wallMs }) };
+    return { event: "run_terminal", phase: null, ...prOf(rec), payload: compact({ reason: rec.reason, code: rec.code, wall_ms: wallMs,
+      milestone: rec.milestone, conformance_mode: rec.conformance_mode, shadow_would: rec.shadow_would }) };
+  }
+  if (rec.event === "SANDBOX_USAGE") {
+    return { event: "sandbox_usage", phase: null, payload: compact({ provider: rec.provider, sandbox_id: rec.sandbox_id, minutes: rec.minutes,
+      cost_usd: Number.isFinite(rec.cost_usd) ? Math.round(rec.cost_usd * 1e6) / 1e6 : undefined, batch: rec.batch }) };
   }
   if (rec.event === "STRUCTURE_DELTA") {
     const bit = (v) => (typeof v === "boolean" ? Number(v) : v);
-    return { event: "structure_delta", phase: rec.phase ?? "implement", payload: compact({ edges_added: rec.edges_added, edges_removed: rec.edges_removed,
-      import_files_added: rec.import_files_added, import_files_removed: rec.import_files_removed, boundary_changed: bit(rec.boundary_changed), boundary_loosened: bit(rec.boundary_loosened) }) };
+    return { event: "structure_delta", phase: rec.phase ?? "implement", ...prOf(rec), payload: compact({ edges_added: rec.edges_added, edges_removed: rec.edges_removed,
+      import_files_added: rec.import_files_added, import_files_removed: rec.import_files_removed, boundary_changed: bit(rec.boundary_changed), boundary_loosened: bit(rec.boundary_loosened),
+      ...structureLists(rec) }) };
   }
   if (rec.event === "CI_PENDING_WAIT") {
     return { event: "ci_pending_wait", phase: rec.phase ?? "review", payload: compact({ pr: rec.pr, minutes_waited: rec.minutes_waited, result: rec.result, polls: rec.polls, rate_limit_sleeps: rec.rate_limit_sleeps }) };
@@ -65,7 +93,8 @@ export function makeEmitTelemetry({ repo = null, emitFn = emit, now = Date.now,
       if (!tried) { tried = true; resolved = resolveRepo() || null; }
       if (!resolved) return; // no destination: the Node emitter would drop it anyway
       // ctx carries repo/issue/run/phase: emit() drops those reserved keys from the payload.
-      emitFn(mapped.event, mapped.payload, { repo: resolved, issue: rec.issue, run: `r_${rec.issue}_${Math.floor(startedAt / 1000)}`, phase: mapped.phase ?? undefined });
+      emitFn(mapped.event, mapped.payload, { repo: resolved, issue: rec.issue, run: `r_${rec.issue}_${Math.floor(startedAt / 1000)}`, phase: mapped.phase ?? undefined,
+        ...(Number.isSafeInteger(mapped.pr) ? { pr: mapped.pr } : {}) });
     } catch { /* telemetry never fails a run */ }
   };
 }

package/bin/engine/telemetry-seam.test.mjs

@@ -14,6 +14,12 @@ function harness(over = {}) {
 }
 
 describe("engine telemetry seam", () => {
+  it("forge#4257: phase_attempt carries the backend, omitted when unreported", () => {
+    assert.equal(mapRecord({ event: "PHASE_ATTEMPT_USAGE", phase: "build", attempt: 1, backend: "api" }, 0).payload.backend, "api");
+    assert.equal("backend" in mapRecord({ event: "PHASE_ATTEMPT_USAGE", phase: "build", attempt: 1, backend: null }, 0).payload, false);
+    assert.equal("backend" in mapRecord({ event: "PHASE_ATTEMPT_USAGE", phase: "build", attempt: 1 }, 0).payload, false);
+  });
+
   it("flattens usage, omits nulls and passes repo/issue/run/phase as ctx (not payload)", async () => {
     const { calls, emit } = harness();
     await emit(attempt);
@@ -41,6 +47,31 @@ describe("engine telemetry seam", () => {
     const m = mapRecord({ event: "STRUCTURE_DELTA", issue: 7, edges_added: 1, edges_removed: 0, import_files_added: 8, import_files_removed: 2, boundary_changed: true, boundary_loosened: false }, 10);
     assert.deepEqual(m, { event: "structure_delta", phase: "implement", payload: { edges_added: 1, edges_removed: 0, import_files_added: 8, import_files_removed: 2, boundary_changed: 1, boundary_loosened: 0 } });
   });
+  it("forge#4165: the merge-time STRUCTURE_DELTA and RUN_TERMINAL carry scalar lists, milestone and the pr in ctx", async () => {
+    const { calls, emit } = harness();
+    await emit({ event: "STRUCTURE_DELTA", issue: 7, pr: 41, phase: "merge", milestone: "M", edges_added: 1, edges_removed: 0, import_files_added: 2, import_files_removed: 0,
+      boundary_changed: false, boundary_loosened: false, deps_added: [{ from: "@s/a", to: "@s/b" }], deps_removed: [], new_edges: [{ from: "@s/a", to: "@s/c", file: "x.ts" }, { from: "@s/d", to: "@s/c" }],
+      removed_edges: [], top_growth: [{ file: "x.ts", lines: 9 }], decisions: [{ id: "K1", source: "lint" }, { id: "K2", source: null }] });
+    await emit({ event: "RUN_TERMINAL", issue: 7, pr: 41, reason: "merged", code: null, milestone: "M", conformance_mode: "shadow", shadow_would: "conforms" });
+    assert.equal(calls.length, 2);
+    const [[e1, p1, c1], [e2, p2, c2]] = calls;
+    assert.deepEqual([e1, c1.pr, c1.phase, e2, c2.pr], ["structure_delta", 41, "merge", "run_terminal", 41]);
+    assert.deepEqual([p1.deps_added, p1.new_edges, p1.removed_edges, p1.top_growth, p1.decisions, p1.decisions_lint, p1.decisions_doc, p1.milestone],
+      ["@s/a:@s/b", "@s/a:@s/c,@s/d:@s/c", "none", "x.ts+9", "K1:lint,K2:none", 1, 0, "M"]);
+    for (const v of [...Object.values(p1), ...Object.values(p2)]) assert.notEqual(typeof v, "object");
+    assert.deepEqual(p2, { reason: "merged", wall_ms: 0, milestone: "M", conformance_mode: "shadow", shadow_would: "conforms" });
+    assert.equal(mapRecord({ event: "RUN_TERMINAL", reason: "merged", pr: 0 }, 1).pr, undefined, "no pr 0 in ctx");
+  });
+  it("forge#4255: SANDBOX_USAGE maps to sandbox_usage with provider, minutes and cost", async () => {
+    const { calls, emit } = harness();
+    await emit({ event: "SANDBOX_USAGE", issue: 7, provider: "container", sandbox_id: "fdsbx-b-7", minutes: 12.5, cost_usd: 0.1234567891, batch: "b1" });
+    const [event, payload, ctx] = calls[0];
+    assert.equal(event, "sandbox_usage");
+    assert.deepEqual(payload, { provider: "container", sandbox_id: "fdsbx-b-7", minutes: 12.5, cost_usd: 0.123457, batch: "b1" });
+    assert.equal(ctx.issue, 7);
+    assert.equal(ctx.phase, undefined);
+  });
+
   it("null usage yields a record with no token fields", () => {
     assert.deepEqual(mapRecord({ event: "PHASE_ATTEMPT_USAGE", phase: "build", attempt: 1, usage: null, duration_ms: 5 }).payload, { attempt: 1, duration_ms: 5 });
   });