forgedock 1.27.0 → 1.29.0

package/scripts/bootstrap-create.mjs

@@ -0,0 +1,393 @@
+#!/usr/bin/env node
+// SPDX-FileCopyrightText: Copyright (c) RapierCraft Studios
+// SPDX-License-Identifier: AGPL-3.0-or-later
+/**
+ * scripts/bootstrap-create.mjs — idempotent, resumable backlog creation for /bootstrap (#4176, part of #4155)
+ *
+ * Takes a plan that passes scripts/bootstrap-validate.mjs and makes it real on GitHub: the spike parent issue,
+ * milestones in build order with their `milestone/<slug>` branches, every sub-issue (guardrails and domain rules
+ * injected into its Context, `#{{KEY}}` placeholders resolved), board items with Status=Todo, Priority and Component,
+ * a FORGE:BOOTSTRAP_PLAN and a FORGE:DECOMPOSED comment on the spike, and `workflow:decomposed` via
+ * scripts/transition-label.sh. Nothing is written when the plan or the spike doc is invalid.
+ *
+ * Re-running is safe: the spike is reused by exact title, milestones by title, branches when present, sub-issues by
+ * exact title plus a `**Parent**: #<spike>` line, comments when a trusted marker comment already exists.
+ *
+ * Plan fields on top of the validator's shape (all optional):
+ *   "spike": "SPIKE"                                  plan key of the spike parent issue (default "SPIKE")
+ *   "spikeDoc": "docs/engineering/technical-spike.md" spike doc, checked for its sections (checkSpikeDoc)
+ *   "guardrails": ["no auto-send"], "domainRules": ["India-only data residency"]
+ *   issues[].priority: "p0".."p3" (label priority:P<n> + board Priority), issues[].component: board Component label,
+ *   issues[].estimatedDiffLines: number (FORGE:BOOTSTRAP_PLAN estimate)
+ *   "costPerIssueUsd": 3                              dry-run cost estimate per build issue (#4177)
+ *
+ * Usage: node scripts/bootstrap-create.mjs <plan.json> [--dry-run] [--json] [--repo OWNER/NAME] [--config forge.yaml]
+ *                                                     [--base <branch>] [--cost-per-issue <usd>]
+ * Cost: plan.costPerIssueUsd, else --cost-per-issue, else DEFAULT_ISSUE_COST_USD, times the build sub-issues.
+ * Repo: --repo, else forge.yaml project.owner/project.repo. Base: --base, else branches.staging, else "staging".
+ * Exit: 0 created (or dry-run preview); 1 plan or spike doc invalid (nothing written); 2 usage; 3 GitHub error.
+ */
+
+import { execFileSync } from "node:child_process";
+import { existsSync, readFileSync } from "node:fs";
+import { fileURLToPath } from "node:url";
+import { validatePlan, formatErrors, resolvePlaceholders, isNeedsHuman } from "./bootstrap-validate.mjs";
+import { milestoneSlug } from "../bin/lean/lane.mjs";
+
+const SELF_DIR = fileURLToPath(new URL(".", import.meta.url));
+const DEFAULT_SPIKE_DOC = "docs/engineering/technical-spike.md";
+const PLACEHOLDER_RE = /#\{\{[^{}]*\}\}/;
+/** Planning estimate of agent spend per build issue (one lean lane) for the dry-run preview; not a measured figure. */
+export const DEFAULT_ISSUE_COST_USD = 3;
+
+/** Spike-doc sections (templates/bootstrap/technical-spike.md), matched as a prefix of a `## ` heading. */
+export const SPIKE_SECTIONS = [
+  "Stack", "Repo layout", "Runtime topology", "Data model", "Security and privacy",
+  "Design system and quality gates", "Route/API map", "Credentials", "Milestone plan",
+];
+
+/** `## ` sections of a markdown text, fenced code ignored: [{ heading, lines }]. */
+function h2Sections(text) {
+  const out = [];
+  let cur = null, fence = null;
+  for (const line of String(text ?? "").replace(/\r\n/g, "\n").split("\n")) {
+    const f = line.match(/^[ \t]*(`{3,}|~{3,})/);
+    if (f) fence = fence ? (f[1][0] === fence ? null : fence) : f[1][0];
+    const h = !fence && !f && line.match(/^##(?!#)[ \t]+(.+?)[ \t]*$/);
+    if (h) { cur = { heading: h[1], lines: [] }; out.push(cur); continue; }
+    if (cur) cur.lines.push(line);
+  }
+  return out;
+}
+
+/**
+ * Check a spike doc: every SPIKE_SECTIONS section present, each with at least one ADR-style `**Rationale**:` line.
+ * @returns {{ ok: boolean, missing: string[], noRationale: string[] }}
+ */
+export function checkSpikeDoc(text) {
+  const secs = h2Sections(text);
+  const missing = [], noRationale = [];
+  for (const name of SPIKE_SECTIONS) {
+    const s = secs.find((x) => x.heading.toLowerCase().startsWith(name.toLowerCase()));
+    if (!s) missing.push(name);
+    else if (!s.lines.some((l) => /\*\*Rationale\*\*:/.test(l))) noRationale.push(name);
+  }
+  return { ok: missing.length === 0 && noRationale.length === 0, missing, noRationale };
+}
+
+/**
+ * Append the plan's guardrails and domain rules to the end of a body's `## Context` section (idempotent).
+ * Bodies without a Context section are returned unchanged (the validator rejects those).
+ */
+export function injectGuardrails(body, guardrails = [], domainRules = []) {
+  const g = (guardrails ?? []).filter((x) => typeof x === "string" && x.trim());
+  const d = (domainRules ?? []).filter((x) => typeof x === "string" && x.trim());
+  if ((!g.length && !d.length) || /^\*\*Guardrails\*\*:/m.test(body)) return body;
+  const lines = String(body).split("\n");
+  let start = -1, end = lines.length, fence = null;
+  for (let i = 0; i < lines.length; i++) {
+    const f = lines[i].match(/^[ \t]*(`{3,}|~{3,})/);
+    if (f) { fence = fence ? (f[1][0] === fence ? null : fence) : f[1][0]; continue; }
+    if (fence || !/^##(?!#)[ \t]+/.test(lines[i])) continue;
+    if (start < 0 && /^##[ \t]+context\b/i.test(lines[i])) start = i;
+    else if (start >= 0) { end = i; break; }
+  }
+  if (start < 0) return body;
+  while (end > start + 1 && lines[end - 1].trim() === "") end--;
+  const block = [""];
+  if (g.length) block.push("**Guardrails**:", ...g.map((x) => `- ${x}`));
+  if (d.length) block.push(...(g.length ? [""] : []), "**Domain rules**:", ...d.map((x) => `- ${x}`));
+  lines.splice(end, 0, ...block);
+  return lines.join("\n");
+}
+
+/** Sub-issues (every plan issue but the spike) in dependency order: milestone order, then a stable topo sort. */
+export function subIssueOrder(plan) {
+  const spikeKey = plan.spike ?? "SPIKE";
+  const mIdx = new Map(plan.milestones.map((m, i) => [m.key, i]));
+  const pending = plan.issues.filter((it) => it.key !== spikeKey)
+    .map((it, i) => ({ it, i })).sort((a, b) => mIdx.get(a.it.milestone) - mIdx.get(b.it.milestone) || a.i - b.i)
+    .map((x) => x.it);
+  const keys = new Set(pending.map((it) => it.key));
+  const done = new Set(), out = [];
+  while (pending.length) {
+    const at = pending.findIndex((it) => (it.deps ?? []).every((d) => !keys.has(d) || done.has(d)));
+    const [next] = pending.splice(at < 0 ? 0 : at, 1);
+    done.add(next.key);
+    out.push(next);
+  }
+  return out;
+}
+
+/** The priority label for an issue (`p1` -> `priority:P1`), or null. */
+export function priorityLabel(issue) {
+  const m = String(issue?.priority ?? "").match(/^p([0-3])$/i);
+  return m ? `priority:P${m[1]}` : null;
+}
+
+/** Plan preview for --dry-run: milestones, titles, DAG summary, estimated agent cost. No GitHub access. */
+export function planPreview(plan, { costPerIssueUsd } = {}) {
+  const spikeKey = plan.spike ?? "SPIKE";
+  const spike = plan.issues.find((it) => it.key === spikeKey);
+  const subs = subIssueOrder(plan);
+  const edges = subs.reduce((n, it) => n + (it.deps ?? []).filter((d) => d !== spikeKey).length, 0);
+  const human = subs.filter(isNeedsHuman);
+  const lines = ["## Bootstrap plan preview (dry run: nothing written to GitHub)", "",
+    `Spike: ${spike ? spike.title : "(missing)"} — doc \`${plan.spikeDoc ?? DEFAULT_SPIKE_DOC}\``, ""];
+  plan.milestones.forEach((m, i) => {
+    const its = subs.filter((it) => it.milestone === m.key);
+    lines.push(`### ${i + 1}. ${m.title} (\`milestone/${milestoneSlug(m.title)}\`, ${its.length} issues)`);
+    for (const it of its) {
+      const deps = (it.deps ?? []).filter((d) => d !== spikeKey);
+      lines.push(`- ${it.title}${isNeedsHuman(it) ? " [needs-human]" : ""}${deps.length ? ` — depends on ${deps.join(", ")}` : ""}`);
+    }
+    lines.push("");
+  });
+  lines.push(`DAG: ${subs.length} sub-issues (${subs.length - human.length} build, ${human.length} needs-human), ${edges} dependency edges, ${plan.milestones.length} milestones.`);
+  const rate = [plan.costPerIssueUsd, costPerIssueUsd].find((r) => Number.isFinite(r) && r > 0) ?? DEFAULT_ISSUE_COST_USD;
+  const build = subs.length - human.length;
+  lines.push(`Estimated agent cost: ~$${(build * rate).toFixed(2)} (${build} build issues × $${rate}/issue; needs-human and spike excluded).`);
+  return lines.join("\n");
+}
+
+/** `<!-- FORGE:BOOTSTRAP_PLAN -->` body, the shape commands/work-on/decompose.md Phase D0 reads. */
+export function bootstrapPlanComment(plan) {
+  const subs = subIssueOrder(plan);
+  const pos = new Map(subs.map((it, i) => [it.key, i + 1]));
+  const items = subs.map((it, i) => {
+    const deps = (it.deps ?? []).filter((d) => pos.has(d)).map((d) => pos.get(d));
+    const est = Number.isFinite(it.estimatedDiffLines) ? ` — ESTIMATED_DIFF_LINES: ${it.estimatedDiffLines}` : "";
+    return `${i + 1}. ${it.title}${est}${deps.length ? ` _(depends on ${deps.join(", ")})_` : ""}`;
+  });
+  return ["<!-- FORGE:BOOTSTRAP_PLAN -->", "## Bootstrap Plan", "", `Spike: \`${plan.spikeDoc ?? DEFAULT_SPIKE_DOC}\``, "",
+    "### Decomposition Assessment", "", "**YES**", "", ...items, ""].join("\n");
+}
+
+/** `<!-- FORGE:DECOMPOSED -->` body in the decompose Phase D5 format (`- #N: TITLE _(depends on #M)_`). */
+export function decomposedComment(plan, numbers, { attribution = true } = {}) {
+  const spikeKey = plan.spike ?? "SPIKE";
+  const subs = subIssueOrder(plan);
+  const items = subs.map((it) => {
+    const deps = (it.deps ?? []).filter((d) => d !== spikeKey && numbers.has(d)).map((d) => `#${numbers.get(d)}`);
+    return `- #${numbers.get(it.key)}: ${it.title}${deps.length ? ` _(depends on ${deps.join(", ")})_` : ""}${isNeedsHuman(it) ? " — needs-human" : ""}`;
+  });
+  return ["<!-- FORGE:DECOMPOSED -->", "## Decomposition Complete", "",
+    "This spike has been broken into sub-issues by /bootstrap. Each sub-issue runs through its own /work-on pipeline independently.", "",
+    "**Plan source**: bootstrap", "", "### Sub-Issues Created", "", ...items, "", "### Decomposition Rationale", "",
+    `Milestones in build order: ${plan.milestones.map((m) => m.title).join(" → ")}. Needs-human issues are referenced by code issues but never block them; mocks keep dev and CI green.`,
+    ...(attribution ? ["", "> Pipeline powered by [ForgeDock](https://github.com/RapierCraftStudios/ForgeDock)"] : []),
+    "<!-- FORGE:DECOMPOSED:COMPLETE -->"].join("\n");
+}
+
+/** Human hand-off: created links, needs-human links, `/orchestrate` per milestone in build order. */
+export function handoffText(result) {
+  const link = (x) => `- #${x.number} ${x.title} — ${x.url}${x.reused ? " (reused)" : ""}`;
+  const lines = ["## Bootstrap hand-off", "", `Spike: #${result.spike.number} ${result.spike.title} — ${result.spike.url}`, "", "### Created"];
+  lines.push(...result.issues.filter((x) => !x.needsHuman).map(link));
+  lines.push("", "### Needs a human");
+  const human = result.issues.filter((x) => x.needsHuman);
+  lines.push(...(human.length ? human.map(link) : ["- none"]));
+  lines.push("", "### Next commands (in build order)");
+  result.milestones.forEach((m, i) => lines.push(`${i + 1}. ${m.orchestrate}`));
+  if (result.notes.length) lines.push("", "### Notes", ...result.notes.map((n) => `- ${n}`));
+  return lines.join("\n");
+}
+
+const parseIssueNumber = (out) => {
+  const m = String(out).match(/\/issues\/(\d+)/);
+  if (!m) throw new Error(`could not read the created issue number from: ${String(out).trim().slice(0, 200)}`);
+  return Number(m[1]);
+};
+
+/**
+ * Create (or reuse) the whole backlog. All GitHub access goes through `io`:
+ *   io.gh(args, input?) -> stdout string (throws on failure)
+ *   io.sh(cmd, args, input?) -> stdout string (bash helpers: transition-label.sh, trusted-comments.sh)
+ * @returns {object} result for handoffText (spike, issues, milestones, notes)
+ */
+export function createBacklog(plan, { io, repo, config = {}, base }) {
+  const spikeKey = plan.spike ?? "SPIKE";
+  const spikePlan = plan.issues.find((it) => it.key === spikeKey);
+  if (!spikePlan) throw new Error(`plan has no spike issue with key ${spikeKey}`);
+  const notes = [];
+  const board = config.project_board ?? {};
+  const pattern = config.branches?.feature_pattern || "milestone/{slug}";
+  const baseBranch = base || config.branches?.staging || "staging";
+  const gh = (args, input) => io.gh(args, input);
+  const json = (args) => JSON.parse(gh(args) || "null");
+
+  // Labels: create only the missing ones (never overwrite an existing label's colour or description).
+  const haveLabels = new Set((json(["label", "list", "-R", repo, "--limit", "1000", "--json", "name"]) ?? []).map((l) => l.name));
+  const wanted = new Set(plan.issues.flatMap((it) => [...(it.labels ?? []), priorityLabel(it)]).filter(Boolean));
+  for (const l of wanted) if (!haveLabels.has(l)) gh(["label", "create", l, "-R", repo]);
+
+  // Milestones in build order, each with its lane branch.
+  const ghMilestones = json(["api", `repos/${repo}/milestones?state=all&per_page=100`]) ?? [];
+  let baseSha = null;
+  const milestones = plan.milestones.map((m) => {
+    let reused = ghMilestones.some((g) => g.title === m.title);
+    if (!reused) gh(["api", `repos/${repo}/milestones`, "--method", "POST", "-f", `title=${m.title}`, "-f", `description=${m.description ?? ""}`, "-f", "state=open"]);
+    const slug = milestoneSlug(m.title);
+    if (!slug) throw new Error(`milestone "${m.title}" gives an empty branch slug`);
+    const branch = pattern.replace("{slug}", slug);
+    let exists = true;
+    try { gh(["api", `repos/${repo}/git/ref/heads/${branch}`]); } catch { exists = false; }
+    if (!exists) {
+      baseSha ??= gh(["api", `repos/${repo}/git/ref/heads/${baseBranch}`, "--jq", ".object.sha"]).trim();
+      gh(["api", `repos/${repo}/git/refs`, "--method", "POST", "-f", `ref=refs/heads/${branch}`, "-f", `sha=${baseSha}`]);
+    }
+    return { key: m.key, title: m.title, branch, reused, orchestrate: `/orchestrate milestone "${m.title}"` };
+  });
+  const milestoneTitle = new Map(plan.milestones.map((m) => [m.key, m.title]));
+
+  // Existing issues, read once (no search-index lag on a resume right after a create).
+  const existing = json(["issue", "list", "-R", repo, "--state", "all", "--limit", "2000", "--json", "number,title,body,url,labels"]) ?? [];
+  const numbers = new Map(Object.entries(plan.numbers ?? {}).filter(([, n]) => Number.isSafeInteger(n) && n > 0));
+  const finalBody = (it) => injectGuardrails(resolvePlaceholders(it.body, numbers), plan.guardrails, plan.domainRules);
+  const records = new Map();
+  const ensure = (it, match) => {
+    const found = existing.find((x) => x.title === it.title && match(x));
+    if (found) {
+      numbers.set(it.key, found.number);
+      records.set(it.key, { number: found.number, url: found.url, body: found.body ?? "", labels: (found.labels ?? []).map((l) => l.name), reused: true });
+      return;
+    }
+    const args = ["issue", "create", "-R", repo, "--title", it.title, "--body-file", "-", "--milestone", milestoneTitle.get(it.milestone)];
+    for (const l of [...(it.labels ?? []), priorityLabel(it)].filter(Boolean)) args.push("--label", l);
+    const body = finalBody(it);
+    const out = gh(args, body);
+    const number = parseIssueNumber(out);
+    numbers.set(it.key, number);
+    records.set(it.key, { number, url: String(out).trim().split("\n").pop(), body, labels: [], reused: false });
+  };
+
+  ensure(spikePlan, () => true);
+  const spikeNo = numbers.get(spikeKey);
+  const parentRe = new RegExp(`\\*\\*Parent\\*\\*:?\\s*#${spikeNo}(?!\\d)`);
+  const subs = subIssueOrder(plan);
+  for (const it of subs) ensure(it, (x) => parentRe.test(x.body ?? ""));
+
+  // Edit pass: resolve forward references left as placeholders at create time (also finishes an interrupted run).
+  for (const it of [spikePlan, ...subs]) {
+    const rec = records.get(it.key);
+    if (!PLACEHOLDER_RE.test(rec.body)) continue;
+    const body = finalBody(it);
+    if (body === rec.body) continue;
+    gh(["issue", "edit", String(rec.number), "-R", repo, "--body-file", "-"], body);
+    rec.body = body;
+  }
+
+  // Board: Status=Todo, Priority, Component (best effort per field; skipped when the board is not configured).
+  if (board.project_number && board.project_id) {
+    const owner = board.owner || config.project?.owner || repo.split("/")[0];
+    const fields = board.field_ids ?? {}, options = board.option_ids ?? {};
+    const components = Array.isArray(board.components) ? board.components : [];
+    for (const it of [spikePlan, ...subs]) {
+      const rec = records.get(it.key);
+      const itemId = gh(["project", "item-add", String(board.project_number), "--owner", owner, "--url", rec.url, "--format", "json", "--jq", ".id"]).trim();
+      if (!itemId) { notes.push(`#${rec.number}: board item id not returned; fields not set`); continue; }
+      const prio = String(it.priority ?? "").toLowerCase();
+      const comp = components.find((c) => it.component && String(c.label ?? "").toLowerCase() === String(it.component).toLowerCase())
+        ?? components.find((c) => c.repo === repo);
+      const sets = [[fields.status, options.status?.todo], [fields.priority, options.priority?.[prio]], [fields.component, comp?.option_id]];
+      for (const [field, option] of sets) {
+        if (!field || !option) continue;
+        try {
+          gh(["project", "item-edit", "--project-id", board.project_id, "--id", itemId, "--field-id", field, "--single-select-option-id", option]);
+        } catch (e) { notes.push(`#${rec.number}: board field ${field} not set (${String(e?.message ?? e).slice(0, 120)})`); }
+      }
+    }
+  } else {
+    notes.push("project_board not configured in forge.yaml (project_number/project_id): board sync skipped");
+  }
+
+  // Spike comments, posted once: a trusted marker comment already present is reused.
+  const comments = gh(["api", "--paginate", `repos/${repo}/issues/${spikeNo}/comments`]);
+  const trusted = (re) => io.sh("bash", [`${SELF_DIR}trusted-comments.sh`, "count", re], comments).trim() !== "0";
+  const attribution = String(config.branding?.show_attribution ?? "true") !== "false";
+  if (!trusted("^<!-- FORGE:BOOTSTRAP_PLAN -->")) gh(["issue", "comment", String(spikeNo), "-R", repo, "--body-file", "-"], bootstrapPlanComment(plan));
+  if (!trusted("^<!-- FORGE:DECOMPOSED -->")) gh(["issue", "comment", String(spikeNo), "-R", repo, "--body-file", "-"], decomposedComment(plan, numbers, { attribution }));
+  if (!records.get(spikeKey).labels.includes("workflow:decomposed")) {
+    io.sh("bash", [`${SELF_DIR}transition-label.sh`, String(spikeNo), "-R", repo, "decomposed"]);
+  }
+
+  const spikeRec = records.get(spikeKey);
+  return {
+    repo,
+    spike: { key: spikeKey, number: spikeNo, title: spikePlan.title, url: spikeRec.url, reused: spikeRec.reused },
+    issues: subs.map((it) => ({ key: it.key, number: numbers.get(it.key), title: it.title, url: records.get(it.key).url,
+      needsHuman: isNeedsHuman(it), milestone: milestoneTitle.get(it.milestone), reused: records.get(it.key).reused })),
+    milestones,
+    notes,
+  };
+}
+
+/** Plan + spike-doc preflight shared by the CLI and tests: [] when creation may proceed. */
+export function preflight(plan, spikeText) {
+  const errors = [];
+  const v = validatePlan(plan);
+  if (!v.ok) errors.push(formatErrors(v.errors));
+  else if (!plan.issues.some((it) => it.key === (plan.spike ?? "SPIKE"))) errors.push(`(plan): [invalid-plan] no spike issue with key ${plan.spike ?? "SPIKE"}`);
+  if (spikeText == null) errors.push(`(spike): [missing-spike-doc] ${plan?.spikeDoc ?? DEFAULT_SPIKE_DOC} not found`);
+  else {
+    const s = checkSpikeDoc(spikeText);
+    if (s.missing.length) errors.push(`(spike): [missing-section] ${s.missing.join(", ")}`);
+    if (s.noRationale.length) errors.push(`(spike): [missing-rationale] no **Rationale**: line in ${s.noRationale.join(", ")}`);
+  }
+  return errors;
+}
+
+const defaultIo = {
+  gh: (args, input) => execFileSync("gh", args, { encoding: "utf8", input, stdio: ["pipe", "pipe", "pipe"], maxBuffer: 64 << 20 }),
+  sh: (cmd, args, input) => execFileSync(cmd, args, { encoding: "utf8", input, stdio: ["pipe", "pipe", "inherit"] }),
+};
+
+function readConfig(path) {
+  if (!existsSync(path)) return {};
+  try { return JSON.parse(execFileSync("yq", ["-o=json", ".", path], { encoding: "utf8" })) ?? {}; } catch { return {}; }
+}
+
+export function main(argv, { io = defaultIo, out = process.stdout, err = process.stderr } = {}) {
+  let planPath = null, dryRun = false, asJson = false, repo = "", configPath = process.env.FORGE_CONFIG || "forge.yaml", base = "", costPerIssueUsd;
+  for (let i = 0; i < argv.length; i++) {
+    const a = argv[i];
+    const val = () => { if (i + 1 >= argv.length) throw new Error(`${a} requires a value`); return argv[++i]; };
+    try {
+      if (a === "--dry-run") dryRun = true;
+      else if (a === "--json") asJson = true;
+      else if (a === "--repo") repo = val();
+      else if (a === "--config") configPath = val();
+      else if (a === "--base") base = val();
+      else if (a === "--cost-per-issue") {
+        costPerIssueUsd = Number(val());
+        if (!Number.isFinite(costPerIssueUsd) || costPerIssueUsd <= 0) throw new Error("--cost-per-issue must be a positive number");
+      }
+      else if (!planPath && !a.startsWith("--")) planPath = a;
+      else throw new Error(`unknown argument ${a}`);
+    } catch (e) { err.write(`bootstrap-create: ${e.message}\n`); return 2; }
+  }
+  if (!planPath) { err.write("usage: node scripts/bootstrap-create.mjs <plan.json> [--dry-run] [--json] [--repo OWNER/NAME] [--config forge.yaml] [--base <branch>] [--cost-per-issue <usd>]\n"); return 2; }
+  let plan;
+  try { plan = JSON.parse(readFileSync(planPath, "utf8")); } catch (e) { err.write(`bootstrap-create: cannot read plan: ${e.message}\n`); return 2; }
+  const spikePath = plan?.spikeDoc ?? DEFAULT_SPIKE_DOC;
+  const spikeText = existsSync(spikePath) ? readFileSync(spikePath, "utf8") : null;
+  const errors = preflight(plan, spikeText);
+  if (errors.length) { err.write(errors.join("\n") + "\nbootstrap-create: not creating anything; fix the plan or spike doc\n"); return 1; }
+  if (dryRun) { out.write(planPreview(plan, { costPerIssueUsd }) + "\n"); return 0; }
+
+  const config = readConfig(configPath);
+  repo ||= config.project?.owner && config.project?.repo ? `${config.project.owner}/${config.project.repo}` : "";
+  if (!/^[^/\s]+\/[^/\s]+$/.test(repo)) { err.write("bootstrap-create: no target repository: pass --repo OWNER/NAME or a forge.yaml with project.owner/project.repo\n"); return 2; }
+  let result;
+  try { result = createBacklog(plan, { io, repo, config, base }); } catch (e) {
+    err.write(`bootstrap-create: GitHub error: ${String(e?.stderr || e?.message || e).trim().slice(0, 500)}\nRe-run the same command to resume; existing issues are reused.\n`);
+    return 3;
+  }
+  out.write((asJson ? JSON.stringify(result, null, 2) : handoffText(result)) + "\n");
+  return 0;
+}
+
+// Run as CLI only when invoked directly, not when imported by tests.
+if (process.argv[1] && fileURLToPath(import.meta.url) === process.argv[1]) {
+  process.exitCode = main(process.argv.slice(2));
+}

package/scripts/bootstrap-validate.mjs

@@ -11,12 +11,14 @@
  *     "labels":     ["backend", ...],                      // optional: when given, every issue label must be in it
  *     "milestones": [{ "key": "M1", "title": "..." }, ...],  // build order: foundation first
  *     "issues": [{ "key": "API", "title": "...", "milestone": "M1", "labels": ["backend"],
- *                  "deps": ["DB"], "needsHuman": false, "body": "## Problem\n..." }],
+ *                  "deps": ["DB"], "needsHuman": false, "ui": false, "hero": false, "body": "## Problem\n..." }],
  *     "numbers":    { "DB": 12 }                             // optional: real issue numbers once created
  *   }
  *
  * Rejects: invalid-plan, unknown-dep, cycle, forward-milestone-dep, depends-on-mismatch, unresolved-placeholder,
- * missing-section, missing-type-tag, missing-parent, missing-labels, unknown-label, code-depends-on-needs-human.
+ * missing-section, missing-type-tag, missing-parent, missing-labels, unknown-label, code-depends-on-needs-human,
+ * missing-design-spec (a `ui` issue lacks a design-spec item), hero-not-routed (a `hero` issue never names `/design`),
+ * missing-needs-human-section (a needs-human issue lacks setup steps, env var names or secret handling) (#4176).
  * `Depends on` lines are read with the engine's own parser (bin/engine/phases.mjs parseDependsOn) after placeholder
  * resolution, so the plan and engine admission agree on what blocks what.
  *
@@ -45,6 +47,25 @@ const SYNTHETIC_BASE = 1_000_000_000;
 // ignoring a trailing parenthetical such as "Root Cause (if known)".
 const SECTIONS = ["Problem", "Root Cause", "Affected Files", "Acceptance Criteria", "Context", "Dependencies"];
 
+// UI issues carry a full design spec (#4176): each item must be named somewhere in the body.
+const DESIGN_SPEC = [
+  ["breakpoints", /breakpoint/i],
+  ["empty state", /\bempty\b/i],
+  ["loading state", /\bloading\b/i],
+  ["error state", /\berror\b/i],
+  ["offline state", /\boffline\b/i],
+  ["motion", /\bmotion\b/i],
+  ["accessibility", /\b(?:accessibility|a11y)\b/i],
+  ["dark mode", /\bdark[ -]mode\b/i],
+];
+// Needs-human issues (#4176): headings for setup steps, env vars and secret handling, plus at least one env var name.
+const NEEDS_HUMAN_PARTS = [
+  ["setup steps heading", /^#{2,6}[ \t]+.*\bsetup steps\b/im],
+  ["environment variables heading", /^#{2,6}[ \t]+.*\b(?:environment variables|env vars)\b/im],
+  ["secret handling heading", /^#{2,6}[ \t]+.*\bsecrets?\b/im],
+  ["env var name (e.g. `API_KEY`)", /`[A-Z][A-Z0-9]*_[A-Z0-9_]+`/],
+];
+
 const isObj = (v) => v !== null && typeof v === "object" && !Array.isArray(v);
 const isNonEmptyString = (v) => typeof v === "string" && v.trim() !== "";
 const normHeading = (h) => h.replace(/\([^)]*\)\s*$/, "").trim().toLowerCase();
@@ -125,6 +146,7 @@ function checkStructure(plan) {
       at("`labels` must be an array of strings");
     }
     if (typeof it.body !== "string") at("needs a string `body`");
+    for (const f of ["ui", "hero"]) if (it[f] !== undefined && typeof it[f] !== "boolean") at(`\`${f}\` must be a boolean`);
   });
   return errors;
 }
@@ -242,6 +264,18 @@ export function validatePlan(plan) {
     }
     const ctx = secs.get("context");
     if (ctx && !ctx.some((l) => l.includes("**Parent**"))) err(it.key, "missing-parent", "`## Context` has no `**Parent**` line");
+
+    if (it.ui === true) {
+      const lacking = DESIGN_SPEC.filter(([, re]) => !re.test(it.body)).map(([name]) => name);
+      if (lacking.length) err(it.key, "missing-design-spec", `UI issue design spec lacks: ${lacking.join(", ")}`);
+    }
+    if (it.hero === true && !/(^|[^\w/])\/design\b/.test(it.body)) {
+      err(it.key, "hero-not-routed", "hero surface issue does not route to `/design`");
+    }
+    if (isNeedsHuman(it)) {
+      const lacking = NEEDS_HUMAN_PARTS.filter(([, re]) => !re.test(it.body)).map(([name]) => name);
+      if (lacking.length) err(it.key, "missing-needs-human-section", `needs-human issue lacks: ${lacking.join(", ")}`);
+    }
   }
   return { ok: errors.length === 0, errors };
 }

package/scripts/calibration.mjs

@@ -36,6 +36,12 @@
  *   --dry-run                      Compute table(s) but do not write any files
  *   --verbose                      Verbose logging
  *   --issue <number>               Compute outcome for a single issue only (debugging)
+ *   --shadow                       Instead: the per-repo conformance-gate shadow verdict distribution (forge#4160),
+ *                                  read from FORGE:REVIEW comments, and whether to switch engine.conformance to enforce
+ *                                  (--repo accepts a comma-separated list)
+ *   --threshold <share>            --shadow: recommend enforce while would-block-on-incomplete stays under this share (default 0.10)
+ *   --min-runs <n>                 --shadow: minimum shadow reviews before recommending enforce (default 20)
+ *   --json                         --shadow: print JSON instead of the text table
  *
  * Exit codes:
  *   0  — success
@@ -75,6 +81,11 @@ const SCHEMA_VERSION = 1;
 // larger and cells accumulate samples more slowly.
 const DEFAULT_PROVENANCE_MIN_SAMPLES = 5;
 
+// forge#4160: the conformance gate graduates from shadow to enforce once incomplete reviews are rare.
+const DEFAULT_SHADOW_THRESHOLD = 0.10;
+const DEFAULT_SHADOW_MIN_RUNS = 20;
+const SHADOW_VERDICTS = ['pass', 'nonconforming', 'review.incomplete', 'review.inconsistent', 'review.unparseable'];
+
 // Survival rate threshold above which a cell is classified as PROVEN (eligible
 // for optional-agent de-escalation).
 const PROVENANCE_PROVEN_THRESHOLD = 0.90;
@@ -105,6 +116,10 @@ export function parseArgs(argv) {
     dryRun: false,
     verbose: false,
     issue: null,
+    shadow: false,
+    threshold: DEFAULT_SHADOW_THRESHOLD,
+    minRuns: DEFAULT_SHADOW_MIN_RUNS,
+    json: false,
   };
 
   for (let i = 2; i < argv.length; i++) {
@@ -120,6 +135,10 @@ export function parseArgs(argv) {
     else if (arg === '--dry-run') { args.dryRun = true; }
     else if (arg === '--verbose') { args.verbose = true; }
     else if (arg === '--issue' && argv[i + 1]) { args.issue = parseInt(argv[++i], 10); }
+    else if (arg === '--shadow') { args.shadow = true; }
+    else if (arg === '--threshold' && argv[i + 1]) { args.threshold = parseFloat(argv[++i]); }
+    else if (arg === '--min-runs' && argv[i + 1]) { args.minRuns = parseInt(argv[++i], 10); }
+    else if (arg === '--json') { args.json = true; }
   }
 
   return args;
@@ -812,6 +831,98 @@ export function publishTable(table, repoPath, dryRun = false) {
 // Main pipeline
 // ---------------------------------------------------------------------------
 
+// ---------------------------------------------------------------------------
+// Shadow verdict distribution (forge#4160)
+// ---------------------------------------------------------------------------
+
+/**
+ * The shadow gate verdict in one FORGE:REVIEW comment body: "pass", or the would-block state
+ * (nonconforming | review.incomplete | review.inconsistent | review.unparseable). Null for anything else,
+ * including an enforced review.
+ * @param {string} body
+ * @returns {string|null}
+ */
+export function parseShadowVerdict(body) {
+  const b = String(body ?? '');
+  if (!b.includes('FORGE:REVIEW')) return null;
+  const m = /\*\*Gate verdict:\*\*\s*would (PASS|BLOCK \(([^)]+)\))/.exec(b);
+  if (!m) return null;
+  return m[1] === 'PASS' ? 'pass' : m[2].trim();
+}
+
+/**
+ * Per-repo counts and shares of shadow verdicts.
+ * @param {{repo: string, verdict: string}[]} records
+ * @returns {Record<string, {runs: number, counts: Record<string, number>, shares: Record<string, number>}>}
+ */
+export function shadowDistribution(records) {
+  const out = {};
+  for (const r of records ?? []) {
+    if (!r?.repo || !r?.verdict) continue;
+    const e = out[r.repo] ??= { runs: 0, counts: Object.fromEntries(SHADOW_VERDICTS.map((v) => [v, 0])), shares: {} };
+    e.runs++;
+    e.counts[r.verdict] = (e.counts[r.verdict] ?? 0) + 1;
+  }
+  for (const e of Object.values(out)) e.shares = Object.fromEntries(Object.entries(e.counts).map(([k, n]) => [k, e.runs ? Math.round((n / e.runs) * 1000) / 1000 : 0]));
+  return out;
+}
+
+/**
+ * enforce once there are at least minRuns shadow reviews and the would-block-on-incomplete share is under threshold.
+ * @param {{runs: number, counts: Record<string, number>}} entry
+ * @param {{threshold?: number, minRuns?: number}} [o]
+ * @returns {{mode: 'enforce'|'shadow', incompleteShare: number, reason: string}}
+ */
+export function recommendEnforce(entry, { threshold = DEFAULT_SHADOW_THRESHOLD, minRuns = DEFAULT_SHADOW_MIN_RUNS } = {}) {
+  const runs = entry?.runs ?? 0;
+  const share = runs ? (entry.counts?.['review.incomplete'] ?? 0) / runs : 0;
+  const pct = (x) => `${Math.round(x * 1000) / 10}%`;
+  if (runs < minRuns) return { mode: 'shadow', incompleteShare: share, reason: `only ${runs} shadow review(s); need ${minRuns}` };
+  if (share >= threshold) return { mode: 'shadow', incompleteShare: share, reason: `would-block-on-incomplete ${pct(share)} is not under ${pct(threshold)}` };
+  return { mode: 'enforce', incompleteShare: share, reason: `would-block-on-incomplete ${pct(share)} < ${pct(threshold)} over ${runs} runs` };
+}
+
+/**
+ * Shadow FORGE:REVIEW verdicts for one repo, from its issue comments.
+ * @param {string} repo
+ * @param {(path: string, flags: string[]) => string} [gh]
+ * @returns {{repo: string, issue: number|null, verdict: string}[]}
+ */
+export function fetchShadowRecords(repo, gh = ghApi) {
+  const out = gh(`repos/${repo}/issues/comments?per_page=100`, ['--paginate', '--jq', '.[] | select(.body | contains("<!-- FORGE:REVIEW -->")) | {issue_url, body}']);
+  const records = [];
+  for (const l of String(out).split('\n')) {
+    let c; try { c = JSON.parse(l); } catch { continue; }
+    const verdict = parseShadowVerdict(c?.body);
+    if (verdict) records.push({ repo, issue: Number(/\/issues\/(\d+)$/.exec(String(c.issue_url ?? ''))?.[1]) || null, verdict });
+  }
+  return records;
+}
+
+/** Text table: one row per repo with the verdict mix and the recommendation. */
+export function formatShadowReport(report) {
+  const lines = ['| Repo | Runs | ' + SHADOW_VERDICTS.join(' | ') + ' | Recommend |', '| --- | --- | ' + SHADOW_VERDICTS.map(() => '---').join(' | ') + ' | --- |'];
+  for (const [repo, e] of Object.entries(report.repos)) {
+    lines.push(`| ${repo} | ${e.runs} | ${SHADOW_VERDICTS.map((v) => `${e.counts[v] ?? 0} (${Math.round((e.shares[v] ?? 0) * 100)}%)`).join(' | ')} | ${e.recommendation.mode}: ${e.recommendation.reason} |`);
+  }
+  return lines.join('\n');
+}
+
+/**
+ * --shadow entry: per-repo distribution plus an enforce recommendation each.
+ * @returns {{shadow: true, repos: object}}
+ */
+export function runShadow(args, repoPath, gh = ghApi) {
+  const repos = String(args.repo || readForgeYaml(repoPath) || '').split(',').map((r) => r.trim()).filter(Boolean);
+  if (!repos.length) throw new Error('Cannot determine repo: no --repo flag and forge.yaml not found or missing project.owner/repo');
+  const records = repos.flatMap((r) => fetchShadowRecords(r, gh));
+  const dist = shadowDistribution(records);
+  const zero = () => Object.fromEntries(SHADOW_VERDICTS.map((v) => [v, 0]));
+  for (const r of repos) dist[r] ??= { runs: 0, counts: zero(), shares: zero() };
+  for (const e of Object.values(dist)) e.recommendation = recommendEnforce(e, { threshold: args.threshold, minRuns: args.minRuns });
+  return { shadow: true, repos: dist };
+}
+
 /**
  * Main entry point (exported for testing).
  *
@@ -821,6 +932,7 @@ export function publishTable(table, repoPath, dryRun = false) {
  */
 export async function run(args, repoPath) {
   _verbose = args.verbose;
+  if (args.shadow) return runShadow(args, repoPath);
 
   // Resolve repo name
   const repo = args.repo || readForgeYaml(repoPath);
@@ -965,6 +1077,10 @@ if (process.argv[1] === fileURLToPath(import.meta.url)) {
 
   run(args, repoPath)
     .then(result => {
+      if (result?.shadow) {
+        process.stdout.write((args.json ? JSON.stringify(result, null, 2) : formatShadowReport(result)) + '\n');
+        process.exit(0);
+      }
       const table = result.table || result;
       const provenanceTable = result.provenanceTable || null;
       const errorCount = result.errorCount || 0;

package/scripts/check-devdocs-freshness.sh

@@ -10,14 +10,19 @@
 # Exempt: a `docs-not-needed: <reason>` line (non-empty reason) in the PR body.
 # Deterministic, no LLM. bash 3.2 portable (no bash-4 builtins, no PCRE grep).
 #
-# Usage: bash scripts/check-devdocs-freshness.sh <base-ref> [--body-file <file>]
+# Usage: bash scripts/check-devdocs-freshness.sh [<base-ref>] [--body-file <file>]
+#   No base-ref: the current branch's upstream (@{upstream}), else origin/HEAD (the remote default branch).
 #   PR body: --body-file, else env PR_BODY (empty when unset). Diff: <base-ref>...HEAD, run from the repo root.
 # Exit: 0 fresh / exempt / nothing to check | 1 docs missing | 2 bad usage or unknown base ref
 
 set -u
-BASE="${1:-}"
-[ -n "$BASE" ] || { echo "usage: check-devdocs-freshness.sh <base-ref> [--body-file <file>]" >&2; exit 2; }
-shift
+BASE=""
+case "${1:-}" in
+  ""|--*) BASE=$(git rev-parse --abbrev-ref --symbolic-full-name '@{upstream}' 2>/dev/null) \
+            || BASE=$(git rev-parse --abbrev-ref origin/HEAD 2>/dev/null) || BASE="" ;;
+  *) BASE="$1"; shift ;;
+esac
+[ -n "$BASE" ] || { echo "usage: check-devdocs-freshness.sh [<base-ref>] [--body-file <file>] (no upstream or origin/HEAD to default to)" >&2; exit 2; }
 BODY="${PR_BODY:-}"
 while [ $# -gt 0 ]; do
   case "$1" in