@bugmole/cli 0.4.0

package/src/runtime/project-spec.ts

@@ -0,0 +1,140 @@
+import fs from "node:fs";
+import path from "node:path";
+import YAML from "yaml";
+import { createProjectObjectStore } from "../storage/create-object-store.js";
+import { projectPrefix } from "../storage/keys.js";
+
+/**
+ * Publish a project's spec (roles, blockers) into its own artifact store.
+ *
+ * The dashboard reads project-scoped artifacts through the local bridge; it
+ * has no access to the repo's spec/ directory. Without this, its blocker
+ * list fell back to hardcoded fixtures that described a different
+ * application entirely, and the richer fields the spec actually carries —
+ * a blocker's type, its selectors, its resolution strategy — had nowhere
+ * to come from and were simply not rendered.
+ *
+ * Publishing per project also stops spec/ being implicitly repo-global:
+ * each project points at its own spec_dir, so two projects no longer have
+ * to share one roles.yaml.
+ */
+
+export type PublishedBlocker = {
+  id: string;
+  title: string;
+  description: string;
+  hardStop: boolean;
+  source: string;
+  type?: string;
+  selectors: string[];
+  resolutionStrategy?: string;
+  onEncounter?: string;
+};
+
+export type PublishedRole = {
+  name: string;
+  description?: string;
+  permissions: string[];
+};
+
+export type PublishedProjectSpec = {
+  schemaVersion: 1;
+  projectId: string;
+  publishedAt: string;
+  specDir: string;
+  roles: PublishedRole[];
+  blockers: PublishedBlocker[];
+};
+
+function readYaml(filePath: string): any {
+  try {
+    return YAML.parse(fs.readFileSync(filePath, "utf8"));
+  } catch {
+    return null;
+  }
+}
+
+function asStrings(value: unknown): string[] {
+  return Array.isArray(value) ? value.filter((item): item is string => typeof item === "string") : [];
+}
+
+/** Human title from a snake_case blocker name, without inventing wording. */
+function titleFromName(name: string): string {
+  const spaced = name.replace(/[_-]+/g, " ").trim();
+  return spaced.charAt(0).toUpperCase() + spaced.slice(1);
+}
+
+export function readProjectSpec(specDir: string, projectId: string): PublishedProjectSpec {
+  const rolesDoc = readYaml(path.join(specDir, "roles.yaml"));
+  const blockersDoc = readYaml(path.join(specDir, "blockers.yaml"));
+
+  const roles: PublishedRole[] = Array.isArray(rolesDoc?.roles)
+    ? rolesDoc.roles
+      .filter((role: any) => typeof role?.name === "string")
+      .map((role: any) => ({
+        name: role.name,
+        description: typeof role.description === "string" ? role.description : undefined,
+        permissions: asStrings(role.permissions),
+      }))
+    : [];
+
+  const blockers: PublishedBlocker[] = Array.isArray(blockersDoc?.blockers)
+    ? blockersDoc.blockers
+      .filter((entry: any) => typeof entry?.name === "string")
+      .map((entry: any) => ({
+        id: entry.name,
+        title: titleFromName(entry.name),
+        description: typeof entry.description === "string" ? entry.description : "",
+        hardStop: entry.hard_stop === true,
+        source: "spec/blockers.yaml",
+        type: typeof entry.type === "string" ? entry.type : undefined,
+        selectors: asStrings(entry.selectors),
+        resolutionStrategy: typeof entry.resolution?.strategy === "string" ? entry.resolution.strategy : undefined,
+        onEncounter: typeof entry.on_encounter === "string" ? entry.on_encounter : undefined,
+      }))
+    : [];
+
+  // Top-level hard_stops are policy conditions rather than encounterable
+  // blockers, but they do stop a run, so they belong in the same list
+  // instead of being invisible to the dashboard.
+  const policyStops: PublishedBlocker[] = Array.isArray(blockersDoc?.hard_stops)
+    ? blockersDoc.hard_stops
+      .filter((entry: any) => typeof entry?.name === "string")
+      .map((entry: any) => ({
+        id: entry.name,
+        title: titleFromName(entry.name),
+        description: typeof entry.description === "string" ? entry.description : "",
+        hardStop: true,
+        source: "spec/blockers.yaml",
+        type: "policy",
+        selectors: [],
+        resolutionStrategy: typeof entry.action === "string" ? entry.action : undefined,
+        onEncounter: typeof entry.condition === "string" ? entry.condition : undefined,
+      }))
+    : [];
+
+  const seen = new Set(blockers.map((blocker) => blocker.id));
+  return {
+    schemaVersion: 1,
+    projectId,
+    publishedAt: new Date().toISOString(),
+    specDir,
+    roles,
+    blockers: [...blockers, ...policyStops.filter((entry) => !seen.has(entry.id))],
+  };
+}
+
+export function projectSpecKey(projectId: string): string {
+  return `${projectPrefix(projectId)}/spec/project-spec.json`;
+}
+
+/** Write the project's spec so the dashboard can read it via the bridge. */
+export async function publishProjectSpec(cfg: any): Promise<PublishedProjectSpec | null> {
+  const projectId = cfg.project?.id ?? "default";
+  const specDir = cfg.runtime?.spec_dir || "./spec";
+  const spec = readProjectSpec(specDir, projectId);
+  if (spec.roles.length === 0 && spec.blockers.length === 0) return null;
+  const store = createProjectObjectStore(cfg);
+  await store.put(projectSpecKey(projectId), JSON.stringify(spec, null, 2) + "\n", "application/json");
+  return spec;
+}

package/src/runtime/record-run-verdicts.ts

@@ -0,0 +1,68 @@
+/**
+ * Persisting the verdicts a run produced.
+ *
+ * Kept beside the pure judging logic rather than inside the worker so the
+ * artifact shape stays in one place: the dashboard, the pipeline's run stage
+ * and the agent-facing bugmole_write_test_case_results tool all read the
+ * same contract, and a second writer that drifted from it would silently stop
+ * counting.
+ */
+
+import { createProjectObjectStore } from "../storage/create-object-store.js";
+import { projectPrefix } from "../storage/keys.js";
+import { verdictsFromRun, type NodeOutcome } from "./test-case-verdicts.js";
+
+function store(cfg: any) {
+  return createProjectObjectStore(cfg, "./artifacts");
+}
+
+/**
+ * Judge and record every case written for `journeyId` from what the run saw.
+ *
+ * Returns the number of cases judged, or null when the flow has no case set
+ * to judge — that is not a failure, it just means nothing was written for
+ * this flow yet and the testcases stage has more to do.
+ */
+export async function recordRunVerdicts(
+  cfg: any,
+  projectId: string,
+  journeyId: string,
+  nodes: NodeOutcome[],
+  runId?: string,
+  unreachedReason?: string,
+): Promise<{ caseSetId: string; judged: number } | null> {
+  const objects = await store(cfg).list(`${projectPrefix(projectId)}/test-cases/`);
+  for (const object of objects) {
+    if (!object.key.endsWith(".json")) continue;
+    let caseSet: any;
+    try {
+      caseSet = JSON.parse(await store(cfg).getText(object.key));
+    } catch {
+      continue;
+    }
+    if (caseSet?.journeyId !== journeyId) continue;
+    const cases = Array.isArray(caseSet.cases)
+      ? caseSet.cases.filter((item: any) => typeof item?.caseId === "string" && typeof item?.nodeId === "string")
+      : [];
+    if (cases.length === 0) return null;
+
+    const results = verdictsFromRun(cases, nodes, unreachedReason);
+    const record = {
+      schemaVersion: 1,
+      caseSetId: caseSet.caseSetId,
+      projectId,
+      journeyId,
+      // Stamped from the case set, so a verdict always says which revision of
+      // the assumptions it judged — a run against superseded assumptions is
+      // discarded rather than shown as proof.
+      journeyRevision: caseSet.journeyRevision,
+      runId,
+      recordedAt: new Date().toISOString(),
+      results,
+    };
+    const key = `${projectPrefix(projectId)}/test-case-results/${caseSet.caseSetId}.json`;
+    await store(cfg).put(key, JSON.stringify(record, null, 2) + "\n", "application/json");
+    return { caseSetId: caseSet.caseSetId, judged: results.length };
+  }
+  return null;
+}

package/src/runtime/reporter.test.ts

@@ -0,0 +1,56 @@
+import { strict as assert } from "node:assert";
+import test from "node:test";
+import fs from "node:fs/promises";
+import os from "node:os";
+import path from "node:path";
+import { runKey } from "../storage/keys.js";
+
+async function makeTestConfig() {
+  const artifactsDir = await fs.mkdtemp(path.join(os.tmpdir(), "bugmole-reporter-test-"));
+  return { project: { id: "demo" }, runtime: { artifacts_dir: artifactsDir } };
+}
+
+test("report includes real per-step detail and evidence links, and never a fabricated drift metric", async () => {
+  const { generateReport } = await import("./reporter.js");
+  const cfg = await makeTestConfig();
+  const resultsKey = runKey("demo", "run-1", "results.json");
+  await fs.mkdir(path.dirname(path.join(cfg.runtime.artifacts_dir, resultsKey)), { recursive: true });
+  await fs.writeFile(
+    path.join(cfg.runtime.artifacts_dir, resultsKey),
+    JSON.stringify({
+      runId: "run-1",
+      journeyId: "checkout",
+      planId: "checkout",
+      timestamp: "2026-08-27T00:00:00.000Z",
+      results: [
+        {
+          nodeId: "step-1",
+          action: "launch",
+          outcome: "PASS",
+          reason: { code: "verified", message: "launchAppCommand: COMPLETED" },
+          screenshot: { source: "runtime", status: "captured", artifactKey: "projects/demo/journeys/evidence/checkout/step-1.png" },
+        },
+        {
+          nodeId: "step-2",
+          action: "assert-home",
+          outcome: "FAIL",
+          reason: { code: "execution_failed", message: "assertVisibleCommand: FAILED" },
+        },
+      ],
+    }),
+    "utf8",
+  );
+
+  const result = await generateReport(cfg, { runId: "run-1", format: "markdown" });
+
+  assert.equal(result.status, "success");
+  assert.equal(result.summary?.passed, 1);
+  assert.equal(result.summary?.failed, 1);
+  assert.ok(result.summary && !("driftDetected" in result.summary));
+  assert.doesNotMatch(result.report ?? "", /Drift Detected/);
+  assert.match(result.report ?? "", /launchAppCommand: COMPLETED/);
+  assert.match(result.report ?? "", /assertVisibleCommand: FAILED/);
+  assert.match(result.report ?? "", /projects\/demo\/journeys\/evidence\/checkout\/step-1\.png/);
+
+  await fs.rm(cfg.runtime.artifacts_dir, { recursive: true, force: true });
+});

package/src/runtime/reporter.ts

@@ -0,0 +1,158 @@
+import { createProjectObjectStore } from "../storage/create-object-store.js";
+import { json, type ObjectStore } from "../storage/object-store.js";
+import { projectPrefix } from "../storage/keys.js";
+import { bugCount, molesFound } from "./mole-voice.js";
+
+interface ReportOptions {
+  runId?: string;
+  format?: "json" | "markdown";
+}
+
+interface ReportResult {
+  status: "success" | "error";
+  reportPath?: string;
+  report?: string;
+  summary?: {
+    totalRuns: number;
+    totalJourneys: number;
+    passed: number;
+    failed: number;
+    unknown: number;
+  };
+  message?: string;
+  error?: string;
+}
+
+function evidenceLine(result: any): string | undefined {
+  const artifactKey = result?.screenshot?.artifactKey;
+  if (!artifactKey) return undefined;
+  return `evidence: ${artifactKey}`;
+}
+
+/**
+ * Reporter: Produces health reports with evidence links and plan diffs
+ */
+export async function generateReport(cfg: any, options: ReportOptions): Promise<ReportResult> {
+  try {
+    const { runId, format = "markdown" } = options;
+    const artifactsDir = cfg.runtime.artifacts_dir || "./runs";
+    const store: ObjectStore = createProjectObjectStore(cfg, artifactsDir);
+
+    // Collect all run results
+    const runs: any[] = [];
+    const projectId = cfg.project?.id ?? "default";
+    const objects = await store.list(`${projectPrefix(projectId)}/runs/`);
+    for (const object of objects.filter((item) => item.key.endsWith("/results.json"))) {
+      const results = JSON.parse(await store.getText(object.key));
+      if (!runId || results.runId === runId) runs.push(results);
+    }
+
+    // Calculate summary
+    let totalJourneys = 0;
+    let passed = 0;
+    let failed = 0;
+    let unknown = 0;
+
+    for (const run of runs) {
+      if (run.results) {
+        for (const result of run.results) {
+          totalJourneys++;
+          if (result.outcome === "PASS") passed++;
+          else if (result.outcome === "FAIL") failed++;
+          else unknown++;
+        }
+      }
+    }
+
+    const summary = {
+      totalRuns: runs.length,
+      totalJourneys,
+      passed,
+      failed,
+      unknown,
+    };
+
+    // Generate report
+    let report: string;
+    const timestamp = new Date().toISOString();
+
+    if (format === "json") {
+      report = JSON.stringify(
+        {
+          timestamp,
+          summary,
+          runs: runs.map((r) => ({
+            runId: r.runId,
+            journeyId: r.journeyId,
+            planId: r.planId,
+            timestamp: r.timestamp,
+            results: r.results,
+          })),
+        },
+        null,
+        2,
+      );
+    } else {
+      // Markdown format
+      report = `# Bugmole report
+
+Generated: ${timestamp}
+
+## Summary
+
+**${molesFound(summary.failed)}.**
+
+- **Total Runs**: ${summary.totalRuns}
+- **Total Journeys**: ${summary.totalJourneys}
+- **Passed**: ${summary.passed}
+- **Failed**: ${summary.failed}
+- **Unknown**: ${summary.unknown}
+
+## Runs
+
+${runs
+  .map(
+    (run) => `### Run: ${run.runId}
+
+- **Journey**: ${run.journeyId}
+- **Plan**: ${run.planId}
+- **Timestamp**: ${run.timestamp}
+- **Results**: ${run.results?.length || 0} steps
+
+${
+  run.results
+    ?.map((r: any) => {
+      const evidence = evidenceLine(r);
+      const detail = r.reason?.message ? ` — ${r.reason.message}` : "";
+      return `- ${r.action}: ${r.outcome}${detail}${evidence ? ` (${evidence})` : ""}`;
+    })
+    .join("\n") || "No results"
+}`,
+  )
+  .join("\n\n")}
+
+## Next Steps
+
+${failed > 0 ? `- Review the ${bugCount(failed)} our moles found` : ""}
+${unknown > 0 ? `- Investigate ${unknown} unknown outcome(s)` : ""}
+`;
+    }
+
+    // Save report
+    const reportKey = `${projectPrefix(projectId)}/reports/report_${Date.now()}.${format === "json" ? "json" : "md"}`;
+    await store.put(reportKey, format === "json" ? json(JSON.parse(report)) : report, format === "json" ? "application/json" : "text/markdown");
+
+    return {
+      status: "success",
+      reportPath: reportKey,
+      report,
+      summary,
+      message: `Report generated: ${reportKey}`,
+    };
+  } catch (error: any) {
+    return {
+      status: "error",
+      error: error.message || "Report generation failed",
+    };
+  }
+}

package/src/runtime/reset.test.ts

@@ -0,0 +1,44 @@
+import assert from "node:assert/strict";
+import fs from "node:fs/promises";
+import os from "node:os";
+import path from "node:path";
+import test from "node:test";
+import { resetProjectData } from "./reset.js";
+import { LocalObjectStore } from "../storage/object-store.js";
+
+test("reset removes all current-project generated data but preserves other projects", async () => {
+  const directory = await fs.mkdtemp(path.join(os.tmpdir(), "bugmole-reset-"));
+  const store = new LocalObjectStore(directory);
+  await store.put("projects/demo/runs/run-1/results.json", "{}");
+  await store.put("projects/demo/reports/report.json", "{}");
+  await store.put("projects/demo/progress/progress.json", "{}");
+  await store.put("projects/demo/plans/plan.flow.yaml", "version: 1");
+  await store.put("projects/demo/journeys/graph.json", "{}");
+  await store.put("projects/demo/ui-audits/audit.json", "{}");
+  await store.put("projects/demo/ui-parity-audits/audit.json", "{}");
+  await store.put("projects/demo/ui-reverse-engineering/job.json", "{}");
+  await store.put("projects/other/runs/run-2/results.json", "{}");
+  await fs.mkdir(path.join(directory, "spec", "plans"), { recursive: true });
+  await fs.writeFile(path.join(directory, "spec", "journeys.graph.json"), "{}");
+  await fs.writeFile(path.join(directory, "spec", "plans", "generated.flow.yaml"), "version: 1");
+  try {
+    await resetProjectData({
+      project: { id: "demo" },
+      runtime: { artifacts_dir: directory, spec_dir: path.join(directory, "spec") },
+      storage: { provider: "local", localDirectory: directory },
+    });
+    assert.equal(await store.has("projects/demo/runs/run-1/results.json"), false);
+    assert.equal(await store.has("projects/demo/reports/report.json"), false);
+    assert.equal(await store.has("projects/demo/progress/progress.json"), false);
+    assert.equal(await store.has("projects/demo/plans/plan.flow.yaml"), false);
+    assert.equal(await store.has("projects/demo/journeys/graph.json"), false);
+    assert.equal(await store.has("projects/demo/ui-audits/audit.json"), false);
+    assert.equal(await store.has("projects/demo/ui-parity-audits/audit.json"), false);
+    assert.equal(await store.has("projects/demo/ui-reverse-engineering/job.json"), false);
+    assert.equal(await fs.stat(path.join(directory, "spec", "journeys.graph.json")).catch(() => null), null);
+    assert.equal(await fs.stat(path.join(directory, "spec", "plans")).catch(() => null), null);
+    assert.equal(await store.has("projects/other/runs/run-2/results.json"), true);
+  } finally {
+    await fs.rm(directory, { recursive: true, force: true });
+  }
+});

package/src/runtime/reset.ts

@@ -0,0 +1,61 @@
+import readline from "node:readline/promises";
+import fs from "node:fs/promises";
+import path from "node:path";
+import { stdin as input, stdout as output } from "node:process";
+import { createProjectObjectStore } from "../storage/create-object-store.js";
+import { projectPrefix } from "../storage/keys.js";
+
+export async function confirmProjectReset(projectId: string): Promise<boolean> {
+  const rl = readline.createInterface({ input, output });
+  try {
+    const answer = await rl.question(
+      `This permanently deletes execution data for project "${projectId}". Type "RESET ${projectId}" to continue: `,
+    );
+    return answer.trim() === `RESET ${projectId}`;
+  } finally {
+    rl.close();
+  }
+}
+
+export async function resetProjectData(cfg: any): Promise<void> {
+  const projectId = cfg.project?.id ?? "default";
+  const artifactsDir = cfg.runtime.artifacts_dir || "./runs";
+  const store = createProjectObjectStore(cfg, artifactsDir);
+  const prefix = projectPrefix(projectId);
+  for (const segment of [
+    "runs",
+    "reports",
+    "progress",
+    "cursor-commits",
+    "plans",
+    "journeys",
+    "ui-audits",
+    "ui-parity-audits",
+    "ui-reverse-engineering",
+  ]) {
+    await store.deletePrefix(`${prefix}/${segment}/`);
+  }
+
+  // These are generated workspace views, not configuration or contracts.
+  // Remove them so the next exploration cannot silently reuse stale state.
+  const specDir = path.resolve(cfg.runtime?.spec_dir || "./spec");
+  await fs.rm(path.join(specDir, "journeys.graph.json"), { force: true });
+  await fs.rm(path.join(specDir, "plans"), { recursive: true, force: true });
+
+  const registryUrl = process.env.BUGMOLE_REGISTRY_URL?.trim();
+  const apiKey = process.env.BUGMOLE_API_KEY?.trim();
+  if (registryUrl && apiKey) {
+    const response = await fetch(
+      `${registryUrl.replace(/\/$/, "")}/api/projects/${encodeURIComponent(projectId)}/reset`,
+      {
+        method: "POST",
+        headers: {
+          authorization: `Bearer ${apiKey}`,
+          "content-type": "application/json",
+        },
+        body: JSON.stringify({ confirmation: `RESET ${projectId}` }),
+      },
+    );
+    if (!response.ok) throw new Error(`Registry reset failed (${response.status})`);
+  }
+}

package/src/runtime/reviewer.test.ts

@@ -0,0 +1,73 @@
+import { strict as assert } from "node:assert";
+import test from "node:test";
+import fs from "node:fs";
+import path from "node:path";
+import os from "node:os";
+
+function makeTestConfig(testDir: string) {
+  const specDir = path.join(testDir, "spec");
+  const artifactsDir = path.join(testDir, "artifacts");
+  fs.mkdirSync(specDir, { recursive: true });
+  fs.mkdirSync(artifactsDir, { recursive: true });
+  fs.writeFileSync(path.join(specDir, "journey_graph.yaml"), "journeys: []\n", "utf8");
+  return { runtime: { spec_dir: specDir, artifacts_dir: artifactsDir } };
+}
+
+test("does not approve a change on the strength of an evidence object alone", async () => {
+  const { review } = await import("./reviewer.js");
+  const testDir = fs.mkdtempSync(path.join(os.tmpdir(), "bugmole-reviewer-test-"));
+  const cfg = makeTestConfig(testDir);
+
+  const result = await review(cfg, {
+    filePath: "journey_graph.yaml",
+    changeSummary: "Widened the selector for the sign-in button.",
+    diff: "-tapOn: '#old-button'\n+tapOn: '#new-button'\n",
+    // Non-empty, but references nothing that exists on disk.
+    evidence: { screenshot: "not-a-real-file.png", note: "trust me" },
+  });
+
+  assert.equal(result.status, "success");
+  assert.equal(result.verdict, "REQUIRE_CLARIFICATION");
+  assert.match(result.reasoning ?? "", /none found on disk/);
+  assert.equal(result.riskAssessment?.couldMaskDefect, true);
+  fs.rmSync(testDir, { recursive: true, force: true });
+});
+
+test("approves a low-risk change once its referenced evidence is verified on disk", async () => {
+  const { review } = await import("./reviewer.js");
+  const testDir = fs.mkdtempSync(path.join(os.tmpdir(), "bugmole-reviewer-test-"));
+  const cfg = makeTestConfig(testDir);
+  const screenshotPath = path.join(cfg.runtime.artifacts_dir, "sign-in-after.png");
+  fs.writeFileSync(screenshotPath, Buffer.from([0]));
+
+  const result = await review(cfg, {
+    filePath: "journey_graph.yaml",
+    changeSummary: "Widened the selector for the sign-in button.",
+    diff: "-tapOn: '#old-button'\n+tapOn: '#new-button'\n",
+    evidence: { screenshot: "sign-in-after.png" },
+  });
+
+  assert.equal(result.verdict, "APPROVED");
+  assert.equal(result.riskAssessment?.level, "low");
+  assert.match(result.reasoning ?? "", /1\/1 referenced artifacts verified/);
+  fs.rmSync(testDir, { recursive: true, force: true });
+});
+
+test("rejects a diff that removes a journey even with verified evidence", async () => {
+  const { review } = await import("./reviewer.js");
+  const testDir = fs.mkdtempSync(path.join(os.tmpdir(), "bugmole-reviewer-test-"));
+  const cfg = makeTestConfig(testDir);
+  const screenshotPath = path.join(cfg.runtime.artifacts_dir, "proof.png");
+  fs.writeFileSync(screenshotPath, Buffer.from([0]));
+
+  const result = await review(cfg, {
+    filePath: "journey_graph.yaml",
+    changeSummary: "Removed a stale journey no longer used by any plan.",
+    diff: "-  - id: checkout\n     journey: checkout\n",
+    evidence: { screenshot: "proof.png" },
+  });
+
+  assert.equal(result.verdict, "REJECTED");
+  assert.ok(result.riskAssessment?.failureModes?.includes("Journey removed"));
+  fs.rmSync(testDir, { recursive: true, force: true });
+});