@bugmole/cli 0.4.0 → 0.6.0

package/src/runtime/record-run-verdicts.ts

@@ -10,6 +10,7 @@
 
 import { createProjectObjectStore } from "../storage/create-object-store.js";
 import { projectPrefix } from "../storage/keys.js";
+import { putTestCaseResults, testCaseHistoryKey, testCaseResultsKey } from "../storage/test-case-results.js";
 import { verdictsFromRun, type NodeOutcome } from "./test-case-verdicts.js";
 
 function store(cfg: any) {
@@ -30,6 +31,12 @@ export async function recordRunVerdicts(
   nodes: NodeOutcome[],
   runId?: string,
   unreachedReason?: string,
+  /**
+   * A retest of some cases only (a Jira ticket's case). Just those cases are
+   * judged; the other cases keep their latest verdicts, and the history entry
+   * holds only what this run judged.
+   */
+  scope?: { caseSetId: string; caseIds: string[] } | null,
 ): Promise<{ caseSetId: string; judged: number } | null> {
   const objects = await store(cfg).list(`${projectPrefix(projectId)}/test-cases/`);
   for (const object of objects) {
@@ -40,9 +47,10 @@ export async function recordRunVerdicts(
     } catch {
       continue;
     }
-    if (caseSet?.journeyId !== journeyId) continue;
+    if (scope ? caseSet?.caseSetId !== scope.caseSetId : caseSet?.journeyId !== journeyId) continue;
     const cases = Array.isArray(caseSet.cases)
-      ? caseSet.cases.filter((item: any) => typeof item?.caseId === "string" && typeof item?.nodeId === "string")
+      ? caseSet.cases.filter((item: any) => typeof item?.caseId === "string" && typeof item?.nodeId === "string"
+        && (!scope || scope.caseIds.includes(item.caseId)))
       : [];
     if (cases.length === 0) return null;
 
@@ -60,9 +68,35 @@ export async function recordRunVerdicts(
       recordedAt: new Date().toISOString(),
       results,
     };
-    const key = `${projectPrefix(projectId)}/test-case-results/${caseSet.caseSetId}.json`;
-    await store(cfg).put(key, JSON.stringify(record, null, 2) + "\n", "application/json");
+    if (scope) {
+      await putScopedResults(cfg, projectId, record, scope.caseIds);
+      return { caseSetId: caseSet.caseSetId, judged: results.length };
+    }
+    await putTestCaseResults(store(cfg), projectId, record);
     return { caseSetId: caseSet.caseSetId, judged: results.length };
   }
   return null;
 }
+
+/** A scoped retest: history gets this run's verdicts, the latest file keeps every other case's. */
+async function putScopedResults(
+  cfg: any,
+  projectId: string,
+  record: { caseSetId: string; recordedAt: string; runId?: string; results: Array<{ caseId: string }> } & Record<string, unknown>,
+  caseIds: string[],
+): Promise<void> {
+  const objects = store(cfg);
+  const latestKey = testCaseResultsKey(projectId, record.caseSetId);
+  let previous: any = null;
+  try {
+    previous = JSON.parse(await objects.getText(latestKey));
+  } catch {
+    previous = null;
+  }
+  const kept = Array.isArray(previous?.results)
+    ? previous.results.filter((item: any) => typeof item?.caseId === "string" && !caseIds.includes(item.caseId))
+    : [];
+  const body = (value: unknown) => JSON.stringify(value, null, 2) + "\n";
+  await objects.put(testCaseHistoryKey(projectId, record.caseSetId, record.recordedAt, record.runId), body(record), "application/json");
+  await objects.put(latestKey, body({ ...record, results: [...kept, ...record.results] }), "application/json");
+}

package/src/runtime/reporter.ts

@@ -100,7 +100,7 @@ Generated: ${timestamp}
 
 ## Summary
 
-**${molesFound(summary.failed)}.**
+**${summary.failed > 0 || summary.passed > 0 && summary.unknown === 0 ? molesFound(summary.failed) : summary.totalRuns === 0 ? "Nothing has run yet" : `${summary.passed} passed, ${summary.unknown} with no verdict`}.**
 
 - **Total Runs**: ${summary.totalRuns}
 - **Total Journeys**: ${summary.totalJourneys}

package/src/runtime/reset.ts

@@ -41,6 +41,8 @@ export async function resetProjectData(cfg: any): Promise<void> {
   const specDir = path.resolve(cfg.runtime?.spec_dir || "./spec");
   await fs.rm(path.join(specDir, "journeys.graph.json"), { force: true });
   await fs.rm(path.join(specDir, "plans"), { recursive: true, force: true });
+  // Saved sign-in sessions (bugmole_sign_in) hold the app's cookies.
+  await fs.rm(path.resolve(artifactsDir, projectId, "sessions"), { recursive: true, force: true });
 
   const registryUrl = process.env.BUGMOLE_REGISTRY_URL?.trim();
   const apiKey = process.env.BUGMOLE_API_KEY?.trim();

package/src/runtime/roles-cli.test.ts

@@ -0,0 +1,47 @@
+import assert from "node:assert/strict";
+import fs from "node:fs/promises";
+import os from "node:os";
+import path from "node:path";
+import test from "node:test";
+import { parseRolesCommand, runRolesCommand } from "./roles-cli.js";
+import { parseArgs, resolveMode } from "../../scripts/bugmole.js";
+
+const argv = (...rest: string[]) => ["node", "bugmole", "roles", ...rest];
+
+test("`bugmole roles` lists; `suggest` needs a name and evidence, or --from-evidence", () => {
+  assert.equal(resolveMode(parseArgs(argv())), "roles");
+  assert.deepEqual(parseRolesCommand(argv(), parseArgs(argv())), { action: "list" });
+  const fromEvidence = argv("suggest", "--from-evidence");
+  assert.deepEqual(parseRolesCommand(fromEvidence, parseArgs(fromEvidence)), { action: "suggest", fromEvidence: true });
+  const noEvidence = argv("suggest", "admin");
+  assert.match((parseRolesCommand(noEvidence, parseArgs(noEvidence)) as { error: string }).error, /--evidence/);
+  const full = argv("suggest", "Admin", "--evidence", "route /admin", "--source", "role_gated_route", "--description", "Admin area");
+  assert.deepEqual(parseRolesCommand(full, parseArgs(full)), {
+    action: "suggest", fromEvidence: false, name: "admin", description: "Admin area", evidence: "route /admin", source: "role_gated_route",
+  });
+  const asPerson = argv("suggest", "admin", "--evidence", "x", "--source", "person");
+  assert.equal((parseRolesCommand(asPerson, parseArgs(asPerson)) as { source: string }).source, "agent", "the CLI can't claim to be a person");
+  assert.ok("error" in parseRolesCommand(argv("confirm"), {}));
+});
+
+test("the CLI lists confirmed and suggested roles and can only suggest", async () => {
+  const directory = await fs.mkdtemp(path.join(os.tmpdir(), "bugmole-roles-cli-"));
+  await fs.writeFile(path.join(directory, "roles.yaml"), "roles:\n  - name: user\n  - name: admin\n    status: suggested\n");
+  const cfg = { project: { id: "demo" }, runtime: { spec_dir: directory } };
+  const listed = await runRolesCommand(cfg, { action: "list" }, null);
+  assert.match(listed[0], /Confirmed .*: user/);
+  assert.match(listed[1], /Suggested .*: admin/);
+
+  await assert.rejects(runRolesCommand(cfg, { action: "suggest", fromEvidence: true }, null), /BUGMOLE_API_KEY/);
+  const sent: string[] = [];
+  const client = {
+    listRoles: async () => ({ roles: [] }),
+    proposeRole: async (input: { name: string; description: string; source: string; evidence: string }) => {
+      sent.push(input.name);
+      return { role: { id: "r", name: input.name, description: input.description, status: "suggested" as const, source: input.source, evidence: input.evidence } };
+    },
+  };
+  const out = await runRolesCommand(cfg, { action: "suggest", fromEvidence: false, name: "guest", description: "", evidence: "sign-in at /login", source: "sign_in_wall" }, client);
+  assert.deepEqual(sent, ["guest"]);
+  assert.match(out[0], /guest: suggested — a person confirms/);
+});

package/src/runtime/roles-cli.ts

@@ -0,0 +1,60 @@
+// `bugmole roles`: list the project's roles, or suggest one (or every role
+// the project's evidence points at) for a person to confirm. The CLI, like
+// an agent, can only suggest; confirming happens on the dashboard's Roles page.
+//
+//   bugmole roles                                   list roles and their status
+//   bugmole roles suggest --from-evidence           suggest what the graph, blockers and roles.yaml imply
+//   bugmole roles suggest <name> --evidence "…" [--description "…"] [--source role_gated_route]
+import type { ControlPlaneClient } from "../registry/control-plane-client.js";
+import { ROLE_SOURCES, normalizeRoleName } from "../registry/roles.js";
+import { collectRoleSuggestions, projectRoles, proposeRoleSuggestions, rolesClient, type ProposedRole } from "./project-roles.js";
+
+type RolesClient = Pick<ControlPlaneClient, "listRoles" | "proposeRole">;
+
+export type RolesCommand =
+  | { action: "list" }
+  | { action: "suggest"; fromEvidence: true }
+  | { action: "suggest"; fromEvidence: false; name: string; description: string; evidence: string; source: string };
+
+/** Reads `bugmole roles …` arguments; returns an error message for bad input. */
+export function parseRolesCommand(argv: string[], args: Record<string, unknown>): RolesCommand | { error: string } {
+  const after = argv.slice(argv.indexOf("roles") + 1).filter((arg, index, all) => !arg.startsWith("--") && !(index > 0 && all[index - 1].startsWith("--") && all[index - 1] !== "--from-evidence"));
+  const sub = after[0] ?? "list";
+  if (sub === "list") return { action: "list" };
+  if (sub !== "suggest") return { error: "Usage: bugmole roles [list | suggest]" };
+  if (args.fromevidence === true) return { action: "suggest", fromEvidence: true };
+  const name = normalizeRoleName(after[1]);
+  if (!name) return { error: "Usage: bugmole roles suggest <name> --evidence \"what points at it\" [--description \"…\"], or --from-evidence" };
+  const evidence = typeof args.evidence === "string" ? args.evidence.trim() : "";
+  if (!evidence) return { error: "Say what points at this role with --evidence (a sign-in wall, a gated route or menu, a spec entry)" };
+  const source = typeof args.source === "string" && (ROLE_SOURCES as readonly string[]).includes(args.source) && args.source !== "person" ? args.source : "agent";
+  return { action: "suggest", fromEvidence: false, name, description: typeof args.description === "string" ? args.description : "", evidence, source };
+}
+
+/** Runs the command and returns the lines to print. */
+export async function runRolesCommand(cfg: any, command: RolesCommand, client: RolesClient | null = rolesClient(cfg)): Promise<string[]> {
+  if (command.action === "list") {
+    const roles = await projectRoles(cfg, client);
+    const lines = [
+      `Confirmed (used for test cases and runs): ${roles.usable.join(", ") || "none"}`,
+      `Suggested (waiting for a person on the Roles page): ${roles.suggested.join(", ") || "none"}`,
+    ];
+    if (client && !roles.registryRead) lines.push("Bugmole couldn't reach your account, so these are the roles in spec/roles.yaml only.");
+    return lines;
+  }
+  if (!client) throw new Error("BUGMOLE_API_KEY and a project id are required to suggest roles; or add the role to roles.yaml with status: suggested");
+  let proposed: ProposedRole[];
+  if (command.fromEvidence) {
+    const roles = await projectRoles(cfg, client);
+    proposed = await proposeRoleSuggestions(collectRoleSuggestions(cfg), client, roles.usable);
+    if (!proposed.length) return ["Nothing new to suggest: the evidence points at no role that isn't already confirmed."];
+  } else {
+    const { role, alreadyDecided } = await client.proposeRole({ name: command.name, description: command.description, source: command.source, evidence: command.evidence });
+    proposed = [{ name: role.name, source: command.source, status: role.status, ...(alreadyDecided ? { alreadyDecided } : {}) }];
+  }
+  return proposed.map((role) => role.error
+    ? `${role.name}: not sent (${role.error})`
+    : role.alreadyDecided
+      ? `${role.name}: a person already ${role.status} it; unchanged`
+      : `${role.name}: ${role.status === "suggested" ? "suggested — a person confirms, renames or rejects it on the Roles page" : role.status}`);
+}

package/src/runtime/run-job.test.ts

@@ -0,0 +1,21 @@
+import assert from "node:assert/strict";
+import test from "node:test";
+import { executeRegistryRun } from "./run-job.js";
+
+test("only a passing run promotes the journey it ran, and before it reports done", async () => {
+  for (const status of ["success", "partial", "error"] as const) {
+    const order: string[] = [];
+    const client = { updateRun: async (_id: string, update: Record<string, unknown>) => { if (update.eventType === "completed") order.push("completed"); return {}; } };
+    const run = { id: `run_${status}`, planId: "discovered-home", journeyId: "discovered-home", status: "claimed" } as never;
+    await executeRegistryRun({}, client as never, run, {
+      projectId: "demo",
+      workerId: "w1",
+      execute: (async () => ({ status, stepsExecuted: 1, stepsPassed: 1, stepsFailed: 0 })) as never,
+      stabilize: async (_cfg, promoted) => {
+        order.push(`stabilize:${promoted.id}`);
+        return promoted.journeyId ?? null;
+      },
+    });
+    assert.deepEqual(order, status === "success" ? ["stabilize:run_success", "completed"] : ["completed"], status);
+  }
+});

package/src/runtime/run-job.ts

@@ -8,6 +8,7 @@ import { normalizeTargets } from "./browser-matrix.js";
 import { execute } from "./executor.js";
 import type { FailureAnalysis, FailureContext } from "./failure-analysis.js";
 import { loadDeviceCloudAccess, type DeviceCloudAccess } from "./device-clouds/index.js";
+import { stabilizeAfterPassingRun } from "./journey-editor.js";
 
 /** The registry expires a claim 60s after its last update, so a running run renews it well inside that. */
 export const RUN_HEARTBEAT_MS = 30_000;
@@ -56,6 +57,8 @@ export type ExecuteRunOptions = {
   heartbeatMs?: number;
   /** Loads device-cloud credentials and app builds, for runs that target cloud devices. */
   loadDeviceAccess?: () => Promise<DeviceCloudAccess>;
+  /** Promotes the journey a passing run proved; replaceable in tests. */
+  stabilize?: typeof stabilizeAfterPassingRun;
 };
 
 /** How a worker fetches a run's device access from the registry. */
@@ -112,6 +115,11 @@ export async function executeRegistryRun(
       },
     });
     clearInterval(heartbeat);
+    // Before reporting done, so a dashboard that reloads the graph on
+    // completion already sees the flow this run proved.
+    if (result.status === "success") {
+      await (options.stabilize ?? stabilizeAfterPassingRun)(cfg, run).catch(() => null);
+    }
     await client.updateRun(run.id, {
       workerId,
       status: result.status === "success" ? "success" : result.status === "partial" ? "partial" : "error",

package/src/runtime/run-once.ts

@@ -9,7 +9,11 @@ import { ControlPlaneClient, type RegistryRun, type RegistryTask } from "../regi
 import { createProjectObjectStore } from "../storage/create-object-store.js";
 import { projectPrefix } from "../storage/keys.js";
 import { executeRegistryRun, registryDeviceAccess, type StopSignal } from "./run-job.js";
-import { runDiscoveryTask } from "./discovery-task.js";
+import { recordRunVerdicts } from "./record-run-verdicts.js";
+import { discoveryUrl, runDiscoveryTask } from "./discovery-task.js";
+import { fetchRunnerSecrets, injectFlowSecrets, resolveVaultReferences, signInWithFlow, type ReleasedSecrets } from "./cloud-secrets.js";
+import { registryIdentityTokens } from "./vault-federation.js";
+import { redactSecrets } from "./secret-redaction.js";
 import type { execute } from "./executor.js";
 import type { discoverSite } from "./site-discovery.js";
 
@@ -19,7 +23,13 @@ type JobBundle = {
   maxMinutes?: number;
 };
 export type RunBundle = JobBundle & { run: RegistryRun };
-export type TaskBundle = JobBundle & { task: RegistryTask };
+/** `signInFlow` is the project's sign-in flow, run before cloud exploration. */
+export type TaskBundle = JobBundle & {
+  task: RegistryTask;
+  signInFlow?: string | null;
+  /** How far a discovery task may explore, from the workspace's plan. */
+  discovery?: { maxPages: number; maxDepth: number };
+};
 
 type Env = Record<string, string | undefined>;
 
@@ -31,6 +41,8 @@ export type RunOnceDeps = {
   discover?: typeof discoverSite;
   client?: RunnerClient;
   log?: (message: string) => void;
+  /** Runs the sign-in flow before exploring; replaceable in tests. */
+  signIn?: typeof signInWithFlow;
 };
 
 /** The config cloud work executes with: web, headless, Bugmole storage, no local spec. */
@@ -121,7 +133,24 @@ export async function runOnce(env: Env = process.env, deps: RunOnceDeps = {}): P
     ? client.updateRun(job.id, update)
     : client.updateTask(job.id, { workerId, status: update.status, errorMessage: update.errorMessage, progressMessage: update.errorMessage });
 
+  // The project's sign-in secrets, in this process's environment only while
+  // this job runs, and redacted from everything it reports.
+  let removeSecrets = () => undefined as void;
   try {
+    let released: ReleasedSecrets = { secrets: {}, unreadable: [], available: false };
+    try {
+      released = await resolveVaultReferences(await fetchRunnerSecrets(doFetch, registryUrl, token, job), {
+        fetch: doFetch,
+        identityToken: registryIdentityTokens(doFetch, registryUrl, token, job),
+        sessionName: `bugmole-${job.kind}-${job.id}`,
+      });
+    } catch (error) {
+      log(`Couldn't load the project's secrets: ${error instanceof Error ? error.message : error}`);
+    }
+    for (const [name, problem] of Object.entries(released.vaultErrors ?? {})) log(`Couldn't read ${name} from the project's vault: ${problem}`);
+    removeSecrets = injectFlowSecrets(released.secrets);
+    const secretCount = Object.keys(released.secrets).length;
+    if (secretCount) log(`Loaded ${secretCount} project secret${secretCount === 1 ? "" : "s"} for this ${job.kind}`);
     let work: Promise<string>;
     if (job.kind === "run") {
       log(`Running ${bundle.run.planId} for ${projectId} (${job.id})`);
@@ -132,11 +161,43 @@ export async function runOnce(env: Env = process.env, deps: RunOnceDeps = {}): P
         execute: deps.execute,
         loadDeviceAccess: registryDeviceAccess(cfg, registryUrl, token, projectId, job.id),
       })
-        .then((result) => result.status);
+        .then(async (result) => {
+          // Judge the flow's cases like `bugmole serve` does, so a cloud run
+          // gives each case (and its Jira ticket) a verdict. A scoped retest
+          // judges only its own cases. Bookkeeping never fails the run.
+          if (bundle.run.journeyId) {
+            await recordRunVerdicts(
+              cfg,
+              projectId,
+              bundle.run.journeyId,
+              (result.results ?? []).map((item) => ({
+                nodeId: item.nodeId,
+                outcome: item.outcome,
+                status: item.status,
+                reason: (item as { reason?: { code?: string; message?: string } }).reason,
+              })),
+              job.id,
+              undefined,
+              bundle.run.caseScope,
+            ).catch((error: unknown) => log(`Couldn't record case verdicts: ${error instanceof Error ? error.message : error}`));
+          }
+          return result.status;
+        });
     } else {
       log(`Exploring for ${projectId} (${job.id})`);
       await restoreJourneyGraph(cfg);
-      work = runDiscoveryTask(cfg, client, bundle.task, workerId, deps.discover, { claimed: true, onStop: stop })
+      const signInFlow = bundle.signInFlow?.trim() ? bundle.signInFlow : null;
+      const signIn = signInFlow
+        ? (url: string) => (deps.signIn ?? signInWithFlow)({
+          flow: signInFlow,
+          baseUrl: discoveryUrl(bundle.task) ?? url,
+          workdir,
+          projectId,
+          jobId: job.id,
+          released,
+        })
+        : undefined;
+      work = runDiscoveryTask(cfg, client, bundle.task, workerId, deps.discover, { claimed: true, onStop: stop, signIn, ...(bundle.discovery ? { limits: bundle.discovery } : {}) })
         .then(() => "done");
     }
     const outcome = await Promise.race([
@@ -157,12 +218,14 @@ export async function runOnce(env: Env = process.env, deps: RunOnceDeps = {}): P
     log(`Finished: ${outcome.status}`);
     return 0;
   } catch (error) {
-    const message = error instanceof Error ? error.message : "Cloud work failed";
+    const message = redactSecrets(error instanceof Error ? error.message : "Cloud work failed");
     log(`Failed: ${message}`);
     await report({ workerId, status: "error", eventType: "failed", errorMessage: message }).catch(() => undefined);
     return 1;
   } finally {
     if (deadline) clearTimeout(deadline);
+    removeSecrets();
+    // Holds the signed-in session and anything the sign-in left behind.
     fs.rmSync(workdir, { recursive: true, force: true });
   }
 }

package/src/runtime/scenario-matrix.test.ts

@@ -0,0 +1,41 @@
+import assert from "node:assert/strict";
+import { readFile } from "node:fs/promises";
+import test from "node:test";
+import { SCENARIO_COPY_HEADER, scenarioCopy, scenarioSource } from "../../scripts/sync-plan-catalog.mjs";
+import { normalizeScenarioExclusions, scenarioMatrix } from "./scenario-matrix.js";
+
+test("the dashboard's copy of the scenario matrix matches the source", async () => {
+  const [source, copy] = await Promise.all([readFile(scenarioSource, "utf8"), readFile(scenarioCopy, "utf8")]);
+  assert.equal(copy, SCENARIO_COPY_HEADER + source, "run `npm run sync:catalog`");
+});
+
+test("a happy path is asked of every step, other categories once per journey", () => {
+  const matrix = scenarioMatrix(["a", "b"], [
+    { caseId: "a-ok", nodeId: "a", category: "happy_path" },
+    { caseId: "b-invalid", nodeId: "b", category: "validation" },
+    { caseId: "old", nodeId: "b" },
+    { caseId: "elsewhere", nodeId: "z", category: "edge_case" },
+  ], [
+    { category: "permission", reason: "One role" },
+    { category: "error_state", nodeId: "a", reason: "Static page" },
+  ]);
+  assert.deepEqual(matrix.rows[0].cells.happy_path, { state: "covered", caseIds: ["a-ok"] });
+  assert.deepEqual(matrix.rows[0].cells.error_state, { state: "excluded", reason: "Static page" });
+  assert.deepEqual(matrix.rows[1].cells.error_state, { state: "empty" });
+  assert.deepEqual(matrix.rows[1].cells.permission, { state: "excluded", reason: "One role" });
+  assert.deepEqual(matrix.uncategorized, ["old"]);
+  // error_state is excluded on a; that answers it for the journey.
+  assert.deepEqual(matrix.missing, [{ category: "happy_path", nodeId: "b" }, { category: "edge_case" }]);
+  assert.deepEqual(scenarioMatrix([], []).missing, []);
+});
+
+test("keeps only well-formed exclusions", () => {
+  assert.deepEqual(normalizeScenarioExclusions([
+    { category: "permission", reason: " One role " },
+    { category: "smoke", reason: "x" },
+    { category: "validation" },
+    { category: "edge_case", nodeId: "a", reason: "No input" },
+    null,
+  ]), [{ category: "permission", reason: "One role" }, { category: "edge_case", nodeId: "a", reason: "No input" }]);
+  assert.deepEqual(normalizeScenarioExclusions("nope"), []);
+});

package/src/runtime/scenario-matrix.ts

@@ -0,0 +1,98 @@
+// Which kinds of scenario a journey's test cases cover. "Is every scenario
+// covered?" is answerable only if each case says what kind of scenario it
+// is: one happy-path case per step proves the step works, not that it
+// rejects bad input or survives an error. This lays a journey's cases out as
+// steps × categories and names what is missing, so a gap is a missing kind
+// of scenario, not only a missing page.
+//
+// Keep this file dependency-free and type-strippable: `npm run sync:catalog`
+// copies it verbatim into the dashboard (apps/qa/src/lib/scenario-matrix.ts).
+
+export const SCENARIO_CATEGORIES = ["happy_path", "validation", "error_state", "permission", "edge_case"] as const;
+export type ScenarioCategory = (typeof SCENARIO_CATEGORIES)[number];
+
+export const SCENARIO_LABEL: Record<ScenarioCategory, string> = {
+  happy_path: "Happy path",
+  validation: "Invalid input",
+  error_state: "Error or empty state",
+  permission: "Permissions",
+  edge_case: "Edge case",
+};
+
+/** What each category asks for, in the words an agent or a person writes cases from. */
+export const SCENARIO_MEANING: Record<ScenarioCategory, string> = {
+  happy_path: "the step works for the intended user with valid input",
+  validation: "bad, missing or out-of-range input is rejected with a clear message",
+  error_state: "a failing backend, a network error or no data is handled without breaking the page",
+  permission: "a signed-out user or a role without access is kept out",
+  edge_case: "limits and unusual but valid input: long text, many items, double submits, back and refresh",
+};
+
+export function isScenarioCategory(value: unknown): value is ScenarioCategory {
+  return typeof value === "string" && (SCENARIO_CATEGORIES as readonly string[]).includes(value);
+}
+
+/**
+ * A case set says a category doesn't apply, with the reason, instead of
+ * leaving a silent hole: a step with no input has nothing to validate, an
+ * app with one role has no permission scenarios. Without nodeId it covers
+ * the whole journey.
+ */
+export type ScenarioExclusion = { category: ScenarioCategory; nodeId?: string; reason: string };
+
+export type ScenarioCase = { caseId: string; nodeId: string; category?: string; status?: string };
+
+export type ScenarioCell =
+  | { state: "covered"; caseIds: string[] }
+  | { state: "excluded"; reason: string }
+  | { state: "empty" };
+
+export type ScenarioMatrix = {
+  rows: Array<{ nodeId: string; cells: Record<ScenarioCategory, ScenarioCell> }>;
+  /** Cases written before categories existed, which the matrix can't place. */
+  uncategorized: string[];
+  /**
+   * What is missing, in order: a step with no happy-path case, then each
+   * category no step of the journey covers or excludes.
+   */
+  missing: Array<{ category: ScenarioCategory; nodeId?: string }>;
+};
+
+/**
+ * The happy path is asked of every step; the other categories once per
+ * journey, on whichever step they belong to. Asking all five of every step
+ * would fill the matrix with "not applicable" and bury the real holes.
+ */
+export function scenarioMatrix(nodeIds: readonly string[], cases: readonly ScenarioCase[], exclusions: readonly ScenarioExclusion[] = []): ScenarioMatrix {
+  const steps = new Set(nodeIds);
+  const uncategorized: string[] = [];
+  const rows = nodeIds.map((nodeId) => {
+    const cells = {} as Record<ScenarioCategory, ScenarioCell>;
+    for (const category of SCENARIO_CATEGORIES) {
+      const caseIds = cases.filter((item) => item.nodeId === nodeId && item.category === category).map((item) => item.caseId);
+      const excluded = exclusions.find((item) => item.category === category && (item.nodeId === nodeId || item.nodeId === undefined));
+      cells[category] = caseIds.length ? { state: "covered", caseIds } : excluded ? { state: "excluded", reason: excluded.reason } : { state: "empty" };
+    }
+    return { nodeId, cells };
+  });
+  for (const item of cases) if (steps.has(item.nodeId) && !isScenarioCategory(item.category)) uncategorized.push(item.caseId);
+
+  const missing: ScenarioMatrix["missing"] = [];
+  for (const row of rows) if (row.cells.happy_path.state === "empty") missing.push({ category: "happy_path", nodeId: row.nodeId });
+  for (const category of SCENARIO_CATEGORIES) {
+    if (category === "happy_path" || !rows.length) continue;
+    if (rows.every((row) => row.cells[category].state === "empty")) missing.push({ category });
+  }
+  return { rows, uncategorized, missing };
+}
+
+/** Exclusions as stored, keeping only well-formed ones. */
+export function normalizeScenarioExclusions(input: unknown): ScenarioExclusion[] {
+  if (!Array.isArray(input)) return [];
+  return input.flatMap((item) => {
+    if (!item || typeof item !== "object") return [];
+    const record = item as Record<string, unknown>;
+    if (!isScenarioCategory(record.category) || typeof record.reason !== "string" || !record.reason.trim()) return [];
+    return [{ category: record.category, reason: record.reason.trim(), ...(typeof record.nodeId === "string" ? { nodeId: record.nodeId } : {}) }];
+  });
+}

package/src/runtime/secret-driver.ts

@@ -0,0 +1,77 @@
+// Gives a flow the secrets it types, from wherever the project keeps them
+// (see secret-sources.ts), for exactly as long as the flow runs: each value is
+// read, attached to that flow's driver context as BUGMOLE_FLOW_SECRET_<NAME>,
+// and registered for redaction. Keeping resolved values off process.env makes
+// concurrent browser/device targets safe.
+
+import fs from "node:fs/promises";
+import type { AppDriver, DriverContext, DriverResult } from "./driver.js";
+import { FLOW_SECRET_ENV_PREFIX, referencedSecretNames } from "./flow-language.js";
+import { redactSecrets, registerSecretValue } from "./secret-redaction.js";
+import { defaultLocalReference, describeReference, resolveSecret, SOURCE_LABEL, type SecretDeps, type SecretReference } from "./secret-sources.js";
+
+export type FlowSecretOptions = {
+  map: Record<string, SecretReference>;
+  projectId: string;
+  deps?: SecretDeps;
+  /**
+   * Look in this machine's secure store for names the map doesn't list (default
+   * true). Off on a cloud runner, which has none: the driver reports
+   * the missing secret itself.
+   */
+  localFallback?: boolean;
+};
+
+function blocked(message: string): DriverResult {
+  return { status: "blocked", message, artifacts: [], failureCode: "secret_unavailable", recoverable: false };
+}
+
+/**
+ * Wraps a driver so each flow gets the secrets it references. A name already
+ * set in the process environment (BUGMOLE_FLOW_SECRET_<NAME>, or a Bugmole
+ * Cloud runner's released secrets) is used as is; resolved names are carried
+ * in the individual DriverContext.
+ */
+export function withFlowSecrets(driver: AppDriver, options: FlowSecretOptions): AppDriver {
+  return {
+    id: driver.id,
+    supports: (platform) => driver.supports(platform),
+    async run(context: DriverContext): Promise<DriverResult> {
+      let content: string;
+      try {
+        content = await fs.readFile(context.flowPath, "utf8");
+      } catch {
+        return driver.run(context);
+      }
+      const env = options.deps?.env ?? process.env;
+      const needed = referencedSecretNames(content).filter((name) => !env[`${FLOW_SECRET_ENV_PREFIX}${name}`]);
+      if (!needed.length) return driver.run(context);
+
+      const values: Record<string, string> = {};
+      for (const name of needed) {
+        const mapped = options.map[name];
+        if (!mapped && options.localFallback === false) continue;
+        const reference = mapped ?? defaultLocalReference(options.projectId, name);
+        try {
+          values[name] = await resolveSecret(reference, options.deps);
+        } catch (error) {
+          const reason = error instanceof Error ? error.message : String(error);
+          return blocked(
+            `The flow types {{secret.${name}}}, but ${SOURCE_LABEL[reference.kind]} (${describeReference(reference)}) couldn't supply it: ${redactSecrets(reason)}.`
+            + (reference.kind === "local" ? ` Store it with: bugmole secrets set ${name}` : ""),
+          );
+        }
+      }
+
+      // Keep resolved values on this flow's context instead of mutating the
+      // process environment. Browser/device matrices run flows concurrently;
+      // global variables would let one target remove another target's secret.
+      const environment = { ...(context.environment ?? {}) };
+      for (const [name, value] of Object.entries(values)) {
+        environment[`${FLOW_SECRET_ENV_PREFIX}${name}`] = value;
+        registerSecretValue(value);
+      }
+      return driver.run({ ...context, environment });
+    },
+  };
+}