@bugmole/cli 0.5.0 → 0.6.0

package/src/runtime/propose-cli.ts

@@ -0,0 +1,66 @@
+// `bugmole propose <file>`: a developer proposes test-case changes from a
+// file, the way an agent does with bugmole_propose_test_case.
+//
+//   { "caseSetId": "login", "summary": "Why, for every change below",
+//     "changes": [ { "caseId": "TC-login", "action": "change", "case": { ... } },
+//                  { "caseId": "TC-old", "action": "retire", "summary": "Why this one" } ] }
+//
+// Each change names its case; the revision being changed is read from the
+// project's storage, so the file never has to track revision numbers.
+import fs from "node:fs";
+import { ControlPlaneClient, type RegistryProposal } from "../registry/control-plane-client.js";
+import { activeRevision, caseProblem, revisionOf, type CaseSetDocument } from "../registry/test-case-revisions.js";
+import { createProjectObjectStore } from "../storage/create-object-store.js";
+import { projectPrefix } from "../storage/keys.js";
+import { applyAcceptedProposals } from "./apply-proposals.js";
+
+type ProposalFile = {
+  caseSetId?: unknown;
+  summary?: unknown;
+  changes?: Array<{ caseId?: unknown; action?: unknown; case?: Record<string, unknown>; summary?: unknown }>;
+};
+
+export function registryClient(cfg: any, env: Record<string, string | undefined> = process.env): ControlPlaneClient {
+  const apiKey = env.BUGMOLE_API_KEY ?? cfg.registry?.api_key;
+  if (!apiKey) throw new Error("BUGMOLE_API_KEY is required to propose changes");
+  if (!cfg.project?.id) throw new Error("A project id is required to propose changes");
+  return new ControlPlaneClient({ registryUrl: env.BUGMOLE_REGISTRY_URL ?? cfg.registry?.url ?? "https://api.bugmole.com", apiKey, projectId: cfg.project.id });
+}
+
+/** Proposes every change in the file; stops at the first invalid one before sending anything. */
+export async function proposeFromFile(cfg: any, filePath: string, client: Pick<ControlPlaneClient, "proposeTestCaseChange" | "listUnappliedProposals" | "reportProposalApplied"> = registryClient(cfg)): Promise<RegistryProposal[]> {
+  const file = JSON.parse(fs.readFileSync(filePath, "utf8")) as ProposalFile;
+  const caseSetId = typeof file.caseSetId === "string" ? file.caseSetId : "";
+  if (!caseSetId) throw new Error(`${filePath}: caseSetId is required`);
+  if (!Array.isArray(file.changes) || !file.changes.length) throw new Error(`${filePath}: changes must list at least one change`);
+  const store = createProjectObjectStore(cfg);
+  const key = `${projectPrefix(cfg.project?.id ?? "default")}/test-cases/${caseSetId}.json`;
+  if (!(await store.has(key))) throw new Error(`There's no case set ${caseSetId} in this project's storage`);
+  const caseSet = JSON.parse(await store.getText(key)) as CaseSetDocument;
+
+  const requests = file.changes.map((change, index) => {
+    const where = `${filePath}: change ${index + 1}`;
+    const caseId = typeof change.caseId === "string" ? change.caseId : "";
+    const action = change.action as "add" | "change" | "retire";
+    const summary = typeof change.summary === "string" && change.summary.trim() ? change.summary : typeof file.summary === "string" ? file.summary : "";
+    if (action !== "add" && action !== "change" && action !== "retire") throw new Error(`${where}: action must be add, change or retire`);
+    if (!summary.trim()) throw new Error(`${where}: say why in summary`);
+    const current = activeRevision(caseSet, caseId);
+    if (action !== "add" && !current) throw new Error(`${where}: ${caseId} has no current revision`);
+    if (action === "add" && current) throw new Error(`${where}: ${caseId} already exists; use change`);
+    if (action !== "retire") {
+      const problem = caseProblem({ ...(change.case ?? {}), caseId } as never);
+      if (problem) throw new Error(`${where}: ${problem}`);
+    }
+    return {
+      caseSetId, caseId, action, summary, source: "cli" as const,
+      ...(current ? { baseRevision: revisionOf(current) } : {}),
+      ...(action !== "retire" ? { case: change.case } : {}),
+    };
+  });
+  const proposals: RegistryProposal[] = [];
+  for (const request of requests) proposals.push((await client.proposeTestCaseChange(request)).proposal);
+  // Accepted on arrival (extreme mode) on the project's own storage: write them now.
+  if (proposals.some((proposal) => proposal.status === "accepted" && !proposal.appliedAt)) await applyAcceptedProposals(cfg, client);
+  return proposals;
+}

package/src/runtime/record-run-verdicts.test.ts

@@ -0,0 +1,58 @@
+import assert from "node:assert/strict";
+import fs from "node:fs";
+import os from "node:os";
+import path from "node:path";
+import test from "node:test";
+import { recordRunVerdicts } from "./record-run-verdicts.js";
+
+function setup() {
+  const directory = fs.mkdtempSync(path.join(os.tmpdir(), "bugmole-verdicts-"));
+  const cfg = { storage: { provider: "local", local: { directory } } };
+  const base = path.join(directory, "projects", "shop");
+  fs.mkdirSync(path.join(base, "test-cases"), { recursive: true });
+  fs.writeFileSync(path.join(base, "test-cases", "checkout-cases.json"), JSON.stringify({
+    caseSetId: "checkout-cases",
+    journeyId: "checkout",
+    cases: [
+      { caseId: "TC-login", nodeId: "screen:/login", title: "Sign in" },
+      { caseId: "TC-logout", nodeId: "screen:/logout", title: "Sign out" },
+    ],
+  }));
+  const read = (relative: string) => JSON.parse(fs.readFileSync(path.join(base, relative), "utf8"));
+  const history = () => fs.readdirSync(path.join(base, "test-case-history", "checkout-cases"));
+  return { cfg, directory, read, history };
+}
+
+test("a scoped retest judges only its cases and keeps every other case's latest verdict", async () => {
+  const { cfg, directory, read, history } = setup();
+  try {
+    const nodes = [
+      { nodeId: "screen:/login", outcome: "PASS" as const, status: "success" as const },
+      { nodeId: "screen:/logout", outcome: "FAIL" as const, status: "failed" as const, reason: { message: "Nothing happened" } },
+    ];
+    const full = await recordRunVerdicts(cfg, "shop", "checkout", nodes, "run_full");
+    assert.deepEqual(full, { caseSetId: "checkout-cases", judged: 2 });
+
+    const retest = await recordRunVerdicts(
+      cfg,
+      "shop",
+      "checkout",
+      [{ nodeId: "screen:/login", outcome: "FAIL", status: "failed" }, { nodeId: "screen:/logout", outcome: "PASS", status: "success" }],
+      "run_retest",
+      undefined,
+      { caseSetId: "checkout-cases", caseIds: ["TC-logout"] },
+    );
+    assert.deepEqual(retest, { caseSetId: "checkout-cases", judged: 1 });
+    const latest = read("test-case-results/checkout-cases.json");
+    assert.equal(latest.runId, "run_retest");
+    const byCase = Object.fromEntries(latest.results.map((item: { caseId: string; status: string }) => [item.caseId, item.status]));
+    assert.deepEqual(byCase, { "TC-login": "passed", "TC-logout": "passed" }, "the login verdict is the full run's, not the retest's");
+
+    const entries = history();
+    assert.equal(entries.length, 2);
+    const retestEntry = JSON.parse(fs.readFileSync(path.join(directory, "projects", "shop", "test-case-history", "checkout-cases", entries.find((name) => name.endsWith("-run_retest.json"))!), "utf8"));
+    assert.deepEqual(retestEntry.results.map((item: { caseId: string }) => item.caseId), ["TC-logout"], "history holds only what the retest judged");
+  } finally {
+    fs.rmSync(directory, { recursive: true, force: true });
+  }
+});

package/src/runtime/record-run-verdicts.ts

@@ -10,6 +10,7 @@
 
 import { createProjectObjectStore } from "../storage/create-object-store.js";
 import { projectPrefix } from "../storage/keys.js";
+import { putTestCaseResults, testCaseHistoryKey, testCaseResultsKey } from "../storage/test-case-results.js";
 import { verdictsFromRun, type NodeOutcome } from "./test-case-verdicts.js";
 
 function store(cfg: any) {
@@ -30,6 +31,12 @@ export async function recordRunVerdicts(
   nodes: NodeOutcome[],
   runId?: string,
   unreachedReason?: string,
+  /**
+   * A retest of some cases only (a Jira ticket's case). Just those cases are
+   * judged; the other cases keep their latest verdicts, and the history entry
+   * holds only what this run judged.
+   */
+  scope?: { caseSetId: string; caseIds: string[] } | null,
 ): Promise<{ caseSetId: string; judged: number } | null> {
   const objects = await store(cfg).list(`${projectPrefix(projectId)}/test-cases/`);
   for (const object of objects) {
@@ -40,9 +47,10 @@ export async function recordRunVerdicts(
     } catch {
       continue;
     }
-    if (caseSet?.journeyId !== journeyId) continue;
+    if (scope ? caseSet?.caseSetId !== scope.caseSetId : caseSet?.journeyId !== journeyId) continue;
     const cases = Array.isArray(caseSet.cases)
-      ? caseSet.cases.filter((item: any) => typeof item?.caseId === "string" && typeof item?.nodeId === "string")
+      ? caseSet.cases.filter((item: any) => typeof item?.caseId === "string" && typeof item?.nodeId === "string"
+        && (!scope || scope.caseIds.includes(item.caseId)))
       : [];
     if (cases.length === 0) return null;
 
@@ -60,9 +68,35 @@ export async function recordRunVerdicts(
       recordedAt: new Date().toISOString(),
       results,
     };
-    const key = `${projectPrefix(projectId)}/test-case-results/${caseSet.caseSetId}.json`;
-    await store(cfg).put(key, JSON.stringify(record, null, 2) + "\n", "application/json");
+    if (scope) {
+      await putScopedResults(cfg, projectId, record, scope.caseIds);
+      return { caseSetId: caseSet.caseSetId, judged: results.length };
+    }
+    await putTestCaseResults(store(cfg), projectId, record);
     return { caseSetId: caseSet.caseSetId, judged: results.length };
   }
   return null;
 }
+
+/** A scoped retest: history gets this run's verdicts, the latest file keeps every other case's. */
+async function putScopedResults(
+  cfg: any,
+  projectId: string,
+  record: { caseSetId: string; recordedAt: string; runId?: string; results: Array<{ caseId: string }> } & Record<string, unknown>,
+  caseIds: string[],
+): Promise<void> {
+  const objects = store(cfg);
+  const latestKey = testCaseResultsKey(projectId, record.caseSetId);
+  let previous: any = null;
+  try {
+    previous = JSON.parse(await objects.getText(latestKey));
+  } catch {
+    previous = null;
+  }
+  const kept = Array.isArray(previous?.results)
+    ? previous.results.filter((item: any) => typeof item?.caseId === "string" && !caseIds.includes(item.caseId))
+    : [];
+  const body = (value: unknown) => JSON.stringify(value, null, 2) + "\n";
+  await objects.put(testCaseHistoryKey(projectId, record.caseSetId, record.recordedAt, record.runId), body(record), "application/json");
+  await objects.put(latestKey, body({ ...record, results: [...kept, ...record.results] }), "application/json");
+}

package/src/runtime/reporter.ts

@@ -100,7 +100,7 @@ Generated: ${timestamp}
 
 ## Summary
 
-**${molesFound(summary.failed)}.**
+**${summary.failed > 0 || summary.passed > 0 && summary.unknown === 0 ? molesFound(summary.failed) : summary.totalRuns === 0 ? "Nothing has run yet" : `${summary.passed} passed, ${summary.unknown} with no verdict`}.**
 
 - **Total Runs**: ${summary.totalRuns}
 - **Total Journeys**: ${summary.totalJourneys}

package/src/runtime/reset.ts

@@ -41,6 +41,8 @@ export async function resetProjectData(cfg: any): Promise<void> {
   const specDir = path.resolve(cfg.runtime?.spec_dir || "./spec");
   await fs.rm(path.join(specDir, "journeys.graph.json"), { force: true });
   await fs.rm(path.join(specDir, "plans"), { recursive: true, force: true });
+  // Saved sign-in sessions (bugmole_sign_in) hold the app's cookies.
+  await fs.rm(path.resolve(artifactsDir, projectId, "sessions"), { recursive: true, force: true });
 
   const registryUrl = process.env.BUGMOLE_REGISTRY_URL?.trim();
   const apiKey = process.env.BUGMOLE_API_KEY?.trim();

package/src/runtime/roles-cli.test.ts

@@ -0,0 +1,47 @@
+import assert from "node:assert/strict";
+import fs from "node:fs/promises";
+import os from "node:os";
+import path from "node:path";
+import test from "node:test";
+import { parseRolesCommand, runRolesCommand } from "./roles-cli.js";
+import { parseArgs, resolveMode } from "../../scripts/bugmole.js";
+
+const argv = (...rest: string[]) => ["node", "bugmole", "roles", ...rest];
+
+test("`bugmole roles` lists; `suggest` needs a name and evidence, or --from-evidence", () => {
+  assert.equal(resolveMode(parseArgs(argv())), "roles");
+  assert.deepEqual(parseRolesCommand(argv(), parseArgs(argv())), { action: "list" });
+  const fromEvidence = argv("suggest", "--from-evidence");
+  assert.deepEqual(parseRolesCommand(fromEvidence, parseArgs(fromEvidence)), { action: "suggest", fromEvidence: true });
+  const noEvidence = argv("suggest", "admin");
+  assert.match((parseRolesCommand(noEvidence, parseArgs(noEvidence)) as { error: string }).error, /--evidence/);
+  const full = argv("suggest", "Admin", "--evidence", "route /admin", "--source", "role_gated_route", "--description", "Admin area");
+  assert.deepEqual(parseRolesCommand(full, parseArgs(full)), {
+    action: "suggest", fromEvidence: false, name: "admin", description: "Admin area", evidence: "route /admin", source: "role_gated_route",
+  });
+  const asPerson = argv("suggest", "admin", "--evidence", "x", "--source", "person");
+  assert.equal((parseRolesCommand(asPerson, parseArgs(asPerson)) as { source: string }).source, "agent", "the CLI can't claim to be a person");
+  assert.ok("error" in parseRolesCommand(argv("confirm"), {}));
+});
+
+test("the CLI lists confirmed and suggested roles and can only suggest", async () => {
+  const directory = await fs.mkdtemp(path.join(os.tmpdir(), "bugmole-roles-cli-"));
+  await fs.writeFile(path.join(directory, "roles.yaml"), "roles:\n  - name: user\n  - name: admin\n    status: suggested\n");
+  const cfg = { project: { id: "demo" }, runtime: { spec_dir: directory } };
+  const listed = await runRolesCommand(cfg, { action: "list" }, null);
+  assert.match(listed[0], /Confirmed .*: user/);
+  assert.match(listed[1], /Suggested .*: admin/);
+
+  await assert.rejects(runRolesCommand(cfg, { action: "suggest", fromEvidence: true }, null), /BUGMOLE_API_KEY/);
+  const sent: string[] = [];
+  const client = {
+    listRoles: async () => ({ roles: [] }),
+    proposeRole: async (input: { name: string; description: string; source: string; evidence: string }) => {
+      sent.push(input.name);
+      return { role: { id: "r", name: input.name, description: input.description, status: "suggested" as const, source: input.source, evidence: input.evidence } };
+    },
+  };
+  const out = await runRolesCommand(cfg, { action: "suggest", fromEvidence: false, name: "guest", description: "", evidence: "sign-in at /login", source: "sign_in_wall" }, client);
+  assert.deepEqual(sent, ["guest"]);
+  assert.match(out[0], /guest: suggested — a person confirms/);
+});

package/src/runtime/roles-cli.ts

@@ -0,0 +1,60 @@
+// `bugmole roles`: list the project's roles, or suggest one (or every role
+// the project's evidence points at) for a person to confirm. The CLI, like
+// an agent, can only suggest; confirming happens on the dashboard's Roles page.
+//
+//   bugmole roles                                   list roles and their status
+//   bugmole roles suggest --from-evidence           suggest what the graph, blockers and roles.yaml imply
+//   bugmole roles suggest <name> --evidence "…" [--description "…"] [--source role_gated_route]
+import type { ControlPlaneClient } from "../registry/control-plane-client.js";
+import { ROLE_SOURCES, normalizeRoleName } from "../registry/roles.js";
+import { collectRoleSuggestions, projectRoles, proposeRoleSuggestions, rolesClient, type ProposedRole } from "./project-roles.js";
+
+type RolesClient = Pick<ControlPlaneClient, "listRoles" | "proposeRole">;
+
+export type RolesCommand =
+  | { action: "list" }
+  | { action: "suggest"; fromEvidence: true }
+  | { action: "suggest"; fromEvidence: false; name: string; description: string; evidence: string; source: string };
+
+/** Reads `bugmole roles …` arguments; returns an error message for bad input. */
+export function parseRolesCommand(argv: string[], args: Record<string, unknown>): RolesCommand | { error: string } {
+  const after = argv.slice(argv.indexOf("roles") + 1).filter((arg, index, all) => !arg.startsWith("--") && !(index > 0 && all[index - 1].startsWith("--") && all[index - 1] !== "--from-evidence"));
+  const sub = after[0] ?? "list";
+  if (sub === "list") return { action: "list" };
+  if (sub !== "suggest") return { error: "Usage: bugmole roles [list | suggest]" };
+  if (args.fromevidence === true) return { action: "suggest", fromEvidence: true };
+  const name = normalizeRoleName(after[1]);
+  if (!name) return { error: "Usage: bugmole roles suggest <name> --evidence \"what points at it\" [--description \"…\"], or --from-evidence" };
+  const evidence = typeof args.evidence === "string" ? args.evidence.trim() : "";
+  if (!evidence) return { error: "Say what points at this role with --evidence (a sign-in wall, a gated route or menu, a spec entry)" };
+  const source = typeof args.source === "string" && (ROLE_SOURCES as readonly string[]).includes(args.source) && args.source !== "person" ? args.source : "agent";
+  return { action: "suggest", fromEvidence: false, name, description: typeof args.description === "string" ? args.description : "", evidence, source };
+}
+
+/** Runs the command and returns the lines to print. */
+export async function runRolesCommand(cfg: any, command: RolesCommand, client: RolesClient | null = rolesClient(cfg)): Promise<string[]> {
+  if (command.action === "list") {
+    const roles = await projectRoles(cfg, client);
+    const lines = [
+      `Confirmed (used for test cases and runs): ${roles.usable.join(", ") || "none"}`,
+      `Suggested (waiting for a person on the Roles page): ${roles.suggested.join(", ") || "none"}`,
+    ];
+    if (client && !roles.registryRead) lines.push("Bugmole couldn't reach your account, so these are the roles in spec/roles.yaml only.");
+    return lines;
+  }
+  if (!client) throw new Error("BUGMOLE_API_KEY and a project id are required to suggest roles; or add the role to roles.yaml with status: suggested");
+  let proposed: ProposedRole[];
+  if (command.fromEvidence) {
+    const roles = await projectRoles(cfg, client);
+    proposed = await proposeRoleSuggestions(collectRoleSuggestions(cfg), client, roles.usable);
+    if (!proposed.length) return ["Nothing new to suggest: the evidence points at no role that isn't already confirmed."];
+  } else {
+    const { role, alreadyDecided } = await client.proposeRole({ name: command.name, description: command.description, source: command.source, evidence: command.evidence });
+    proposed = [{ name: role.name, source: command.source, status: role.status, ...(alreadyDecided ? { alreadyDecided } : {}) }];
+  }
+  return proposed.map((role) => role.error
+    ? `${role.name}: not sent (${role.error})`
+    : role.alreadyDecided
+      ? `${role.name}: a person already ${role.status} it; unchanged`
+      : `${role.name}: ${role.status === "suggested" ? "suggested — a person confirms, renames or rejects it on the Roles page" : role.status}`);
+}

package/src/runtime/run-once.ts

@@ -9,7 +9,11 @@ import { ControlPlaneClient, type RegistryRun, type RegistryTask } from "../regi
 import { createProjectObjectStore } from "../storage/create-object-store.js";
 import { projectPrefix } from "../storage/keys.js";
 import { executeRegistryRun, registryDeviceAccess, type StopSignal } from "./run-job.js";
-import { runDiscoveryTask } from "./discovery-task.js";
+import { recordRunVerdicts } from "./record-run-verdicts.js";
+import { discoveryUrl, runDiscoveryTask } from "./discovery-task.js";
+import { fetchRunnerSecrets, injectFlowSecrets, resolveVaultReferences, signInWithFlow, type ReleasedSecrets } from "./cloud-secrets.js";
+import { registryIdentityTokens } from "./vault-federation.js";
+import { redactSecrets } from "./secret-redaction.js";
 import type { execute } from "./executor.js";
 import type { discoverSite } from "./site-discovery.js";
 
@@ -19,7 +23,13 @@ type JobBundle = {
   maxMinutes?: number;
 };
 export type RunBundle = JobBundle & { run: RegistryRun };
-export type TaskBundle = JobBundle & { task: RegistryTask };
+/** `signInFlow` is the project's sign-in flow, run before cloud exploration. */
+export type TaskBundle = JobBundle & {
+  task: RegistryTask;
+  signInFlow?: string | null;
+  /** How far a discovery task may explore, from the workspace's plan. */
+  discovery?: { maxPages: number; maxDepth: number };
+};
 
 type Env = Record<string, string | undefined>;
 
@@ -31,6 +41,8 @@ export type RunOnceDeps = {
   discover?: typeof discoverSite;
   client?: RunnerClient;
   log?: (message: string) => void;
+  /** Runs the sign-in flow before exploring; replaceable in tests. */
+  signIn?: typeof signInWithFlow;
 };
 
 /** The config cloud work executes with: web, headless, Bugmole storage, no local spec. */
@@ -121,7 +133,24 @@ export async function runOnce(env: Env = process.env, deps: RunOnceDeps = {}): P
     ? client.updateRun(job.id, update)
     : client.updateTask(job.id, { workerId, status: update.status, errorMessage: update.errorMessage, progressMessage: update.errorMessage });
 
+  // The project's sign-in secrets, in this process's environment only while
+  // this job runs, and redacted from everything it reports.
+  let removeSecrets = () => undefined as void;
   try {
+    let released: ReleasedSecrets = { secrets: {}, unreadable: [], available: false };
+    try {
+      released = await resolveVaultReferences(await fetchRunnerSecrets(doFetch, registryUrl, token, job), {
+        fetch: doFetch,
+        identityToken: registryIdentityTokens(doFetch, registryUrl, token, job),
+        sessionName: `bugmole-${job.kind}-${job.id}`,
+      });
+    } catch (error) {
+      log(`Couldn't load the project's secrets: ${error instanceof Error ? error.message : error}`);
+    }
+    for (const [name, problem] of Object.entries(released.vaultErrors ?? {})) log(`Couldn't read ${name} from the project's vault: ${problem}`);
+    removeSecrets = injectFlowSecrets(released.secrets);
+    const secretCount = Object.keys(released.secrets).length;
+    if (secretCount) log(`Loaded ${secretCount} project secret${secretCount === 1 ? "" : "s"} for this ${job.kind}`);
     let work: Promise<string>;
     if (job.kind === "run") {
       log(`Running ${bundle.run.planId} for ${projectId} (${job.id})`);
@@ -132,11 +161,43 @@ export async function runOnce(env: Env = process.env, deps: RunOnceDeps = {}): P
         execute: deps.execute,
         loadDeviceAccess: registryDeviceAccess(cfg, registryUrl, token, projectId, job.id),
       })
-        .then((result) => result.status);
+        .then(async (result) => {
+          // Judge the flow's cases like `bugmole serve` does, so a cloud run
+          // gives each case (and its Jira ticket) a verdict. A scoped retest
+          // judges only its own cases. Bookkeeping never fails the run.
+          if (bundle.run.journeyId) {
+            await recordRunVerdicts(
+              cfg,
+              projectId,
+              bundle.run.journeyId,
+              (result.results ?? []).map((item) => ({
+                nodeId: item.nodeId,
+                outcome: item.outcome,
+                status: item.status,
+                reason: (item as { reason?: { code?: string; message?: string } }).reason,
+              })),
+              job.id,
+              undefined,
+              bundle.run.caseScope,
+            ).catch((error: unknown) => log(`Couldn't record case verdicts: ${error instanceof Error ? error.message : error}`));
+          }
+          return result.status;
+        });
     } else {
       log(`Exploring for ${projectId} (${job.id})`);
       await restoreJourneyGraph(cfg);
-      work = runDiscoveryTask(cfg, client, bundle.task, workerId, deps.discover, { claimed: true, onStop: stop })
+      const signInFlow = bundle.signInFlow?.trim() ? bundle.signInFlow : null;
+      const signIn = signInFlow
+        ? (url: string) => (deps.signIn ?? signInWithFlow)({
+          flow: signInFlow,
+          baseUrl: discoveryUrl(bundle.task) ?? url,
+          workdir,
+          projectId,
+          jobId: job.id,
+          released,
+        })
+        : undefined;
+      work = runDiscoveryTask(cfg, client, bundle.task, workerId, deps.discover, { claimed: true, onStop: stop, signIn, ...(bundle.discovery ? { limits: bundle.discovery } : {}) })
         .then(() => "done");
     }
     const outcome = await Promise.race([
@@ -157,12 +218,14 @@ export async function runOnce(env: Env = process.env, deps: RunOnceDeps = {}): P
     log(`Finished: ${outcome.status}`);
     return 0;
   } catch (error) {
-    const message = error instanceof Error ? error.message : "Cloud work failed";
+    const message = redactSecrets(error instanceof Error ? error.message : "Cloud work failed");
     log(`Failed: ${message}`);
     await report({ workerId, status: "error", eventType: "failed", errorMessage: message }).catch(() => undefined);
     return 1;
   } finally {
     if (deadline) clearTimeout(deadline);
+    removeSecrets();
+    // Holds the signed-in session and anything the sign-in left behind.
     fs.rmSync(workdir, { recursive: true, force: true });
   }
 }

package/src/runtime/scenario-matrix.test.ts

@@ -0,0 +1,41 @@
+import assert from "node:assert/strict";
+import { readFile } from "node:fs/promises";
+import test from "node:test";
+import { SCENARIO_COPY_HEADER, scenarioCopy, scenarioSource } from "../../scripts/sync-plan-catalog.mjs";
+import { normalizeScenarioExclusions, scenarioMatrix } from "./scenario-matrix.js";
+
+test("the dashboard's copy of the scenario matrix matches the source", async () => {
+  const [source, copy] = await Promise.all([readFile(scenarioSource, "utf8"), readFile(scenarioCopy, "utf8")]);
+  assert.equal(copy, SCENARIO_COPY_HEADER + source, "run `npm run sync:catalog`");
+});
+
+test("a happy path is asked of every step, other categories once per journey", () => {
+  const matrix = scenarioMatrix(["a", "b"], [
+    { caseId: "a-ok", nodeId: "a", category: "happy_path" },
+    { caseId: "b-invalid", nodeId: "b", category: "validation" },
+    { caseId: "old", nodeId: "b" },
+    { caseId: "elsewhere", nodeId: "z", category: "edge_case" },
+  ], [
+    { category: "permission", reason: "One role" },
+    { category: "error_state", nodeId: "a", reason: "Static page" },
+  ]);
+  assert.deepEqual(matrix.rows[0].cells.happy_path, { state: "covered", caseIds: ["a-ok"] });
+  assert.deepEqual(matrix.rows[0].cells.error_state, { state: "excluded", reason: "Static page" });
+  assert.deepEqual(matrix.rows[1].cells.error_state, { state: "empty" });
+  assert.deepEqual(matrix.rows[1].cells.permission, { state: "excluded", reason: "One role" });
+  assert.deepEqual(matrix.uncategorized, ["old"]);
+  // error_state is excluded on a; that answers it for the journey.
+  assert.deepEqual(matrix.missing, [{ category: "happy_path", nodeId: "b" }, { category: "edge_case" }]);
+  assert.deepEqual(scenarioMatrix([], []).missing, []);
+});
+
+test("keeps only well-formed exclusions", () => {
+  assert.deepEqual(normalizeScenarioExclusions([
+    { category: "permission", reason: " One role " },
+    { category: "smoke", reason: "x" },
+    { category: "validation" },
+    { category: "edge_case", nodeId: "a", reason: "No input" },
+    null,
+  ]), [{ category: "permission", reason: "One role" }, { category: "edge_case", nodeId: "a", reason: "No input" }]);
+  assert.deepEqual(normalizeScenarioExclusions("nope"), []);
+});

package/src/runtime/scenario-matrix.ts

@@ -0,0 +1,98 @@
+// Which kinds of scenario a journey's test cases cover. "Is every scenario
+// covered?" is answerable only if each case says what kind of scenario it
+// is: one happy-path case per step proves the step works, not that it
+// rejects bad input or survives an error. This lays a journey's cases out as
+// steps × categories and names what is missing, so a gap is a missing kind
+// of scenario, not only a missing page.
+//
+// Keep this file dependency-free and type-strippable: `npm run sync:catalog`
+// copies it verbatim into the dashboard (apps/qa/src/lib/scenario-matrix.ts).
+
+export const SCENARIO_CATEGORIES = ["happy_path", "validation", "error_state", "permission", "edge_case"] as const;
+export type ScenarioCategory = (typeof SCENARIO_CATEGORIES)[number];
+
+export const SCENARIO_LABEL: Record<ScenarioCategory, string> = {
+  happy_path: "Happy path",
+  validation: "Invalid input",
+  error_state: "Error or empty state",
+  permission: "Permissions",
+  edge_case: "Edge case",
+};
+
+/** What each category asks for, in the words an agent or a person writes cases from. */
+export const SCENARIO_MEANING: Record<ScenarioCategory, string> = {
+  happy_path: "the step works for the intended user with valid input",
+  validation: "bad, missing or out-of-range input is rejected with a clear message",
+  error_state: "a failing backend, a network error or no data is handled without breaking the page",
+  permission: "a signed-out user or a role without access is kept out",
+  edge_case: "limits and unusual but valid input: long text, many items, double submits, back and refresh",
+};
+
+export function isScenarioCategory(value: unknown): value is ScenarioCategory {
+  return typeof value === "string" && (SCENARIO_CATEGORIES as readonly string[]).includes(value);
+}
+
+/**
+ * A case set says a category doesn't apply, with the reason, instead of
+ * leaving a silent hole: a step with no input has nothing to validate, an
+ * app with one role has no permission scenarios. Without nodeId it covers
+ * the whole journey.
+ */
+export type ScenarioExclusion = { category: ScenarioCategory; nodeId?: string; reason: string };
+
+export type ScenarioCase = { caseId: string; nodeId: string; category?: string; status?: string };
+
+export type ScenarioCell =
+  | { state: "covered"; caseIds: string[] }
+  | { state: "excluded"; reason: string }
+  | { state: "empty" };
+
+export type ScenarioMatrix = {
+  rows: Array<{ nodeId: string; cells: Record<ScenarioCategory, ScenarioCell> }>;
+  /** Cases written before categories existed, which the matrix can't place. */
+  uncategorized: string[];
+  /**
+   * What is missing, in order: a step with no happy-path case, then each
+   * category no step of the journey covers or excludes.
+   */
+  missing: Array<{ category: ScenarioCategory; nodeId?: string }>;
+};
+
+/**
+ * The happy path is asked of every step; the other categories once per
+ * journey, on whichever step they belong to. Asking all five of every step
+ * would fill the matrix with "not applicable" and bury the real holes.
+ */
+export function scenarioMatrix(nodeIds: readonly string[], cases: readonly ScenarioCase[], exclusions: readonly ScenarioExclusion[] = []): ScenarioMatrix {
+  const steps = new Set(nodeIds);
+  const uncategorized: string[] = [];
+  const rows = nodeIds.map((nodeId) => {
+    const cells = {} as Record<ScenarioCategory, ScenarioCell>;
+    for (const category of SCENARIO_CATEGORIES) {
+      const caseIds = cases.filter((item) => item.nodeId === nodeId && item.category === category).map((item) => item.caseId);
+      const excluded = exclusions.find((item) => item.category === category && (item.nodeId === nodeId || item.nodeId === undefined));
+      cells[category] = caseIds.length ? { state: "covered", caseIds } : excluded ? { state: "excluded", reason: excluded.reason } : { state: "empty" };
+    }
+    return { nodeId, cells };
+  });
+  for (const item of cases) if (steps.has(item.nodeId) && !isScenarioCategory(item.category)) uncategorized.push(item.caseId);
+
+  const missing: ScenarioMatrix["missing"] = [];
+  for (const row of rows) if (row.cells.happy_path.state === "empty") missing.push({ category: "happy_path", nodeId: row.nodeId });
+  for (const category of SCENARIO_CATEGORIES) {
+    if (category === "happy_path" || !rows.length) continue;
+    if (rows.every((row) => row.cells[category].state === "empty")) missing.push({ category });
+  }
+  return { rows, uncategorized, missing };
+}
+
+/** Exclusions as stored, keeping only well-formed ones. */
+export function normalizeScenarioExclusions(input: unknown): ScenarioExclusion[] {
+  if (!Array.isArray(input)) return [];
+  return input.flatMap((item) => {
+    if (!item || typeof item !== "object") return [];
+    const record = item as Record<string, unknown>;
+    if (!isScenarioCategory(record.category) || typeof record.reason !== "string" || !record.reason.trim()) return [];
+    return [{ category: record.category, reason: record.reason.trim(), ...(typeof record.nodeId === "string" ? { nodeId: record.nodeId } : {}) }];
+  });
+}