@bugmole/cli 0.5.0 → 0.6.0

package/src/runtime/site-discovery.ts

@@ -14,7 +14,7 @@ import { launcherFor } from "./browser-matrix.js";
 
 export type DiscoveryEvent = {
   at: string;
-  kind: "visit" | "click" | "form" | "page" | "flows" | "done" | "skip" | "error";
+  kind: "visit" | "click" | "form" | "page" | "flows" | "done" | "skip" | "error" | "limit";
   text: string;
   route?: string;
 };
@@ -52,13 +52,45 @@ export type DiscoveredFlow = {
   steps: string[];
 };
 
+/**
+ * The bounds this exploration ran with and whether they cut it short. A crawl
+ * that stops at a limit has seen part of the app, and says how much it left.
+ */
+export type DiscoveryLimits = {
+  maxPages: number;
+  maxDepth: number;
+  /** It stopped with pages still queued because it reached maxPages. */
+  stoppedAtPageLimit: boolean;
+  /** Pages it found links to but never opened: queued past maxPages, or deeper than maxDepth. */
+  notFollowed: string[];
+};
+
 export type DiscoveryResult = {
   baseUrl: string;
   pages: DiscoveredPage[];
   flows: DiscoveredFlow[];
   events: DiscoveryEvent[];
+  limits?: DiscoveryLimits;
 };
 
+/** Discovery's bounds when nobody sets them: the Free plan's. */
+export const DEFAULT_DISCOVERY_LIMITS = { maxPages: 12, maxDepth: 2 } as const;
+
+/**
+ * Limits from the environment of the machine that explores
+ * (BUGMOLE_DISCOVERY_MAX_PAGES / BUGMOLE_DISCOVERY_MAX_DEPTH), for a project's
+ * own worker. Anything not a positive whole number is ignored.
+ */
+export function discoveryLimitsFromEnv(env: Record<string, string | undefined> = process.env): { maxPages?: number; maxDepth?: number } {
+  const read = (name: string) => {
+    const value = Number(env[name]);
+    return Number.isInteger(value) && value > 0 ? value : undefined;
+  };
+  const maxPages = read("BUGMOLE_DISCOVERY_MAX_PAGES");
+  const maxDepth = read("BUGMOLE_DISCOVERY_MAX_DEPTH");
+  return { ...(maxPages ? { maxPages } : {}), ...(maxDepth ? { maxDepth } : {}) };
+}
+
 export type DiscoveryOptions = {
   maxPages?: number;
   maxDepth?: number;
@@ -223,7 +255,8 @@ const pageProbe = `() => {
     const label = text(link) || link.getAttribute("aria-label") || "";
     // Card-style links carry a title and a sentence; only whole paragraphs
     // wrapped in a link are left out.
-    if (label && label.length <= 200) links.push({ label, href: link.href });
+    // A download (an export, a report file) isn't a page a visitor reads.
+    if (label && label.length <= 200) links.push({ label, href: link.href, ...(link.hasAttribute("download") ? { download: true } : {}) });
   }
   const labelOf = (field) => {
     const own = field.labels && field.labels[0] ? text(field.labels[0]) : "";
@@ -255,7 +288,7 @@ const pageProbe = `() => {
   return { title: document.title, heading: heading ? text(heading) : undefined, links, forms, controls };
 }`;
 
-type Probe = { title: string; heading?: string; links: Array<{ label: string; href: string }>; forms: DiscoveredForm[]; controls?: string[] };
+type Probe = { title: string; heading?: string; links: Array<{ label: string; href: string; download?: boolean }>; forms: DiscoveredForm[]; controls?: string[] };
 
 async function readPage(page: Page): Promise<Probe> {
   return page.evaluate((source: string) => (new Function(`return (${source})`)() as () => Probe)(), pageProbe);
@@ -332,12 +365,19 @@ export function stoppedAtSignIn(result: Pick<DiscoveryResult, "pages">): boolean
 export async function discoverSite(baseUrl: string, options: DiscoveryOptions = {}): Promise<DiscoveryResult> {
   const start = new URL(baseUrl);
   const root = `${start.origin}${start.pathname}`;
-  const maxPages = options.maxPages ?? 12;
-  const maxDepth = options.maxDepth ?? 2;
+  const maxPages = options.maxPages ?? DEFAULT_DISCOVERY_LIMITS.maxPages;
+  const maxDepth = options.maxDepth ?? DEFAULT_DISCOVERY_LIMITS.maxDepth;
   const timeout = options.navigationTimeoutMs ?? 15_000;
   const envDwell = Number(process.env.BUGMOLE_DISCOVERY_DWELL_MS);
   const dwellMs = options.dwellMs ?? (Number.isFinite(envDwell) && envDwell >= 0 ? envDwell : 600);
   const result: DiscoveryResult = { baseUrl: root, pages: [], flows: [], events: [] };
+  // The same link in a shared menu is skipped on every page; say so once.
+  const skipped = new Set<string>();
+  const skipOnce = async (text: string, route: string) => {
+    if (skipped.has(text)) return;
+    skipped.add(text);
+    await emit("skip", text, route);
+  };
   const emit = async (kind: DiscoveryEvent["kind"], text: string, route?: string) => {
     const event: DiscoveryEvent = { at: new Date().toISOString(), kind, text, ...(route ? { route } : {}) };
     result.events.push(event);
@@ -354,6 +394,8 @@ export async function discoverSite(baseUrl: string, options: DiscoveryOptions =
     const page = await context.newPage();
     page.setDefaultTimeout(timeout);
     const seen = new Set<string>();
+    // Links seen on a page at the depth limit, so never queued.
+    const tooDeep = new Set<string>();
     // Breadth-first, so pages close to home are found first.
     const queue: Array<{ route: string; path: string[]; parent?: string; menu?: string }> = [{ route: discoverableRoute(root, root) ?? "/", path: [] }];
 
@@ -373,7 +415,10 @@ export async function discoverSite(baseUrl: string, options: DiscoveryOptions =
           const before = await currentHeading(page);
           if (next.menu) {
             await emit("click", `Opening “${next.menu}”`, next.route);
-            await page.getByRole("button", { name: next.menu, exact: true }).first().click();
+            // A disclosure can remember being open (the dashboard's Advanced
+            // group does); clicking it then would close it and hide the link.
+            const menuButton = page.getByRole("button", { name: next.menu, exact: true }).first();
+            if ((await menuButton.getAttribute("aria-expanded").catch(() => null)) !== "true") await menuButton.click();
           }
           await emit("click", `Clicking “${label}”`, next.route);
           await page.getByRole("link", { name: label, exact: true }).first().click();
@@ -402,14 +447,24 @@ export async function discoverSite(baseUrl: string, options: DiscoveryOptions =
         }
         if (route !== next.route && seen.has(route)) continue;
         seen.add(route);
+        // A link to an API endpoint or a file opens JSON or a viewer, not a page of the app.
+        const contentType = await page.evaluate("document.contentType").then(String).catch(() => "text/html");
+        if (!/html/i.test(contentType)) {
+          await emit("skip", `“${next.path[next.path.length - 1] ?? route}” opens a file (${contentType}), not a page`, route);
+          continue;
+        }
         const probe = await readPage(page);
         const inMenus = await linksInMenus(page, new Set(probe.links.map((link) => link.href)));
         const links: DiscoveredPage["links"] = [];
-        for (const link of [...probe.links, ...inMenus] as Array<{ label: string; href: string; menu?: string }>) {
+        for (const link of [...probe.links, ...inMenus] as Array<{ label: string; href: string; menu?: string; download?: boolean }>) {
           const target = discoverableRoute(link.href, root);
           if (!target || target === route || links.some((existing) => existing.route === target)) continue;
           if (UNSAFE_LINK.test(link.label) || UNSAFE_LINK.test(target.replace(/[/_-]+/g, " "))) {
-            await emit("skip", `Skipping “${link.label}” (it could change data)`, target);
+            await skipOnce(`Skipping “${link.label}” (it could change data)`, target);
+            continue;
+          }
+          if (link.download || /^\/api(\/|$)/.test(target)) {
+            await skipOnce(`Skipping “${link.label}” (it downloads a file or calls the API)`, target);
             continue;
           }
           links.push({ label: link.label, route: target, ...(link.menu ? { menu: link.menu } : {}) });
@@ -430,7 +485,9 @@ export async function discoverSite(baseUrl: string, options: DiscoveryOptions =
           await emit("form", `Found a form: ${form.name} (${form.fields.map((field) => field.label).join(", ")})`, route);
         }
         if (dwellMs > 0) await page.waitForTimeout(dwellMs);
-        if (next.path.length < maxDepth) {
+        if (next.path.length >= maxDepth) {
+          for (const link of links) tooDeep.add(link.route);
+        } else {
           for (const link of links) {
             if (!seen.has(link.route)) {
               queue.push({
@@ -447,6 +504,19 @@ export async function discoverSite(baseUrl: string, options: DiscoveryOptions =
         await emit("error", `Could not open ${next.route}: ${error instanceof Error ? error.message.split("\n")[0] : String(error)}`, next.route);
       }
     }
+    const opened = new Set(result.pages.map((found) => found.route));
+    const queued = [...new Set(queue.map((item) => item.route))].filter((route) => !seen.has(route) && !opened.has(route));
+    const notFollowed = [...new Set([...queued, ...tooDeep])].filter((route) => !opened.has(route) && !seen.has(route));
+    const stoppedAtPageLimit = result.pages.length >= maxPages && queued.length > 0;
+    result.limits = { maxPages, maxDepth, stoppedAtPageLimit, notFollowed };
+    if (notFollowed.length) {
+      await emit(
+        "limit",
+        stoppedAtPageLimit
+          ? `Stopped at the ${maxPages}-page limit; ${notFollowed.length} linked page${notFollowed.length === 1 ? " was" : "s were"} not opened`
+          : `${notFollowed.length} linked page${notFollowed.length === 1 ? " is" : "s are"} deeper than ${maxDepth} click${maxDepth === 1 ? "" : "s"} from home and ${notFollowed.length === 1 ? "was" : "were"} not opened`,
+      );
+    }
     result.flows = flowsFor(`${root}${start.search}`, result.pages, { openEachPage: options.openEachPage });
     await emit("flows", `Discovered ${result.flows.length} flow${result.flows.length === 1 ? "" : "s"} across ${result.pages.length} page${result.pages.length === 1 ? "" : "s"}`);
     await emit("done", "Exploration complete");

package/src/runtime/vault-federation.test.ts

@@ -0,0 +1,37 @@
+import assert from "node:assert/strict";
+import test from "node:test";
+import { awsWebIdentityCredentials, googleFederatedToken, registryIdentityTokens } from "./vault-federation.js";
+
+test("asks the registry for a token for the runner's own job, with the audience given", async () => {
+  const seen: Array<{ url: string; auth: string | null }> = [];
+  const source = registryIdentityTokens((async (url: string, init?: RequestInit) => {
+    seen.push({ url, auth: new Headers(init?.headers).get("authorization") });
+    return new Response(JSON.stringify({ token: "jwt" }), { status: 200 });
+  }) as typeof fetch, "https://api.bugmole.com", "vrt_runner", { kind: "task", id: "task 1" });
+  assert.equal(await source("sts.amazonaws.com"), "jwt");
+  assert.deepEqual(seen, [{ url: "https://api.bugmole.com/api/runner/tasks/task%201/identity-token?audience=sts.amazonaws.com", auth: "Bearer vrt_runner" }]);
+  const off = registryIdentityTokens((async () => new Response("{}", { status: 503 })) as unknown as typeof fetch, "https://r", "t", { kind: "run", id: "r" });
+  await assert.rejects(off("x"), /doesn't issue identity tokens/);
+});
+
+test("refuses settings that aren't a Google provider or an IAM role, before asking anyone", async () => {
+  const never = async () => { throw new Error("not called"); };
+  await assert.rejects(googleFederatedToken({ provider: "my-pool", identityToken: never }), /must look like projects\/NUMBER/);
+  await assert.rejects(awsWebIdentityCredentials({ roleArn: "bugmole-reader", sessionName: "s", identityToken: never }), /must be an IAM role ARN/);
+});
+
+test("passes on what the cloud said when it refuses", async () => {
+  const google = googleFederatedToken({
+    provider: "projects/1/locations/global/workloadIdentityPools/p/providers/q",
+    identityToken: async () => "jwt",
+    fetch: (async () => new Response(JSON.stringify({ error: "invalid_grant", error_description: "The audience in ID Token does not match" }), { status: 400 })) as unknown as typeof fetch,
+  });
+  await assert.rejects(google, /refused Bugmole's token \(400\): The audience in ID Token does not match/);
+  const aws = awsWebIdentityCredentials({
+    roleArn: "arn:aws:iam::123456789012:role/r",
+    sessionName: "bugmole run/1",
+    identityToken: async () => "jwt",
+    fetch: (async () => new Response("<ErrorResponse><Error><Message>Not authorized to perform sts:AssumeRoleWithWebIdentity</Message></Error></ErrorResponse>", { status: 403 })) as unknown as typeof fetch,
+  });
+  await assert.rejects(aws, /\(403\): Not authorized to perform sts:AssumeRoleWithWebIdentity/);
+});

package/src/runtime/vault-federation.ts

@@ -0,0 +1,128 @@
+// A Bugmole Cloud runner reading a project's own vault without a stored key.
+// The registry signs a short-lived identity token for the runner's job
+// (registry-worker/identity/oidc.ts); the project's cloud trusts Bugmole as an
+// identity provider and trades that token for credentials limited to what
+// the project allowed:
+//
+//   Google: Workload Identity Federation (STS token exchange), optionally
+//           impersonating a service account
+//   AWS:    an IAM role assumed with AssumeRoleWithWebIdentity
+//
+// What to trust is configured with three non-secret project settings, kept
+// with the project's secrets: VAULT_GCP_WORKLOAD_PROVIDER (the provider's
+// resource name), VAULT_GCP_SERVICE_ACCOUNT (optional) and VAULT_AWS_ROLE_ARN.
+
+import type { AwsCredentials } from "../vendor/byok.js";
+
+/** Asks the registry for an identity token with this audience, for the runner's job. */
+export type IdentityTokenSource = (audience: string) => Promise<string>;
+
+export function registryIdentityTokens(doFetch: typeof fetch, registryUrl: string, token: string, job: { kind: "run" | "task"; id: string }): IdentityTokenSource {
+  return async (audience) => {
+    const reply = await doFetch(`${registryUrl}/api/runner/${job.kind}s/${encodeURIComponent(job.id)}/identity-token?audience=${encodeURIComponent(audience)}`, {
+      headers: { authorization: `Bearer ${token}` },
+    });
+    if (reply.status === 503) throw new Error("This Bugmole registry doesn't issue identity tokens");
+    if (!reply.ok) throw new Error(`The registry refused an identity token (${reply.status})`);
+    const body = await reply.json() as { token?: unknown };
+    if (typeof body.token !== "string") throw new Error("The registry returned no identity token");
+    return body.token;
+  };
+}
+
+async function failure(reply: Response, what: string): Promise<Error> {
+  const text = (await reply.text().catch(() => "")).slice(0, 400);
+  let detail = text;
+  try {
+    const parsed = JSON.parse(text) as { error_description?: string; error?: { message?: string } | string };
+    detail = parsed.error_description ?? (typeof parsed.error === "string" ? parsed.error : parsed.error?.message) ?? text;
+  } catch {
+    // Not JSON (AWS answers in XML).
+    detail = /<Message>([^<]*)<\/Message>/.exec(text)?.[1] ?? text;
+  }
+  return new Error(`${what} (${reply.status}): ${detail}`);
+}
+
+/** A Workload Identity provider's resource name, as Google shows it. */
+const PROVIDER = /^projects\/\d+\/locations\/global\/workloadIdentityPools\/[a-z0-9-]+\/providers\/[a-z0-9-]+$/;
+
+/**
+ * A Google access token for the project's Workload Identity provider: the
+ * federated token itself, or a service account's when one is named.
+ */
+export async function googleFederatedToken(options: {
+  provider: string;
+  serviceAccount?: string;
+  identityToken: IdentityTokenSource;
+  fetch?: typeof fetch;
+}): Promise<string> {
+  const doFetch = options.fetch ?? fetch;
+  const provider = options.provider.replace(/^\/\/iam\.googleapis\.com\//, "");
+  if (!PROVIDER.test(provider)) {
+    throw new Error("VAULT_GCP_WORKLOAD_PROVIDER must look like projects/NUMBER/locations/global/workloadIdentityPools/POOL/providers/PROVIDER");
+  }
+  const audience = `//iam.googleapis.com/${provider}`;
+  const subjectToken = await options.identityToken(audience);
+  const exchanged = await doFetch("https://sts.googleapis.com/v1/token", {
+    method: "POST",
+    headers: { "content-type": "application/json" },
+    body: JSON.stringify({
+      grantType: "urn:ietf:params:oauth:grant-type:token-exchange",
+      audience,
+      scope: "https://www.googleapis.com/auth/cloud-platform",
+      requestedTokenType: "urn:ietf:params:oauth:token-type:access_token",
+      subjectTokenType: "urn:ietf:params:oauth:token-type:jwt",
+      subjectToken,
+    }),
+  });
+  if (!exchanged.ok) throw await failure(exchanged, "Google's Workload Identity Federation refused Bugmole's token");
+  const federated = (await exchanged.json() as { access_token?: string }).access_token;
+  if (!federated) throw new Error("Google's token exchange returned no access token");
+  if (!options.serviceAccount) return federated;
+  const impersonated = await doFetch(
+    `https://iamcredentials.googleapis.com/v1/projects/-/serviceAccounts/${encodeURIComponent(options.serviceAccount)}:generateAccessToken`,
+    {
+      method: "POST",
+      headers: { authorization: `Bearer ${federated}`, "content-type": "application/json" },
+      body: JSON.stringify({ scope: ["https://www.googleapis.com/auth/cloud-platform"], lifetime: "600s" }),
+    },
+  );
+  if (!impersonated.ok) throw await failure(impersonated, `Google refused impersonating ${options.serviceAccount}`);
+  const token = (await impersonated.json() as { accessToken?: string }).accessToken;
+  if (!token) throw new Error("Google returned no service account token");
+  return token;
+}
+
+/** Temporary AWS credentials for the project's role, assumed with Bugmole's token. */
+export async function awsWebIdentityCredentials(options: {
+  roleArn: string;
+  sessionName: string;
+  identityToken: IdentityTokenSource;
+  fetch?: typeof fetch;
+}): Promise<AwsCredentials> {
+  if (!/^arn:aws[a-z-]*:iam::\d{12}:role\/[\w+=,.@/-]+$/.test(options.roleArn)) {
+    throw new Error("VAULT_AWS_ROLE_ARN must be an IAM role ARN (arn:aws:iam::ACCOUNT:role/NAME)");
+  }
+  const webIdentityToken = await options.identityToken("sts.amazonaws.com");
+  // AssumeRoleWithWebIdentity is authorized by the token itself: no signature.
+  const reply = await (options.fetch ?? fetch)("https://sts.amazonaws.com/", {
+    method: "POST",
+    headers: { "content-type": "application/x-www-form-urlencoded" },
+    body: new URLSearchParams({
+      Action: "AssumeRoleWithWebIdentity",
+      Version: "2011-06-15",
+      RoleArn: options.roleArn,
+      RoleSessionName: options.sessionName.replace(/[^\w+=,.@-]/g, "-").slice(0, 64),
+      WebIdentityToken: webIdentityToken,
+      DurationSeconds: "900",
+    }).toString(),
+  });
+  if (!reply.ok) throw await failure(reply, `AWS refused assuming ${options.roleArn} with Bugmole's token`);
+  const xml = await reply.text();
+  const field = (name: string) => new RegExp(`<${name}>([^<]+)</${name}>`).exec(xml)?.[1];
+  const accessKeyId = field("AccessKeyId");
+  const secretAccessKey = field("SecretAccessKey");
+  const sessionToken = field("SessionToken");
+  if (!accessKeyId || !secretAccessKey || !sessionToken) throw new Error("AWS returned no credentials for the role");
+  return { accessKeyId, secretAccessKey, sessionToken };
+}

package/src/runtime/web-suite.ts

@@ -1,7 +1,7 @@
 import fs from "node:fs/promises";
 import path from "node:path";
 import { PlaywrightDriver } from "./playwright-driver.js";
-import type { DriverResult } from "./driver.js";
+import type { AppDriver, DriverResult } from "./driver.js";
 import { targetLabel, type RunTarget } from "./browser-matrix.js";
 import { resolveTargetConcurrency, runPool } from "./target-runner.js";
 
@@ -13,6 +13,8 @@ export interface WebSuiteOptions {
   runId?: string;
   headless?: boolean;
   timeoutMs?: number;
+  /** Wraps the suite's driver, e.g. to give flows their secrets (secret-driver.ts). */
+  wrapDriver?: (driver: AppDriver) => AppDriver;
   /**
    * Comma-separated substrings matched against flow file names, for re-running
    * part of a suite. Include the sign-in flow when the suite shares a session.
@@ -96,7 +98,8 @@ export async function runWebSuite(options: WebSuiteOptions): Promise<WebSuiteRep
     throw new Error(`No flow files found in ${options.flowsDir}${options.grep ? ` matching "${options.grep}"` : ""}.`);
   }
 
-  const driver = new PlaywrightDriver({ headless: options.headless ?? true });
+  const base = new PlaywrightDriver({ headless: options.headless ?? true });
+  const driver = options.wrapDriver ? options.wrapDriver(base) : base;
   const startedAt = new Date().toISOString();
   const targets = options.targets?.length ? options.targets : undefined;
 

package/src/storage/create-object-store.ts

@@ -1,5 +1,5 @@
 import { Storage } from "@google-cloud/storage";
-import { LocalObjectStore, type ObjectStore, type StorageProvider, type StoredObject } from "./object-store.js";
+import { LocalObjectStore, withSecretRedaction, type ObjectStore, type StorageProvider, type StoredObject } from "./object-store.js";
 import { S3ObjectStore } from "./remote-object-store.js";
 import { RegistryObjectStore } from "./registry-object-store.js";
 import { resolveLocalStorageDirectory, type StorageSettings } from "./storage-directory.js";
@@ -106,6 +106,10 @@ export function createProjectObjectStore(cfg: StorageSettings, fallback = "./run
 }
 
 export function createObjectStore(config: ObjectStoreConfig): ObjectStore {
+  return withSecretRedaction(openObjectStore(config));
+}
+
+function openObjectStore(config: ObjectStoreConfig): ObjectStore {
   const nested = (config as Record<string, unknown>)[config.provider] as Record<string, string | undefined> | undefined;
   const value = (name: string, fallback?: string) => nested?.[name] ?? fallback;
   if (config.provider === "local") {

package/src/storage/object-store.ts

@@ -1,5 +1,6 @@
 import fs from "node:fs/promises";
 import path from "node:path";
+import { redactBody, redactSecrets } from "../runtime/secret-redaction.js";
 
 /** "bugmole" is Bugmole-managed storage, reached through the registry. */
 export type StorageProvider = "local" | "gcs" | "s3" | "r2" | "bugmole";
@@ -22,6 +23,16 @@ export type ObjectStore = {
   publicUrl?(key: string, expiresInSeconds?: number): Promise<string>;
 };
 
+/**
+ * Wraps a store so text uploads never carry a flow secret the worker typed:
+ * logs, results, manifests and failure context are redacted on the way out.
+ */
+export function withSecretRedaction(store: ObjectStore): ObjectStore {
+  const put = store.put.bind(store);
+  store.put = (key, body, contentType) => put(key, redactBody(body, contentType), contentType);
+  return store;
+}
+
 export function normalizeKey(key: string): string {
   const normalized = key.replaceAll("\\", "/").replace(/^\/+/, "");
   if (!normalized || normalized.split("/").some((part) => part === ".." || part === ".")) {
@@ -97,5 +108,5 @@ export class LocalObjectStore implements ObjectStore {
 }
 
 export function json(value: unknown): string {
-  return JSON.stringify(value, null, 2);
+  return redactSecrets(JSON.stringify(value, null, 2));
 }

package/src/storage/test-case-results.test.ts

@@ -0,0 +1,64 @@
+import assert from "node:assert/strict";
+import test from "node:test";
+import type { ObjectStore, StoredObject } from "./object-store.js";
+import { putTestCaseResults, testCaseHistoryKey, testCaseHistoryPrefix, testCaseResultsKey } from "./test-case-results.js";
+
+function memoryStore(): ObjectStore & { objects: Map<string, string> } {
+  const objects = new Map<string, string>();
+  const stored = (key: string): StoredObject => ({ key, size: objects.get(key)!.length, modifiedAt: new Date(0).toISOString() });
+  return {
+    provider: "local",
+    objects,
+    async put(key, body) {
+      objects.set(key, typeof body === "string" ? body : new TextDecoder().decode(body));
+      return stored(key);
+    },
+    async get(key) {
+      return new TextEncoder().encode(objects.get(key)!);
+    },
+    async getText(key) {
+      if (!objects.has(key)) throw new Error("missing");
+      return objects.get(key)!;
+    },
+    async has(key) {
+      return objects.has(key);
+    },
+    async list(prefix) {
+      return [...objects.keys()].filter((key) => key.startsWith(prefix)).map(stored);
+    },
+    async deletePrefix(prefix) {
+      for (const key of [...objects.keys()]) if (key.startsWith(prefix)) objects.delete(key);
+    },
+    async publicUrl(key) {
+      return key;
+    },
+  } as ObjectStore & { objects: Map<string, string> };
+}
+
+test("latest and history keys stay inside the project and apart from each other", () => {
+  assert.equal(testCaseResultsKey("qa framework", "login"), "projects/qa%20framework/test-case-results/login.json");
+  assert.equal(testCaseHistoryPrefix("qa framework", "login"), "projects/qa%20framework/test-case-history/login/");
+  assert.ok(!testCaseHistoryPrefix("demo").startsWith("projects/demo/test-case-results/"));
+});
+
+test("history keys sort by recording time and name the run", () => {
+  const early = testCaseHistoryKey("demo", "login", "2026-09-01T09:05:03.007Z", "run-b");
+  const late = testCaseHistoryKey("demo", "login", "2026-09-10T00:00:00.000Z", "run-a");
+  assert.equal(early, "projects/demo/test-case-history/login/20260901T090503007Z-run-b.json");
+  assert.ok(early < late);
+  // A run id that isn't safe in a key is replaced, not copied into the path.
+  const unsafe = testCaseHistoryKey("demo", "login", "2026-09-01T00:00:00.000Z", "../other");
+  assert.match(unsafe, /^projects\/demo\/test-case-history\/login\/20260901T000000000Z-[0-9a-f]{8}\.json$/);
+});
+
+test("each write replaces the latest file and adds one history entry", async () => {
+  const store = memoryStore();
+  await putTestCaseResults(store, "demo", { caseSetId: "login", recordedAt: "2026-09-01T00:00:00.000Z", runId: "r1", results: [{ caseId: "a", status: "passed" }] });
+  const second = await putTestCaseResults(store, "demo", { caseSetId: "login", recordedAt: "2026-09-02T00:00:00.000Z", results: [{ caseId: "a", status: "failed" }] });
+  assert.equal(second.objectKey, "projects/demo/test-case-results/login.json");
+  assert.equal(JSON.parse(store.objects.get(second.objectKey)!).results[0].status, "failed");
+  const history = await store.list(testCaseHistoryPrefix("demo", "login"));
+  assert.equal(history.length, 2);
+  assert.ok(history.some((object) => object.key === second.historyKey));
+  assert.equal((await store.list("projects/demo/test-case-results/")).length, 1);
+});

package/src/storage/test-case-results.ts

@@ -0,0 +1,49 @@
+import { randomBytes } from "node:crypto";
+import type { ObjectStore } from "./object-store.js";
+import { projectPrefix } from "./keys.js";
+
+/**
+ * Where test case verdicts live.
+ *
+ * `latest` is the one file per case set every reader already knows; each
+ * write replaces it. `history` keeps one file per write beside it so the
+ * dashboard can show a case's pass/fail trail. History sits under its own
+ * prefix rather than inside `test-case-results/`: readers list that prefix
+ * and treat every JSON file in it as the latest verdicts for a case set.
+ */
+export function testCaseResultsKey(projectId: string, caseSetId: string): string {
+  return `${projectPrefix(projectId)}/test-case-results/${caseSetId}.json`;
+}
+
+export function testCaseHistoryPrefix(projectId: string, caseSetId?: string): string {
+  const base = `${projectPrefix(projectId)}/test-case-history/`;
+  return caseSetId ? `${base}${caseSetId}/` : base;
+}
+
+/**
+ * One history file name per write. It starts with the recording time in a
+ * fixed-width form so names sort oldest to newest, then the run id (or a
+ * random suffix) so two writes in the same millisecond never collide.
+ */
+export function testCaseHistoryKey(projectId: string, caseSetId: string, recordedAt: string, runId?: string): string {
+  const parsed = typeof recordedAt === "string" ? Date.parse(recordedAt) : Number.NaN;
+  const stamp = new Date(Number.isFinite(parsed) ? parsed : Date.now()).toISOString().replace(/[-:.]/g, "");
+  const suffix = typeof runId === "string" && /^[a-zA-Z0-9._-]+$/.test(runId) ? runId : randomBytes(4).toString("hex");
+  return `${testCaseHistoryPrefix(projectId, caseSetId)}${stamp}-${suffix}.json`;
+}
+
+/** Write a result set as the latest verdicts and as one more history entry. */
+export async function putTestCaseResults(
+  store: ObjectStore,
+  projectId: string,
+  record: { caseSetId: string; recordedAt: string; runId?: string } & Record<string, unknown>,
+): Promise<{ objectKey: string; historyKey: string }> {
+  const body = JSON.stringify(record, null, 2) + "\n";
+  const objectKey = testCaseResultsKey(projectId, record.caseSetId);
+  const historyKey = testCaseHistoryKey(projectId, record.caseSetId, record.recordedAt, record.runId);
+  // History first: if the second write fails the latest file is unchanged,
+  // rather than showing a verdict that never reached the history.
+  await store.put(historyKey, body, "application/json");
+  await store.put(objectKey, body, "application/json");
+  return { objectKey, historyKey };
+}