@bugmole/cli 0.4.0 → 0.5.0

package/src/runtime/site-discovery.ts

@@ -29,8 +29,15 @@ export type DiscoveredPage = {
   heading?: string;
   /** The link labels a visitor clicks, from the home page, to reach this page. */
   path: string[];
-  links: Array<{ label: string; route: string }>;
+  /** `menu` names the button that must be opened first to see the link. */
+  links: Array<{ label: string; route: string; menu?: string }>;
   forms: DiscoveredForm[];
+  /**
+   * Visible buttons outside forms ("Continue with Aegis", "Add to cart"), by
+   * name. Discovery never presses them; they tell AI exploration what is
+   * really on the page, so it doesn't invent controls.
+   */
+  controls?: string[];
 };
 
 export type DiscoveredFlow = {
@@ -62,10 +69,21 @@ export type DiscoveryOptions = {
    * on the app gentle. Defaults to 600ms (BUGMOLE_DISCOVERY_DWELL_MS).
    */
   dwellMs?: number;
+  /**
+   * A Playwright storage-state file to start from, for exploring an app that
+   * needs sign-in. Links that sign out are already never followed.
+   */
+  storageState?: string;
+  /**
+   * Discovered flows open each page at its address instead of repeating the
+   * clicks that reached it. Use it for apps whose navigation hides behind a
+   * menu on phones, where the desktop clicks would not exist.
+   */
+  openEachPage?: boolean;
   onEvent?: (event: DiscoveryEvent, snapshot: DiscoveryResult) => void | Promise<void>;
 };
 
-const UNSAFE_LINK = /\b(log ?out|sign ?out|delete|remove|destroy|unsubscribe|deactivate|cancel (?:my )?(?:account|subscription))\b/i;
+export const UNSAFE_LINK = /\b(log ?out|sign ?out|delete|remove|destroy|unsubscribe|deactivate|cancel (?:my )?(?:account|subscription))\b/i;
 
 export function slugify(value: string): string {
   return value.toLowerCase().normalize("NFKD").replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 48) || "home";
@@ -97,7 +115,7 @@ function sampleValue(field: DiscoveredField): string {
   if (hint.includes("email")) return "qa@example.com";
   if (hint.includes("phone") || field.type === "tel") return "+1 555 010 0100";
   if (field.type === "number") return "1";
-  if (field.type === "url" || hint.includes("website")) return "https://example.com";
+  if (field.type === "url" || /\b(url|website|link)\b/.test(hint)) return "https://example.com";
   if (hint.includes("name")) return "Bugmole";
   return "Test";
 }
@@ -105,7 +123,11 @@ function sampleValue(field: DiscoveredField): string {
 const FILLABLE = new Set(["text", "email", "password", "tel", "number", "url", "search", "textarea"]);
 
 /** Flows a visitor could run for each discovered page and form. */
-export function flowsFor(baseUrl: string, pages: readonly DiscoveredPage[]): DiscoveredFlow[] {
+export function flowsFor(
+  baseUrl: string,
+  pages: readonly DiscoveredPage[],
+  options: { openEachPage?: boolean } = {},
+): DiscoveredFlow[] {
   const flows: DiscoveredFlow[] = [];
   const used = new Set<string>();
   const uniqueId = (base: string) => {
@@ -114,22 +136,27 @@ export function flowsFor(baseUrl: string, pages: readonly DiscoveredPage[]): Dis
     used.add(id);
     return id;
   };
-  const header = (name: string) => `url: ${baseUrl}\nname: ${JSON.stringify(name)}\n---\n`;
-  const pathSteps = (page: DiscoveredPage) => page.path.map((label) => ({ tapOn: label }));
+  // A query on the base URL is app context (a selected project, say) that
+  // every page needs, so each opened page carries it.
+  const context = new URL(baseUrl).search;
+  const header = (name: string, page: DiscoveredPage) =>
+    `url: ${options.openEachPage ? `${new URL(page.route, baseUrl).origin}${page.route}${context}` : baseUrl}\nname: ${JSON.stringify(name)}\n---\n`;
+  const pathSteps = (page: DiscoveredPage) => (options.openEachPage ? [] : page.path.map((label) => ({ tapOn: label })));
 
   for (const page of pages) {
     const pageName = page.route === "/" ? "Open the home page" : `Visit ${describeRoute(page)}`;
     const landmark = page.heading ?? page.title;
     if (landmark) {
       const id = uniqueId(`discovered-${slugify(page.route === "/" ? "home" : describeRoute(page))}`);
-      const steps: unknown[] = ["launchApp", ...pathSteps(page), { assertVisible: landmark }];
+      // Then the page itself: it fits the screen and its links lead somewhere.
+      const steps: unknown[] = ["launchApp", ...pathSteps(page), { assertVisible: landmark }, "assertLayout", "assertLinks"];
       flows.push({
         id,
         name: pageName,
         kind: "page",
         route: page.route,
         file: `flows/discovered/${id}.yaml`,
-        content: header(pageName) + YAML.stringify(steps),
+        content: header(pageName, page) + YAML.stringify(steps),
         steps: steps.map(stepLabel),
       });
     }
@@ -141,7 +168,9 @@ export function flowsFor(baseUrl: string, pages: readonly DiscoveredPage[]): Dis
       const steps: unknown[] = [
         "launchApp",
         ...pathSteps(page),
-        ...fields.flatMap((field) => [{ tapOn: field.label }, { inputText: sampleValue(field) }]),
+        // Cleared first: a field may already hold a value (an app's own URL,
+        // say), and typing into the middle of it makes nonsense.
+        ...fields.flatMap((field) => [{ tapOn: field.label }, "eraseText", { inputText: sampleValue(field) }]),
         // Discovery never submits; the flow proves the form is ready to.
         ...(form.submit ? [{ assertClickable: form.submit }] : []),
       ];
@@ -151,7 +180,7 @@ export function flowsFor(baseUrl: string, pages: readonly DiscoveredPage[]): Dis
         kind: "form",
         route: page.route,
         file: `flows/discovered/${id}.yaml`,
-        content: header(name) + YAML.stringify(steps),
+        content: header(name, page) + YAML.stringify(steps),
         steps: steps.map(stepLabel),
       });
     }
@@ -160,7 +189,13 @@ export function flowsFor(baseUrl: string, pages: readonly DiscoveredPage[]): Dis
 }
 
 function stepLabel(step: unknown): string {
-  if (typeof step === "string") return step === "launchApp" ? "Open the app" : step;
+  if (typeof step === "string") {
+    if (step === "launchApp") return "Open the app";
+    if (step === "assertLayout") return "Nothing spills off the screen";
+    if (step === "assertLinks") return "Every link on the site works";
+    if (step === "eraseText") return "Clear the field";
+    return step;
+  }
   const [[key, value]] = Object.entries(step as Record<string, unknown>);
   const text = String(value);
   switch (key) {
@@ -186,7 +221,9 @@ const pageProbe = `() => {
   for (const link of document.querySelectorAll("a[href]")) {
     if (!visible(link)) continue;
     const label = text(link) || link.getAttribute("aria-label") || "";
-    if (label && label.length <= 60) links.push({ label, href: link.href });
+    // Card-style links carry a title and a sentence; only whole paragraphs
+    // wrapped in a link are left out.
+    if (label && label.length <= 200) links.push({ label, href: link.href });
   }
   const labelOf = (field) => {
     const own = field.labels && field.labels[0] ? text(field.labels[0]) : "";
@@ -208,15 +245,90 @@ const pageProbe = `() => {
     const legend = form.getAttribute("aria-label") || (form.querySelector("legend, h1, h2, h3") ? text(form.querySelector("legend, h1, h2, h3")) : "");
     forms.push({ name: legend || submitText || "form", fields, submit: submitText || undefined });
   }
-  return { title: document.title, heading: heading ? text(heading) : undefined, links, forms };
+  const controls = [];
+  for (const control of document.querySelectorAll("button, [role=button], input[type=submit], input[type=button]")) {
+    if (!visible(control) || control.closest("form") || control.closest("a[href]")) continue;
+    const name = (text(control) || control.getAttribute("aria-label") || control.value || "").trim();
+    if (name && name.length <= 60 && !controls.includes(name)) controls.push(name);
+    if (controls.length >= 30) break;
+  }
+  return { title: document.title, heading: heading ? text(heading) : undefined, links, forms, controls };
 }`;
 
-type Probe = { title: string; heading?: string; links: Array<{ label: string; href: string }>; forms: DiscoveredForm[] };
+type Probe = { title: string; heading?: string; links: Array<{ label: string; href: string }>; forms: DiscoveredForm[]; controls?: string[] };
 
 async function readPage(page: Page): Promise<Probe> {
   return page.evaluate((source: string) => (new Function(`return (${source})`)() as () => Probe)(), pageProbe);
 }
 
+/**
+ * Menus a person can open — a closed disclosure button (aria-expanded="false")
+ * such as an account menu — and the links inside each. Opening one only
+ * reveals links; it is closed again before moving on.
+ */
+async function linksInMenus(page: Page, known: ReadonlySet<string>): Promise<Array<{ menu: string; label: string; href: string }>> {
+  const found: Array<{ menu: string; label: string; href: string }> = [];
+  const startUrl = page.url();
+  const buttons = page.locator('button[aria-expanded="false"]');
+  const count = Math.min(await buttons.count(), 8);
+  const names: string[] = [];
+  for (let index = 0; index < count; index += 1) {
+    const button = buttons.nth(index);
+    if (!(await button.isVisible().catch(() => false))) continue;
+    // The accessible name, exactly as getByRole matches it, from the
+    // accessibility tree itself.
+    const snapshot = await button.ariaSnapshot().catch(() => "");
+    const quoted = /^- button "((?:[^"\\]|\\.)*)"/.exec(snapshot.trim());
+    const name = quoted ? (JSON.parse(`"${quoted[1]}"`) as string) : "";
+    if (name && name.length <= 60 && !UNSAFE_LINK.test(name) && !names.includes(name)) names.push(name);
+  }
+  for (const name of names) {
+    const button = page.getByRole("button", { name, exact: true }).first();
+    try {
+      await button.click({ timeout: 3_000 });
+      await page.waitForTimeout(200);
+      if (page.url() !== startUrl) {
+        // Not a menu after all: put the page back and stop opening things.
+        await page.goto(startUrl, { waitUntil: "load" });
+        break;
+      }
+      for (const link of (await readPage(page)).links) {
+        if (!known.has(link.href) && !found.some((item) => item.href === link.href)) found.push({ menu: name, ...link });
+      }
+    } catch {
+      // A button that can't be pressed hides nothing we can reach.
+    } finally {
+      await page.keyboard.press("Escape").catch(() => undefined);
+      if ((await button.getAttribute("aria-expanded").catch(() => null)) === "true") await button.click({ timeout: 3_000 }).catch(() => undefined);
+    }
+  }
+  return found;
+}
+
+/** The first visible h1/h2, to tell when a client-side navigation has rendered. */
+async function currentHeading(page: Page): Promise<string | undefined> {
+  return (await readPage(page).catch(() => undefined))?.heading;
+}
+
+const SIGN_IN_WORDS = /\b(sign ?in|log ?in|welcome back)\b/i;
+const SIGN_IN_BUTTON = /^(continue|sign in|log in) with\b/i;
+
+function looksLikeSignIn(page: DiscoveredPage): boolean {
+  if (page.forms.some((form) => form.fields.some((field) => field.type === "password"))) return true;
+  if (SIGN_IN_WORDS.test(page.heading ?? "") || SIGN_IN_WORDS.test(page.title)) return true;
+  return Boolean(page.controls?.some((control) => SIGN_IN_BUTTON.test(control)));
+}
+
+/**
+ * The crawl stopped at a sign-in page: everything it could reach is a sign-in
+ * screen, so what lies behind it is unexplored. Discovery doesn't sign in on
+ * its own; the caller should say so rather than present the one page it saw
+ * as the whole app.
+ */
+export function stoppedAtSignIn(result: Pick<DiscoveryResult, "pages">): boolean {
+  return result.pages.length > 0 && result.pages.length <= 2 && result.pages.every(looksLikeSignIn);
+}
+
 export async function discoverSite(baseUrl: string, options: DiscoveryOptions = {}): Promise<DiscoveryResult> {
   const start = new URL(baseUrl);
   const root = `${start.origin}${start.pathname}`;
@@ -235,12 +347,15 @@ export async function discoverSite(baseUrl: string, options: DiscoveryOptions =
   const { type, channel } = launcherFor("chromium");
   const browser = await type.launch({ headless: options.headless ?? true, ...(channel ? { channel } : {}) });
   try {
-    const context = await browser.newContext({ viewport: { width: 1280, height: 800 } });
+    const context = await browser.newContext({
+      viewport: { width: 1280, height: 800 },
+      ...(options.storageState ? { storageState: options.storageState } : {}),
+    });
     const page = await context.newPage();
     page.setDefaultTimeout(timeout);
     const seen = new Set<string>();
     // Breadth-first, so pages close to home are found first.
-    const queue: Array<{ route: string; path: string[]; parent?: string }> = [{ route: discoverableRoute(root, root) ?? "/", path: [] }];
+    const queue: Array<{ route: string; path: string[]; parent?: string; menu?: string }> = [{ route: discoverableRoute(root, root) ?? "/", path: [] }];
 
     while (queue.length > 0 && result.pages.length < maxPages) {
       const next = queue.shift()!;
@@ -249,30 +364,55 @@ export async function discoverSite(baseUrl: string, options: DiscoveryOptions =
       try {
         if (next.path.length === 0) {
           await emit("visit", "Visiting home page", next.route);
-          await page.goto(root, { waitUntil: "load" });
+          await page.goto(`${root}${start.search}`, { waitUntil: "load" });
         } else {
           // Reach the page the way a visitor would: from its parent, by clicking.
-          const parentUrl = new URL(next.parent ?? "/", root).toString();
+          const parentUrl = `${new URL(next.parent ?? "/", root).toString()}${start.search}`;
           if (discoverableRoute(page.url(), root) !== next.parent) await page.goto(parentUrl, { waitUntil: "load" });
           const label = next.path[next.path.length - 1];
+          const before = await currentHeading(page);
+          if (next.menu) {
+            await emit("click", `Opening “${next.menu}”`, next.route);
+            await page.getByRole("button", { name: next.menu, exact: true }).first().click();
+          }
           await emit("click", `Clicking “${label}”`, next.route);
           await page.getByRole("link", { name: label, exact: true }).first().click();
+          // Apps that route on the client change the address and then render,
+          // some time after the click returns: wait for both, bounded.
+          await page.waitForURL((url) => discoverableRoute(url.toString(), root) !== next.parent, { timeout }).catch(() => undefined);
           await page.waitForLoadState("load").catch(() => undefined);
+          await page.waitForFunction(
+            ([source, previous]) => ((new Function(`return (${source})`)() as () => { heading?: string })().heading ?? "") !== previous,
+            [pageProbe, before ?? ""] as const,
+            { timeout: 5_000 },
+          ).catch(() => undefined);
           await page.waitForTimeout(250);
         }
-        const route = discoverableRoute(page.url(), root) ?? next.route;
+        const landed = discoverableRoute(page.url(), root);
+        if (!landed) {
+          // A same-origin link that hands off elsewhere (an OAuth or install
+          // redirect, say) is not a page of this app.
+          await emit("skip", `“${next.path[next.path.length - 1] ?? next.route}” leaves the app`, next.route);
+          continue;
+        }
+        const route = landed;
+        if (next.path.length > 0 && route === next.parent) {
+          await emit("skip", `“${next.path[next.path.length - 1]}” did not lead anywhere`, next.route);
+          continue;
+        }
         if (route !== next.route && seen.has(route)) continue;
         seen.add(route);
         const probe = await readPage(page);
+        const inMenus = await linksInMenus(page, new Set(probe.links.map((link) => link.href)));
         const links: DiscoveredPage["links"] = [];
-        for (const link of probe.links) {
+        for (const link of [...probe.links, ...inMenus] as Array<{ label: string; href: string; menu?: string }>) {
           const target = discoverableRoute(link.href, root);
           if (!target || target === route || links.some((existing) => existing.route === target)) continue;
-          if (UNSAFE_LINK.test(link.label)) {
+          if (UNSAFE_LINK.test(link.label) || UNSAFE_LINK.test(target.replace(/[/_-]+/g, " "))) {
             await emit("skip", `Skipping “${link.label}” (it could change data)`, target);
             continue;
           }
-          links.push({ label: link.label, route: target });
+          links.push({ label: link.label, route: target, ...(link.menu ? { menu: link.menu } : {}) });
         }
         const discovered: DiscoveredPage = {
           route,
@@ -282,6 +422,7 @@ export async function discoverSite(baseUrl: string, options: DiscoveryOptions =
           path: next.path,
           links,
           forms: probe.forms,
+          ...(probe.controls?.length ? { controls: probe.controls } : {}),
         };
         result.pages.push(discovered);
         await emit("page", `Found ${describeRoute(discovered)} (${links.length} link${links.length === 1 ? "" : "s"})`, route);
@@ -291,14 +432,22 @@ export async function discoverSite(baseUrl: string, options: DiscoveryOptions =
         if (dwellMs > 0) await page.waitForTimeout(dwellMs);
         if (next.path.length < maxDepth) {
           for (const link of links) {
-            if (!seen.has(link.route)) queue.push({ route: link.route, path: [...next.path, link.label], parent: route });
+            if (!seen.has(link.route)) {
+              queue.push({
+                route: link.route,
+                // A flow replays the path as taps, so a menu's button is a step too.
+                path: [...next.path, ...(link.menu ? [link.menu] : []), link.label],
+                parent: route,
+                ...(link.menu ? { menu: link.menu } : {}),
+              });
+            }
           }
         }
       } catch (error) {
         await emit("error", `Could not open ${next.route}: ${error instanceof Error ? error.message.split("\n")[0] : String(error)}`, next.route);
       }
     }
-    result.flows = flowsFor(root, result.pages);
+    result.flows = flowsFor(`${root}${start.search}`, result.pages, { openEachPage: options.openEachPage });
     await emit("flows", `Discovered ${result.flows.length} flow${result.flows.length === 1 ? "" : "s"} across ${result.pages.length} page${result.pages.length === 1 ? "" : "s"}`);
     await emit("done", "Exploration complete");
     return result;

package/src/runtime/web-suite.test.ts

@@ -95,3 +95,68 @@ test("typed secrets never reach the uploaded log, and the shared session is dele
     await fs.rm(dir, { recursive: true, force: true });
   }
 });
+
+test("a flow written for production runs against --base-url, not production", async (t) => {
+  const { chromium } = await import("playwright");
+  try {
+    await fs.access(chromium.executablePath());
+  } catch {
+    t.skip("Chromium is not installed");
+    return;
+  }
+  const http = await import("node:http");
+  const seen: string[] = [];
+  const server = http.createServer((request, response) => {
+    seen.push(request.url ?? "");
+    response.writeHead(200, { "content-type": "text/html" });
+    response.end("<!doctype html><h1>Pricing on staging</h1>");
+  });
+  await new Promise<void>((resolve) => server.listen(0, "127.0.0.1", resolve));
+  const { port } = server.address() as { port: number };
+  const dir = await fs.mkdtemp(path.join(os.tmpdir(), "bugmole-suite-rebase-"));
+  await fs.writeFile(path.join(dir, "pricing.yaml"), "url: https://production.invalid/pricing/\n---\n- launchApp\n- assertVisible: \"Pricing on staging\"\n");
+  try {
+    const report = await runWebSuite({ flowsDir: dir, baseUrl: `http://127.0.0.1:${port}`, artifactsDir: path.join(dir, "artifacts"), projectId: "demo" });
+    assert.equal(report.totals.passed, 1, report.flows[0]?.message);
+    assert.ok(seen.includes("/pricing/"), seen.join(","));
+  } finally {
+    server.close();
+    await fs.rm(dir, { recursive: true, force: true });
+  }
+});
+
+test("waiting for text survives the page navigating away mid-check, as a sign-in redirect does", async (t) => {
+  const { chromium } = await import("playwright");
+  try {
+    await fs.access(chromium.executablePath());
+  } catch {
+    t.skip("Chromium is not installed");
+    return;
+  }
+  const { isNavigationError } = await import("./playwright-driver.js");
+  assert.equal(isNavigationError(new Error("page.evaluate: Execution context was destroyed, most likely because of a navigation")), true);
+  assert.equal(isNavigationError(new Error("Target closed")), false);
+  const http = await import("node:http");
+  const server = http.createServer((request, response) => {
+    response.writeHead(200, { "content-type": "text/html" });
+    if (request.url === "/app") {
+      // The app is slow to answer, so the check runs while the page is between documents.
+      setTimeout(() => response.end("<!doctype html><h1>Quality, as a flow.</h1>"), 700);
+      return;
+    }
+    response.end("<!doctype html><p>Signing you in…</p><script>setTimeout(() => { location.href = '/app'; }, 150)</script>");
+  });
+  await new Promise<void>((resolve) => server.listen(0, "127.0.0.1", resolve));
+  const { port } = server.address() as { port: number };
+  const dir = await fs.mkdtemp(path.join(os.tmpdir(), "bugmole-suite-nav-"));
+  await fs.writeFile(path.join(dir, "sign-in.yaml"), "url: /\n---\n- launchApp\n- assertVisible:\n    text: \"Quality, as a flow.\"\n    timeout: 10000\n");
+  try {
+    for (let attempt = 0; attempt < 3; attempt++) {
+      const report = await runWebSuite({ flowsDir: dir, baseUrl: `http://127.0.0.1:${port}/`, artifactsDir: path.join(dir, `artifacts-${attempt}`), projectId: "demo" });
+      assert.equal(report.totals.passed, 1, report.flows[0]?.message);
+    }
+  } finally {
+    server.close();
+    await fs.rm(dir, { recursive: true, force: true });
+  }
+});

package/src/runtime/web-suite.ts

@@ -125,6 +125,9 @@ export async function runWebSuite(options: WebSuiteOptions): Promise<WebSuiteRep
         flowPath,
         platform: "web",
         target: { baseUrl: options.baseUrl },
+        // The suite runs against --base-url: a flow written for another
+        // address (production, say) is moved onto it, as plan runs do.
+        rebaseOrigin: true,
         timeoutMs: options.timeoutMs ?? 30_000,
         artifactsDir: options.artifactsDir,
         sessionStatePath,