@bugmole/cli 0.4.0 → 0.6.0

package/src/runtime/site-discovery.test.ts

@@ -5,16 +5,16 @@ import os from "node:os";
 import path from "node:path";
 import test from "node:test";
 import { chromium } from "playwright";
-import { discoverableRoute, discoverSite, flowsFor, slugify } from "./site-discovery.js";
-import { discoveryUrl, runDiscoveryTask } from "./discovery-task.js";
+import { discoverableRoute, discoverSite, discoveryLimitsFromEnv, flowsFor, slugify, stoppedAtSignIn } from "./site-discovery.js";
+import { discoveryUrl, progressOf, runDiscoveryTask } from "./discovery-task.js";
 import { PlaywrightDriver } from "./playwright-driver.js";
 import { startLocalRegistryStub } from "./local-registry-stub.js";
 import { ControlPlaneClient } from "../registry/control-plane-client.js";
 
 const pages: Record<string, string> = {
-  "/": `<title>Acme</title><nav><a href="/pricing">Pricing</a> <a href="/signup">Sign up</a> <a href="/logout">Log out</a> <a href="https://elsewhere.example/">Blog</a></nav><h1>Welcome to Acme</h1>`,
+  "/": `<title>Acme</title><nav><a href="/pricing">Pricing</a> <a href="/signup">Sign up</a> <a href="/logout">Log out</a> <a href="https://elsewhere.example/">Blog</a> </nav><h1>Welcome to Acme</h1>`,
   "/pricing": `<title>Pricing · Acme</title><a href="/">Home</a><h1>Plans for every team</h1>`,
-  "/signup": `<title>Sign up · Acme</title><h1>Create your account</h1>
+  "/signup": `<title>Sign up · Acme</title><h1>Create your account</h1><a href="/">Home</a> <a href="/logout">Log out</a> <a href="/api/export">Export</a> <a href="/report.csv" download>Download report</a> <a href="/data">Raw data</a>
 <form aria-label="Registration"><label>Email <input type="email" required></label><label>Password <input type="password" required></label><button type="submit">Create account</button></form>`,
   "/logout": `<title>Bye</title><h1>You are signed out</h1>`,
 };
@@ -24,6 +24,11 @@ async function withSite(run: (baseUrl: string, hits: string[]) => Promise<void>)
   const server = http.createServer((request, response) => {
     const route = new URL(request.url ?? "/", "http://x").pathname;
     hits.push(route);
+    if (route === "/data" || route === "/api/export" || route === "/report.csv") {
+      response.writeHead(200, { "content-type": route === "/report.csv" ? "text/csv" : "application/json" });
+      response.end(route === "/report.csv" ? "a,b\n" : JSON.stringify({ rows: [] }));
+      return;
+    }
     const body = pages[route];
     response.writeHead(body ? 200 : 404, { "content-type": "text/html" });
     response.end(`<!doctype html><html><body>${body ?? "Not found"}</body></html>`);
@@ -86,6 +91,10 @@ test("explores a site from its URL like a visitor, and the flows it writes pass"
     const result = await discoverSite(baseUrl, { dwellMs: 0, onEvent: (event) => { events.push(event.text); } });
     assert.deepEqual(result.pages.map((page) => page.route).sort(), ["/", "/pricing", "/signup"]);
     assert.ok(!hits.includes("/logout"), "never opens a sign-out link");
+    assert.ok(!hits.includes("/api/export") && !hits.includes("/report.csv"), "never opens an API or download link");
+    assert.ok(events.some((text) => text.startsWith("Skipping “Export” (it downloads a file or calls the API)")));
+    assert.ok(events.some((text) => /“Raw data” opens a file \(application\/json\), not a page/.test(text)), "a link that opens JSON isn't a page");
+    assert.equal(events.filter((text) => text.startsWith("Skipping “Log out”")).length, 1, "a skipped link is reported once, not on every page that has it");
     assert.ok(events.includes("Visiting home page"));
     assert.ok(events.includes("Clicking “Pricing”"));
     assert.ok(events.some((text) => text.startsWith("Skipping “Log out”")));
@@ -118,6 +127,114 @@ test("explores a site from its URL like a visitor, and the flows it writes pass"
   });
 });
 
+test("a crawl cut short by its limits says which linked pages it never opened", async (t) => {
+  if (!(await chromiumInstalled())) {
+    t.skip("Chromium is not installed");
+    return;
+  }
+  await withSite(async (baseUrl) => {
+    const events: Array<{ kind: string; text: string }> = [];
+    const capped = await discoverSite(baseUrl, { dwellMs: 0, maxPages: 2, onEvent: (event) => { events.push(event); } });
+    assert.equal(capped.pages.length, 2);
+    assert.equal(capped.limits?.maxPages, 2);
+    assert.equal(capped.limits?.stoppedAtPageLimit, true);
+    assert.equal(capped.limits?.notFollowed.length, 1);
+    assert.ok(!capped.pages.some((page) => page.route === capped.limits?.notFollowed[0]));
+    assert.match(events.find((event) => event.kind === "limit")?.text ?? "", /Stopped at the 2-page limit; 1 linked page was not opened/);
+    const progress = progressOf(baseUrl, capped);
+    assert.equal(progress.limits?.notFollowed, 1);
+    assert.deepEqual(progress.limits?.notFollowedRoutes, capped.limits?.notFollowed);
+
+    const shallow = await discoverSite(baseUrl, { dwellMs: 0, maxDepth: 0 });
+    assert.deepEqual(shallow.pages.map((page) => page.route), ["/"]);
+    assert.equal(shallow.limits?.stoppedAtPageLimit, false);
+    assert.deepEqual([...(shallow.limits?.notFollowed ?? [])].sort(), ["/pricing", "/signup"]);
+
+    const whole = await discoverSite(baseUrl, { dwellMs: 0 });
+    assert.deepEqual(whole.limits, { maxPages: 12, maxDepth: 2, stoppedAtPageLimit: false, notFollowed: [] });
+  });
+});
+
+test("a project's own worker reads its exploration limits from the environment", () => {
+  assert.deepEqual(discoveryLimitsFromEnv({ BUGMOLE_DISCOVERY_MAX_PAGES: "60", BUGMOLE_DISCOVERY_MAX_DEPTH: "4" }), { maxPages: 60, maxDepth: 4 });
+  assert.deepEqual(discoveryLimitsFromEnv({ BUGMOLE_DISCOVERY_MAX_PAGES: "0", BUGMOLE_DISCOVERY_MAX_DEPTH: "two" }), {});
+  assert.deepEqual(discoveryLimitsFromEnv({}), {});
+});
+
+test("explores a signed-in app from a saved session, and stays inside it", async (t) => {
+  if (!(await chromiumInstalled())) {
+    t.skip("Chromium is not installed");
+    return;
+  }
+  const hits: string[] = [];
+  const signedIn: Record<string, string> = {
+    // "Test plans" routes on the client, like a single-page app: the address
+    // and the page change some time after the click. "Billing" and "Sign out"
+    // only exist once the account menu is opened.
+    "/": `<h1>Overview</h1><a href="/plans" id="plans">Test plans</a><a href="/account/sign-out">Account</a><a href="/connect">Connect GitHub</a>
+      <a href="/manage/domains"><strong>Domains</strong> <span>Verify the sites Bugmole Cloud is allowed to run against.</span></a>
+      <button aria-expanded="false" id="account"><span aria-hidden="true">A</span> <strong>Account</strong> <small>menu</small></button><div id="menu"></div>
+      <script>
+        document.getElementById("plans").addEventListener("click", (event) => {
+          event.preventDefault();
+          setTimeout(() => {
+            history.pushState(null, "", "/plans" + location.search);
+            document.querySelector("h1").textContent = "Test plans";
+          }, 600);
+        });
+        const account = document.getElementById("account");
+        account.addEventListener("click", () => {
+          const open = account.getAttribute("aria-expanded") !== "true";
+          account.setAttribute("aria-expanded", String(open));
+          document.getElementById("menu").innerHTML = open ? '<a href="/billing">Billing</a><a href="/sign-out">Sign out</a>' : "";
+        });
+        document.addEventListener("keydown", (event) => { if (event.key === "Escape" && account.getAttribute("aria-expanded") === "true") account.click(); });
+      </script>`,
+    "/plans": `<h1>Test plans</h1><a href="/">Overview</a>`,
+    "/billing": `<h1>Billing</h1><a href="/">Overview</a>`,
+    "/manage/domains": `<h1>Domains</h1><a href="/">Overview</a>`,
+  };
+  const server = http.createServer((request, response) => {
+    const route = new URL(request.url ?? "/", "http://x").pathname;
+    hits.push(route);
+    if (!/session=ok/.test(request.headers.cookie ?? "")) {
+      response.writeHead(307, { location: "/login" });
+      return response.end();
+    }
+    // Same-origin link that hands off to another site, like an install redirect.
+    if (route === "/connect") {
+      response.writeHead(302, { location: `http://localhost:${(server.address() as { port: number }).port}/elsewhere` });
+      return response.end();
+    }
+    const body = signedIn[route];
+    response.writeHead(body ? 200 : 404, { "content-type": "text/html" });
+    response.end(`<!doctype html><html><body>${body ?? "Not found"}</body></html>`);
+  });
+  await new Promise<void>((resolve) => server.listen(0, "127.0.0.1", resolve));
+  const directory = await fs.mkdtemp(path.join(os.tmpdir(), "bugmole-signed-in-"));
+  try {
+    const baseUrl = `http://127.0.0.1:${(server.address() as { port: number }).port}/`;
+    const storageState = path.join(directory, "state.json");
+    await fs.writeFile(storageState, JSON.stringify({
+      cookies: [{ name: "session", value: "ok", domain: "127.0.0.1", path: "/", expires: -1, httpOnly: true, secure: false, sameSite: "Lax" }],
+      origins: [],
+    }));
+    const result = await discoverSite(`${baseUrl}?project=p1`, { dwellMs: 0, storageState, openEachPage: true });
+    assert.deepEqual(result.pages.map((page) => page.route).sort(), ["/", "/billing", "/manage/domains", "/plans"]);
+    assert.equal(result.pages.find((page) => page.route === "/plans")?.heading, "Test plans", "waits for a client-side page to render");
+    assert.deepEqual(result.pages.find((page) => page.route === "/billing")?.path, ["Account menu", "Billing"]);
+    assert.ok(!hits.includes("/account/sign-out"), "a sign-out address is never opened, whatever its label");
+    assert.ok(!hits.includes("/sign-out"), "nor a sign-out link inside a menu");
+    assert.ok(result.events.some((event) => event.text === "“Connect GitHub” leaves the app"));
+    const plans = result.flows.find((flow) => flow.route === "/plans")!;
+    assert.match(plans.content, new RegExp(`^url: ${baseUrl}plans\\?project=p1\\n`), "the base URL's app context is kept");
+    assert.doesNotMatch(plans.content, /tapOn/, "each flow opens its page directly");
+  } finally {
+    server.close();
+    await fs.rm(directory, { recursive: true, force: true });
+  }
+});
+
 test("a discovery task streams progress, then publishes plans and a valid journey graph", async (t) => {
   if (!(await chromiumInstalled())) {
     t.skip("Chromium is not installed");
@@ -166,3 +283,63 @@ test("a discovery task streams progress, then publishes plans and a valid journe
     }
   });
 });
+
+test("a crawl that only reached sign-in screens is reported as stopped at sign-in", () => {
+  const page = (extra: Record<string, unknown>) => ({ route: "/login", url: "https://app.test/login", title: "App", path: [], links: [], forms: [], ...extra });
+  assert.equal(stoppedAtSignIn({ pages: [page({ heading: "Welcome back", controls: ["Continue with Aegis"] })] }), true);
+  assert.equal(stoppedAtSignIn({ pages: [page({ forms: [{ name: "Log in", fields: [{ label: "Email", type: "email", required: true }, { label: "Password", type: "password", required: true }] }] })] }), true);
+  assert.equal(stoppedAtSignIn({ pages: [page({ heading: "Pricing" })] }), false, "a page that isn't a sign-in screen");
+  assert.equal(stoppedAtSignIn({ pages: [] }), false, "nothing explored is a different failure");
+  assert.equal(
+    stoppedAtSignIn({ pages: [page({ heading: "Welcome back" }), page({ route: "/", heading: "Home" }), page({ route: "/a", heading: "A" })] }),
+    false,
+    "a public site that also has a sign-in page",
+  );
+});
+
+test("an app that opens on a sign-in button: discovery records the button, says it stopped, and skips AI", async (t) => {
+  if (!(await chromiumInstalled())) {
+    t.skip("Chromium is not installed");
+    return;
+  }
+  const server = http.createServer((_request, response) => {
+    response.writeHead(200, { "content-type": "text/html; charset=utf-8" });
+    response.end("<!doctype html><html><head><title>Bugmole</title></head><body><h1>Welcome back</h1><p>Sign in or create your account.</p><button>Continue with Aegis →</button></body></html>");
+  });
+  await new Promise<void>((resolve) => server.listen(0, "127.0.0.1", resolve));
+  const baseUrl = `http://127.0.0.1:${(server.address() as { port: number }).port}/`;
+  const directory = await fs.mkdtemp(path.join(os.tmpdir(), "bugmole-sign-in-wall-"));
+  const stub = await startLocalRegistryStub({ directory: path.join(directory, "registry") });
+  try {
+    const seen = await discoverSite(baseUrl, { dwellMs: 0 });
+    assert.deepEqual(seen.pages[0].controls, ["Continue with Aegis →"]);
+    assert.equal(stoppedAtSignIn(seen), true);
+
+    const client = new ControlPlaneClient({ registryUrl: stub.origin, apiKey: "local", projectId: "demo" });
+    const { task } = await client.createTask({ title: "Explore with AI app", description: `Explore ${baseUrl} like a first-time visitor, then have AI write multi-page journeys.`, taskType: "discovery", idempotencyKey: "discover-wall" });
+    const cfg = {
+      project: { id: "demo" },
+      runtime: { spec_dir: path.join(directory, "spec"), artifacts_dir: path.join(directory, "artifacts") },
+      storage: { provider: "local" },
+    };
+    const updates: Array<Record<string, unknown>> = [];
+    await runDiscoveryTask(cfg, {
+      claimTask: client.claimTask.bind(client),
+      publishPlan: client.publishPlan.bind(client),
+      updateTask: async (id: string, update: Record<string, unknown>) => {
+        updates.push(update);
+        return client.updateTask(id, update);
+      },
+    }, task, "worker-1");
+    const final = updates.at(-1)!;
+    assert.equal(final.status, "completed", String(final.errorMessage));
+    assert.match(String(final.completionSummary), /Stopped at a sign-in page/);
+    const texts = (final.continuation as { events: Array<{ text: string }> }).events.map((event) => event.text);
+    assert.ok(texts.some((text) => text.startsWith("Stopped at a sign-in page")), texts.join("\n"));
+    assert.ok(!texts.some((text) => /AI/.test(text) && !text.startsWith("Stopped")), "the AI step never ran");
+  } finally {
+    await stub.close();
+    server.close();
+    await fs.rm(directory, { recursive: true, force: true });
+  }
+});