@bugmole/cli 0.4.0

package/src/runtime/serve-worker.ts

@@ -0,0 +1,278 @@
+import os from "node:os";
+import type WebSocket from "ws";
+import { ControlPlaneClient, RegistryRequestError, type RegistryRun } from "../registry/control-plane-client.js";
+import { executeRegistryRun, registryDeviceAccess } from "./run-job.js";
+import { connectServeGateway } from "./serve-gateway.js";
+import { runNextAgentTask } from "./cursor-driver-run.js";
+import { reconcilePipeline } from "./pipeline.js";
+import { publishProjectSpec } from "./project-spec.js";
+import { recordRunVerdicts } from "./record-run-verdicts.js";
+import { isClaimableTask, MAX_TASK_ATTEMPTS } from "../registry/task-scheduling.js";
+import { DISCOVERY_TASK_TYPE, runDiscoveryTask } from "./discovery-task.js";
+
+export { RUN_HEARTBEAT_MS } from "./run-job.js";
+
+/**
+ * Runs executed at once. Defaults to one: a browser run with video recording is heavy, and an
+ * unbounded worker used to start every queued run simultaneously.
+ */
+export function resolveMaxConcurrentRuns(env: Record<string, string | undefined> = process.env): number {
+  const value = Number(env.BUGMOLE_MAX_CONCURRENT_RUNS);
+  return Number.isInteger(value) && value > 0 ? value : 1;
+}
+
+/**
+ * Agent tasks and the autonomous pipeline need a coding agent. A cloud worker without one sets
+ * BUGMOLE_WORKER_TASKS=false and only executes runs.
+ */
+export function workerTasksEnabled(env: Record<string, string | undefined> = process.env): boolean {
+  return env.BUGMOLE_WORKER_TASKS?.trim().toLowerCase() !== "false";
+}
+
+/** Another worker claimed the run first. That is not a failure of the run. */
+export function isClaimConflict(error: unknown): boolean {
+  return error instanceof RegistryRequestError && error.status === 409;
+}
+
+export async function startServeWorker(cfg: any): Promise<void> {
+  const apiKey = process.env.BUGMOLE_API_KEY?.trim();
+  const registryUrl = process.env.BUGMOLE_REGISTRY_URL?.trim();
+  const projectId = cfg.project?.id;
+  const workspaceId = cfg.project?.workspace_id ?? cfg.project?.workspaceId;
+  if (!apiKey || !registryUrl || !projectId) {
+    console.warn("[Bugmole] Worker idle: registry URL, API key, and project id are required");
+    return;
+  }
+  const workerId = `worker_${os.hostname()}_${process.pid}`;
+  const client = new ControlPlaneClient({ registryUrl, apiKey, projectId, workspaceId });
+  const active = new Set<string>();
+  const maxConcurrentRuns = resolveMaxConcurrentRuns();
+  const tasksEnabled = workerTasksEnabled();
+  if (!tasksEnabled) {
+    console.log("[Bugmole] Agent tasks are off (BUGMOLE_WORKER_TASKS=false); this worker only executes runs.");
+  }
+  // Consecutive failed runs per flow, so a run that keeps crashing stops
+  // being retried forever. See the catch in processRun.
+  const runFailures = new Map<string, number>();
+
+  const processRun = async (run: RegistryRun) => {
+    if (active.has(run.id)) return;
+    active.add(run.id);
+    try {
+      try {
+        await client.claimRun(run.id, workerId);
+      } catch (error) {
+        if (isClaimConflict(error)) return;
+        throw error;
+      }
+      const result = await executeRegistryRun(cfg, client, run, {
+        projectId,
+        workerId,
+        loadDeviceAccess: registryDeviceAccess(cfg, registryUrl, apiKey, projectId, run.id),
+      });
+      // A run that does not judge its cases leaves them "planned", and the
+      // pipeline keeps re-queueing the flow waiting for verdicts that no run
+      // ever writes. Recording here is what lets the run stage ever finish.
+      if (run.journeyId) {
+        // The flow completed a run, so earlier crashes were transient and
+        // should not count toward giving up on it later.
+        runFailures.delete(run.journeyId);
+        try {
+          const judged = await recordRunVerdicts(
+            cfg,
+            projectId,
+            run.journeyId,
+            (result.results ?? []).map((item) => ({
+              nodeId: item.nodeId,
+              outcome: item.outcome,
+              status: item.status,
+              reason: (item as { reason?: { code?: string; message?: string } }).reason,
+            })),
+            run.id,
+          );
+          if (judged) {
+            console.log("[Bugmole] Recorded " + judged.judged + " case verdicts for " + judged.caseSetId);
+          }
+        } catch (error) {
+          // Never fail a completed run over its bookkeeping — the run really
+          // did happen. The flow stays unjudged and gets picked up again.
+          console.warn(
+            "[Bugmole] Could not record verdicts for " + run.journeyId + ":",
+            error instanceof Error ? error.message : error,
+          );
+        }
+      }
+    } catch (error) {
+      const message = error instanceof Error ? error.message : "Worker execution failed";
+      await client.updateRun(run.id, {
+        workerId,
+        status: "error",
+        eventType: "failed",
+        errorMessage: message,
+      }).catch(() => undefined);
+      // A run that throws records no verdicts, so its flow stays unjudged and
+      // the pipeline queues it again — which is right for a flaky driver and
+      // catastrophic for a broken one, because nothing ever breaks the cycle.
+      //
+      // Retry a bounded number of times, then judge the flow blocked with the
+      // real error so the pipeline converges and surfaces it instead of
+      // spinning. Held in memory on purpose: restarting `bugmole serve` is a
+      // deliberate "try again", and should not inherit an old grudge.
+      if (run.journeyId) {
+        const failures = (runFailures.get(run.journeyId) ?? 0) + 1;
+        runFailures.set(run.journeyId, failures);
+        if (failures >= MAX_TASK_ATTEMPTS) {
+          await recordRunVerdicts(
+            cfg,
+            projectId,
+            run.journeyId,
+            [],
+            run.id,
+            `The run failed ${failures} times and never reached this step. Last error: ${message}`,
+          ).catch(() => undefined);
+          console.warn(
+            "[Bugmole] " + run.journeyId + " failed " + failures
+            + " runs; its cases are recorded blocked so the pipeline can move on.",
+          );
+        }
+      }
+    } finally {
+      active.delete(run.id);
+    }
+  };
+
+  // Previously, queuing an agent task only wrote a database row — nothing
+  // claimed or ran it until a human separately went and typed into their own
+  // coding-agent chat. This makes the worker do for tasks what it already
+  // did for runs: claim and execute the next queued one on its own.
+  let taskInFlight = false;
+  let lastPipelineReason = "";
+  const pollTasks = async () => {
+    if (taskInFlight) return;
+    taskInFlight = true;
+    try {
+      const { task } = await runNextAgentTask({
+        registryUrl,
+        apiKey,
+        projectId,
+        workerId,
+        artifactsDir: cfg.runtime?.artifacts_dir || "./artifacts",
+        projectDirectory: cfg.cursor_driver?.project_directory,
+      });
+      if (task) console.log(`[Bugmole] Worker ran agent task ${task.id}`);
+
+      // Advance the pipeline once the queue is idle. Running this inside
+      // the same in-flight guard means a stage is only ever queued when
+      // nothing else is executing, so stages can't race each other.
+      if (!task) {
+        const { state, enqueued, startedRun } = await reconcilePipeline(cfg, client);
+        if (enqueued) {
+          console.log(`[Bugmole] Pipeline queued ${enqueued.stage}${enqueued.journeyId ? ` for ${enqueued.journeyId}` : ""}`);
+          lastPipelineReason = "";
+        } else if (startedRun) {
+          console.log(`[Bugmole] Pipeline started run of ${startedRun.planId} for ${startedRun.journeyId}`);
+          lastPipelineReason = "";
+        } else if (state.reason !== lastPipelineReason) {
+          // Only log a standing state once, or an idle project would print
+          // the same line every 5 seconds.
+          console.log(`[Bugmole] Pipeline ${state.status}: ${state.reason}`);
+          lastPipelineReason = state.reason;
+        }
+      }
+    } catch (error) {
+      console.warn(`[Bugmole] Worker task poll failed: ${error instanceof Error ? error.message : error}`);
+    } finally {
+      taskInFlight = false;
+    }
+  };
+
+  // Discovery needs only a browser, so it runs even where agent tasks are off.
+  let discoveryInFlight = false;
+  const pollDiscovery = async () => {
+    if (discoveryInFlight) return;
+    discoveryInFlight = true;
+    try {
+      const { tasks } = await client.listTasks();
+      const next = tasks.find((task) => task.taskType === DISCOVERY_TASK_TYPE && task.executionMode !== "cloud" && isClaimableTask(task));
+      if (next) {
+        console.log(`[Bugmole] Exploring for discovery task ${next.id}`);
+        await runDiscoveryTask(cfg, client, next, workerId);
+      }
+    } catch (error) {
+      console.warn(`[Bugmole] Discovery poll failed: ${error instanceof Error ? error.message : error}`);
+    } finally {
+      discoveryInFlight = false;
+    }
+  };
+
+  const poll = async () => {
+    try {
+      const { runs } = await client.listRuns();
+      for (const run of runs.filter((item) => item.status === "queued" && item.executionMode !== "cloud")) {
+        if (active.size >= maxConcurrentRuns) break;
+        void processRun(run);
+      }
+    } catch (error) {
+      console.warn(`[Bugmole] Worker registry poll failed: ${error instanceof Error ? error.message : error}`);
+    }
+    void pollDiscovery();
+    if (tasksEnabled) void pollTasks();
+  };
+
+  let socket: WebSocket | undefined;
+  let reconnectTimer: NodeJS.Timeout | undefined;
+  const gatewayUrl = process.env.BUGMOLE_GATEWAY_URL?.trim();
+  const gatewaySecret = process.env.BUGMOLE_GATEWAY_SECRET?.trim();
+  const gatewayAvailable = async (): Promise<boolean> => {
+    if (!gatewayUrl || !gatewaySecret) return false;
+    try {
+      const response = await fetch(`${gatewayUrl.replace(/^ws/, "http")}/health`, {
+        signal: AbortSignal.timeout(500),
+      });
+      return response.ok;
+    } catch {
+      return false;
+    }
+  };
+  const connect = async () => {
+    socket?.close();
+    const onMessage = (message: Record<string, unknown>) => {
+      if (message.type === "job_queued" && typeof message.runId === "string") {
+        void poll();
+      }
+      // A queued discovery starts right away instead of on the next poll.
+      if (message.type === "task_event" && message.status === "queued") {
+        void pollDiscovery();
+      }
+    };
+    if (await gatewayAvailable()) {
+      socket = connectServeGateway(
+        gatewayUrl!,
+        gatewaySecret!,
+        {
+          workerId,
+          projectId,
+          workspaceId,
+          registryUrl,
+          apiKey,
+        },
+        onMessage,
+      );
+    } else {
+      socket = client.connectWorker(workerId, onMessage);
+    }
+    socket.on("close", () => {
+      reconnectTimer = setTimeout(() => void connect(), 2_000);
+    });
+    socket.on("error", () => undefined);
+  };
+  // Publish the project's spec once at startup so the dashboard shows this
+  // project's real roles and blockers rather than falling back to fixtures.
+  await publishProjectSpec(cfg).catch((error) => {
+    console.warn(`[Bugmole] Could not publish project spec: ${error instanceof Error ? error.message : error}`);
+    return null;
+  });
+  void connect();
+  await poll();
+  setInterval(() => void poll(), 5_000);
+}

package/src/runtime/site-discovery.test.ts

@@ -0,0 +1,168 @@
+import assert from "node:assert/strict";
+import fs from "node:fs/promises";
+import http from "node:http";
+import os from "node:os";
+import path from "node:path";
+import test from "node:test";
+import { chromium } from "playwright";
+import { discoverableRoute, discoverSite, flowsFor, slugify } from "./site-discovery.js";
+import { discoveryUrl, runDiscoveryTask } from "./discovery-task.js";
+import { PlaywrightDriver } from "./playwright-driver.js";
+import { startLocalRegistryStub } from "./local-registry-stub.js";
+import { ControlPlaneClient } from "../registry/control-plane-client.js";
+
+const pages: Record<string, string> = {
+  "/": `<title>Acme</title><nav><a href="/pricing">Pricing</a> <a href="/signup">Sign up</a> <a href="/logout">Log out</a> <a href="https://elsewhere.example/">Blog</a></nav><h1>Welcome to Acme</h1>`,
+  "/pricing": `<title>Pricing · Acme</title><a href="/">Home</a><h1>Plans for every team</h1>`,
+  "/signup": `<title>Sign up · Acme</title><h1>Create your account</h1>
+<form aria-label="Registration"><label>Email <input type="email" required></label><label>Password <input type="password" required></label><button type="submit">Create account</button></form>`,
+  "/logout": `<title>Bye</title><h1>You are signed out</h1>`,
+};
+
+async function withSite(run: (baseUrl: string, hits: string[]) => Promise<void>) {
+  const hits: string[] = [];
+  const server = http.createServer((request, response) => {
+    const route = new URL(request.url ?? "/", "http://x").pathname;
+    hits.push(route);
+    const body = pages[route];
+    response.writeHead(body ? 200 : 404, { "content-type": "text/html" });
+    response.end(`<!doctype html><html><body>${body ?? "Not found"}</body></html>`);
+  });
+  await new Promise<void>((resolve) => server.listen(0, "127.0.0.1", resolve));
+  try {
+    await run(`http://127.0.0.1:${(server.address() as { port: number }).port}/`, hits);
+  } finally {
+    server.close();
+  }
+}
+
+async function chromiumInstalled() {
+  try {
+    await fs.access(chromium.executablePath());
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+test("only same-origin page links are followed", () => {
+  const base = "https://app.example.com/";
+  assert.equal(discoverableRoute("/pricing/", base), "/pricing");
+  assert.equal(discoverableRoute("https://app.example.com/?tab=1#x", base), "/");
+  assert.equal(discoverableRoute("https://other.example.com/", base), null);
+  assert.equal(discoverableRoute("mailto:hi@example.com", base), null);
+  assert.equal(discoverableRoute("/logo.png", base), null);
+  assert.equal(slugify("Sign up · Acme!"), "sign-up-acme");
+  assert.equal(discoveryUrl({ description: "Explore https://app.example.com/start please" }), "https://app.example.com/start");
+  assert.equal(discoveryUrl({ description: "javascript:alert(1)" }), null);
+});
+
+test("forms become fill-in flows that never submit, and passwords are referenced", () => {
+  const flows = flowsFor("https://app.example.com/", [{
+    route: "/signup",
+    url: "https://app.example.com/signup",
+    title: "Sign up",
+    heading: "Create your account",
+    path: ["Sign up"],
+    links: [],
+    forms: [{ name: "Registration", submit: "Create account", fields: [{ label: "Email", type: "email", required: true }, { label: "Password", type: "password", required: true }] }],
+  }]);
+  const form = flows.find((flow) => flow.kind === "form")!;
+  assert.match(form.content, /tapOn: Sign up/);
+  assert.match(form.content, /inputText: qa@example\.com/);
+  assert.match(form.content, /\{\{secret\.APP_PASSWORD\}\}/);
+  assert.match(form.content, /assertClickable: Create account/);
+  assert.doesNotMatch(form.content, /tapOn: Create account/);
+  assert.ok(form.steps.includes("Type the password"));
+});
+
+test("explores a site from its URL like a visitor, and the flows it writes pass", async (t) => {
+  if (!(await chromiumInstalled())) {
+    t.skip("Chromium is not installed");
+    return;
+  }
+  await withSite(async (baseUrl, hits) => {
+    const events: string[] = [];
+    const result = await discoverSite(baseUrl, { dwellMs: 0, onEvent: (event) => { events.push(event.text); } });
+    assert.deepEqual(result.pages.map((page) => page.route).sort(), ["/", "/pricing", "/signup"]);
+    assert.ok(!hits.includes("/logout"), "never opens a sign-out link");
+    assert.ok(events.includes("Visiting home page"));
+    assert.ok(events.includes("Clicking “Pricing”"));
+    assert.ok(events.some((text) => text.startsWith("Skipping “Log out”")));
+    assert.ok(events.includes("Found a form: Registration (Email, Password)"));
+    assert.equal(events.at(-1), "Exploration complete");
+    assert.deepEqual(result.pages.find((page) => page.route === "/signup")?.path, ["Sign up"]);
+
+    const directory = await fs.mkdtemp(path.join(os.tmpdir(), "bugmole-discovered-"));
+    process.env.BUGMOLE_FLOW_SECRET_APP_PASSWORD = "discovered-secret";
+    try {
+      for (const flow of result.flows) {
+        const flowPath = path.join(directory, `${flow.id}.yaml`);
+        await fs.writeFile(flowPath, flow.content);
+        const run = await new PlaywrightDriver({ headless: true }).run({
+          runId: "run-discovered",
+          projectId: "demo",
+          journeyId: flow.id,
+          flowPath,
+          platform: "web",
+          target: {},
+          timeoutMs: 20_000,
+          artifactsDir: directory,
+        });
+        assert.equal(run.status, "passed", `${flow.id}: ${run.message}`);
+      }
+    } finally {
+      delete process.env.BUGMOLE_FLOW_SECRET_APP_PASSWORD;
+      await fs.rm(directory, { recursive: true, force: true });
+    }
+  });
+});
+
+test("a discovery task streams progress, then publishes plans and a valid journey graph", async (t) => {
+  if (!(await chromiumInstalled())) {
+    t.skip("Chromium is not installed");
+    return;
+  }
+  await withSite(async (baseUrl) => {
+    const directory = await fs.mkdtemp(path.join(os.tmpdir(), "bugmole-discovery-task-"));
+    const stub = await startLocalRegistryStub({ directory: path.join(directory, "registry") });
+    try {
+      const client = new ControlPlaneClient({ registryUrl: stub.origin, apiKey: "local", projectId: "demo" });
+      const { task } = await client.createTask({ title: "Explore", description: `Explore ${baseUrl}`, taskType: "discovery", idempotencyKey: "discover-1" });
+      const cfg = {
+        project: { id: "demo" },
+        runtime: { spec_dir: path.join(directory, "spec"), artifacts_dir: path.join(directory, "artifacts") },
+        storage: { provider: "local" },
+      };
+      const updates: Array<Record<string, unknown>> = [];
+      const recording = {
+        claimTask: client.claimTask.bind(client),
+        publishPlan: client.publishPlan.bind(client),
+        updateTask: async (id: string, update: Record<string, unknown>) => {
+          updates.push(update);
+          return client.updateTask(id, update);
+        },
+      };
+      await runDiscoveryTask(cfg, recording, task, "worker-1");
+
+      const final = updates.at(-1)!;
+      assert.equal(final.status, "completed", String(final.errorMessage));
+      const progress = final.continuation as { events: Array<{ text: string }>; flows: Array<{ planId?: string }> };
+      assert.equal(progress.events.at(-1)?.text, "Exploration complete");
+      assert.ok(progress.flows.length >= 3 && progress.flows.every((flow) => flow.planId));
+      assert.ok(updates.some((update) => update.status === "running" && String(update.progressMessage).startsWith("Clicking")));
+
+      const { plans } = await client.listPlans();
+      assert.equal(plans.length, progress.flows.length);
+      const graph = JSON.parse(await fs.readFile(path.join(directory, "spec", "journeys.graph.json"), "utf8"));
+      assert.ok(graph.nodes.some((node: { id: string }) => node.id === "screen:/signup"));
+      assert.ok(graph.edges.some((edge: { label?: string }) => edge.label === "Sign up"));
+      const plan = await fs.readFile(path.join(directory, "spec", "plans", "discovered-home.yaml"), "utf8");
+      assert.match(plan, /flow: flows\/discovered\/discovered-home\.yaml/);
+      await fs.access(path.join(directory, "artifacts", "projects", "demo", "spec", "flows", "discovered", "discovered-home.yaml"));
+    } finally {
+      await stub.close();
+      await fs.rm(directory, { recursive: true, force: true });
+    }
+  });
+});