@bugmole/cli 0.4.0

package/src/runtime/executor.test.ts

@@ -0,0 +1,509 @@
+import { strict as assert } from "node:assert";
+import test from "node:test";
+import fs from "node:fs";
+import path from "node:path";
+import os from "node:os";
+import type { AppDriver, DriverContext, DriverResult } from "./driver.js";
+
+function makeTestConfig(testDir: string) {
+  const specDir = path.join(testDir, "spec");
+  const artifactsDir = path.join(testDir, "artifacts");
+  fs.mkdirSync(path.join(specDir, "plans"), { recursive: true });
+  fs.mkdirSync(artifactsDir, { recursive: true });
+  return {
+    project: { id: "demo" },
+    runtime: { spec_dir: specDir, artifacts_dir: artifactsDir },
+    // "false" always exits 1 without touching --test-output-dir, reproducing
+    // a driver that fails before any step executes (e.g. "device offline")
+    // without depending on a real Maestro install or device.
+    execution: { maestro: { command: "false" } },
+  };
+}
+
+// Reproduces run_1786624327747 from a real Bugmole project: Maestro failed
+// to connect to a device before any step ran, yet the old executor reported
+// every declared step as independently "failed" with an identical message.
+// The fix must stop claiming per-step verification it never performed.
+test("does not fabricate per-step verification when the driver fails before any step runs", async () => {
+  const { execute } = await import("./executor.js");
+  const testDir = fs.mkdtempSync(path.join(os.tmpdir(), "bugmole-executor-test-"));
+  const cfg = makeTestConfig(testDir);
+  const planYaml = `
+suites:
+  - journeys:
+      - id: checkout
+        driver: maestro
+        platform: android
+        flow: flows/checkout.yaml
+        steps:
+          - node_id: step-1
+            action: launch
+          - node_id: step-2
+            action: tap-sign-in
+          - node_id: step-3
+            action: enter-email
+`;
+  fs.writeFileSync(path.join(cfg.runtime.spec_dir, "plans", "checkout.yaml"), planYaml, "utf8");
+
+  const result = await execute(cfg, { planId: "checkout", journeyId: "checkout" });
+
+  assert.equal(result.status, "partial");
+  assert.equal(result.stepsExecuted, 3);
+  assert.equal(result.stepsFailed, 3);
+  assert.ok(result.results);
+  for (const step of result.results!) {
+    assert.equal(step.outcome, "FAIL");
+    // The old behavior copied a single opaque "The journey action failed."
+    // onto every step, presenting it as independently verified. The fix
+    // must say plainly that only the aggregate run outcome is known.
+    assert.equal(step.reason.code, "aggregate_only");
+    assert.match(step.reason.message, /not independently verified/);
+  }
+  fs.rmSync(testDir, { recursive: true, force: true });
+});
+
+// The companion, positive case: when the driver's per-step count matches the
+// plan's declared steps, each step must carry its own real observed status
+// instead of the aggregate one.
+test("uses real per-step status when the driver's step count matches the plan", async () => {
+  const { execute } = await import("./executor.js");
+  const testDir = fs.mkdtempSync(path.join(os.tmpdir(), "bugmole-executor-test-"));
+  const cfg = makeTestConfig(testDir);
+  cfg.execution.maestro.command = path.join(import.meta.dirname, "fixtures", "fake-maestro.sh");
+  const planYaml = `
+suites:
+  - journeys:
+      - id: checkout
+        driver: maestro
+        platform: android
+        flow: flows/checkout.yaml
+        steps:
+          - node_id: step-1
+            action: launch
+          - node_id: step-2
+            action: tap-sign-in
+          - node_id: step-3
+            action: assert-home
+`;
+  fs.writeFileSync(path.join(cfg.runtime.spec_dir, "plans", "checkout.yaml"), planYaml, "utf8");
+
+  const result = await execute(cfg, { planId: "checkout", journeyId: "checkout" });
+
+  assert.equal(result.stepsExecuted, 3);
+  assert.equal(result.stepsPassed, 2);
+  assert.equal(result.stepsFailed, 1);
+  assert.ok(result.results);
+  const [first, second, third] = result.results!;
+  assert.equal(first.outcome, "PASS");
+  assert.equal(first.reason.code, "verified");
+  assert.equal(second.outcome, "PASS");
+  assert.equal(third.outcome, "FAIL");
+  assert.equal(third.reason.code, "execution_failed");
+  // Real, distinct per-step timing from Maestro's own commands.json, not a
+  // single timestamp copied onto every step.
+  assert.notEqual(first.startedAt, third.startedAt);
+  fs.rmSync(testDir, { recursive: true, force: true });
+});
+
+// A flow edited after its plan was written has a different step count. The
+// steps then carry only the aggregate verdict, but the run's counts must still
+// be the driver's real ones: one failed command is "1 failed", not "4 failed".
+test("counts the driver's own steps when they do not line up with the plan", async () => {
+  const { execute } = await import("./executor.js");
+  const testDir = fs.mkdtempSync(path.join(os.tmpdir(), "bugmole-executor-test-"));
+  const cfg = makeTestConfig(testDir);
+  cfg.execution.maestro.command = path.join(import.meta.dirname, "fixtures", "fake-maestro.sh");
+  const planYaml = `
+suites:
+  - journeys:
+      - id: checkout
+        driver: maestro
+        platform: android
+        flow: flows/checkout.yaml
+        steps:
+          - node_id: step-1
+            action: launch
+          - node_id: step-2
+            action: tap-sign-in
+          - node_id: step-3
+            action: enter-email
+          - node_id: step-4
+            action: assert-home
+`;
+  fs.writeFileSync(path.join(cfg.runtime.spec_dir, "plans", "checkout.yaml"), planYaml, "utf8");
+
+  const result = await execute(cfg, { planId: "checkout", journeyId: "checkout" });
+
+  assert.equal(result.status, "partial");
+  assert.equal(result.stepsExecuted, 4);
+  assert.equal(result.stepsPassed, 2);
+  assert.equal(result.stepsFailed, 1);
+  assert.ok(result.results!.every((step) => step.reason.code === "aggregate_only"));
+  fs.rmSync(testDir, { recursive: true, force: true });
+});
+
+test("loads the plan and its flow from the object store when they are not on the worker's disk", async () => {
+  const { execute } = await import("./executor.js");
+  const { LocalObjectStore } = await import("../storage/object-store.js");
+  const testDir = fs.mkdtempSync(path.join(os.tmpdir(), "bugmole-executor-store-"));
+  const cfg = makeTestConfig(testDir);
+  const store = new LocalObjectStore(cfg.runtime.artifacts_dir);
+  await store.put("projects/demo/plans/cloud-checkout.flow.yaml", `
+suites:
+  - journeys:
+      - id: cloud-checkout
+        driver: maestro
+        platform: android
+        flow: flows/cloud-checkout.yaml
+        steps:
+          - node_id: step-1
+            action: launch
+`, "text/yaml");
+  await store.put("projects/demo/spec/flows/cloud-checkout.yaml", "appId: demo\n---\n- launchApp\n", "text/yaml");
+
+  const result = await execute(cfg, { planId: "cloud-checkout", journeyId: "cloud-checkout", runId: "run-store" });
+
+  assert.doesNotMatch(result.error ?? "", /Plan not found/);
+  assert.equal(result.stepsExecuted, 1);
+  assert.ok(fs.existsSync(path.join(cfg.runtime.artifacts_dir, "demo", "run-store", "spec", "flows", "cloud-checkout.yaml")));
+  assert.equal(result.manifestKey, "projects/demo/runs/run-store/manifest.json");
+  assert.equal(result.evidencePath, "projects/demo/runs/run-store");
+});
+
+
+test("fans a web run out across browser/device targets in parallel against the selected environment", async () => {
+  const { execute } = await import("./executor.js");
+  const testDir = fs.mkdtempSync(path.join(os.tmpdir(), "bugmole-executor-matrix-"));
+  const cfg = makeTestConfig(testDir);
+  const planYaml = `
+suites:
+  - journeys:
+      - id: sign-in
+        driver: playwright
+        platform: web
+        flow: flows/sign-in.yaml
+        target:
+          base_url: http://localhost:3000
+        steps:
+          - node_id: open
+            action: open
+          - node_id: submit
+            action: submit
+`;
+  fs.writeFileSync(path.join(cfg.runtime.spec_dir, "plans", "sign-in.yaml"), planYaml, "utf8");
+
+  const contexts: DriverContext[] = [];
+  let inFlight = 0;
+  let peak = 0;
+  const driver: AppDriver = {
+    id: "playwright",
+    async run(context): Promise<DriverResult> {
+      contexts.push(context);
+      inFlight += 1;
+      peak = Math.max(peak, inFlight);
+      await new Promise((resolve) => setTimeout(resolve, 40));
+      inFlight -= 1;
+      const target = context.runTarget!;
+      if (target.browser === "msedge") {
+        return { status: "blocked", message: "Edge is not installed", failureCode: "browser_not_installed", artifacts: [] };
+      }
+      const screenshot = path.join(testDir, `${target.id}-final.png`);
+      fs.writeFileSync(screenshot, "png");
+      const failed = target.browser === "firefox";
+      return {
+        status: failed ? "failed" : "passed",
+        message: failed ? "Submit button not found" : "ok",
+        artifacts: [{ name: "playwright-final.png", path: screenshot, contentType: "image/png" }],
+        steps: [
+          { sequenceNumber: 0, status: "passed" },
+          { sequenceNumber: 1, status: failed ? "failed" : "passed" },
+        ],
+        durationMs: 40,
+      };
+    },
+  } as AppDriver;
+
+  const snapshots: string[][] = [];
+  const result = await execute(cfg, {
+    planId: "sign-in",
+    journeyId: "sign-in",
+    runId: "run-matrix",
+    baseUrl: "https://staging.example.com",
+    targets: [
+      { id: "chromium", browser: "chromium" },
+      { id: "firefox", browser: "firefox" },
+      { id: "msedge", browser: "msedge" },
+      { id: "webkit-iphone-15", browser: "webkit", device: "iPhone 15" },
+    ],
+    driverFactory: () => driver,
+    onProgress: (progress) => {
+      if (progress.targetResults) snapshots.push(progress.targetResults.map((entry) => entry.status));
+    },
+  });
+
+  assert.equal(contexts.length, 4);
+  assert.ok(peak > 1, "targets should run concurrently");
+  for (const context of contexts) {
+    assert.equal(context.target.baseUrl, "https://staging.example.com");
+    assert.equal(context.rebaseOrigin, true);
+  }
+  assert.equal(result.status, "partial");
+  assert.deepEqual(result.targetResults?.map((entry) => [entry.targetId, entry.status]), [
+    ["chromium", "passed"],
+    ["firefox", "failed"],
+    ["msedge", "blocked"],
+    ["webkit-iphone-15", "passed"],
+  ]);
+  const chromium = result.targetResults![0];
+  assert.equal(chromium.durationMs, 40);
+  assert.equal(chromium.stepsPassed, 2);
+  assert.equal(chromium.finalScreenshotKey, "projects/demo/runs/run-matrix/artifacts/chromium/playwright-final.png");
+  assert.deepEqual(result.targetResults![2].artifactKeys, []);
+  // The worst status per step across targets that ran.
+  assert.equal(result.stepsFailed, 1);
+  assert.deepEqual(snapshots[0], ["queued", "queued", "queued", "queued"]);
+  assert.ok(snapshots.some((snapshot) => snapshot.includes("running")));
+  assert.equal(typeof result.durationMs, "number");
+  const manifest = JSON.parse(fs.readFileSync(path.join(cfg.runtime.artifacts_dir, "projects", "demo", "runs", "run-matrix", "manifest.json"), "utf8"));
+  assert.equal(manifest.targets.length, 4);
+  fs.rmSync(testDir, { recursive: true, force: true });
+});
+
+test("keeps the single-driver path when no matrix is requested", async () => {
+  const { execute } = await import("./executor.js");
+  const testDir = fs.mkdtempSync(path.join(os.tmpdir(), "bugmole-executor-single-"));
+  const cfg = makeTestConfig(testDir);
+  fs.writeFileSync(path.join(cfg.runtime.spec_dir, "plans", "home.yaml"), `
+suites:
+  - journeys:
+      - id: home
+        driver: playwright
+        platform: web
+        flow: flows/home.yaml
+        steps:
+          - node_id: open
+            action: open
+`, "utf8");
+  const contexts: DriverContext[] = [];
+  const driver = {
+    id: "playwright",
+    async run(context: DriverContext): Promise<DriverResult> {
+      contexts.push(context);
+      return { status: "passed", message: "ok", artifacts: [], steps: [{ sequenceNumber: 0, status: "passed" }] };
+    },
+  } as unknown as AppDriver;
+  const result = await execute(cfg, { planId: "home", journeyId: "home", driverFactory: () => driver });
+  assert.equal(contexts.length, 1);
+  assert.equal(contexts[0].runTarget, undefined);
+  assert.equal(contexts[0].rebaseOrigin, false);
+  assert.equal(result.targetResults, undefined);
+  assert.equal(result.status, "success");
+  fs.rmSync(testDir, { recursive: true, force: true });
+});
+
+test("a failing progress update never stops a browser target from running", async () => {
+  const { execute } = await import("./executor.js");
+  const testDir = fs.mkdtempSync(path.join(os.tmpdir(), "bugmole-executor-progress-"));
+  const cfg = makeTestConfig(testDir);
+  fs.writeFileSync(path.join(cfg.runtime.spec_dir, "plans", "home.yaml"), `
+suites:
+  - journeys:
+      - id: home
+        driver: playwright
+        platform: web
+        flow: flows/home.yaml
+        steps:
+          - node_id: open
+            action: open
+`, "utf8");
+  const ran: string[] = [];
+  const driver = {
+    id: "playwright",
+    async run(context: DriverContext): Promise<DriverResult> {
+      ran.push(context.runTarget!.id);
+      await new Promise((resolve) => setTimeout(resolve, 10));
+      return { status: "passed", message: "ok", artifacts: [], steps: [{ sequenceNumber: 0, status: "passed" }] };
+    },
+  } as unknown as AppDriver;
+  // Reproduces concurrent registry updates failing (HTTP 500) while lanes report.
+  let calls = 0;
+  const result = await execute(cfg, {
+    planId: "home",
+    journeyId: "home",
+    targets: [
+      { id: "chromium", browser: "chromium" },
+      { id: "firefox", browser: "firefox" },
+      { id: "webkit", browser: "webkit" },
+    ],
+    driverFactory: () => driver,
+    onProgress: async () => {
+      calls += 1;
+      if (calls > 1) throw new Error("Registry update failed (500)");
+    },
+  });
+  assert.deepEqual([...ran].sort(), ["chromium", "firefox", "webkit"]);
+  assert.equal(result.status, "success");
+  assert.deepEqual(result.targetResults?.map((entry) => entry.status), ["passed", "passed", "passed"]);
+  fs.rmSync(testDir, { recursive: true, force: true });
+});
+
+test("apply fix patches the flow before running, and refuses a stale or empty fix", async () => {
+  const { execute } = await import("./executor.js");
+  const testDir = fs.mkdtempSync(path.join(os.tmpdir(), "bugmole-executor-fix-"));
+  const cfg = makeTestConfig(testDir);
+  fs.mkdirSync(path.join(cfg.runtime.spec_dir, "flows"), { recursive: true });
+  const flowFile = path.join(cfg.runtime.spec_dir, "flows", "home.yaml");
+  fs.writeFileSync(flowFile, 'url: http://localhost\n---\n- launchApp\n- tapOn: "Sign in"\n', "utf8");
+  fs.writeFileSync(path.join(cfg.runtime.spec_dir, "plans", "home.yaml"), `
+suites:
+  - journeys:
+      - id: home
+        driver: playwright
+        platform: web
+        flow: flows/home.yaml
+        steps:
+          - node_id: open
+            action: open
+`, "utf8");
+  const seen: string[] = [];
+  const driver = {
+    id: "playwright",
+    async run(context: DriverContext): Promise<DriverResult> {
+      seen.push(fs.readFileSync(context.flowPath, "utf8"));
+      return { status: "passed", message: "ok", artifacts: [], steps: [{ sequenceNumber: 0, status: "passed" }] };
+    },
+  } as unknown as AppDriver;
+  const change = { op: "replace" as const, index: 1, expect: '{"type":"tapOn","text":"Sign in","index":0,"optional":false}', step: { tapOn: "Log in" } };
+  const fixed = await execute(cfg, {
+    planId: "home",
+    journeyId: "home",
+    targets: [{ id: "chromium", browser: "chromium" }],
+    applyFix: { changes: [change], sourceRunId: "run-old", targetId: "chromium" },
+    driverFactory: () => driver,
+  });
+  assert.match(seen[0], /Log in/);
+  assert.match(fs.readFileSync(flowFile, "utf8"), /Log in/);
+  assert.deepEqual(fixed.fixApplied, { sourceRunId: "run-old", targetId: "chromium", flow: "flows/home.yaml" });
+
+  // Same fix again: the step no longer says "Sign in", so it is stale.
+  const stale = await execute(cfg, { planId: "home", journeyId: "home", targets: [{ id: "chromium", browser: "chromium" }], applyFix: { changes: [change] }, driverFactory: () => driver });
+  assert.equal(stale.failureCode, "fix_not_applicable");
+  assert.equal(seen.length, 1);
+  const empty = await execute(cfg, { planId: "home", journeyId: "home", targets: [{ id: "chromium", browser: "chromium" }], applyFix: { changes: [] }, driverFactory: () => driver });
+  assert.equal(empty.failureCode, "fix_not_applicable");
+  fs.rmSync(testDir, { recursive: true, force: true });
+});
+
+test("an inserted fix adds matching nodes to a plan that lists one node per flow step", async () => {
+  const { execute } = await import("./executor.js");
+  const testDir = fs.mkdtempSync(path.join(os.tmpdir(), "bugmole-executor-fix-plan-"));
+  const cfg = makeTestConfig(testDir);
+  fs.mkdirSync(path.join(cfg.runtime.spec_dir, "flows"), { recursive: true });
+  fs.writeFileSync(path.join(cfg.runtime.spec_dir, "flows", "form.yaml"), 'url: http://localhost\n---\n- launchApp\n- tapOn: "Submit"\n', "utf8");
+  const planFile = path.join(cfg.runtime.spec_dir, "plans", "form.yaml");
+  fs.writeFileSync(planFile, `
+suites:
+  - journeys:
+      - id: form
+        driver: playwright
+        platform: web
+        flow: flows/form.yaml
+        steps:
+          - { node_id: open, action: Open }
+          - { node_id: submit, action: Submit }
+`, "utf8");
+  const driver = {
+    id: "playwright",
+    async run(): Promise<DriverResult> {
+      return {
+        status: "passed",
+        message: "ok",
+        artifacts: [],
+        steps: [0, 1, 2, 3].map((sequenceNumber) => ({ sequenceNumber, status: "passed" as const })),
+      };
+    },
+  } as unknown as AppDriver;
+  const result = await execute(cfg, {
+    planId: "form",
+    journeyId: "form",
+    targets: [{ id: "chromium", browser: "chromium" }],
+    applyFix: { changes: [{ op: "insert", index: 1, steps: [{ tapOn: "Email" }, { inputText: "qa@example.com" }] }] },
+    driverFactory: () => driver,
+  });
+  const plan = fs.readFileSync(planFile, "utf8");
+  assert.match(plan, /node_id: fix-1[\s\S]*Tap “Email”[\s\S]*node_id: fix-2[\s\S]*node_id: submit/);
+  // Four plan nodes for four flow steps: every step is reported on its own.
+  assert.equal(result.stepsExecuted, 4);
+  assert.equal(result.results?.[0].reason.code, "verified");
+  fs.rmSync(testDir, { recursive: true, force: true });
+});
+
+test("AI repair asks the model for a checked edit, then runs the fixed flow", async () => {
+  const { execute } = await import("./executor.js");
+  const testDir = fs.mkdtempSync(path.join(os.tmpdir(), "bugmole-executor-ai-fix-"));
+  const cfg = makeTestConfig(testDir);
+  fs.mkdirSync(path.join(cfg.runtime.spec_dir, "flows"), { recursive: true });
+  const flowFile = path.join(cfg.runtime.spec_dir, "flows", "home.yaml");
+  fs.writeFileSync(flowFile, 'url: http://localhost\n---\n- launchApp\n- tapOn: "Continue"\n- assertVisible: "Payment"\n', "utf8");
+  fs.writeFileSync(path.join(cfg.runtime.spec_dir, "plans", "home.yaml"), `
+suites:
+  - journeys:
+      - id: home
+        driver: playwright
+        platform: web
+        flow: flows/home.yaml
+`, "utf8");
+  const seen: string[] = [];
+  const driver = {
+    id: "playwright",
+    async run(context: DriverContext): Promise<DriverResult> {
+      seen.push(fs.readFileSync(context.flowPath, "utf8"));
+      return { status: "passed", message: "ok", artifacts: [], steps: [] };
+    },
+  } as unknown as AppDriver;
+  const failureContext = {
+    failedStepIndex: 2, error: "Timed out", pageErrors: [], consoleErrors: [],
+    candidates: [{ role: "textbox", text: "Email" }], alerts: ["Enter your email"], invalidFields: [],
+  };
+  const requests: any[] = [];
+  const originalFetch = globalThis.fetch;
+  const saved = { url: process.env.BUGMOLE_REGISTRY_URL, key: process.env.BUGMOLE_API_KEY };
+  process.env.BUGMOLE_REGISTRY_URL = "https://registry.test";
+  process.env.BUGMOLE_API_KEY = "vk_test";
+  const reply = { summary: "Enter the email first.", changes: [{ op: "insert", index: 1, steps: [{ tapOn: "Email" }, { inputText: "qa@example.com" }] }] };
+  globalThis.fetch = (async (url: string, init: RequestInit) => {
+    requests.push({ url, body: JSON.parse(String(init.body)) });
+    return Response.json({ text: JSON.stringify(reply), model: "claude-sonnet-5" });
+  }) as unknown as typeof fetch;
+  try {
+    const result = await execute(cfg, {
+      planId: "home",
+      journeyId: "home",
+      runId: "run-new",
+      targets: [{ id: "chromium", browser: "chromium" }],
+      applyFix: { changes: [], mode: "ai", sourceRunId: "run-old", targetId: "chromium", failureContext },
+      driverFactory: () => driver,
+    });
+    assert.equal(result.fixApplied?.sourceRunId, "run-old");
+    assert.match(seen[0], /inputText: qa@example.com/);
+    assert.equal(requests.length, 1);
+    assert.equal(requests[0].url, "https://registry.test/api/ai/complete");
+    assert.equal(requests[0].body.purpose, "repair");
+    assert.equal(requests[0].body.runId, "run-new");
+    assert.match(requests[0].body.prompt, /Enter your email/);
+
+    // A model answer the flow checker rejects never reaches the test.
+    reply.changes = [{ op: "insert", index: 42, steps: [{ tapOn: "x" }] }];
+    const rejected = await execute(cfg, {
+      planId: "home", journeyId: "home", runId: "run-3", targets: [{ id: "chromium", browser: "chromium" }],
+      applyFix: { changes: [], mode: "ai", failureContext }, driverFactory: () => driver,
+    });
+    assert.equal(rejected.failureCode, "fix_not_applicable");
+    assert.equal(seen.length, 1);
+  } finally {
+    globalThis.fetch = originalFetch;
+    if (saved.url === undefined) delete process.env.BUGMOLE_REGISTRY_URL; else process.env.BUGMOLE_REGISTRY_URL = saved.url;
+    if (saved.key === undefined) delete process.env.BUGMOLE_API_KEY; else process.env.BUGMOLE_API_KEY = saved.key;
+    fs.rmSync(testDir, { recursive: true, force: true });
+  }
+});