@bugmole/cli 0.4.0

package/src/runtime/failure-analysis.test.ts

@@ -0,0 +1,111 @@
+import assert from "node:assert/strict";
+import test from "node:test";
+import { analyzeFailure, applyFlowChanges, labelSimilarity, rawFlowSteps, type FailureContext } from "./failure-analysis.js";
+import { explainWithModel, analysisModel } from "./ai-analysis.js";
+import { parseMaestroStyleFlow } from "./flow-language.js";
+
+const checkoutFlow = `url: http://localhost:3000/checkout
+name: checkout
+---
+- launchApp
+- tapOn: "Full name"
+- inputText: "Ada Lovelace"
+- tapOn: "Continue to payment"
+- assertVisible: "Payment"
+`;
+
+const baseContext = (overrides: Partial<FailureContext>): FailureContext => ({
+  failedStepIndex: 4,
+  error: 'Timed out after 15000ms waiting for "Payment" to be visible.',
+  pageErrors: [],
+  consoleErrors: [],
+  candidates: [],
+  alerts: [],
+  invalidFields: [],
+  ...overrides,
+});
+
+test("an empty required field behind a validation message is a test-data problem, with a fix", () => {
+  const flow = parseMaestroStyleFlow(checkoutFlow);
+  const analysis = analyzeFailure(flow.steps, rawFlowSteps(checkoutFlow), baseContext({
+    alerts: ["Please enter a valid email"],
+    invalidFields: [{ label: "Email", type: "email", empty: true }],
+  }));
+  assert.equal(analysis.category, "test");
+  assert.match(analysis.title, /email field was left empty/);
+  assert.match(analysis.rootCause, /Please enter a valid email/);
+  assert.match(analysis.suggestedFix!.summary, /valid email address/);
+  // The fill goes in before the submitting tap.
+  const fixed = parseMaestroStyleFlow(applyFlowChanges(checkoutFlow, analysis.suggestedFix!.changes));
+  assert.deepEqual(fixed.steps.slice(3, 6), [
+    { type: "tapOn", text: "Email", index: 0, optional: false },
+    { type: "inputText", text: "qa@example.com" },
+    { type: "tapOn", text: "Continue to payment", index: 0, optional: false },
+  ]);
+  assert.equal(fixed.url, "http://localhost:3000/checkout");
+});
+
+test("renamed UI copy is matched to the closest control and replaced in place", () => {
+  const flow = parseMaestroStyleFlow(checkoutFlow);
+  const analysis = analyzeFailure(flow.steps, rawFlowSteps(checkoutFlow), baseContext({
+    failedStepIndex: 3,
+    error: 'No element matching "Continue to payment" at index 0.',
+    candidates: [{ role: "button", text: "Continue to checkout" }, { role: "link", text: "Help" }],
+  }));
+  assert.equal(analysis.category, "test");
+  assert.equal(analysis.title, "“Continue to payment” is now “Continue to checkout”");
+  assert.deepEqual(analysis.suggestedFix!.preview.map((line) => line.kind), ["remove", "add"]);
+  const fixed = parseMaestroStyleFlow(applyFlowChanges(checkoutFlow, analysis.suggestedFix!.changes));
+  assert.equal((fixed.steps[3] as { text: string }).text, "Continue to checkout");
+  // A fix written for another version of the flow is refused.
+  const edited = checkoutFlow.replace('tapOn: "Continue to payment"', 'tapOn: "Pay now"');
+  assert.throws(() => applyFlowChanges(edited, analysis.suggestedFix!.changes), /has changed since this fix was suggested/);
+});
+
+test("a missing element with no close match is reported as an app problem, without a fix", () => {
+  const flow = parseMaestroStyleFlow(checkoutFlow);
+  const analysis = analyzeFailure(flow.steps, rawFlowSteps(checkoutFlow), baseContext({
+    candidates: [{ role: "heading", text: "Order history" }],
+  }));
+  assert.equal(analysis.category, "app");
+  assert.equal(analysis.suggestedFix, undefined);
+});
+
+test("crashes and unreachable environments are classified, and typed text never leaks", () => {
+  const flow = parseMaestroStyleFlow(checkoutFlow);
+  const crash = analyzeFailure(flow.steps, rawFlowSteps(checkoutFlow), baseContext({ error: "boom", pageErrors: ["TypeError: cart is undefined"] }));
+  assert.equal(crash.category, "app");
+  const down = analyzeFailure(flow.steps, rawFlowSteps(checkoutFlow), baseContext({ error: "page.goto: net::ERR_CONNECTION_REFUSED at http://localhost:3000/" }));
+  assert.equal(down.category, "environment");
+  const typed = analyzeFailure(flow.steps, rawFlowSteps(checkoutFlow), baseContext({ failedStepIndex: 2, error: 'No element matching "x"' }));
+  assert.doesNotMatch(JSON.stringify(typed), /Ada Lovelace/);
+});
+
+test("label similarity tolerates case, punctuation, and small edits", () => {
+  assert.equal(labelSimilarity("Sign in", "sign in!"), 1);
+  assert.ok(labelSimilarity("Continue to payment", "Continue to checkout") >= 0.5);
+  assert.ok(labelSimilarity("Sign in", "Delete account") < 0.5);
+});
+
+test("the AI model only rewrites the explanation, never the fix", async () => {
+  const flow = parseMaestroStyleFlow(checkoutFlow);
+  const context = baseContext({ alerts: ["Please enter a valid email"], invalidFields: [{ label: "Email", type: "email", empty: true }] });
+  const rules = analyzeFailure(flow.steps, rawFlowSteps(checkoutFlow), context);
+  assert.equal(analysisModel({ llm: { provider: "manual" } }), null);
+  assert.equal(analysisModel({ llm: { provider: "openai", api_key: "not-set" } }), null);
+  const cfg = { llm: { provider: "openai", model: "gpt-test", api_key: "sk-test" } };
+  let prompt = "";
+  const explained = await explainWithModel(cfg, rules, context, async (text) => {
+    prompt = text;
+    return '```json\n{"title":"Checkout blocked by the email check","rootCause":"The email field was empty, so validation stopped the form.","fixSummary":"Add a valid email address before submitting the form."}\n```';
+  });
+  assert.equal(explained.source, "ai");
+  assert.equal(explained.model, "gpt-test");
+  assert.equal(explained.title, "Checkout blocked by the email check");
+  assert.equal(explained.suggestedFix!.summary, "Add a valid email address before submitting the form.");
+  assert.deepEqual(explained.suggestedFix!.changes, rules.suggestedFix!.changes);
+  assert.doesNotMatch(prompt, /Ada Lovelace/);
+  const fallback = await explainWithModel(cfg, rules, context, async () => { throw new Error("rate limited"); });
+  assert.equal(fallback, rules);
+  assert.equal(await explainWithModel(cfg, rules, context, async () => "not json"), rules);
+});

package/src/runtime/failure-analysis.ts

@@ -0,0 +1,272 @@
+import YAML from "yaml";
+import { describeStepForLog, parseMaestroStyleFlow, type FlowStep } from "./flow-language.js";
+
+/**
+ * What the page looked like when a step failed, captured by the driver. It is
+ * the evidence the analysis reasons from, and it is uploaded with the run so
+ * a person can check the reasoning.
+ */
+export type FailureContext = {
+  url?: string;
+  title?: string;
+  failedStepIndex: number;
+  /** The failing step as logged: typed text masked. */
+  failedStep?: string;
+  error: string;
+  pageErrors: string[];
+  consoleErrors: string[];
+  /** Things a person could see and act on: buttons, links, labels, headings. */
+  candidates: Array<{ role: string; text: string }>;
+  /** Visible error and validation messages. */
+  alerts: string[];
+  /** Fields that are empty or marked invalid, by their label. */
+  invalidFields: Array<{ label: string; type: string; empty: boolean }>;
+};
+
+/** One edit to a flow's top-level step list. `expect` guards against stale fixes. */
+export type FlowChange =
+  | { op: "replace"; index: number; expect: string; step: unknown }
+  | { op: "insert"; index: number; steps: unknown[] };
+
+export type FailureCategory = "test" | "app" | "environment";
+
+export type FailureAnalysis = {
+  version: 1;
+  source: "rules" | "ai";
+  model?: string;
+  category: FailureCategory;
+  title: string;
+  rootCause: string;
+  evidence: string[];
+  confidence: "high" | "medium" | "low";
+  suggestedFix?: { summary: string; changes: FlowChange[]; preview: Array<{ kind: "add" | "remove"; text: string }> };
+};
+
+const normalize = (value: string) => value.toLowerCase().replace(/[^\p{Letter}\p{Number}]+/gu, " ").trim();
+
+function levenshtein(a: string, b: string): number {
+  const row = Array.from({ length: b.length + 1 }, (_, index) => index);
+  for (let i = 1; i <= a.length; i += 1) {
+    let previous = row[0];
+    row[0] = i;
+    for (let j = 1; j <= b.length; j += 1) {
+      const current = row[j];
+      row[j] = Math.min(row[j] + 1, row[j - 1] + 1, previous + (a[i - 1] === b[j - 1] ? 0 : 1));
+      previous = current;
+    }
+  }
+  return row[b.length];
+}
+
+/** 0–1 similarity between two labels, tolerant of case, punctuation, and small edits. */
+export function labelSimilarity(left: string, right: string): number {
+  const a = normalize(left);
+  const b = normalize(right);
+  if (!a || !b) return 0;
+  if (a === b) return 1;
+  const edit = 1 - levenshtein(a, b) / Math.max(a.length, b.length);
+  const aWords = new Set(a.split(" "));
+  const bWords = new Set(b.split(" "));
+  const shared = [...aWords].filter((word) => bWords.has(word)).length;
+  const overlap = shared / Math.max(aWords.size, bWords.size);
+  const contains = a.includes(b) || b.includes(a) ? 0.75 : 0;
+  return Math.max(edit, overlap, contains);
+}
+
+/** A raw step as it appears in the flow file, for fix previews. Never used for typed text. */
+function describeRaw(raw: unknown): string {
+  return YAML.stringify([raw]).trim().replace(/^- /, "");
+}
+
+function sampleValue(field: { label: string; type: string }): string {
+  const hint = `${field.type} ${field.label}`.toLowerCase();
+  if (hint.includes("email")) return "qa@example.com";
+  if (hint.includes("tel") || hint.includes("phone")) return "+1 555 010 0100";
+  if (hint.includes("zip") || hint.includes("postal")) return "94107";
+  if (hint.includes("number") || hint.includes("quantity")) return "1";
+  if (hint.includes("password")) return "{{secret.APP_PASSWORD}}";
+  return "Bugmole";
+}
+
+const isMissingElement = (error: string) =>
+  /No element matching|waiting for "[^"]+" to be visible|to scroll to|did not become clickable/i.test(error);
+
+/**
+ * Explains a failed flow from the captured page state, and proposes a flow
+ * change when the failure is the test's fault. Rules only: this never guesses
+ * without evidence, and every proposed change is checkable against the flow.
+ */
+export function analyzeFailure(steps: FlowStep[], rawSteps: unknown[], context: FailureContext): FailureAnalysis {
+  const { error, failedStepIndex } = context;
+  const step = steps[failedStepIndex];
+  const evidence: string[] = [`Step ${failedStepIndex + 1} failed: ${error}`];
+  if (context.url) evidence.push(`Page at failure: ${context.url}${context.title ? ` (“${context.title}”)` : ""}`);
+
+  if (/ERR_CONNECTION|ERR_NAME_NOT_RESOLVED|ECONNREFUSED|net::ERR_|getaddrinfo/i.test(error)) {
+    return {
+      version: 1,
+      source: "rules",
+      category: "environment",
+      title: "The app could not be reached",
+      rootCause: "The browser could not connect to the environment, so no step could be checked. The server may be down, or the environment's base URL may be wrong.",
+      evidence,
+      confidence: "high",
+    };
+  }
+
+  if (context.pageErrors.length > 0 && !isMissingElement(error)) {
+    evidence.push(...context.pageErrors.slice(0, 3).map((message) => `Uncaught page error: ${message}`));
+    return {
+      version: 1,
+      source: "rules",
+      category: "app",
+      title: "The page crashed",
+      rootCause: `The application threw an uncaught error: ${context.pageErrors[0]}. The flow is behaving correctly; the app needs a fix.`,
+      evidence,
+      confidence: "high",
+    };
+  }
+
+  // A form refused to submit: the next assertion never came true, and the
+  // page is showing validation messages for fields the flow left empty.
+  const emptyFields = context.invalidFields.filter((field) => field.empty);
+  if (context.alerts.length > 0 && emptyFields.length > 0) {
+    evidence.push(...context.alerts.slice(0, 3).map((alert) => `Visible message: “${alert}”`));
+    const submitIndex = [...steps.keys()].slice(0, failedStepIndex).reverse().find((index) => steps[index].type === "tapOn");
+    const field = emptyFields[0];
+    const value = sampleValue(field);
+    const insertAt = submitIndex ?? failedStepIndex;
+    const inserted = [{ tapOn: field.label }, { inputText: value }];
+    return {
+      version: 1,
+      source: "rules",
+      category: "test",
+      title: `The ${field.label.toLowerCase()} field was left empty`,
+      rootCause: `The form showed “${context.alerts[0]}” and did not continue, because the flow never filled in ${field.label}. The page behaved correctly; the test data was incomplete.`,
+      evidence,
+      confidence: "high",
+      suggestedFix: {
+        summary: `Fill in ${field.label} with a valid ${field.type === "email" ? "email address" : "value"} before submitting the form.`,
+        changes: [{ op: "insert", index: insertAt, steps: inserted }],
+        // Sample values are placeholders, never secrets, so the preview shows them.
+        preview: inserted.map((raw) => ({ kind: "add" as const, text: describeRaw(raw) })),
+      },
+    };
+  }
+
+  // The element the flow looked for is gone, but something very similar is
+  // on the page: the UI copy changed and the flow did not.
+  // Typed text can be a secret; it never becomes part of the analysis.
+  const expected = step && step.type !== "inputText" && "text" in step ? step.text : undefined;
+  if (expected && isMissingElement(error)) {
+    const ranked = context.candidates
+      .map((candidate) => ({ ...candidate, score: labelSimilarity(expected, candidate.text) }))
+      .filter((candidate) => normalize(candidate.text) !== normalize(expected))
+      .sort((left, right) => right.score - left.score);
+    const best = ranked[0];
+    if (best && best.score >= 0.5 && step && rawSteps[failedStepIndex] !== undefined) {
+      const replacement = replaceText(rawSteps[failedStepIndex], expected, best.text);
+      evidence.push(`Closest match on the page: ${best.role} “${best.text}”`);
+      return {
+        version: 1,
+        source: "rules",
+        category: "test",
+        title: `“${expected}” is now “${best.text}”`,
+        rootCause: `The flow looks for “${expected}”, but the page shows a ${best.role} labelled “${best.text}”. The UI copy changed and the test was not updated.`,
+        evidence,
+        confidence: best.score >= 0.75 ? "high" : "medium",
+        suggestedFix: {
+          summary: `Update step ${failedStepIndex + 1} to use “${best.text}”.`,
+          changes: [{ op: "replace", index: failedStepIndex, expect: stepSignature(step), step: replacement }],
+          preview: [
+            { kind: "remove", text: describeRaw(rawSteps[failedStepIndex]) },
+            { kind: "add", text: describeRaw(replacement) },
+          ],
+        },
+      };
+    }
+    return {
+      version: 1,
+      source: "rules",
+      category: "app",
+      title: `“${expected}” is missing from the page`,
+      rootCause: `The flow expected “${expected}”, and nothing similar is on the page. Either the feature is broken or the flow reached the wrong page.`,
+      evidence,
+      confidence: "medium",
+    };
+  }
+
+  return {
+    version: 1,
+    source: "rules",
+    category: "test",
+    title: "The flow stopped",
+    rootCause: error,
+    evidence,
+    confidence: "low",
+  };
+}
+
+/** A stable identity for a step, used to reject a fix written for another version of the flow. */
+export function stepSignature(step: FlowStep): string {
+  return describeStepForLog(step);
+}
+
+function replaceText(raw: unknown, from: string, to: string): unknown {
+  if (typeof raw === "string") return raw;
+  const record = raw as Record<string, unknown>;
+  const [key] = Object.keys(record);
+  const value = record[key];
+  if (typeof value === "string") return { [key]: value === from ? to : value };
+  if (value && typeof value === "object") {
+    const nested = { ...(value as Record<string, unknown>) };
+    for (const field of ["text", "label", "id", "element"]) {
+      if (nested[field] === from) nested[field] = to;
+    }
+    return { [key]: nested };
+  }
+  return raw;
+}
+
+/** Splits a flow file into its header text and raw step list. */
+function splitFlow(content: string): { header: string; steps: unknown[] } {
+  const documents = content.split(/^---\s*$/m);
+  if (documents.length < 2) throw new Error("Flow file must have a header document and a steps document separated by '---'.");
+  const steps: unknown = YAML.parse(documents.slice(1).join("---"));
+  if (!Array.isArray(steps)) throw new Error("Flow steps document must be a YAML list.");
+  return { header: documents[0].trimEnd(), steps };
+}
+
+export function rawFlowSteps(content: string): unknown[] {
+  return splitFlow(content).steps;
+}
+
+/**
+ * Applies a suggested fix to a flow file's text. Every replacement must still
+ * match the step it was written for; otherwise the fix is stale and refused.
+ */
+export function applyFlowChanges(content: string, changes: readonly FlowChange[]): string {
+  const { header, steps } = splitFlow(content);
+  // Flows may take their URL from the environment; only the steps matter here.
+  const validationBase = "http://localhost";
+  const parsed = parseMaestroStyleFlow(content, validationBase).steps;
+  const next = [...steps];
+  // Later indexes first, so earlier inserts do not shift them.
+  const ordered = [...changes].sort((left, right) => right.index - left.index);
+  for (const change of ordered) {
+    if (change.index < 0 || change.index > next.length) throw new Error(`Fix refers to step ${change.index + 1}, which the flow does not have.`);
+    if (change.op === "replace") {
+      const current = parsed[change.index];
+      if (!current || stepSignature(current) !== change.expect) {
+        throw new Error(`Step ${change.index + 1} has changed since this fix was suggested; re-run the test for a new suggestion.`);
+      }
+      next[change.index] = change.step;
+    } else {
+      next.splice(change.index, 0, ...change.steps);
+    }
+  }
+  const updated = `${header}\n---\n${YAML.stringify(next)}`;
+  // Refuse to write a flow the driver would reject.
+  parseMaestroStyleFlow(updated, validationBase);
+  return updated;
+}

package/src/runtime/fixtures/fake-maestro.sh

@@ -0,0 +1,36 @@
+#!/bin/sh
+# Test fixture standing in for the real `maestro` binary: writes a
+# commands.json shaped like Maestro's own --test-output-dir output (one
+# passed step, one passed step, one failed step) and exits non-zero, the way
+# a real flow with a failed assertion would.
+set -e
+outdir=""
+prev=""
+for arg in "$@"; do
+  if [ "$prev" = "--test-output-dir" ]; then
+    outdir="$arg"
+  fi
+  prev="$arg"
+done
+if [ -z "$outdir" ]; then
+  echo "fake-maestro: missing --test-output-dir" >&2
+  exit 2
+fi
+mkdir -p "$outdir"
+cat > "$outdir/commands.json" <<'JSON'
+[
+  {
+    "command": { "launchAppCommand": { "appId": "com.example.app" } },
+    "metadata": { "status": "COMPLETED", "timestamp": 1000, "duration": 3363, "sequenceNumber": 0, "depth": 0 }
+  },
+  {
+    "command": { "tapOnCommand": { "text": "Sign in" } },
+    "metadata": { "status": "COMPLETED", "timestamp": 4400, "duration": 50, "sequenceNumber": 1, "depth": 0 }
+  },
+  {
+    "command": { "assertVisibleCommand": { "text": "Home" } },
+    "metadata": { "status": "FAILED", "timestamp": 4500, "duration": 200, "sequenceNumber": 2, "depth": 0 }
+  }
+]
+JSON
+exit 1