@specific.dev/spectest 0.66.0 → 0.67.0

package/src/locator-errors.test.ts

@@ -1,6 +1,11 @@
 import { describe, expect, test } from "bun:test";
 
-import { formatWaited, locatorFailureMessage, rewriteLocatorError } from "./locator-errors.js";
+import {
+  classifyLocatorFailure,
+  formatWaited,
+  locatorFailureMessage,
+  rewriteLocatorErrorWithHints,
+} from "./locator-errors.js";
 
 /** A playwright actionability timeout, shaped exactly as playwright-core
  *  1.61 raises one: `name`, the message with the call log appended, the same
@@ -83,22 +88,105 @@ describe("locatorFailureMessage", () => {
     );
   });
 
-  test("leaves errors that are not locator timeouts alone", () => {
-    const strict = new Error(
-      "click: Error: strict mode violation: locator('p') resolved to 2 elements",
-    );
-    expect(locatorFailureMessage('css "p"', strict)).toBeUndefined();
+  test("leaves errors that are neither a timeout nor a strict violation alone", () => {
     expect(locatorFailureMessage('css "p"', new Error("page closed"))).toBeUndefined();
     expect(locatorFailureMessage('css "p"', "not an error at all")).toBeUndefined();
   });
+
+  test("an ambiguous locator says how to narrow it, and lists the matches", () => {
+    // Playwright's own shape: the count, then one numbered line per match.
+    const strict = new Error(
+      "click: Error: strict mode violation: getByRole('button') resolved to 2 elements:\n" +
+        "    1) <button>Save</button> aka getByRole('button', { name: 'Save' })\n" +
+        "    2) <button>Save all</button> aka getByRole('button', { name: 'Save all' })\n",
+    );
+    expect(locatorFailureMessage("role button", strict)).toBe(
+      "role button matches 2 elements, so it is ambiguous — narrow it with .first(), " +
+        ".nth(i), .filter({ hasText }) or a more specific query:\n" +
+        "  - <button>Save</button> aka getByRole('button', { name: 'Save' })\n" +
+        "  - <button>Save all</button> aka getByRole('button', { name: 'Save all' })",
+    );
+  });
+
+  test("an ambiguous locator with no listed matches still says what to do", () => {
+    const strict = new Error("strict mode violation: locator('p') resolved to 7 elements");
+    expect(locatorFailureMessage('css "p"', strict)).toBe(
+      'css "p" matches 7 elements, so it is ambiguous — narrow it with .first(), ' +
+        ".nth(i), .filter({ hasText }) or a more specific query",
+    );
+  });
+});
+
+describe("classifyLocatorFailure", () => {
+  test("only a locator that matched nothing is worth asking the page about", () => {
+    const missing = timeoutError("click: Timeout 5000ms exceeded.", [
+      "  - waiting for getByTestId('add')",
+    ]);
+    expect(classifyLocatorFailure('testid "add"', missing)?.kind).toBe("no-match");
+
+    const disabled = timeoutError("click: Timeout 5000ms exceeded.", [
+      "  - waiting for locator('#dis')",
+      '    - locator resolved to <button id="dis" disabled>Nope</button>',
+      "      - element is not enabled",
+    ]);
+    expect(classifyLocatorFailure('css "#dis"', disabled)?.kind).toBe("state");
+
+    const strict = new Error("strict mode violation: locator('p') resolved to 2 elements");
+    expect(classifyLocatorFailure('css "p"', strict)?.kind).toBe("strict");
+  });
+});
+
+describe("rewriteLocatorErrorWithHints", () => {
+  const missing = (): Error =>
+    timeoutError("click: Timeout 5000ms exceeded.", ["  - waiting for getByTestId('add')"]);
+
+  test("appends the near-miss block to a locator that matched nothing", async () => {
+    const err = (await rewriteLocatorErrorWithHints(missing(), 'testid "add"', async () =>
+      "\nClose matches on the page:\n  - testid \"add-todo\"",
+    )) as Error;
+    expect(err.message).toBe(
+      'No element matches testid "add" (waited 5s)\n' +
+        'Close matches on the page:\n  - testid "add-todo"',
+    );
+    // The stack's copy of the message is rewritten too, or the CLI would
+    // print the friendly sentence and the raw timeout under it.
+    expect(err.stack).toContain('Close matches on the page');
+    expect(err.stack).not.toContain("Timeout 5000ms exceeded");
+  });
+
+  test("never asks the page about a failure it could not explain", async () => {
+    const disabled = timeoutError("click: Timeout 5000ms exceeded.", [
+      "  - waiting for locator('#dis')",
+      '    - locator resolved to <button id="dis" disabled>Nope</button>',
+      "      - element is not enabled",
+    ]);
+    let asked = false;
+    const err = (await rewriteLocatorErrorWithHints(disabled, 'css "#dis"', async () => {
+      asked = true;
+      return "should not appear";
+    })) as Error;
+    expect(asked).toBe(false);
+    expect(err.message).toBe('Element css "#dis" is not enabled (waited 5s)');
+  });
+
+  test("a diagnostic that throws leaves the failure standing", async () => {
+    const err = (await rewriteLocatorErrorWithHints(missing(), 'testid "add"', async () => {
+      throw new Error("page closed");
+    })) as Error;
+    expect(err.message).toBe('No element matches testid "add" (waited 5s)');
+  });
 });
 
-describe("rewriteLocatorError", () => {
-  test("rewrites the message AND the copy embedded in the stack", () => {
+describe("rewriteLocatorErrorWithHints (stack)", () => {
+  test("rewrites the message AND the copy embedded in the stack", async () => {
     const err = timeoutError("click: Timeout 5000ms exceeded.", [
       "  - waiting for getByTestId('add')",
     ]);
-    const out = rewriteLocatorError(err, 'testid "add"') as Error;
+    const out = (await rewriteLocatorErrorWithHints(
+      err,
+      'testid "add"',
+      async () => "",
+    )) as Error;
 
     expect(out).toBe(err); // same error: the author's own frames survive
     expect(out.message).toBe('No element matches testid "add" (waited 5s)');
@@ -107,9 +195,9 @@ describe("rewriteLocatorError", () => {
     expect(out.stack).toContain("todo.ts:12:34");
   });
 
-  test("hands back anything it does not understand untouched", () => {
+  test("hands back anything it does not understand untouched", async () => {
     const err = new Error("page closed");
-    expect(rewriteLocatorError(err, 'css "p"')).toBe(err);
+    expect(await rewriteLocatorErrorWithHints(err, 'css "p"', async () => "")).toBe(err);
     expect(err.message).toBe("page closed");
   });
 });

package/src/locator-errors.ts

@@ -8,10 +8,15 @@
 // as an infrastructure problem and says nothing about the page.
 //
 // This module turns those into one sentence that names the element and what
-// was wrong with it. Anything it does not recognise (a strict-mode violation,
-// a closed page, an assertion of ours) is passed through untouched — the rule
-// is that a message is only ever replaced when we have something better to
-// say.
+// was wrong with it. A strict-mode violation — the opposite failure, where the
+// locator matched too much — is rewritten too, keeping playwright's own list
+// of matches, which is the useful half of it. Anything else (a closed page, an
+// assertion of ours) is passed through untouched: the rule is that a message is
+// only ever replaced when we have something better to say.
+//
+// `classifyLocatorFailure` also says WHICH failure it was, because only one of
+// them — a locator that matched nothing — can be explained further by asking
+// the page what it holds (see locator-hints.ts).
 //
 // The call log is read from the error's own `log` array (playwright attaches
 // it) and falls back to parsing the message, since only the message survives
@@ -92,38 +97,87 @@ function actionabilityReason(log: string[]): string | undefined {
   return undefined;
 }
 
+/** How a locator failed. `no-match` is the only kind worth asking the page
+ *  about (see locator-hints.ts) — for every other kind the element was found
+ *  and near misses would be noise. */
+export type LocatorFailureKind = "no-match" | "state" | "strict";
+
+export interface LocatorFailure {
+  message: string;
+  kind: LocatorFailureKind;
+}
+
+/** Playwright's strict-mode violation: the locator matched more than one
+ *  element and refused to act. It is an ordinary `Error`, not a
+ *  `TimeoutError`, and its message already lists the matches — which is the
+ *  useful half, so it is kept and only the wall of text around it is cut. */
+function parseStrictViolation(err: unknown, label: string): LocatorFailure | undefined {
+  const e = err as { message?: unknown } | null;
+  if (!e || typeof e.message !== "string") return undefined;
+  if (!e.message.includes("strict mode violation")) return undefined;
+  const m = /resolved to (\d+) elements/.exec(e.message);
+  if (!m) return undefined;
+  const matches = e.message
+    .split("\n")
+    .filter((l) => /^\s*\d+\)\s/.test(l))
+    .slice(0, 3)
+    .map((l) => `  - ${l.trim().replace(/^\d+\)\s*/, "")}`);
+  const head =
+    `${label} matches ${m[1]} elements, so it is ambiguous` +
+    " — narrow it with .first(), .nth(i), .filter({ hasText }) or a more specific query";
+  return {
+    kind: "strict",
+    message: matches.length ? `${head}:\n${matches.join("\n")}` : head,
+  };
+}
+
 /**
- * The sentence to fail with, or `undefined` when `err` is not a locator
- * timeout we understand (in which case the caller must leave it alone).
+ * What to fail with, or `undefined` when `err` is not a locator failure we
+ * understand (in which case the caller must leave it alone).
  *
  * `label` is the locator's human chain label, the same string the timeline
  * step shows.
  */
-export function locatorFailureMessage(label: string, err: unknown): string | undefined {
+export function classifyLocatorFailure(
+  label: string,
+  err: unknown,
+): LocatorFailure | undefined {
+  const strict = parseStrictViolation(err, label);
+  if (strict) return strict;
   const detail = parseTimeout(err);
   if (!detail) return undefined;
   const waited = `(waited ${formatWaited(detail.timeoutMs)})`;
-  if (!resolved(detail.log)) return `No element matches ${label} ${waited}`;
+  if (!resolved(detail.log)) {
+    return { kind: "no-match", message: `No element matches ${label} ${waited}` };
+  }
 
   // It matched, so the failure is about the element's state.
+  const state = (message: string): LocatorFailure => ({ kind: "state", message });
   switch (waitedForState(detail.log)) {
     case "hidden":
-      return `Element ${label} is still visible ${waited}`;
+      return state(`Element ${label} is still visible ${waited}`);
     case "detached":
-      return `Element ${label} is still attached to the page ${waited}`;
+      return state(`Element ${label} is still attached to the page ${waited}`);
     case "visible":
-      return `Element ${label} is not visible ${waited}`;
+      return state(`Element ${label} is not visible ${waited}`);
     default:
       break;
   }
   const reason = actionabilityReason(detail.log);
-  return reason
-    ? `Element ${label} ${reason} ${waited}`
-    : `Element ${label} never became ready for this action ${waited}`;
+  return state(
+    reason
+      ? `Element ${label} ${reason} ${waited}`
+      : `Element ${label} never became ready for this action ${waited}`,
+  );
+}
+
+/** The sentence alone. See {@link classifyLocatorFailure}. */
+export function locatorFailureMessage(label: string, err: unknown): string | undefined {
+  return classifyLocatorFailure(label, err)?.message;
 }
 
 /**
- * Replace a locator timeout's message in place and hand the error back, so the
+ * Replace a locator failure's message in place and hand the error back, so the
  * caller can `throw` it unchanged in every other respect.
  *
  * The error is mutated rather than wrapped: its stack holds the frames of the
@@ -132,10 +186,8 @@ export function locatorFailureMessage(label: string, err: unknown): string | und
  * too — otherwise the CLI's failure block would print the friendly message and
  * then the timeout wall right under it.
  */
-export function rewriteLocatorError(err: unknown, label: string): unknown {
-  const message = locatorFailureMessage(label, err);
-  if (message === undefined) return err;
-  const e = err as Error;
+/** Put `message` on `err`, in the message and in the stack's copy of it. */
+function setErrorMessage(e: Error, message: string): Error {
   const old = e.message;
   if (typeof e.stack === "string") {
     for (const [header, replacement] of [
@@ -151,3 +203,30 @@ export function rewriteLocatorError(err: unknown, label: string): unknown {
   e.message = message;
   return e;
 }
+
+/**
+ * The rewrite above, plus the near-miss block for the one failure that can
+ * carry one — a locator that matched nothing.
+ *
+ * `hints` is only called in that case, so a page query is never made for a
+ * failure it could not explain (a disabled element, a covered one, an
+ * ambiguous match). It is expected to be best-effort itself; anything it
+ * throws leaves the plain sentence standing.
+ */
+export async function rewriteLocatorErrorWithHints(
+  err: unknown,
+  label: string,
+  hints: () => Promise<string>,
+): Promise<unknown> {
+  const failure = classifyLocatorFailure(label, err);
+  if (!failure) return err;
+  let message = failure.message;
+  if (failure.kind === "no-match") {
+    try {
+      message += await hints();
+    } catch {
+      /* A diagnostic must never replace the failure it explains. */
+    }
+  }
+  return setErrorMessage(err as Error, message);
+}

package/src/locator-hints.test.ts

@@ -0,0 +1,188 @@
+import { describe, expect, test } from "bun:test";
+
+// The ranking and the snapshot parsing are pure, so they are tested here. The
+// collection half needs a real browser, and was verified against one — every
+// fixture below is captured from that run rather than imagined. To repeat it,
+// write a script under `sdk/` (so it resolves this SDK's own playwright-core,
+// not a newer one from bun's cache), `page.setContent(...)` a page with the
+// cases you care about, and call `nearMissHints(page, target)` directly:
+//
+//   cd sdk && LD_LIBRARY_PATH=/tmp/chromium-scratch/root/usr/lib/aarch64-linux-gnu \
+//     FONTCONFIG_FILE=/tmp/chromium-scratch/fonts.conf bun my-harness.ts
+//
+// (that library path is the no-sudo Chromium recipe this box already carries;
+// `chromium.launch({ args: ["--no-sandbox"] })` then works.) Measured there:
+// 3–20 ms per diagnostic, which is why it can run on every failed locator.
+
+import {
+  buildSuggestions,
+  closeness,
+  EMPTY_DOM_FACTS,
+  parseAriaSnapshot,
+  suggestionLine,
+  type DomFacts,
+} from "./locator-hints.js";
+
+/** A page's accessibility tree exactly as `page.ariaSnapshot()` rendered it —
+ *  captured from playwright-core 1.61.1 driving headless Chromium against the
+ *  fixture page in this module's verification harness. Note the shapes a
+ *  hand-written fixture would have missed: a node's own text after the name
+ *  (`: Spara`), property lines (`/placeholder:`), state suffixes
+ *  (`[selected]`) and unnamed nodes (`- text:`). */
+const SNAPSHOT = `- banner:
+  - heading "Fakturor" [level=1]
+  - button "Spara ändringar": Spara
+- main:
+  - combobox "Sortera":
+    - option "Datum" [selected]
+  - button "Ny faktura"
+  - button "Ta bort"
+  - button "Ta bort allt"
+  - textbox "Sök":
+    - /placeholder: Sök fakturor
+  - paragraph: "Summa: 15 000 kr"
+  - button "Skicka till kund": Skicka
+  - text: Skicka till kund`;
+
+describe("parseAriaSnapshot", () => {
+  test("reads the role and accessible name off every named node", () => {
+    expect(parseAriaSnapshot(SNAPSHOT)).toEqual([
+      { role: "heading", name: "Fakturor" },
+      { role: "button", name: "Spara ändringar" },
+      { role: "combobox", name: "Sortera" },
+      { role: "option", name: "Datum" },
+      { role: "button", name: "Ny faktura" },
+      { role: "button", name: "Ta bort" },
+      { role: "button", name: "Ta bort allt" },
+      { role: "textbox", name: "Sök" },
+      { role: "button", name: "Skicka till kund" },
+    ]);
+  });
+
+  test("skips nodes with no name, and lines it does not recognise", () => {
+    // A container with no accessible name cannot be suggested by name, and a
+    // property line ("/url: …") is not a node at all.
+    expect(parseAriaSnapshot("- generic:\n  - /url: /x\nnot yaml at all")).toEqual([]);
+  });
+
+  test("unescapes a quoted name", () => {
+    expect(parseAriaSnapshot('- button "Say \\"hi\\""')).toEqual([
+      { role: "button", name: 'Say "hi"' },
+    ]);
+  });
+});
+
+describe("closeness", () => {
+  test("the same string, whitespace and case aside, is the closest", () => {
+    expect(closeness("Spara", "spara")).toBe(4);
+    expect(closeness("15 000 kr", "15 000 kr")).toBe(4);
+  });
+
+  test("a name that carries a suffix scores above a typo", () => {
+    expect(closeness("Spara", "Spara ändringar")).toBeGreaterThan(
+      closeness("Spara", "Spraa"),
+    );
+  });
+
+  test("an unrelated name scores nothing", () => {
+    expect(closeness("Spara", "Avbryt")).toBe(0);
+    expect(closeness("Spara", "")).toBe(0);
+  });
+});
+
+describe("buildSuggestions", () => {
+  const aria = parseAriaSnapshot(SNAPSHOT);
+
+  test("a role query finds the same role under a longer name", () => {
+    const [first] = buildSuggestions(
+      { kind: "role", role: "button", query: "Spara" },
+      aria,
+      EMPTY_DOM_FACTS,
+    );
+    expect(first?.fact).toBe('button "Spara ändringar"');
+    expect(suggestionLine(first!, 1)).toBe(
+      '  - button "Spara ändringar" → getByRole("button", { name: "Spara ändringar" })',
+    );
+  });
+
+  test("a name under the wrong role says which role it really is", () => {
+    const lines = buildSuggestions(
+      { kind: "role", role: "button", query: "Sortera" },
+      aria,
+      EMPTY_DOM_FACTS,
+    );
+    expect(lines[0]?.fact).toBe('combobox "Sortera" — this is a combobox, not a button');
+    expect(lines[0]?.locator).toEqual({
+      m: "getByRole",
+      role: "combobox",
+      name: "Sortera",
+    });
+  });
+
+  test("an aria-label that hides the visible text is called out by name", () => {
+    const dom: DomFacts = {
+      ...EMPTY_DOM_FACTS,
+      mismatches: [
+        { tag: "button", text: "Spara", name: "Spara ändringar", from: "aria-label" },
+      ],
+    };
+    const lines = buildSuggestions(
+      { kind: "role", role: "button", query: "Spara" },
+      [],
+      dom,
+    );
+    expect(lines[0]?.fact).toBe(
+      '<button> shows "Spara" but its accessible name is "Spara ändringar" (from aria-label)',
+    );
+    expect(lines[0]?.locator).toEqual({
+      m: "getByRole",
+      role: "button",
+      name: "Spara ändringar",
+    });
+  });
+
+  test("a test id suggests the test ids the page really carries", () => {
+    const dom: DomFacts = { ...EMPTY_DOM_FACTS, testIds: ["add-todo", "remove-todo"] };
+    const lines = buildSuggestions({ kind: "testId", query: "add" }, [], dom);
+    expect(lines).toHaveLength(1);
+    expect(lines[0]?.locator).toEqual({ m: "getByTestId", id: "add-todo" });
+  });
+
+  test("a RegExp query suggests nothing — a pattern has no near miss", () => {
+    expect(buildSuggestions({ kind: "role", role: "button" }, aria, EMPTY_DOM_FACTS)).toEqual(
+      [],
+    );
+  });
+
+  test("nothing close means nothing is said", () => {
+    expect(
+      buildSuggestions(
+        { kind: "role", role: "button", query: "Ladda upp kvitto" },
+        aria,
+        EMPTY_DOM_FACTS,
+      ),
+    ).toEqual([]);
+  });
+
+  test("at most three lines, best first, one per proposed locator", () => {
+    const dom: DomFacts = {
+      ...EMPTY_DOM_FACTS,
+      texts: ["Faktura 1", "Faktura 2", "Faktura 3", "Faktura 4", "Faktura 5"],
+    };
+    const lines = buildSuggestions({ kind: "text", query: "Faktura 1" }, aria, dom);
+    expect(lines).toHaveLength(3);
+    expect(lines[0]?.locator).toEqual({ m: "getByText", text: "Faktura 1" });
+    expect(new Set(lines.map((l) => JSON.stringify(l.locator))).size).toBe(3);
+  });
+
+  test("the count rides the line when a suggestion is itself ambiguous", () => {
+    const [first] = buildSuggestions(
+      { kind: "role", role: "button", query: "Ta bort" },
+      aria,
+      EMPTY_DOM_FACTS,
+    );
+    expect(suggestionLine(first!, 4)).toBe(
+      '  - button "Ta bort" → getByRole("button", { name: "Ta bort" }) (matches 4 elements)',
+    );
+  });
+});