@specific.dev/spectest 0.66.0 → 0.67.0

package/src/page-snapshot.test.ts

@@ -0,0 +1,100 @@
+import { describe, expect, test } from "bun:test";
+
+import {
+  formatPageStructure,
+  MAX_DOCUMENT_BYTES,
+  truncate,
+  type PageStructure,
+} from "./page-snapshot.js";
+
+/** The tree shape `page.ariaSnapshot()` really returns, captured from
+ *  playwright-core 1.61.1 against headless Chromium (see the harness recipe in
+ *  locator-hints.test.ts). */
+const TREE = `- heading "Fakturor" [level=1]
+- button "Spara ändringar": Spara
+- combobox "Sortera":
+  - option "Datum" [selected]
+- iframe`;
+
+const base: PageStructure = {
+  session: "",
+  kind: "browser",
+  action: "click",
+  error: 'No element matches role button "Spara" (waited 5s)',
+  url: "http://app.test.local:3000/",
+  title: "Fakturor",
+  tree: TREE,
+  frames: [],
+};
+
+describe("formatPageStructure", () => {
+  test("leads with a legend, the failed step and the page it was on", () => {
+    const doc = formatPageStructure(base);
+    const lines = doc.split("\n");
+    expect(lines[0]).toBe("# Page structure at the moment this step failed.");
+    // The reader did not choose this format, so the file explains it.
+    expect(doc).toContain('`role "accessible name" [state]: own text`');
+    expect(doc).toContain("`getByRole(role, { name })`");
+    expect(doc).toContain("# failed step: click");
+    expect(doc).toContain(
+      '# failure: No element matches role button "Spara" (waited 5s)',
+    );
+    expect(doc).toContain(
+      '## browser session (default) — url: http://app.test.local:3000/ — title: "Fakturor"',
+    );
+    expect(doc).toContain('- button "Spara ändringar": Spara');
+    expect(doc.endsWith("\n")).toBe(true);
+  });
+
+  test("keeps only the first line of a multi-line failure", () => {
+    // A locator failure carries its near-miss block; that is already in the
+    // CLI output, and this file exists to add to it.
+    const doc = formatPageStructure({
+      ...base,
+      error: "No element matches role button \"Spara\"\nClose matches:\n  - …",
+    });
+    expect(doc).toContain('# failure: No element matches role button "Spara"');
+    expect(doc).not.toContain("Close matches");
+  });
+
+  test("names the session when the test named it", () => {
+    const doc = formatPageStructure({ ...base, session: "alice", kind: "mobile" });
+    expect(doc).toContain('## mobile session "alice"');
+  });
+
+  test("renders each child frame under its own heading", () => {
+    const doc = formatPageStructure({
+      ...base,
+      frames: [
+        { name: "pay", url: "https://pay.test/form", tree: '- textbox "Kortnummer"' },
+        { name: "", url: "about:srcdoc", tree: "- button \"OK\"" },
+      ],
+    });
+    expect(doc).toContain('### iframe "pay" — url: https://pay.test/form');
+    expect(doc).toContain('- textbox "Kortnummer"');
+    // An unnamed frame is still worth showing; it just has no name to print.
+    expect(doc).toContain("### iframe — url: about:srcdoc");
+  });
+
+  test("a page with no url or title still renders its tree", () => {
+    const doc = formatPageStructure({ ...base, url: "", title: "" });
+    expect(doc).toContain("## browser session (default)\n");
+    expect(doc).toContain('- heading "Fakturor" [level=1]');
+  });
+});
+
+describe("truncate", () => {
+  test("leaves an ordinary document alone", () => {
+    const doc = "- button \"OK\"\n";
+    expect(truncate(doc)).toBe(doc);
+  });
+
+  test("cuts on a line boundary and says that it did", () => {
+    const doc = `${"- row \"x\"\n".repeat(40_000)}`;
+    const out = truncate(doc);
+    expect(Buffer.byteLength(out)).toBeLessThanOrEqual(MAX_DOCUMENT_BYTES + 120);
+    expect(out.endsWith('- row "x"\n# … truncated at 262144 bytes (400000 bytes captured)\n')).toBe(
+      true,
+    );
+  });
+});

package/src/page-snapshot.ts

@@ -0,0 +1,180 @@
+// The page's structure at the moment a browser step failed, as text an agent
+// can read.
+//
+// A failed browser test is the one failure where the CLI reader has nothing to
+// look at. The message says what did not happen, the timeline says what ran,
+// and the page itself — the thing that would answer the question — is only on
+// the dashboard, inside an rrweb replay. `screenshot()` is no help either: it
+// throws inside a test run, and an image is not something every reader can
+// read.
+//
+// So a failed browser/mobile step captures its page as an **accessibility
+// tree** (`page.ariaSnapshot()`), the same data locators resolve against. It is
+// small (a form page is under 1 KB; a 200-row table is 38 KB, 4 KB gzipped), it
+// is text, and every name in it is directly usable as
+// `getByRole(role, { name })`. It rides out on the step's own recorded event,
+// which is what keeps this entirely inside the SDK: no new daemon method, no
+// side channel, no supervisor involvement. The control plane lifts it into a
+// `case-page` artifact the same way it already lifts an eval screenshot's
+// bytes, so the failure block can hand the reader one command that prints the
+// page they could not see.
+//
+// Riding the event has a second, load-bearing effect: `ctx.poll` truncates the
+// events of every superseded attempt, so a poll whose predicate fails ten
+// times keeps exactly one capture — the attempt the timeline kept — with no
+// bookkeeping here.
+//
+// Two deliberate choices about the format:
+//
+//   * **Default mode, not `mode: "ai"`.** The ai mode adds `[ref=e12]`
+//     handles, and a ref is only resolvable in the live page it came from —
+//     in a downloaded file it is an invitation to write a locator that cannot
+//     work. Iframe content, which ai mode adds and the default mode leaves as
+//     a bare `- iframe`, is recovered by snapshotting each frame instead.
+//   * **No timestamps.** A guest's wall clock resumes frozen at the snapshot
+//     it was restored from, so a time captured here would be a plausible lie.
+//     The artifact row carries a host-side `created_at`.
+
+import type { Frame, Page } from "playwright-core";
+
+/** One child frame's tree. The main frame's tree leads the document. */
+export interface FrameStructure {
+  /** The frame's `name` attribute, or `""` when it has none. */
+  name: string;
+  url: string;
+  tree: string;
+}
+
+/** Everything the document is rendered from. Split out from the capture so the
+ *  formatting is testable without a browser. */
+export interface PageStructure {
+  /** The session's name — `ctx.browser("alice")` — or `""` for the default. */
+  session: string;
+  kind: "browser" | "mobile";
+  /** The step that failed (`click`, `goto`, `toBeVisible`, …). */
+  action: string;
+  /** Its failure message; only the first line is kept. */
+  error: string;
+  url: string;
+  title: string;
+  tree: string;
+  frames: FrameStructure[];
+}
+
+/** The document is a debugging aid, and a pathological page must not grow the
+ *  `/run` reply (the vm-agent caps a proxied daemon response at 16 MB). Real
+ *  pages are far below this: the biggest thing measured while building it was
+ *  38 KB. */
+export const MAX_DOCUMENT_BYTES = 256 * 1024;
+
+/** How many child frames are worth walking. An ad-heavy page can carry
+ *  dozens; the ones a test drives are at the front. */
+const MAX_FRAMES = 10;
+
+/** Per-call deadline. The page has just failed a step, and it may be wedged; a
+ *  diagnostic must not add to the damage. */
+const CAPTURE_TIMEOUT_MS = 3_000;
+
+function withTimeout<T>(work: Promise<T>, fallback: T): Promise<T> {
+  let timer: ReturnType<typeof setTimeout> | undefined;
+  return Promise.race([
+    work.catch(() => fallback),
+    new Promise<T>((resolve) => {
+      timer = setTimeout(() => resolve(fallback), CAPTURE_TIMEOUT_MS);
+    }),
+  ]).finally(() => {
+    if (timer) clearTimeout(timer);
+  });
+}
+
+async function frameStructure(frame: Frame): Promise<FrameStructure | undefined> {
+  // A frame's tree comes from its own body: `page.ariaSnapshot()` stops at
+  // `- iframe` and never descends into it.
+  const tree = await withTimeout(frame.locator("body").ariaSnapshot(), "");
+  if (!tree.trim()) return undefined;
+  return { name: frame.name(), url: frame.url(), tree };
+}
+
+/**
+ * Capture the page behind a failed step, or `undefined` when there is nothing
+ * to show.
+ *
+ * Everything here is best-effort: a page that is closed, navigating or wedged
+ * yields a partial record rather than an error, because the caller is already
+ * reporting a failure and must not report this one instead.
+ */
+export async function capturePageStructure(
+  page: Page,
+  meta: Pick<PageStructure, "session" | "kind" | "action" | "error">,
+): Promise<string | undefined> {
+  const tree = await withTimeout(page.ariaSnapshot(), "");
+  const title = await withTimeout(page.title(), "");
+  let url = "";
+  try {
+    url = page.url();
+  } catch {
+    /* The page is gone; the tree, if we got one, is still worth keeping. */
+  }
+  const frames: FrameStructure[] = [];
+  try {
+    const children = page.frames().filter((f) => f !== page.mainFrame());
+    for (const f of children.slice(0, MAX_FRAMES)) {
+      const s = await frameStructure(f);
+      if (s) frames.push(s);
+    }
+  } catch {
+    /* Frames are a bonus; the main tree is the point. */
+  }
+  if (!tree.trim() && frames.length === 0) return undefined;
+  return formatPageStructure({ ...meta, url, title, tree, frames });
+}
+
+/** The header that tells a reader what they are looking at. Worth its lines:
+ *  the file is read by someone who did not choose its format, hours later,
+ *  with nothing around it. */
+const LEGEND = [
+  "# Page structure at the moment this step failed.",
+  "#",
+  '# One line per accessibility node: `role "accessible name" [state]: own text`,',
+  "# children indented, properties as `/`-prefixed children (/url, /placeholder).",
+  "# The names here are exactly what `getByRole(role, { name })` matches on.",
+  "# Elements hidden from assistive technology are absent, as they are from locators.",
+];
+
+function quote(s: string): string {
+  return JSON.stringify(s);
+}
+
+function sessionHeading(s: PageStructure): string {
+  const name = s.session === "" ? "(default)" : quote(s.session);
+  const parts = [`## ${s.kind} session ${name}`];
+  if (s.url) parts.push(`url: ${s.url}`);
+  if (s.title) parts.push(`title: ${quote(s.title)}`);
+  return parts.join(" — ");
+}
+
+/** Render the document. Pure, so its shape is testable without a browser. */
+export function formatPageStructure(s: PageStructure): string {
+  const lines = [...LEGEND, "#", `# failed step: ${s.action}`];
+  if (s.error) {
+    // The first line only: the near-miss block under a locator failure is
+    // already in the CLI output, and this file exists to add to it.
+    lines.push(`# failure: ${s.error.split("\n")[0]}`);
+  }
+  lines.push("", sessionHeading(s), s.tree.trimEnd());
+  for (const f of s.frames) {
+    const name = f.name === "" ? "" : ` ${quote(f.name)}`;
+    lines.push("", `### iframe${name} — url: ${f.url}`, f.tree.trimEnd());
+  }
+  return truncate(`${lines.join("\n")}\n`);
+}
+
+/** Cut the document to {@link MAX_DOCUMENT_BYTES}, on a line boundary, and say
+ *  so — a silently short tree reads as a page that ends there. */
+export function truncate(doc: string): string {
+  const bytes = Buffer.byteLength(doc, "utf8");
+  if (bytes <= MAX_DOCUMENT_BYTES) return doc;
+  const text = Buffer.from(doc, "utf8").subarray(0, MAX_DOCUMENT_BYTES).toString("utf8");
+  const cut = text.slice(0, text.lastIndexOf("\n") + 1);
+  return `${cut}# … truncated at ${MAX_DOCUMENT_BYTES} bytes (${bytes} bytes captured)\n`;
+}

package/src/recorder.ts

@@ -488,6 +488,22 @@ export interface BrowserEvent extends BaseEvent {
   attempts?: number;
   durationMs: number;
   error?: string;
+  /**
+   * The page's accessibility tree at the moment this step failed — set only
+   * on a failed browser/mobile step (see `page-snapshot.ts`). It rides the
+   * event because that is the only channel out of the harness the SDK owns
+   * end to end, and because `ctx.poll` truncates a superseded attempt's
+   * events, which drops its capture with it.
+   *
+   * **The control plane lifts this out**: it stores the text as a
+   * `case-page` artifact and replaces the field with `pageArtifactId`,
+   * exactly as it does for an eval screenshot's inline bytes. So a
+   * persisted event carries the id, never the document.
+   */
+  pageStructure?: string;
+  /** The `art_0…` the control plane stored {@link pageStructure} as. Never
+   *  set by the SDK. */
+  pageArtifactId?: string;
   /**
    * Session this op belonged to. Set whenever the Browser was opened
    * with a `BrowserSessionRecorder` attached (the daemon always does).

package/src/text-match.test.ts

@@ -0,0 +1,132 @@
+import { describe, expect, test } from "bun:test";
+
+import {
+  containsText,
+  containsTextArray,
+  escapeInvisible,
+  matchesText,
+  matchesTextArray,
+  normalizeWhiteSpace,
+  textDifferenceNote,
+} from "./text-match.js";
+
+// Every invisible character in this file is written as an escape on purpose:
+// a literal one is unreadable in a diff, and a test whose subject cannot be
+// seen in the source is a test nobody can maintain.
+
+/** What `Intl.NumberFormat("sv-SE")` produces: a NO-BREAK SPACE between the
+ *  thousands. It prints exactly like the space an author types, which is the
+ *  whole reason this module exists. */
+const SEK = "15\u00a0000 kr";
+
+describe("normalizeWhiteSpace", () => {
+  test("folds every space character onto a plain space", () => {
+    expect(normalizeWhiteSpace(SEK)).toBe("15 000 kr");
+    expect(normalizeWhiteSpace("15 000 kr")).toBe("15 000 kr");
+    expect(normalizeWhiteSpace("  a \n\t b  ")).toBe("a b");
+  });
+
+  test("drops the characters that print as nothing", () => {
+    expect(normalizeWhiteSpace("a\u200bb\u00adc")).toBe("abc");
+  });
+});
+
+describe("matchesText", () => {
+  test("a typed space matches a no-break space", () => {
+    expect(matchesText(SEK, "15 000 kr")).toBe(true);
+  });
+
+  test("markup that wraps the value over two lines still matches", () => {
+    expect(matchesText("\n  Total:\n  15 000 kr\n", "Total: 15 000 kr")).toBe(true);
+  });
+
+  test("is a whole-string comparison", () => {
+    expect(matchesText(SEK, "15 000")).toBe(false);
+    expect(containsText(SEK, "15 000")).toBe(true);
+  });
+
+  test("a RegExp is tested against the normalized text", () => {
+    expect(matchesText(SEK, /^15 000 kr$/)).toBe(true);
+    // The same pattern against the raw text would have to spell U+00A0 out.
+    expect(/^15 000 kr$/.test(SEK)).toBe(false);
+  });
+
+  test("ignoreCase covers both a string and a RegExp", () => {
+    expect(matchesText("SPARA", "spara")).toBe(false);
+    expect(matchesText("SPARA", "spara", { ignoreCase: true })).toBe(true);
+    expect(matchesText("SPARA", /spara/, { ignoreCase: true })).toBe(true);
+  });
+});
+
+describe("containsText", () => {
+  test("normalizes both sides", () => {
+    expect(containsText(`Summa: ${SEK} inkl. moms`, "15 000 kr")).toBe(true);
+  });
+
+  test("takes a RegExp, which the string-only signature could not", () => {
+    expect(containsText(SEK, /\d{2} \d{3}/)).toBe(true);
+  });
+});
+
+describe("array forms", () => {
+  const rows = ["Item one", "Item two", "Item three"];
+
+  test("toHaveText's array is position for position, and the counts must agree", () => {
+    expect(matchesTextArray(rows, ["Item one", "Item two", "Item three"])).toBe(true);
+    expect(matchesTextArray(rows, ["Item one", "Item three"])).toBe(false);
+    expect(matchesTextArray(rows, ["Item one", /two/, "Item three"])).toBe(true);
+  });
+
+  test("toContainText's array allows elements in between, but not out of order", () => {
+    expect(containsTextArray(rows, ["one", "three"])).toBe(true);
+    expect(containsTextArray(rows, ["three", "one"])).toBe(false);
+    expect(containsTextArray(rows, ["four"])).toBe(false);
+  });
+});
+
+describe("escapeInvisible", () => {
+  test("spells out the characters that print as nothing", () => {
+    expect(escapeInvisible(JSON.stringify(SEK))).toBe('"15\\u00a0000 kr"');
+    expect(escapeInvisible(JSON.stringify("a\u200bb"))).toBe('"a\\u200bb"');
+  });
+
+  test("leaves ordinary text, and JSON's own escapes, alone", () => {
+    expect(escapeInvisible(JSON.stringify("Spara ändringar"))).toBe('"Spara ändringar"');
+    expect(escapeInvisible(JSON.stringify("a\nb"))).toBe('"a\\nb"');
+  });
+});
+
+describe("textDifferenceNote", () => {
+  test("names the invisible character when that is the whole difference", () => {
+    const note = textDifferenceNote(SEK, "15 000 kr", "equal");
+    expect(note).toContain("they differ only in invisible or look-alike characters");
+    expect(note).toContain("NO-BREAK SPACE at index 2");
+  });
+
+  test("covers a substring comparison too", () => {
+    const note = textDifferenceNote(`Summa: ${SEK}`, "15 000 kr", "contains");
+    expect(note).toContain("NO-BREAK SPACE");
+  });
+
+  test("names a look-alike that is not invisible at all", () => {
+    const note = textDifferenceNote("Fri–Sun", "Fri-Sun", "equal");
+    expect(note).toContain("EN DASH");
+  });
+
+  test("reports the expected side when the actual is the plain one", () => {
+    const note = textDifferenceNote("15 000 kr", SEK, "equal");
+    expect(note).toContain("the expected string has NO-BREAK SPACE");
+  });
+
+  test("says so when only the case differs", () => {
+    expect(textDifferenceNote("Spara", "spara", "equal")).toBe(
+      " — they differ only in letter case",
+    );
+  });
+
+  test("stays quiet for a difference the reader can see", () => {
+    expect(textDifferenceNote("Spara", "Avbryt", "equal")).toBe("");
+    expect(textDifferenceNote("15 000 kr", "15 000 kr", "equal")).toBe("");
+    expect(textDifferenceNote(42, "42", "equal")).toBe("");
+  });
+});