debutly 0.2.0 → 0.3.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "debutly",
-  "version": "0.2.0",
+  "version": "0.3.0",
   "description": "Rehearse your launch: AI agents use your web app like your first customers and report what breaks.",
   "type": "module",
   "license": "AGPL-3.0-only",

package/src/agent/effect.ts

@@ -73,19 +73,25 @@ export function describeEffect(action: Action, before: Observation, after: Obser
 
   const parts: string[] = [];
   if (result.enterIgnored) parts.push("Enter did nothing on it, so Space was pressed instead");
+  if (result.invalid?.length) parts.push(`the browser stopped the form and said: ${result.invalid.join("; ")}`);
   const changed = whatChanged(before, after, (e) => controlLabel(e, view.screenReader));
   if (pathOf(after.url) !== pathOf(before.url)) parts.push(`now on ${pathOf(after.url)}`);
   else if (changed) parts.push(changed);
   else if (after.text !== before.text) parts.push("the page changed");
-  else if (action.type !== "fill" && !signals.some((s) => s.kind === "download")) parts.push("nothing visibly changed");
+  else if (action.type !== "fill" && !result.invalid?.length && !signals.some((s) => s.kind === "download")) parts.push("nothing visibly changed");
 
   for (const s of signals) if (s.kind === "download") parts.push(`a file downloaded: ${s.filename}`);
 
   for (const s of signals) if (s.kind === "dialog") parts.push(`a ${s.dialogType} dialog said "${s.message}" (answered OK)`);
   if (signals.some((s) => s.kind === "page_error")) parts.push("a JavaScript error occurred");
+  // Only the server failing: a person never sees a status code, and apps answer 404 or 401 as part
+  // of working (a setting never saved yet). Told "a request failed (HTTP 404)" after each click, an
+  // agent decided onboarding was broken (2026-10-25). The report still lists those answers.
   const failed = [
     ...new Set(
-      signals.filter((s) => s.kind === "http_error" && pathOf(s.requestUrl) !== "/favicon.ico").map((s) => (s.kind === "http_error" ? s.status : 0)),
+      signals
+        .filter((s) => s.kind === "http_error" && s.status >= 500 && pathOf(s.requestUrl) !== "/favicon.ico")
+        .map((s) => (s.kind === "http_error" ? s.status : 0)),
     ),
   ];
   for (const status of failed) parts.push(`a request failed (HTTP ${status})`);

package/src/agent/intent.ts

@@ -0,0 +1,51 @@
+import type { Action, InteractiveElement, Observation } from "../types";
+
+const norm = (s: string) => s.toLowerCase().replace(/\s+/g, " ").trim();
+
+/**
+ * How a control is named in an agent's words: whole when short, by its first words when its name
+ * runs on into a description ("Email alerts Get notified of hiring changes" is "email alerts").
+ */
+function mentionOf(name: string): string {
+  const n = norm(name);
+  if (n.length <= 30) return n;
+  const words = n.split(" ");
+  const two = words.slice(0, 2).join(" ");
+  return two.length >= 8 ? two : words.slice(0, 3).join(" ");
+}
+
+const escape = (s: string) => s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+
+/**
+ * Whether the agent's words name the control. A one-word name counts only as written on the
+ * control (its capitals), so "go back" in passing isn't the "Back" button.
+ */
+function said(reasoning: string, name: string): boolean {
+  const m = mentionOf(name);
+  if (m.length < 3) return false;
+  if (m.includes(" ")) return norm(reasoning).includes(m);
+  return new RegExp(`(^|[^\\p{L}\\p{N}])${escape(name.trim())}($|[^\\p{L}\\p{N}])`, "u").test(reasoning);
+}
+
+const nameOf = (e: InteractiveElement, screenReader: boolean) => (screenReader && e.ax ? e.ax.name : e.name || e.ax?.name || "");
+
+/**
+ * When an agent's reasoning names controls on the page but its action points at another one: the
+ * controls it named. An agent in a product's onboarding wrote "I will click the button for email
+ * alerts" and then "select 'Competitor'", and both times sent the id of the logo, which restarted
+ * onboarding; it gave up blaming the app (2026-10-25). Undefined when the action matches what it
+ * said, or when there is nothing to compare (no id, or a control without a name).
+ */
+export function saidOtherControl(reasoning: string, action: Action, obs: Observation, screenReader = false): string[] | undefined {
+  if (action.type !== "click" && action.type !== "fill") return undefined;
+  const chosen = obs.elements.find((e) => e.id === action.id);
+  if (!chosen) return undefined;
+  const chosenName = nameOf(chosen, screenReader);
+  if (!chosenName.trim() || said(reasoning, chosenName)) return undefined;
+  // A field filled with text it names ("type Agency into Industry") is fine either way.
+  if (action.type === "fill" && norm(reasoning).includes(norm(action.text))) return undefined;
+  const meant = obs.elements
+    .filter((e) => e.id !== chosen.id && nameOf(e, screenReader).trim().length >= 4 && said(reasoning, nameOf(e, screenReader)))
+    .map((e) => nameOf(e, screenReader).trim());
+  return meant.length ? [...new Set(meant)].slice(0, 3) : undefined;
+}

package/src/agent/loop.ts

@@ -12,6 +12,8 @@ import type { ActionResult, Persona, Session, Step } from "../types";
 import { describeAction } from "./describe";
 import { describeEffect } from "./effect";
 import { nothingToPressYet, untriedWaysOn } from "./give-up";
+import { saidOtherControl } from "./intent";
+import { controlLabel } from "./label";
 import { NoActionError, parseAgentReply } from "./parse-reply";
 import { buildStepMessage, buildSystemPrompt, type AgentIdentity } from "./prompt";
 
@@ -55,12 +57,35 @@ export interface RunSessionOptions {
   firstLoad?: { timeoutMs?: number; slowMs?: number };
 }
 
-function describeInbox(messages: EmailMessage[]): string {
+/** The inbox as the agent reads it, each link numbered ("email link 2") so it can be opened without retyping it. */
+export function describeInbox(messages: EmailMessage[], numberOf: (link: string) => number): string {
   if (!messages.length) return "checked email → no emails yet";
-  const items = messages.map((m) => `"${m.subject}"${m.links.length ? ` links: ${m.links.slice(0, LINKS_PER_EMAIL).join(" ")}` : ""}`);
+  const items = messages.map((m) => {
+    const links = m.links.slice(0, LINKS_PER_EMAIL).map((l) => `[email link ${numberOf(l)}] ${l}`);
+    return `"${m.subject}"${links.length ? ` links: ${links.join(" ")}` : ""}`;
+  });
   return `checked email → ${messages.length} email${messages.length === 1 ? "" : "s"}: ${items.join("; ")}`;
 }
 
+/**
+ * The emailed link a goto means: "email link 2" by its number, or a mistyped copy of one, the same
+ * path and query under a garbled host. An agent retyped a sign-in link as http://127.0.0.0.0.1:…,
+ * got "Invalid URL" and gave up (2026-10-25). Undefined when it means no emailed link.
+ */
+export function emailedLinkMeant(path: string, links: string[]): string | undefined {
+  const numbered = /^email ?link ?(\d+)$/i.exec(path.trim());
+  if (numbered) return links[Number(numbered[1]) - 1];
+  const tail = (u: string) => {
+    const start = u.indexOf("//");
+    const slash = u.indexOf("/", start < 0 ? 0 : start + 2);
+    return slash < 0 ? "" : u.slice(slash);
+  };
+  const typed = tail(path.trim());
+  // Long enough to be a link's own path and token, not a common page like "/login".
+  if (typed.length < 20) return undefined;
+  return links.find((l) => tail(l) === typed && l !== path.trim());
+}
+
 /**
  * Open the app the way a person does: wait for the page, but never fail the session because it
  * is slow. Waiting for every image and font ("load") within Playwright's 30 s ended the
@@ -209,6 +234,22 @@ async function takeSteps(o: RunSessionOptions, { signals, system, steps, history
     if (!reply) {
       return end("error", `${problem} | last reply: ${raw.slice(0, 300) || "(empty)"}`);
     }
+    // One more question about the reply just given; an unreadable answer leaves the reply standing.
+    const askAgain = async (question: string) => {
+      const again = await o.llm.complete({
+        system,
+        messages: [...messages, { role: "assistant", content: raw }, { role: "user", content: question }],
+        maxTokens: 4096,
+        onUsage: meter.add,
+      });
+      try {
+        const answer = parseAgentReply(again);
+        raw = again;
+        return answer;
+      } catch {
+        return undefined;
+      }
+    };
     // Giving up with a way on in plain sight that it never pressed here, or on a page with nothing
     // to press yet: asked once per page. Still giving up, it gives up.
     if (reply.action.type === "done" && !reply.action.success && !askedBeforeGivingUp.has(observation.url)) {
@@ -220,26 +261,16 @@ async function takeSteps(o: RunSessionOptions, { signals, system, steps, history
           : "";
       if (ask) {
         askedBeforeGivingUp.add(observation.url);
-        const again = await o.llm.complete({
-          system,
-          messages: [
-            ...messages,
-            { role: "assistant", content: raw },
-            {
-              role: "user",
-              content: `${ask} Reply again with one JSON object: go on, or give up if you are sure.`,
-            },
-          ],
-          maxTokens: 4096,
-          onUsage: meter.add,
-        });
-        try {
-          reply = parseAgentReply(again);
-        } catch {
-          // an unreadable second answer leaves the first one standing
-        }
+        reply = (await askAgain(`${ask} Reply again with one JSON object: go on, or give up if you are sure.`)) ?? reply;
       }
     }
+    // Its words name one control and its action points at another: asked once to check the id.
+    const meant = saidOtherControl(reply.reasoning, reply.action, observation, screenReader);
+    const chosen = "id" in reply.action ? observation.elements.find((e) => e.id === (reply!.action as { id: number }).id) : undefined;
+    if (meant && chosen) {
+      const question = `You wrote about ${meant.map((n) => `"${n}"`).join(" or ")}, but your action points at [${chosen.id}] ${controlLabel(chosen, screenReader)}. Check the id: reply again with one JSON object.`;
+      reply = (await askAgain(question)) ?? reply;
+    }
 
     let result: ActionResult;
     let historyLine = describeAction(reply.action, observation);
@@ -250,14 +281,22 @@ async function takeSteps(o: RunSessionOptions, { signals, system, steps, history
         try {
           const mail = await o.inbox.latest(o.identity.email);
           for (const m of mail) for (const link of m.links) emailedLinks.add(link);
-          historyLine = describeInbox(mail);
+          const numbered = [...emailedLinks];
+          historyLine = describeInbox(mail, (link) => numbered.indexOf(link) + 1);
           result = {};
         } catch (err) {
           result = { error: err instanceof Error ? err.message : String(err) };
         }
       }
     } else {
-      result = await execute(o.page, reply.action, observation, o.origin, (href) => emailedLinks.has(href), { keyboard: screenReader });
+      // An emailed link, by its number or as a mistyped copy, is opened as it was sent.
+      const link = reply.action.type === "goto" ? emailedLinkMeant(reply.action.path, [...emailedLinks]) : undefined;
+      if (reply.action.type === "goto" && /^email ?link ?\d+$/i.test(reply.action.path.trim()) && !link) {
+        result = { error: `there is no ${reply.action.path.trim()}: check your email to see the links` };
+      } else {
+        const action = link ? { type: "goto" as const, path: link } : reply.action;
+        result = await execute(o.page, action, observation, o.origin, (href) => emailedLinks.has(href), { keyboard: screenReader });
+      }
     }
 
     let screenshotPath: string | undefined;

package/src/agent/prompt.ts

@@ -14,7 +14,7 @@ export interface AgentIdentity {
 export function buildSystemPrompt(persona: Persona, goal: string, identity?: AgentIdentity, brief?: string): string {
   const email = identity
     ? `Your email address is ${identity.email}. Use it whenever the site asks for your email.
-To read your inbox (for example a login link or a confirmation), reply {"type":"check_email"}. To open a link from an email, use {"type":"goto","path":"<the full link>"}.`
+To read your inbox (for example a login link or a confirmation), reply {"type":"check_email"}. To open a link from an email, use its number: {"type":"goto","path":"email link 1"}.`
     : `Use emails at example.com.`;
   const about = [
     `You are ${persona.name}. ${persona.background}`,
@@ -47,8 +47,10 @@ Action is one of:
 }
 
 /** A control's state, as a person sees it (and a screen reader announces it). */
-function stateOf(e: InteractiveElement): string {
+function stateOf(e: InteractiveElement, behindOpenLayer = false): string {
   const states = [
+    // Hidden from assistive technology while something is open: what a modal does to the page behind it.
+    behindOpenLayer && e.ax?.hidden ? "behind what is open" : "",
     e.disabled ? "disabled" : "",
     e.readOnly ? "read-only" : "",
     e.checked === undefined ? "" : e.checked ? "checked" : "not checked",
@@ -59,19 +61,20 @@ function stateOf(e: InteractiveElement): string {
   return states.length ? ` (${states.join(", ")})` : "";
 }
 
-function describeElement(e: InteractiveElement, screenReader = false): string {
-  const line = `[${e.id}] ${controlLabel(e, screenReader)}${stateOf(e)}`;
+function describeElement(e: InteractiveElement, screenReader = false, behindOpenLayer = false): string {
+  const line = `[${e.id}] ${controlLabel(e, screenReader)}${stateOf(e, behindOpenLayer)}`;
   if (!e.options) return line;
   return `${line} set to "${e.value ?? ""}", options: ${e.options.map((o) => `"${o}"`).join(" | ")}`;
 }
 
 export function buildStepMessage(obs: Observation, history: string[], view: StepView = {}): string {
   const shown = view.screenReader ? obs.elements.filter((e) => !e.ax?.hidden) : obs.elements;
-  const elements = shown.map((e) => describeElement(e, view.screenReader)).join("\n");
+  const elements = shown.map((e) => describeElement(e, view.screenReader, !!obs.openLayer)).join("\n");
   const past = history.length ? history.map((h, i) => `${i + 1}. ${h}`).join("\n") : "(none yet)";
+  const open = obs.openLayer ? `\nOpen now: ${obs.openLayer}. It takes clicks outside it: finish it, choose from it or close it (Escape, or its close button) before using what is behind it.\n` : "";
   return `URL: ${obs.url}
 Title: ${obs.title}
-
+${open}
 Interactive elements:
 ${elements}
 

package/src/browser/ax.ts

@@ -67,6 +67,7 @@ export async function accessibleNames(page: Page): Promise<Map<number, Accessibl
         role: node.role?.value ?? "",
         hidden: node.ignored,
         ...(description ? { description } : {}),
+        ...(node.ignoredReasons?.some((r) => r.name === "ariaHiddenElement" || r.name === "ariaHiddenSubtree") ? { ariaHidden: true as const } : {}),
       });
     }
     return out;

package/src/browser/execute.ts

@@ -1,5 +1,6 @@
 import type { Locator, Page } from "playwright";
 import type { Action, ActionResult, InteractiveElement, Observation } from "../types";
+import { OPEN_LAYER_SCRIPT } from "./observe";
 import { sampleFiles, sampleFor, type SampleKind } from "./samples";
 
 const DESTRUCTIVE = /\b(delete|remove|destroy|deactivate|close account|cancel (my )?(subscription|account))\b/i;
@@ -85,6 +86,8 @@ const FOCUS_ONLY = /^(input:(text|email|password|number|search|tel|url|date|time
 const focusCoveredScript = (id: number) => `(() => {
   const el = document.querySelector('[data-debutly-id="${id}"]');
   if (!el) return null;
+  // A dropdown's own open list may be drawn over it, by design (2026-11-01).
+  if (el.getAttribute('aria-expanded') === 'true') return null;
   const r = el.getBoundingClientRect();
   const points = [[0.5, 0.5], [0.1, 0.1], [0.9, 0.1], [0.1, 0.9], [0.9, 0.9]].map(([fx, fy]) => [r.left + r.width * fx, r.top + r.height * fy]);
   let cover = null;
@@ -94,7 +97,7 @@ const focusCoveredScript = (id: number) => `(() => {
     if (!hit || hit === el || el.contains(hit) || hit.contains(el)) return null;
     cover = cover || hit;
   }
-  if (!cover) return 'the edge of the window';
+  if (!cover) return '${EDGE_OF_WINDOW}';
   const classes = typeof cover.className === 'string' ? cover.className.trim().split(/\\s+/).filter(Boolean).slice(0, 2) : [];
   const text = (cover.innerText || '').replace(/\\s+/g, ' ').trim().slice(0, 50);
   return cover.tagName.toLowerCase() + (cover.id ? '#' + cover.id : '') + classes.map((c) => '.' + c).join('') + (text ? ' ("' + text + '")' : '');
@@ -121,9 +124,65 @@ async function focusByKeyboard(page: Page, locator: Locator, id: number): Promis
   if (reach === "disabled") throw new Unusable(UNUSABLE.disabled);
   if (reach === "untabbable") throw new NotByKeyboard(`${NOT_BY_KEYBOARD}: it isn't in the tab order`);
   await locator.focus({ timeout: 5_000 });
+  const hidden = (await page.evaluate(focusCoveredScript(id))) as string | null;
+  // A page that scrolls smoothly brings the focused control into view a moment later: measured at
+  // once, a Continue button in plain sight was reported hidden by the edge of the window.
+  if (hidden !== EDGE_OF_WINDOW) return hidden ?? undefined;
+  await page.waitForTimeout(SMOOTH_SCROLL_MS);
   return ((await page.evaluate(focusCoveredScript(id))) as string | null) ?? undefined;
 }
 
+const EDGE_OF_WINDOW = "the edge of the window";
+
+// What the browser itself says when a submit finds a field invalid ("Please lengthen this text to
+// 8 characters or more"): a person sees that bubble; it isn't in the page's text, so an agent with
+// a 7-letter password saw nothing happen, and gave up saying sign-up was broken (2026-10-25).
+// Listened for during each action; a string, like SCAN_SCRIPT.
+const LISTEN_INVALID = `(() => {
+  if (window.__debutlyInvalid) return;
+  window.__debutlyInvalid = [];
+  document.addEventListener('invalid', (e) => {
+    const el = e.target;
+    const label = (el.labels && el.labels[0] && el.labels[0].textContent) || el.getAttribute('aria-label') || el.getAttribute('placeholder') || el.name || 'a field';
+    window.__debutlyInvalid.push('"' + label.replace(/\\s+/g, ' ').trim().slice(0, 60) + '": ' + el.validationMessage);
+  }, true);
+})()`;
+const READ_INVALID = `(() => {
+  const said = window.__debutlyInvalid || [];
+  window.__debutlyInvalid = [];
+  return [...new Set(said)].slice(0, 3);
+})()`;
+/** Long enough for a smooth scroll to finish. */
+const SMOOTH_SCROLL_MS = 600;
+
+/**
+ * Whether a click on the control was taken by something open over the page, which a person uses
+ * or closes first: a dialog, tour step, menu or dropdown list, or the backdrop that dims the page
+ * behind one (an open dropdown can also make the page itself take every click outside it). Says
+ * what is open, or "" when the cover is part of the page: a fixed bar, or an invisible layer over
+ * everything, which a person can't see to close and so stays a finding.
+ */
+const OPEN_OVER_SCRIPT = (id: number) => `(() => {
+  const el = document.querySelector('[data-debutly-id="${id}"]');
+  if (!el) return '';
+  const r = el.getBoundingClientRect();
+  const hit = document.elementFromPoint(Math.min(Math.max(r.left + r.width / 2, 0), innerWidth - 1), Math.min(Math.max(r.top + r.height / 2, 0), innerHeight - 1));
+  const open = ${OPEN_LAYER_SCRIPT};
+  if (!hit || hit === document.documentElement || hit === document.body) return open;
+  if (hit.closest('[role="dialog"], [role="alertdialog"], dialog, [aria-modal="true"], [role="listbox"], [role="menu"]')) return open || 'something';
+  const b = hit.getBoundingClientRect();
+  const s = getComputedStyle(hit);
+  const fills = (s.position === 'fixed' || s.position === 'absolute') && b.width >= innerWidth * 0.9 && b.height >= innerHeight * 0.9;
+  const bg = s.backgroundColor;
+  const alpha = bg === 'transparent' ? 0 : bg.startsWith('rgba') ? parseFloat(bg.slice(bg.lastIndexOf(',') + 1)) : 1;
+  const dims = alpha > 0 || (s.backdropFilter && s.backdropFilter !== 'none');
+  return fills && dims ? open || 'something (its backdrop dims the page)' : '';
+})()`;
+
+async function openOver(page: Page, id: number): Promise<string> {
+  return ((await page.evaluate(OPEN_OVER_SCRIPT(id)).catch(() => "")) as string) || "";
+}
+
 // Playwright's timeout message hides the cause in its call log; say it the way a person would notice it.
 export function describeFailure(err: unknown): string {
   const message = err instanceof Error ? err.message : String(err);
@@ -331,6 +390,7 @@ export async function execute(
   opts: ExecuteOptions = {},
 ): Promise<ActionResult> {
   const result: ActionResult = {};
+  await page.evaluate(LISTEN_INVALID).catch(() => {});
   try {
     switch (action.type) {
       case "click":
@@ -351,7 +411,13 @@ export async function execute(
           try {
             await locator.click({ timeout: 5_000 });
           } catch (err) {
-            if (!describeFailure(err).startsWith(COVERED) || !(await clickVisiblePart(page, action.id))) throw err;
+            if (!describeFailure(err).startsWith(COVERED) || !(await clickVisiblePart(page, action.id))) {
+              // Something open over the page took the click: the agent is told what to do, as a
+              // person sees it, and it is no defect of the page behind.
+              const open = describeFailure(err).startsWith(COVERED) ? await openOver(page, action.id) : "";
+              if (open) throw new Unusable(`${open} is open over the page and takes the click: finish it, choose from it or close it (Escape, or its close button) first`);
+              throw err;
+            }
           }
         } else if (el.tag === "select") await choose(locator, action.text);
         else if (customDropdown) await pick(page, locator, action.text);
@@ -391,7 +457,9 @@ export async function execute(
     await page.waitForLoadState("domcontentloaded");
     // Let async handlers (fetch after submit, deferred errors) surface their signals.
     await page.waitForTimeout(300);
-    return result.focusObscured || result.enterIgnored ? result : {};
+    const invalid = ((await page.evaluate(READ_INVALID).catch(() => [])) as string[]) ?? [];
+    if (invalid.length) result.invalid = invalid;
+    return result;
   } catch (err) {
     if (err instanceof Refused) return { blocked: err.message };
     if (err instanceof NotByKeyboard || err instanceof Unusable) return { error: err.message };

package/src/browser/observe.ts

@@ -43,6 +43,36 @@ const LANGUAGE_SCRIPT = `(() => {
   };
 })()`;
 
+/**
+ * What is open over the page and takes clicks outside it: a dialog (or a tour step, a bottom sheet),
+ * or a dropdown list or menu. Agents clicked behind a welcome dialog, an open menu and an open
+ * dropdown on three apps in one week (2026-10-25, 2026-11-01), and each was reported as
+ * "something covers it". "" when nothing is open.
+ */
+export const OPEN_LAYER_SCRIPT = `(() => {
+  const shown = (e) => {
+    const b = e.getBoundingClientRect();
+    const s = getComputedStyle(e);
+    return b.width > 0 && b.height > 0 && s.visibility !== 'hidden' && s.display !== 'none' && Number(s.opacity) !== 0;
+  };
+  const nameOf = (e) => {
+    const label = e.getAttribute('aria-label');
+    if (label) return label;
+    const by = (e.getAttribute('aria-labelledby') || '').split(/\\s+/)[0];
+    const titled = by && document.getElementById(by);
+    if (titled) return titled.textContent;
+    const heading = e.querySelector('h1, h2, h3, h4, [role="heading"]');
+    return heading ? heading.textContent : '';
+  };
+  const dialog = [...document.querySelectorAll('[role="dialog"], [role="alertdialog"], dialog[open], [aria-modal="true"]')].find(shown);
+  const list = [...document.querySelectorAll('[role="listbox"], [role="menu"]')].find((e) => shown(e) && document.querySelector('[aria-expanded="true"]'));
+  const layer = dialog || list;
+  if (!layer) return '';
+  const kind = layer === dialog ? 'a dialog' : layer.getAttribute('role') === 'menu' ? 'a menu' : 'a dropdown list';
+  const name = (nameOf(layer) || '').replace(/\\s+/g, ' ').trim().slice(0, 60);
+  return kind + (name ? ' ("' + name + '")' : '');
+})()`;
+
 export async function observe(page: Page): Promise<Observation> {
   const elements = (await page.evaluate(SCAN_SCRIPT)) as InteractiveElement[];
   const names = await accessibleNames(page);
@@ -58,5 +88,6 @@ export async function observe(page: Page): Promise<Observation> {
     .join("\n")
     .slice(0, 2000);
   const language = (await page.evaluate(LANGUAGE_SCRIPT)) as Pick<Observation, "lang" | "dir" | "arabicShare" | "zoomBlocked">;
-  return { url: page.url(), title: await page.title(), elements, text, ...language };
+  const openLayer = (await page.evaluate(OPEN_LAYER_SCRIPT)) as string;
+  return { url: page.url(), title: await page.title(), elements, text, ...language, ...(openLayer ? { openLayer } : {}) };
 }

package/src/detect/findings.ts

@@ -17,9 +17,10 @@ const BASE_SEVERITY: Record<FindingKind, Severity> = {
   reflow: "minor",
   language: "minor",
   zoom: "major",
+  hidden_focus: "major",
 };
 // What the browser measures, not what an agent made of the page: one measurement is enough.
-export const MEASURED: ReadonlySet<FindingKind> = new Set(["a11y_unlabeled", "covered", "keyboard", "focus_obscured", "reflow", "language", "zoom"]);
+export const MEASURED: ReadonlySet<FindingKind> = new Set(["a11y_unlabeled", "covered", "keyboard", "focus_obscured", "reflow", "language", "zoom", "hidden_focus"]);
 /** Sideways overflow this small is reported, but never raised above minor. */
 const SLIGHT_OVERFLOW_PX = 20;
 /** A page whose letters are at least this share Arabic is an Arabic page. */
@@ -34,6 +35,8 @@ const VERY_SLOW_MS = 30_000;
 // "no" and "API key", and the variable it wants.
 const MISSING_KEY =
   /no (?:[\w-]+ ){0,2}api[ _-]?key|api[ _-]?key (?:is )?(?:not set|missing|not configured|required)|no ai api configured|not configured|configure [a-z0-9_]*key\b|\bset [a-z0-9]+_[a-z0-9_]*key\b|\b[a-z0-9]+_api_key (?:is )?(?:not set|missing|undefined)/i;
+// How an auth server refuses a sign-in provider it has no keys for.
+const PROVIDER_OFF = /provider is not enabled|unsupported provider|provider (?:is )?(?:disabled|not configured)/i;
 // A web address typed in on its own, as an agent gives its company's site ("https://www.pulsehr.io/").
 const TYPED_SITE = /^(?:https?:\/\/)?(?:www\.)?((?:[a-z0-9-]+\.)+[a-z]{2,})(?:[/?#]\S*)?$/i;
 // How an app says it couldn't get at a site: "Could not read that website", "DNS lookup failed".
@@ -50,6 +53,14 @@ const SANDBOX_SLACK_MS = 2000;
 // /app/accounts/<their account>/onboarding, which listed the same unlabeled button once per agent.
 const ID_SEGMENT = /^(?:\d+|[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}|[0-9a-f]{16,}|(?=[a-z0-9]*\d)[a-z0-9]{20,})$/i;
 
+const originOf = (url: string) => {
+  try {
+    return new URL(url).origin;
+  } catch {
+    return "";
+  }
+};
+
 const pathOf = (url: string) => {
   try {
     return new URL(url).pathname
@@ -211,6 +222,39 @@ function placeOf(elements: InteractiveElement[], i: number): string {
   return after ? `before ${q(after)}` : "";
 }
 
+/**
+ * Controls hidden from screen readers (aria-hidden) that Tab still reaches: a closed mobile menu
+ * left in the page, its ~15 links aria-hidden but focusable (2026-10-25). Keyboard focus goes into
+ * them unseen, and a screen reader says nothing or something misleading. Not while something is
+ * open over the page: a modal hides the page behind it on purpose, and keeps focus inside itself.
+ */
+function hiddenButReachable(obs: Observation, pagePath: string): Candidate | undefined {
+  if (obs.openLayer) return undefined;
+  const reached = obs.elements.filter((e) => e.ax?.ariaHidden && !e.disabled && e.tabbable !== false && e.role !== "input:file");
+  if (!reached.length) return undefined;
+  const names = [...new Set(reached.map((e) => e.name.trim()).filter(Boolean))].slice(0, 3).map((n) => `"${n.slice(0, 40)}"`);
+  const count = reached.length === 1 ? "A control" : `${reached.length} controls`;
+  return {
+    signature: `hidden-focus:${pagePath}`,
+    kind: "hidden_focus",
+    title: `${count} on ${pagePath} ${reached.length === 1 ? "is" : "are"} hidden from screen readers but reachable with Tab`,
+    detail: `${count}${names.length ? ` (${names.join(", ")}${reached.length > names.length ? ", …" : ""})` : ""} ${reached.length === 1 ? "is" : "are"} marked aria-hidden, often a closed menu or panel left in the page, yet Tab still lands on ${reached.length === 1 ? "it" : "them"}: keyboard focus disappears from view, and a screen reader says nothing or something misleading.`,
+    pagePath,
+    severity: severityFor("hidden_focus", [pagePath]),
+    wcag: ["4.1.2", "2.4.3"],
+  };
+}
+
+/** Whether the agent pressed the control without trouble later on the same page. */
+function pressedLater(steps: Step[], after: number, pagePath: string, control: string): boolean {
+  return steps.some((st) => {
+    if (st.index <= after || st.error || st.blocked || st.action.type !== "click" || pathOf(st.observation.url) !== pagePath) return false;
+    const id = st.action.id;
+    const el = st.observation.elements.find((e) => e.id === id);
+    return !!el && `${el.role} "${el.name}"` === control;
+  });
+}
+
 /** One finding per page, kind of control and kind of problem, naming up to three places. */
 function unnamedControls(elements: InteractiveElement[], pagePath: string): Candidate[] {
   const groups = new Map<string, { control: string; role: string; problem: Unnamed; places: string[]; said: string; count: number }>();
@@ -282,12 +326,12 @@ function unnamedControls(elements: InteractiveElement[], pagePath: string): Cand
 
 const capitalize = (s: string) => s.charAt(0).toUpperCase() + s.slice(1);
 
-/** What a server error said, for quoting: its JSON `error` or `message` if it has one. */
+/** What a server error said, for quoting: its JSON `error`, `message` or `msg` (Supabase's auth) if it has one. */
 function said(body: string): string {
   let text = body;
   try {
-    const parsed = JSON.parse(body) as { error?: unknown; message?: unknown };
-    const field = parsed.error ?? parsed.message;
+    const parsed = JSON.parse(body) as { error?: unknown; message?: unknown; msg?: unknown };
+    const field = typeof parsed.error === "string" ? parsed.error : (parsed.message ?? parsed.msg);
     if (typeof field === "string") text = field;
   } catch {
     // not JSON: quote it as it is
@@ -298,8 +342,12 @@ function said(body: string): string {
 
 /** In a test environment, the app's own words when a server error says a key is missing. */
 function missingKey(signal: Signal, sandbox: boolean): string | undefined {
-  if (!sandbox || signal.kind !== "http_error" || signal.status < 500 || !signal.body || !MISSING_KEY.test(signal.body)) return undefined;
-  return said(signal.body);
+  if (!sandbox || signal.kind !== "http_error" || !signal.body) return undefined;
+  if (signal.status >= 500 && MISSING_KEY.test(signal.body)) return said(signal.body);
+  // A sign-in provider the test environment's auth server has no keys for (Google, Microsoft…)
+  // refuses with a 400: "Unsupported provider: provider is not enabled" (2026-10-25).
+  if (signal.status >= 400 && PROVIDER_OFF.test(signal.body)) return said(signal.body);
+  return undefined;
 }
 
 /** The site an agent typed in for its company, which is made up like the company. */
@@ -455,6 +503,8 @@ export function detectFindings(sessions: Session[], opts: DetectOptions = {}): F
   };
 
   for (const s of sessions) {
+    // The app's own pages are where it started; a sign-in server's raw error page is not one of them.
+    const appOrigin = originOf(s.steps[0]?.observation.url ?? "");
     let repeatKey = "";
     let repeats = 0;
     // Per control: how often a click on it was refused because another element lay on top.
@@ -506,8 +556,10 @@ export function detectFindings(sessions: Session[], opts: DetectOptions = {}): F
       }
 
       for (const c of unnamedControls(step.observation.elements, pagePath)) record(c, s.agentId, step.index);
+      const hiddenFocus = hiddenButReachable(step.observation, pagePath);
+      if (hiddenFocus) record(hiddenFocus, s.agentId, step.index);
 
-      for (const c of languageFindings(step.observation, pagePath)) record(c, s.agentId, step.index);
+      if (originOf(step.observation.url) === appOrigin) for (const c of languageFindings(step.observation, pagePath)) record(c, s.agentId, step.index);
 
       // Keyboard only (the screen-reader agent): a control Tab never reaches, and focus that
       // something entirely hides.
@@ -565,12 +617,14 @@ export function detectFindings(sessions: Session[], opts: DetectOptions = {}): F
 
       // A control something else keeps covering: the person can't press it either. The agent
       // works round it in other ways (keys, closing things), so this is not always "stuck".
+      // Unless the agent presses it later on the same page: then what lay over it went away once
+      // the agent dealt with it (a dropdown list chosen from, 2026-11-04), and it is no defect.
       const target = step.action.type === "click" && step.error?.startsWith(COVERED) ? step.observation.elements.find((e) => e.id === (step.action as { id: number }).id) : undefined;
       if (target) {
         const control = `${target.role} "${target.name}"`;
         const times = (covered.get(control) ?? 0) + 1;
         covered.set(control, times);
-        if (times === COVERED_REPEATS) {
+        if (times === COVERED_REPEATS && !pressedLater(s.steps, step.index, pagePath, control)) {
           const by = step.error!.slice(COVERED.length).replace(/^:\s*/, "");
           record(
             {

package/src/report/fix-first.ts

@@ -51,6 +51,8 @@ export function nextStep(f: Finding): string {
       if (f.signature.startsWith("dir:")) return 'Set dir="rtl" on Arabic pages (on <html>, or on the part that holds the Arabic).';
       if (f.signature === "lang:missing") return 'Add a lang attribute to the <html> element, e.g. lang="en".';
       return 'Set lang="ar" on Arabic pages.';
+    case "hidden_focus":
+      return 'Make the hidden part unreachable while hidden: add inert to it (or tabindex="-1" to each control), or stop hiding it from screen readers.';
     case "zoom":
       return "Remove user-scalable=no and maximum-scale from the page's viewport meta tag.";
   }
@@ -69,6 +71,8 @@ const quote = (s: string) => `"${s.length > MAX_QUOTE ? `${s.slice(0, MAX_QUOTE)
 /** A screenshot as the report's reader finds it: next to the report, not on the machine that ran it. */
 export function shotPath(path: string): string {
   if (/^https?:/.test(path)) return path;
+  // Kept in the run's database too (db:<run>/<file>), and under the same name beside the report.
+  if (path.startsWith("db:")) return `screenshots/${path.split("/").pop()}`;
   const at = path.replace(/\\/g, "/").lastIndexOf("/screenshots/");
   return at >= 0 ? path.replace(/\\/g, "/").slice(at + 1) : path;
 }

package/src/types.ts

@@ -67,6 +67,8 @@ export interface AccessibleName {
   hidden: boolean;
   /** What a screen reader reads after the role (aria-describedby, a tooltip); absent when there's none. */
   description?: string;
+  /** Hidden by aria-hidden on it or around it, rather than for another reason. */
+  ariaHidden?: true;
 }
 
 export interface InteractiveElement {
@@ -109,6 +111,8 @@ export interface Observation {
   arabicShare?: number;
   /** The page's viewport setting when it stops pinch-zoom (user-scalable=no, maximum-scale under 2), else "". */
   zoomBlocked?: string;
+  /** What is open over the page and takes clicks outside it, e.g. 'a dialog ("Welcome")' or "a dropdown list". */
+  openLayer?: string;
 }
 
 export interface ActionResult {
@@ -118,6 +122,8 @@ export interface ActionResult {
   focusObscured?: string;
   /** Keyboard only: Enter did nothing on the button, and Space pressed it. */
   enterIgnored?: true;
+  /** What the browser's own bubble said about fields a submit found invalid: '"Password": Please lengthen…'. */
+  invalid?: string[];
 }
 
 export interface Step extends ActionResult {
@@ -174,7 +180,8 @@ export type FindingKind =
   | "focus_obscured"
   | "reflow"
   | "language"
-  | "zoom";
+  | "zoom"
+  | "hidden_focus";
 
 export interface Finding {
   signature: string;