debutly 0.1.0 → 0.3.0

package/README.md

@@ -24,7 +24,7 @@ Website and docs: [debutly.dev](https://debutly.dev) ([quick start](https://debu
 
 ## Quick start
 
-Needs Node.js 20.12 or later. Until the first npm release, type `npx github:sroay/debutly` wherever this says `npx debutly`. In your app's folder:
+Needs Node.js 20.12 or later. In your app's folder:
 
 ```bash
 npx debutly init     # a few questions, then a debutly.json

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "debutly",
-  "version": "0.1.0",
+  "version": "0.3.0",
   "description": "Rehearse your launch: AI agents use your web app like your first customers and report what breaks.",
   "type": "module",
   "license": "AGPL-3.0-only",

package/src/agent/effect.ts

@@ -1,4 +1,5 @@
-import type { Action, ActionResult, Observation, Signal } from "../types";
+import type { Action, ActionResult, InteractiveElement, Observation, Signal } from "../types";
+import { controlLabel, type StepView } from "./label";
 
 const pathOf = (url: string) => {
   try {
@@ -9,26 +10,88 @@ const pathOf = (url: string) => {
   }
 };
 
+const MAX_LISTED = 4;
+const MAX_LINE = 80;
+
+type Label = (e: InteractiveElement) => string;
+
+/** Controls in `a` beyond those in `b`, counting copies: a second "Untitled" row is new. */
+function beyond(a: InteractiveElement[], b: InteractiveElement[], controlOf: Label): InteractiveElement[] {
+  const left = new Map<string, number>();
+  for (const e of b) left.set(controlOf(e), (left.get(controlOf(e)) ?? 0) + 1);
+  return a.filter((e) => {
+    const n = left.get(controlOf(e)) ?? 0;
+    if (n > 0) left.set(controlOf(e), n - 1);
+    return n === 0;
+  });
+}
+
+/** Controls still there whose state changed: a Save button now enabled, a box now checked. */
+function stateChanges(before: InteractiveElement[], after: InteractiveElement[], controlOf: Label): string[] {
+  const out: string[] = [];
+  for (const a of after) {
+    const b = before.find((e) => controlOf(e) === controlOf(a));
+    if (!b) continue;
+    if (!!b.disabled !== !!a.disabled) out.push(`now ${a.disabled ? "disabled" : "enabled"} ${controlOf(a)}`);
+    if (b.checked !== undefined && a.checked !== undefined && b.checked !== a.checked) out.push(`now ${a.checked ? "checked" : "unchecked"} ${controlOf(a)}`);
+    if (b.expanded !== undefined && a.expanded !== undefined && b.expanded !== a.expanded) out.push(`now ${a.expanded ? "open" : "closed"} ${controlOf(a)}`);
+  }
+  return out;
+}
+
+const listed = (items: string[]) => items.slice(0, MAX_LISTED).join(", ") + (items.length > MAX_LISTED ? `, and ${items.length - MAX_LISTED} more` : "");
+
+/**
+ * What changed on the page, as a person would notice it: controls that appeared or went, state
+ * that changed, and new lines of text. Told only "the page changed", agents missed a password
+ * field that had appeared, pressed "Create company" seven times without seeing seven new rows,
+ * and kept retyping values while waiting for a Save button to wake up.
+ */
+function whatChanged(before: Observation, after: Observation, controlOf: Label): string {
+  const appeared = beyond(after.elements, before.elements, controlOf);
+  const gone = beyond(before.elements, after.elements, controlOf);
+  const names = new Set(after.elements.map((e) => e.name));
+  const had = new Set(before.text.split("\n").map((l) => l.trim()));
+  const newText = [...new Set(after.text.split("\n").map((l) => l.trim()))].filter((l) => l && !had.has(l) && !names.has(l)).map((l) => `"${l.slice(0, MAX_LINE)}"`);
+  const parts = [
+    appeared.length ? `new ${listed(appeared.map(controlOf))}` : "",
+    gone.length ? `gone ${listed(gone.map(controlOf))}` : "",
+    ...stateChanges(before.elements, after.elements, controlOf).slice(0, MAX_LISTED),
+    newText.length ? `new text ${listed(newText)}` : "",
+  ].filter(Boolean);
+  return parts.length ? `the page changed: ${parts.join("; ")}` : "";
+}
+
 // What a person would notice after acting: where they are now, whether anything changed,
 // and whether something visibly broke. Without this, agents retry dead buttons forever.
-export function describeEffect(action: Action, before: Observation, after: Observation, signals: Signal[], result: ActionResult): string {
+// Controls are named as the agent meets them, as in its step message.
+export function describeEffect(action: Action, before: Observation, after: Observation, signals: Signal[], result: ActionResult, view: StepView = {}): string {
   if (result.blocked) return `blocked: ${result.blocked}`;
   if (result.error) return `failed: ${result.error}`;
   // check_email reports its own result; done ends the session.
   if (action.type === "done" || action.type === "check_email") return "";
 
   const parts: string[] = [];
+  if (result.enterIgnored) parts.push("Enter did nothing on it, so Space was pressed instead");
+  if (result.invalid?.length) parts.push(`the browser stopped the form and said: ${result.invalid.join("; ")}`);
+  const changed = whatChanged(before, after, (e) => controlLabel(e, view.screenReader));
   if (pathOf(after.url) !== pathOf(before.url)) parts.push(`now on ${pathOf(after.url)}`);
+  else if (changed) parts.push(changed);
   else if (after.text !== before.text) parts.push("the page changed");
-  else if (action.type !== "fill" && !signals.some((s) => s.kind === "download")) parts.push("nothing visibly changed");
+  else if (action.type !== "fill" && !result.invalid?.length && !signals.some((s) => s.kind === "download")) parts.push("nothing visibly changed");
 
   for (const s of signals) if (s.kind === "download") parts.push(`a file downloaded: ${s.filename}`);
 
   for (const s of signals) if (s.kind === "dialog") parts.push(`a ${s.dialogType} dialog said "${s.message}" (answered OK)`);
   if (signals.some((s) => s.kind === "page_error")) parts.push("a JavaScript error occurred");
+  // Only the server failing: a person never sees a status code, and apps answer 404 or 401 as part
+  // of working (a setting never saved yet). Told "a request failed (HTTP 404)" after each click, an
+  // agent decided onboarding was broken (2026-10-25). The report still lists those answers.
   const failed = [
     ...new Set(
-      signals.filter((s) => s.kind === "http_error" && pathOf(s.requestUrl) !== "/favicon.ico").map((s) => (s.kind === "http_error" ? s.status : 0)),
+      signals
+        .filter((s) => s.kind === "http_error" && s.status >= 500 && pathOf(s.requestUrl) !== "/favicon.ico")
+        .map((s) => (s.kind === "http_error" ? s.status : 0)),
     ),
   ];
   for (const status of failed) parts.push(`a request failed (HTTP ${status})`);

package/src/agent/give-up.ts

@@ -0,0 +1,42 @@
+import type { Observation, Step } from "../types";
+
+// What a person presses before giving up: to go on, join, sign in or start, or to add or create
+// what they came to add (agents said adding a person was "missing" beside "Add person").
+// Arabic too, as agents often use Arabic interfaces.
+const WAY_ON =
+  /^(continue|next|proceed|join\b.*|accept\b.*|sign ?in|log ?in|sign ?up|create (an )?account|get started|start|confirm|verify|submit|(add|create|new)\b.*|استمر|متابعة|التالي|انضم.*|قبول.*|تسجيل الدخول|ابدأ.*|تأكيد|إرسال|(إنشاء|أضف|إضافة).*)$/i;
+// Buttons only: a page's own way on. A "Sign up" link in the site's menu is not one from where it stands.
+const PRESSABLE = /^(button|input:(submit|button))$/;
+
+const pathOf = (url: string) => {
+  try {
+    return new URL(url).pathname;
+  } catch {
+    return url;
+  }
+};
+
+/**
+ * The ways on that an agent about to give up hasn't tried on the page it is on: enabled buttons
+ * named like Continue, Join or Sign in that it hasn't pressed since it came to this page.
+ * A teammate agent gave up on Twenty's invite page saying there was "no way into the workspace",
+ * with a Continue button under its eyes (2026-10-25), and the report blamed the app.
+ */
+export function untriedWaysOn(obs: Observation, steps: Step[], screenReader = false): string[] {
+  const path = pathOf(obs.url);
+  const pressed = new Set<string>();
+  for (let i = steps.length - 1; i >= 0 && pathOf(steps[i].observation.url) === path; i--) {
+    const { action, observation } = steps[i];
+    if (action.type !== "click") continue;
+    const el = observation.elements.find((e) => e.id === action.id);
+    if (el) pressed.add((el.ax?.name || el.name).trim().toLowerCase());
+  }
+  const names = obs.elements
+    .filter((e) => PRESSABLE.test(e.role) && !e.disabled && !(screenReader && e.ax?.hidden))
+    .map((e) => (screenReader && e.ax ? e.ax.name : e.name || e.ax?.name || "").trim())
+    .filter((name) => WAY_ON.test(name) && !pressed.has(name.toLowerCase()));
+  return [...new Set(names)].slice(0, 3);
+}
+
+/** A page with nothing to press is often still arriving: one agent on a slow connection gave up the moment an invite page opened. */
+export const nothingToPressYet = (obs: Observation) => obs.elements.length === 0;

package/src/agent/intent.ts

@@ -0,0 +1,51 @@
+import type { Action, InteractiveElement, Observation } from "../types";
+
+const norm = (s: string) => s.toLowerCase().replace(/\s+/g, " ").trim();
+
+/**
+ * How a control is named in an agent's words: whole when short, by its first words when its name
+ * runs on into a description ("Email alerts Get notified of hiring changes" is "email alerts").
+ */
+function mentionOf(name: string): string {
+  const n = norm(name);
+  if (n.length <= 30) return n;
+  const words = n.split(" ");
+  const two = words.slice(0, 2).join(" ");
+  return two.length >= 8 ? two : words.slice(0, 3).join(" ");
+}
+
+const escape = (s: string) => s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+
+/**
+ * Whether the agent's words name the control. A one-word name counts only as written on the
+ * control (its capitals), so "go back" in passing isn't the "Back" button.
+ */
+function said(reasoning: string, name: string): boolean {
+  const m = mentionOf(name);
+  if (m.length < 3) return false;
+  if (m.includes(" ")) return norm(reasoning).includes(m);
+  return new RegExp(`(^|[^\\p{L}\\p{N}])${escape(name.trim())}($|[^\\p{L}\\p{N}])`, "u").test(reasoning);
+}
+
+const nameOf = (e: InteractiveElement, screenReader: boolean) => (screenReader && e.ax ? e.ax.name : e.name || e.ax?.name || "");
+
+/**
+ * When an agent's reasoning names controls on the page but its action points at another one: the
+ * controls it named. An agent in a product's onboarding wrote "I will click the button for email
+ * alerts" and then "select 'Competitor'", and both times sent the id of the logo, which restarted
+ * onboarding; it gave up blaming the app (2026-10-25). Undefined when the action matches what it
+ * said, or when there is nothing to compare (no id, or a control without a name).
+ */
+export function saidOtherControl(reasoning: string, action: Action, obs: Observation, screenReader = false): string[] | undefined {
+  if (action.type !== "click" && action.type !== "fill") return undefined;
+  const chosen = obs.elements.find((e) => e.id === action.id);
+  if (!chosen) return undefined;
+  const chosenName = nameOf(chosen, screenReader);
+  if (!chosenName.trim() || said(reasoning, chosenName)) return undefined;
+  // A field filled with text it names ("type Agency into Industry") is fine either way.
+  if (action.type === "fill" && norm(reasoning).includes(norm(action.text))) return undefined;
+  const meant = obs.elements
+    .filter((e) => e.id !== chosen.id && nameOf(e, screenReader).trim().length >= 4 && said(reasoning, nameOf(e, screenReader)))
+    .map((e) => nameOf(e, screenReader).trim());
+  return meant.length ? [...new Set(meant)].slice(0, 3) : undefined;
+}

package/src/agent/label.ts

@@ -0,0 +1,19 @@
+import type { InteractiveElement } from "../types";
+
+// Apart from prompt.ts, which needs the browser (Playwright) through the inclusion briefs: the
+// dashboard imports effect.ts, and its build has no Playwright.
+
+export interface StepView {
+  /** As a screen reader presents the page: the browser's names and roles, without what is hidden from it. */
+  screenReader?: boolean;
+}
+
+/**
+ * A control as the agent meets it. A screen reader announces the role and name the browser gives
+ * it; a sighted person looking at a control with no text sees its icon.
+ */
+export function controlLabel(e: InteractiveElement, screenReader = false): string {
+  const heard = screenReader && e.ax ? e.ax : e;
+  const icon = !screenReader && !e.name && e.icon ? ` (${e.icon} icon)` : "";
+  return `${heard.role} "${heard.name}"${icon}`;
+}

package/src/agent/loop.ts

@@ -1,4 +1,4 @@
-import type { Page } from "playwright";
+import type { Page, Request } from "playwright";
 import type { ArtifactStore } from "../artifacts/store";
 import { execute } from "../browser/execute";
 import { observe } from "../browser/observe";
@@ -11,15 +11,22 @@ import type { LlmClient, LlmMessage } from "../llm/types";
 import type { ActionResult, Persona, Session, Step } from "../types";
 import { describeAction } from "./describe";
 import { describeEffect } from "./effect";
-import { parseAgentReply } from "./parse-reply";
+import { nothingToPressYet, untriedWaysOn } from "./give-up";
+import { saidOtherControl } from "./intent";
+import { controlLabel } from "./label";
+import { NoActionError, parseAgentReply } from "./parse-reply";
 import { buildStepMessage, buildSystemPrompt, type AgentIdentity } from "./prompt";
 
 const MAX_REPLY_ATTEMPTS = 3;
+/** Steps a session may wait for want of an action in the reply, before that is an engine error. */
+const MAX_NO_ACTION_WAITS = 3;
 const LINKS_PER_EMAIL = 3;
 /** How long the first page may take to arrive; a slow-connection agent needs more than 30 s. */
 const FIRST_LOAD_MS = 90_000;
 /** A first page slower than this is reported. */
 const SLOW_LOAD_MS = 10_000;
+/** How long a slow first page's scripts and styles may take to be weighed before it is reported without. */
+const WEIGH_MS = 2_000;
 /** How long a step's screenshot may take before the step goes without one. */
 const SCREENSHOT_MS = 15_000;
 
@@ -50,12 +57,35 @@ export interface RunSessionOptions {
   firstLoad?: { timeoutMs?: number; slowMs?: number };
 }
 
-function describeInbox(messages: EmailMessage[]): string {
+/** The inbox as the agent reads it, each link numbered ("email link 2") so it can be opened without retyping it. */
+export function describeInbox(messages: EmailMessage[], numberOf: (link: string) => number): string {
   if (!messages.length) return "checked email → no emails yet";
-  const items = messages.map((m) => `"${m.subject}"${m.links.length ? ` links: ${m.links.slice(0, LINKS_PER_EMAIL).join(" ")}` : ""}`);
+  const items = messages.map((m) => {
+    const links = m.links.slice(0, LINKS_PER_EMAIL).map((l) => `[email link ${numberOf(l)}] ${l}`);
+    return `"${m.subject}"${links.length ? ` links: ${links.join(" ")}` : ""}`;
+  });
   return `checked email → ${messages.length} email${messages.length === 1 ? "" : "s"}: ${items.join("; ")}`;
 }
 
+/**
+ * The emailed link a goto means: "email link 2" by its number, or a mistyped copy of one, the same
+ * path and query under a garbled host. An agent retyped a sign-in link as http://127.0.0.0.0.1:…,
+ * got "Invalid URL" and gave up (2026-10-25). Undefined when it means no emailed link.
+ */
+export function emailedLinkMeant(path: string, links: string[]): string | undefined {
+  const numbered = /^email ?link ?(\d+)$/i.exec(path.trim());
+  if (numbered) return links[Number(numbered[1]) - 1];
+  const tail = (u: string) => {
+    const start = u.indexOf("//");
+    const slash = u.indexOf("/", start < 0 ? 0 : start + 2);
+    return slash < 0 ? "" : u.slice(slash);
+  };
+  const typed = tail(path.trim());
+  // Long enough to be a link's own path and token, not a common page like "/login".
+  if (typed.length < 20) return undefined;
+  return links.find((l) => tail(l) === typed && l !== path.trim());
+}
+
 /**
  * Open the app the way a person does: wait for the page, but never fail the session because it
  * is slow. Waiting for every image and font ("load") within Playwright's 30 s ended the
@@ -73,14 +103,37 @@ async function openFirstPage(
 ): Promise<void> {
   const startedAt = Date.now();
   let finished = true;
+  // What its scripts and styles weigh as sent, and how much came uncompressed: a test copy served
+  // straight from its container may skip the compression its real hosting does (Twenty's sent
+  // 10 MB uncompressed, 2026-10-24), and then a slow page says more about the copy than the app.
+  const assets = { files: 0, bytes: 0, uncompressedBytes: 0 };
+  const weighing = new Set<Promise<void>>();
+  const weigh = (request: Request) => {
+    if (request.resourceType() !== "script" && request.resourceType() !== "stylesheet") return;
+    const done = (async () => {
+      const [{ responseBodySize }, encoding] = await Promise.all([request.sizes(), request.response().then((r) => r?.headerValue("content-encoding"))]);
+      assets.files++;
+      assets.bytes += responseBodySize;
+      if (!encoding || encoding === "identity") assets.uncompressedBytes += responseBodySize;
+    })()
+      .catch(() => {}) // the page moved on before it could be weighed
+      .finally(() => weighing.delete(done));
+    weighing.add(done);
+  };
+  page.on("requestfinished", weigh);
   try {
     await page.goto(origin, { waitUntil: "domcontentloaded", timeout: timeoutMs });
   } catch (err) {
     if (!(err instanceof Error && err.name === "TimeoutError")) throw err;
     finished = false;
+  } finally {
+    page.off("requestfinished", weigh);
   }
   const ms = Date.now() - startedAt;
-  if (!finished || ms >= slowMs) signals.note({ kind: "slow_load", url: origin, ms, finished, slowNetwork });
+  if (!finished || ms >= slowMs) {
+    await Promise.race([Promise.allSettled([...weighing]), new Promise((resolve) => setTimeout(resolve, WEIGH_MS))]);
+    signals.note({ kind: "slow_load", url: origin, ms, finished, slowNetwork, ...(assets.files ? { assets: { ...assets } } : {}) });
+  }
 }
 
 export async function runSession(o: RunSessionOptions): Promise<Session> {
@@ -129,6 +182,8 @@ interface SessionState {
 async function takeSteps(o: RunSessionOptions, { signals, system, steps, history, emailedLinks, meter, startedAt, end, reflow }: SessionState): Promise<Session> {
   // The screen-reader agent hears what the browser tells assistive technology and works by keyboard.
   const screenReader = o.persona.inclusion === "screen_reader";
+  let noActionWaits = 0;
+  const askedBeforeGivingUp = new Set<string>();
   for (let index = 0; index < o.maxSteps; index++) {
     if (o.shouldStop && (await o.shouldStop())) return end("cancelled", "run cancelled");
     const { inputTokens, outputTokens } = meter.total;
@@ -144,13 +199,14 @@ async function takeSteps(o: RunSessionOptions, { signals, system, steps, history
     // Now that we can see the page after the previous action, tell the agent what it did.
     const prev = steps.at(-1);
     if (prev) {
-      const effect = describeEffect(prev.action, prev.observation, observation, prev.signals, prev);
+      const effect = describeEffect(prev.action, prev.observation, observation, prev.signals, prev, { screenReader });
       if (effect) history[history.length - 1] += ` → ${effect}`;
     }
     const messages: LlmMessage[] = [{ role: "user", content: buildStepMessage(observation, history, { screenReader }) }];
     let reply: ReturnType<typeof parseAgentReply> | undefined;
     let raw = "";
     let problem = "";
+    let noAction: NoActionError | undefined;
     // Real models occasionally send prose, an empty reply or malformed JSON; say what was wrong and ask again.
     for (let attempt = 0; attempt < MAX_REPLY_ATTEMPTS && !reply; attempt++) {
       raw = await o.llm.complete({ system, messages: [...messages], maxTokens: 4096, onUsage: meter.add });
@@ -158,6 +214,7 @@ async function takeSteps(o: RunSessionOptions, { signals, system, steps, history
         reply = parseAgentReply(raw);
       } catch (err) {
         problem = err instanceof Error ? err.message : String(err);
+        noAction = err instanceof NoActionError ? err : undefined;
         messages.push(
           { role: "assistant", content: raw || "(empty reply)" },
           {
@@ -167,9 +224,53 @@ async function takeSteps(o: RunSessionOptions, { signals, system, steps, history
         );
       }
     }
+    // Asked three times, GLM 5.3 Flash went on sending only its reasoning, often "I will wait", and
+    // the agent ended on an engine error (2026-10-24). It waits instead, as someone unsure what to
+    // do next would; not for ever.
+    if (!reply && noAction && noActionWaits < MAX_NO_ACTION_WAITS) {
+      noActionWaits++;
+      reply = { reasoning: noAction.reasoning, action: { type: "wait" } };
+    }
     if (!reply) {
       return end("error", `${problem} | last reply: ${raw.slice(0, 300) || "(empty)"}`);
     }
+    // One more question about the reply just given; an unreadable answer leaves the reply standing.
+    const askAgain = async (question: string) => {
+      const again = await o.llm.complete({
+        system,
+        messages: [...messages, { role: "assistant", content: raw }, { role: "user", content: question }],
+        maxTokens: 4096,
+        onUsage: meter.add,
+      });
+      try {
+        const answer = parseAgentReply(again);
+        raw = again;
+        return answer;
+      } catch {
+        return undefined;
+      }
+    };
+    // Giving up with a way on in plain sight that it never pressed here, or on a page with nothing
+    // to press yet: asked once per page. Still giving up, it gives up.
+    if (reply.action.type === "done" && !reply.action.success && !askedBeforeGivingUp.has(observation.url)) {
+      const wayOn = untriedWaysOn(observation, steps, screenReader);
+      const ask = wayOn.length
+        ? `Before you give up: on this page you haven't tried ${wayOn.map((n) => `"${n}"`).join(", ")}. Someone who wanted to get on would try that first.`
+        : nothingToPressYet(observation)
+          ? "Before you give up: nothing on this page can be pressed yet, so it may still be loading. Someone who wanted to get on would wait a little first."
+          : "";
+      if (ask) {
+        askedBeforeGivingUp.add(observation.url);
+        reply = (await askAgain(`${ask} Reply again with one JSON object: go on, or give up if you are sure.`)) ?? reply;
+      }
+    }
+    // Its words name one control and its action points at another: asked once to check the id.
+    const meant = saidOtherControl(reply.reasoning, reply.action, observation, screenReader);
+    const chosen = "id" in reply.action ? observation.elements.find((e) => e.id === (reply!.action as { id: number }).id) : undefined;
+    if (meant && chosen) {
+      const question = `You wrote about ${meant.map((n) => `"${n}"`).join(" or ")}, but your action points at [${chosen.id}] ${controlLabel(chosen, screenReader)}. Check the id: reply again with one JSON object.`;
+      reply = (await askAgain(question)) ?? reply;
+    }
 
     let result: ActionResult;
     let historyLine = describeAction(reply.action, observation);
@@ -180,14 +281,22 @@ async function takeSteps(o: RunSessionOptions, { signals, system, steps, history
         try {
           const mail = await o.inbox.latest(o.identity.email);
           for (const m of mail) for (const link of m.links) emailedLinks.add(link);
-          historyLine = describeInbox(mail);
+          const numbered = [...emailedLinks];
+          historyLine = describeInbox(mail, (link) => numbered.indexOf(link) + 1);
           result = {};
         } catch (err) {
           result = { error: err instanceof Error ? err.message : String(err) };
         }
       }
     } else {
-      result = await execute(o.page, reply.action, observation, o.origin, (href) => emailedLinks.has(href), { keyboard: screenReader });
+      // An emailed link, by its number or as a mistyped copy, is opened as it was sent.
+      const link = reply.action.type === "goto" ? emailedLinkMeant(reply.action.path, [...emailedLinks]) : undefined;
+      if (reply.action.type === "goto" && /^email ?link ?\d+$/i.test(reply.action.path.trim()) && !link) {
+        result = { error: `there is no ${reply.action.path.trim()}: check your email to see the links` };
+      } else {
+        const action = link ? { type: "goto" as const, path: link } : reply.action;
+        result = await execute(o.page, action, observation, o.origin, (href) => emailedLinks.has(href), { keyboard: screenReader });
+      }
     }
 
     let screenshotPath: string | undefined;

package/src/agent/parse-reply.ts

@@ -19,7 +19,56 @@ export function parseAgentReply(raw: string): { reasoning: string; action: Actio
   const start = raw.indexOf("{");
   const end = raw.lastIndexOf("}");
   if (start === -1 || end <= start) throw new Error("no JSON object in agent reply");
-  return ReplySchema.parse(actionAsObject(parseQuotedAction(raw.slice(start, end + 1))));
+  const reply = actionAsObject(parseQuotedAction(raw.slice(start, end + 1)));
+  const parsed = ReplySchema.safeParse(reply);
+  if (!parsed.success) throw new Error(problemWith(reply, parsed.error));
+  return parsed.data;
+}
+
+/** A reply with reasoning and no action. It carries the reasoning, so the agent can be left to wait. */
+export class NoActionError extends Error {
+  constructor(readonly reasoning: string) {
+    super(`the reply has no "action". Add one, for example {"reasoning":"…","action":{"type":"wait"}}`);
+  }
+}
+
+// Said back to the agent when it is asked again, so in words it can act on. Shown the validator's
+// own JSON for a reply with reasoning and no action, one model sent the same reply three times.
+function problemWith(reply: unknown, error: z.ZodError): string {
+  if (reply && typeof reply === "object" && !("action" in reply)) {
+    const reasoning = (reply as { reasoning?: unknown }).reasoning;
+    throw new NoActionError(typeof reasoning === "string" ? reasoning : "");
+  }
+  return error.issues.map((i) => `${i.path.join(".") || "reply"}: ${i.message}`).join("; ");
+}
+
+/**
+ * The reply split into objects one after another, merged: {"reasoning":"…"} and on the next line
+ * {"action":"press","key":"Enter"} (GLM 5.3 Flash, 2026-10-24, twice in one run). When two whole
+ * replies come, the first one counts. Undefined when there is only one object or one doesn't parse.
+ */
+function splitReply(text: string): Record<string, unknown> | undefined {
+  const objects: string[] = [];
+  let depth = 0;
+  let from = 0;
+  let inString = false;
+  for (let i = 0; i < text.length; i++) {
+    const c = text[i];
+    if (inString) {
+      if (c === "\\") i++;
+      else if (c === '"') inString = false;
+    } else if (c === '"') inString = true;
+    else if (c === "{" && depth++ === 0) from = i;
+    else if (c === "}" && depth > 0 && --depth === 0) objects.push(text.slice(from, i + 1));
+  }
+  if (objects.length < 2) return undefined;
+  try {
+    const parsed = objects.map((o) => JSON.parse(o) as unknown);
+    if (!parsed.every((p) => p && typeof p === "object" && !Array.isArray(p))) return undefined;
+    return Object.assign({}, ...(parsed as Record<string, unknown>[]).reverse());
+  } catch {
+    return undefined;
+  }
 }
 
 // Some models (GLM 5.3 Flash) send the action as a string. Unescaped,
@@ -31,6 +80,8 @@ function parseQuotedAction(text: string): unknown {
   try {
     return JSON.parse(text);
   } catch (err) {
+    const split = splitReply(text);
+    if (split) return split;
     const unquoted = text.replace(/("action"\s*:\s*)"(\{[^{}]*\})"/, "$1$2");
     if (unquoted !== text) {
       try {

package/src/agent/prompt.ts

@@ -1,6 +1,9 @@
 import { inclusionBrief } from "../persona/inclusion";
 import { languageBrief } from "../persona/language";
 import type { InteractiveElement, Observation, Persona } from "../types";
+import { controlLabel, type StepView } from "./label";
+
+export type { StepView } from "./label";
 
 // Stays constant for the whole session so providers can cache it; everything that changes goes in the user message.
 export interface AgentIdentity {
@@ -11,7 +14,7 @@ export interface AgentIdentity {
 export function buildSystemPrompt(persona: Persona, goal: string, identity?: AgentIdentity, brief?: string): string {
   const email = identity
     ? `Your email address is ${identity.email}. Use it whenever the site asks for your email.
-To read your inbox (for example a login link or a confirmation), reply {"type":"check_email"}. To open a link from an email, use {"type":"goto","path":"<the full link>"}.`
+To read your inbox (for example a login link or a confirmation), reply {"type":"check_email"}. To open a link from an email, use its number: {"type":"goto","path":"email link 1"}.`
     : `Use emails at example.com.`;
   const about = [
     `You are ${persona.name}. ${persona.background}`,
@@ -38,31 +41,40 @@ Action is one of:
 {"type":"fill","id":number,"text":string}  (on a select or dropdown, text is the option to choose)
 {"type":"upload","id":number,"file":"photo"|"document"}  (attach a file through an upload button or file field: you have a sample photo and a sample CV/document)
 {"type":"goto","path":string}
-{"type":"press","key":"Escape"|"Enter"|"Tab"|"ArrowDown"|"ArrowUp"}  (e.g. Escape closes a popup that blocks the page)
+{"type":"press","key":"Escape"|"Enter"|"Space"|"Tab"|"ArrowDown"|"ArrowUp"}  (e.g. Escape closes a popup that blocks the page; Space presses the focused button or ticks a box)
 {"type":"wait"}  (the page is still loading, e.g. a spinner or a form that has not appeared yet: wait a few seconds rather than leave)${identity ? `\n{"type":"check_email"}` : ""}
 {"type":"done","success":boolean,"note":string}`;
 }
 
-function describeElement(e: InteractiveElement, screenReader = false): string {
-  // A screen reader announces the role and name the browser gives it.
-  const heard = screenReader && e.ax ? e.ax : e;
-  const line = `[${e.id}] ${heard.role} "${heard.name}"`;
-  if (!e.options) return line;
-  return `${line} set to "${e.value ?? ""}", options: ${e.options.map((o) => `"${o}"`).join(" | ")}`;
+/** A control's state, as a person sees it (and a screen reader announces it). */
+function stateOf(e: InteractiveElement, behindOpenLayer = false): string {
+  const states = [
+    // Hidden from assistive technology while something is open: what a modal does to the page behind it.
+    behindOpenLayer && e.ax?.hidden ? "behind what is open" : "",
+    e.disabled ? "disabled" : "",
+    e.readOnly ? "read-only" : "",
+    e.checked === undefined ? "" : e.checked ? "checked" : "not checked",
+    e.expanded === undefined ? "" : e.expanded ? "open" : "closed",
+    e.filled ? "filled in" : "",
+    !e.options && e.value ? `holds "${e.value}"` : "",
+  ].filter(Boolean);
+  return states.length ? ` (${states.join(", ")})` : "";
 }
 
-export interface StepView {
-  /** As a screen reader presents the page: the browser's names and roles, without what is hidden from it. */
-  screenReader?: boolean;
+function describeElement(e: InteractiveElement, screenReader = false, behindOpenLayer = false): string {
+  const line = `[${e.id}] ${controlLabel(e, screenReader)}${stateOf(e, behindOpenLayer)}`;
+  if (!e.options) return line;
+  return `${line} set to "${e.value ?? ""}", options: ${e.options.map((o) => `"${o}"`).join(" | ")}`;
 }
 
 export function buildStepMessage(obs: Observation, history: string[], view: StepView = {}): string {
   const shown = view.screenReader ? obs.elements.filter((e) => !e.ax?.hidden) : obs.elements;
-  const elements = shown.map((e) => describeElement(e, view.screenReader)).join("\n");
+  const elements = shown.map((e) => describeElement(e, view.screenReader, !!obs.openLayer)).join("\n");
   const past = history.length ? history.map((h, i) => `${i + 1}. ${h}`).join("\n") : "(none yet)";
+  const open = obs.openLayer ? `\nOpen now: ${obs.openLayer}. It takes clicks outside it: finish it, choose from it or close it (Escape, or its close button) before using what is behind it.\n` : "";
   return `URL: ${obs.url}
 Title: ${obs.title}
-
+${open}
 Interactive elements:
 ${elements}
 

package/src/browser/ax.ts

@@ -67,6 +67,7 @@ export async function accessibleNames(page: Page): Promise<Map<number, Accessibl
         role: node.role?.value ?? "",
         hidden: node.ignored,
         ...(description ? { description } : {}),
+        ...(node.ignoredReasons?.some((r) => r.name === "ariaHiddenElement" || r.name === "ariaHiddenSubtree") ? { ariaHidden: true as const } : {}),
       });
     }
     return out;