debutly 0.1.0 → 0.3.0

package/src/init.ts

@@ -103,15 +103,15 @@ async function guessCustomers(url: string): Promise<string | undefined> {
   }
 }
 
-export async function init(cwd: string = process.cwd()): Promise<number> {
-  const rl = createInterface({ input: process.stdin, output: process.stdout });
+export async function init(cwd: string = process.cwd(), input: NodeJS.ReadableStream & { isTTY?: boolean } = process.stdin): Promise<number> {
+  const rl = createInterface({ input, output: process.stdout });
   // Lines are buffered, so answers piped in by a script aren't lost between questions; at the end
   // of the input every question takes its default.
   const lines = rl[Symbol.asyncIterator]();
   const ask = async (question: string, fallback = "") => {
     process.stdout.write(fallback ? `${question} [${fallback}] ` : `${question} `);
     const { value, done } = await lines.next();
-    if (done || !process.stdin.isTTY) process.stdout.write("\n");
+    if (done || !input.isTTY) process.stdout.write("\n");
     return (done ? "" : String(value).trim()) || fallback;
   };
   try {
@@ -126,8 +126,9 @@ export async function init(cwd: string = process.cwd()): Promise<number> {
     console.log("\nWhat should the agents try to do?");
     GOALS.forEach((g, i) => console.log(`  ${i + 1}. ${g}`));
     console.log("  4. Something else");
-    const pick = await ask("Choose 1-4:", "1");
-    const goal = GOALS[Number(pick) - 1] ?? (await ask("Describe it in a sentence:"));
+    // A goal typed here instead of a number is the goal.
+    const pick = await ask("Choose 1-4, or type the goal:", "1");
+    const goal = GOALS[Number(pick) - 1] ?? (/^\d*$/.test(pick) ? await ask("Describe it in a sentence:") : pick);
 
     const guess = await guessCustomers(url);
     const icp = await ask("\nWho are your customers? One sentence, e.g. who they are and how big.", guess);

package/src/persona/people.ts

@@ -0,0 +1,32 @@
+import { readFile } from "node:fs/promises";
+import type { CustomerUnit } from "../types";
+
+/** Each run writes its people here, next to the report, so a later run can meet the same ones. */
+export const PEOPLE_FILE = "people.json";
+
+/**
+ * The people of an earlier run, to run the same agents again: after a fix, whether it worked is
+ * only clear with the same people, needs and languages. Generated afresh, a run had a different
+ * crew every day (needs rotate daily), and two runs of one app could not be compared.
+ */
+export async function readPeople(path: string): Promise<CustomerUnit[]> {
+  let parsed: unknown;
+  try {
+    parsed = JSON.parse(await readFile(path, "utf8"));
+  } catch (err) {
+    throw new Error(`${path} can't be read: ${err instanceof Error ? err.message : err}`);
+  }
+  const ok = (u: unknown) => {
+    const unit = u as Partial<CustomerUnit> | null;
+    if (!unit || typeof unit !== "object" || typeof unit.id !== "string") return false;
+    if (unit.kind === "individual") return !!unit.person?.id;
+    return unit.kind === "business" && !!unit.owner?.id && Array.isArray(unit.employees);
+  };
+  if (!Array.isArray(parsed) || !parsed.length || !parsed.every(ok)) {
+    throw new Error(`${path} doesn't hold a run's people: use the ${PEOPLE_FILE} from an earlier run's report folder.`);
+  }
+  return parsed as CustomerUnit[];
+}
+
+/** How many agents the units make: everyone in a crew is one. */
+export const peopleCount = (units: CustomerUnit[]) => units.reduce((n, u) => n + (u.kind === "individual" ? 1 : 1 + u.employees.length), 0);

package/src/report/fix-first.ts

@@ -0,0 +1,125 @@
+import { UNCOMPRESSED } from "../detect/findings";
+import type { Finding, Session } from "../types";
+
+/** At most this many items, so the section reads in a couple of minutes. */
+const MAX_ITEMS = 5;
+const MAX_QUOTE = 200;
+const RANK: Record<Finding["severity"], number> = { blocker: 0, major: 1, minor: 2 };
+
+// Agents that didn't reach the goal because of the app, or seemingly so. An engine error or a
+// cancelled run says nothing about the app; a run cut short for credit has its own section.
+const STOPPED: Partial<Record<Session["outcome"], string>> = {
+  gave_up: "gave up",
+  budget_exhausted: "ran out of steps",
+  not_invited: "never got an invite",
+};
+
+/** What to do about a finding, in a line. */
+export function nextStep(f: Finding): string {
+  switch (f.kind) {
+    case "http_error": {
+      const status = Number(/^HTTP (\d{3})/.exec(f.title)?.[1] ?? 0);
+      return status >= 500
+        ? "The server failed: its log for this request says why."
+        : "The server refused the request: compare what the page sent with what the server expects.";
+    }
+    case "page_error":
+      return "Fix the JavaScript error; the reproduction below shows the step that sets it off.";
+    case "console_error":
+      return "Read the error in the browser console on that page.";
+    case "a11y_unlabeled":
+      if (f.control?.problem === "placeholder") return "Give the field a visible <label>, and keep the placeholder as an example.";
+      if (f.control?.problem === "described") return "Name it: visible text or an aria-label, with the description kept as a hint.";
+      if (f.control && !/^(button|link|menuitem|tab|summary)/.test(f.control.role)) return "Give each one a <label> (or an aria-label) that says what goes in it.";
+      return 'Give each one visible text, or an aria-label that says what it does (aria-label="Open menu" on a menu icon).';
+    case "stuck":
+      return "Watch the reproduction below: the agent repeated an action that seemed to do nothing. Make the page show that it worked, or why it didn't.";
+    case "slow_load":
+      return f.detail.includes(UNCOMPRESSED)
+        ? "Turn on compression (gzip or Brotli) for scripts and styles, or check that the real site has it."
+        : "Find what holds the page up: the agents sat waiting for it.";
+    case "covered":
+      return "Move or close whatever lies over the control, so it can be clicked.";
+    case "keyboard":
+      if (f.signature.startsWith("enter:")) return "Let Enter press it too: look for a key handler that catches Enter and stops it (a page-wide shortcut, a form's own Enter handling).";
+      return 'Use a real <button> or <a>, or add tabindex="0" and handle Enter and Space.';
+    case "focus_obscured":
+      return "Keep the control with keyboard focus in view: move or shrink what covers it.";
+    case "reflow":
+      return "Let the page wrap at narrow widths instead of scrolling sideways: look for fixed widths.";
+    case "language":
+      if (f.signature.startsWith("dir:")) return 'Set dir="rtl" on Arabic pages (on <html>, or on the part that holds the Arabic).';
+      if (f.signature === "lang:missing") return 'Add a lang attribute to the <html> element, e.g. lang="en".';
+      return 'Set lang="ar" on Arabic pages.';
+    case "hidden_focus":
+      return 'Make the hidden part unreachable while hidden: add inert to it (or tabindex="-1" to each control), or stop hiding it from screen readers.';
+    case "zoom":
+      return "Remove user-scalable=no and maximum-scale from the page's viewport meta tag.";
+  }
+}
+
+const lastPath = (s: Session) => {
+  try {
+    return new URL(s.steps.at(-1)?.observation.url ?? "").pathname;
+  } catch {
+    return "";
+  }
+};
+
+const quote = (s: string) => `"${s.length > MAX_QUOTE ? `${s.slice(0, MAX_QUOTE)}…` : s}"`;
+
+/** A screenshot as the report's reader finds it: next to the report, not on the machine that ran it. */
+export function shotPath(path: string): string {
+  if (/^https?:/.test(path)) return path;
+  // Kept in the run's database too (db:<run>/<file>), and under the same name beside the report.
+  if (path.startsWith("db:")) return `screenshots/${path.split("/").pop()}`;
+  const at = path.replace(/\\/g, "/").lastIndexOf("/screenshots/");
+  return at >= 0 ? path.replace(/\\/g, "/").slice(at + 1) : path;
+}
+
+function stoppedItem(s: Session, needs: string, environment: boolean): string {
+  const who = [s.role, needs !== "—" ? needs : ""].filter(Boolean).join("; ");
+  const where = lastPath(s) ? ` on ${lastPath(s)}` : "";
+  const shot = [...s.steps].reverse().find((st) => st.screenshotPath)?.screenshotPath;
+  // Out of steps, the note only says so; the agent's last thought says where it was.
+  const said = s.outcome === "budget_exhausted" && s.steps.at(-1)?.reasoning ? `Its last thought: ${quote(s.steps.at(-1)!.reasoning)}` : `In the agent's words: ${quote(s.outcomeNote)}`;
+  return [
+    `**${s.personaName}${who ? ` (${who})` : ""} ${STOPPED[s.outcome]}${where}.** ${said}`,
+    environment ? "Part of it may be the test environment: see the end of the report." : "",
+    shot ? `Start with the last screenshot, ${shotPath(shot)}.` : "",
+  ]
+    .filter(Boolean)
+    .join(" ");
+}
+
+/**
+ * The few things to look at first: agents the app seemingly stopped, in their own words, then the
+ * blocker and major findings, those that stopped an agent first, each with what to do.
+ */
+export function renderFixFirst(sessions: Session[], findings: Finding[], needLabel: (s: Session) => string): string[] {
+  const stopped = sessions.filter((s) => STOPPED[s.outcome]);
+  const failed = new Set(stopped.map((s) => s.agentId));
+  const caused = findings.filter((f) => f.sandboxCause);
+  const serious = findings
+    .filter((f) => !f.sandboxCause && f.confidence !== "low" && f.severity !== "minor")
+    .map((f, order) => ({ f, order, stoppedSome: f.hitBy.some((id) => failed.has(id)) }))
+    // Most severe first; within that, one an agent didn't get past.
+    .sort((a, b) => RANK[a.f.severity] - RANK[b.f.severity] || Number(b.stoppedSome) - Number(a.stoppedSome) || a.order - b.order)
+    .map(({ f }) => f);
+  const reached = sessions.filter((s) => s.outcome === "success").length;
+  const stops = stopped.slice(0, MAX_ITEMS).map((s) => stoppedItem(s, needLabel(s), caused.some((f) => f.hitBy.includes(s.agentId))));
+  const fixes = serious.slice(0, MAX_ITEMS - stops.length).map((f) => `**${f.title}** (${f.severity}). ${nextStep(f)}`);
+  const shown = [...stops, ...fixes];
+  const rest = findings.filter((f) => !f.sandboxCause).length - fixes.length;
+  return [
+    `## Fix these first`,
+    "",
+    `${reached} of ${sessions.length} agents reached the goal.`,
+    "",
+    ...(shown.length
+      ? shown.map((item, i) => `${i + 1}. ${item}`)
+      : ["Nothing stopped an agent, and nothing serious was found."]),
+    ...(rest > 0 ? ["", `${rest === 1 ? "One" : rest}${fixes.length ? " more" : ""} ${rest === 1 ? "finding is" : "findings are"} below.`] : []),
+    "",
+  ];
+}

package/src/report/markdown.ts

@@ -4,6 +4,7 @@ import { describeCriterion } from "../detect/wcag";
 import { OUT_OF_CREDITS } from "../llm/types";
 import { languageName } from "../persona/language";
 import type { CustomerUnit, Finding, Session } from "../types";
+import { renderFixFirst, shotPath } from "./fix-first";
 
 export interface ReportInput {
   targetUrl: string;
@@ -40,7 +41,7 @@ function renderFinding(f: Finding, sessions: Session[]): string {
       lines.push(`${step.index + 1}. ${describeAction(step.action, step.observation)} — "${step.reasoning}"`);
     }
     const shot = session.steps[ev.stepIndex]?.screenshotPath;
-    if (shot) lines.push("", `Screenshot: ${shot}`);
+    if (shot) lines.push("", `Screenshot: ${shotPath(shot)}`);
   }
   return lines.join("\n");
 }
@@ -164,6 +165,7 @@ export function renderReport(input: ReportInput): string {
     `Findings: ${count("blocker")} blocker, ${count("major")} major, ${count("minor")} minor (${low.length} low-confidence${caused.length ? `, ${caused.length} caused by the test environment` : ""})`,
     "",
     ...renderOutOfCredits(input.sessions),
+    ...renderFixFirst(input.sessions, input.findings, needLabel),
     ...renderUnits(input.units ?? [], input.sessions),
     `## Agent outcomes`,
     "",

package/src/run.ts

@@ -7,6 +7,7 @@ import { runUnit, type RunAgent } from "./crew/run-unit";
 import { detectFindings } from "./detect/findings";
 import type { Inbox } from "./inbox/types";
 import type { LlmClient } from "./llm/types";
+import { PEOPLE_FILE } from "./persona/people";
 import { renderReport } from "./report/markdown";
 import type { ModelPrice } from "./llm/meter";
 import { blockedCalls, type SandboxOutbox } from "./sandbox/outbox";
@@ -43,8 +44,8 @@ export interface SimulationOptions {
   price?: ModelPrice | null;
   /** In-memory kill switch when there is no store (e.g. Ctrl+C). */
   shouldStop?: () => boolean | Promise<boolean>;
-  /** Where to keep each agent's browser state (cookies, storage) at the end, for `debutly hear`. */
-  browserStateDir?: string;
+  /** Where to keep each agent's browser state (cookies, storage) at the end, for `debutly hear`. */
+  browserStateDir?: string;
   /** Sends each agent somewhere else to run (e.g. a queue of workers) instead of this process. */
   dispatch?: (ctx: { runId?: string; runTag: string }) => RunAgent;
 }
@@ -120,6 +121,7 @@ export async function runSimulation(o: SimulationOptions): Promise<SimulationRes
     const reportPath = join(o.outDir, "report.md");
     await writeFile(join(o.outDir, "sessions.json"), JSON.stringify(sessions, null, 2));
     await writeFile(join(o.outDir, "findings.json"), JSON.stringify(findings, null, 2));
+    await writeFile(join(o.outDir, PEOPLE_FILE), JSON.stringify(o.units, null, 2));
     await writeFile(reportPath, reportMd);
     if (store && runId) {
       const status = sessions.some((s) => s.outcome === "cancelled") ? "cancelled" : "finished";

package/src/types.ts

@@ -33,7 +33,7 @@ export type CustomerUnit =
   | { kind: "business"; id: string; business: Business; owner: Persona; employees: Persona[] };
 
 /** Keys a person uses to drive popups and forms; nothing that could close tabs or run shortcuts. */
-export const PRESS_KEYS = ["Escape", "Enter", "Tab", "ArrowDown", "ArrowUp"] as const;
+export const PRESS_KEYS = ["Escape", "Enter", "Space", "Tab", "ArrowDown", "ArrowUp"] as const;
 export type PressKey = (typeof PRESS_KEYS)[number];
 
 export type Action =
@@ -55,7 +55,8 @@ export type Signal =
   | { kind: "http_error"; status: number; requestUrl: string; url: string; startedAt?: number; at?: number; body?: string }
   | { kind: "dialog"; dialogType: string; message: string; url: string }
   // A page that took long to arrive. finished: false when the agent's wait ran out first.
-  | { kind: "slow_load"; url: string; ms: number; finished: boolean; slowNetwork: boolean }
+  // assets: the scripts and styles received by then, as sent, and how much of that came uncompressed.
+  | { kind: "slow_load"; url: string; ms: number; finished: boolean; slowNetwork: boolean; assets?: { files: number; bytes: number; uncompressedBytes: number } }
   // A file the page handed over (a report, an export): the page itself often doesn't change.
   | { kind: "download"; filename: string; url: string };
 
@@ -66,6 +67,8 @@ export interface AccessibleName {
   hidden: boolean;
   /** What a screen reader reads after the role (aria-describedby, a tooltip); absent when there's none. */
   description?: string;
+  /** Hidden by aria-hidden on it or around it, rather than for another reason. */
+  ariaHidden?: true;
 }
 
 export interface InteractiveElement {
@@ -73,9 +76,18 @@ export interface InteractiveElement {
   role: string;
   name: string;
   tag: string;
-  /** A native select's chosen option and its options, by label. */
+  /** What a field holds: a native select's chosen option, or a text field's text (never a password's). */
   value?: string;
+  /** A native select's options, by label. */
   options?: string[];
+  /** A password field with something typed in it. */
+  filled?: true;
+  /** Shown but not editable. */
+  readOnly?: true;
+  /** A checkbox, radio button or switch: on or off. */
+  checked?: boolean;
+  /** A control that opens something (a menu, a dropdown): open or closed. */
+  expanded?: boolean;
   /** The browser's own accessible name and role (Chromium), as a screen reader would announce them. */
   ax?: AccessibleName;
   /** Named only by its placeholder: no label, aria-label or aria-labelledby. */
@@ -84,6 +96,8 @@ export interface InteractiveElement {
   disabled?: true;
   /** Out of the tab order (tabindex -1, or inside an inert part of the page). */
   tabbable?: false;
+  /** A control with no text: the icon it shows, as its markup names it ("menu", "trash"). What a sighted person sees. */
+  icon?: string;
 }
 
 export interface Observation {
@@ -97,6 +111,8 @@ export interface Observation {
   arabicShare?: number;
   /** The page's viewport setting when it stops pinch-zoom (user-scalable=no, maximum-scale under 2), else "". */
   zoomBlocked?: string;
+  /** What is open over the page and takes clicks outside it, e.g. 'a dialog ("Welcome")' or "a dropdown list". */
+  openLayer?: string;
 }
 
 export interface ActionResult {
@@ -104,6 +120,10 @@ export interface ActionResult {
   error?: string;
   /** Keyboard only: what entirely hid the control once it had focus. */
   focusObscured?: string;
+  /** Keyboard only: Enter did nothing on the button, and Space pressed it. */
+  enterIgnored?: true;
+  /** What the browser's own bubble said about fields a submit found invalid: '"Password": Please lengthen…'. */
+  invalid?: string[];
 }
 
 export interface Step extends ActionResult {
@@ -160,7 +180,8 @@ export type FindingKind =
   | "focus_obscured"
   | "reflow"
   | "language"
-  | "zoom";
+  | "zoom"
+  | "hidden_focus";
 
 export interface Finding {
   signature: string;