debutly 0.0.0-stage → 0.1.0

package/src/browser/observe.ts

@@ -0,0 +1,56 @@
+import type { Page } from "playwright";
+import type { InteractiveElement, Observation } from "../types";
+import { accessibleNames } from "./ax";
+import { SCAN_SCRIPT } from "./scan-script";
+
+// The page's language, how much of its visible text is Arabic and the direction that Arabic is
+// laid out in, and whether it lets people zoom. The direction is measured where the Arabic is, as
+// an app may turn its own container right to left rather than <body>: one that did was reported
+// as showing Arabic left to right. A string, like SCAN_SCRIPT: tsx injects helpers into functions
+// the page lacks.
+const LANGUAGE_SCRIPT = `(() => {
+  const body = document.body;
+  const text = (body && body.innerText) || '';
+  const letters = (text.match(/\\p{L}/gu) || []).length;
+  const arabic = (text.match(/\\p{Script=Arabic}/gu) || []).length;
+  let dir = body ? getComputedStyle(body).direction : 'ltr';
+  if (arabic && body) {
+    let rtl = 0, ltr = 0;
+    const walker = document.createTreeWalker(body, NodeFilter.SHOW_TEXT);
+    for (let node = walker.nextNode(), seen = 0; node && seen < 5000; node = walker.nextNode(), seen++) {
+      const count = (node.nodeValue.match(/\\p{Script=Arabic}/gu) || []).length;
+      if (!count || !node.parentElement) continue;
+      if (getComputedStyle(node.parentElement).direction === 'rtl') rtl += count;
+      else ltr += count;
+    }
+    if (rtl + ltr) dir = rtl >= ltr ? 'rtl' : 'ltr';
+  }
+  // Pinch-zoom, which people with low vision rely on on phones (WCAG 1.4.4 asks for 200%).
+  const meta = document.querySelector('meta[name="viewport"]');
+  const viewport = meta ? meta.getAttribute('content') || '' : '';
+  const setting = {};
+  for (const part of viewport.split(/[,;]/)) {
+    const [key, value] = part.split('=').map((s) => s.trim().toLowerCase());
+    if (key) setting[key] = value || '';
+  }
+  const noScaling = setting['user-scalable'] === 'no' || setting['user-scalable'] === '0';
+  const maxScale = parseFloat(setting['maximum-scale']);
+  return {
+    lang: document.documentElement.getAttribute('lang') || '',
+    dir,
+    arabicShare: letters ? Math.round((arabic / letters) * 100) / 100 : 0,
+    zoomBlocked: noScaling || maxScale < 2 ? viewport : '',
+  };
+})()`;
+
+export async function observe(page: Page): Promise<Observation> {
+  const elements = (await page.evaluate(SCAN_SCRIPT)) as InteractiveElement[];
+  const names = await accessibleNames(page);
+  if (names) for (const el of elements) {
+    const ax = names.get(el.id);
+    if (ax) el.ax = ax;
+  }
+  const text = (await page.locator("body").innerText()).replace(/\s+/g, " ").trim().slice(0, 2000);
+  const language = (await page.evaluate(LANGUAGE_SCRIPT)) as Pick<Observation, "lang" | "dir" | "arabicShare" | "zoomBlocked">;
+  return { url: page.url(), title: await page.title(), elements, text, ...language };
+}

package/src/browser/reflow.ts

@@ -0,0 +1,73 @@
+import type { Page } from "playwright";
+import type { ReflowCheck } from "../types";
+
+// WCAG 1.4.10 Reflow: at 320 CSS pixels wide (a 1280-pixel screen zoomed to 400%) a page should
+// read without scrolling sideways. Each page the low-vision agent arrives on is measured where it
+// is: its window is narrowed to that width, measured and widened again before the agent looks.
+// Nothing is requested twice, so no link or form is ever replayed to measure it.
+
+export const REFLOW_WIDTH = 320;
+const MAX_PAGES = 12;
+/** Time for the page to lay itself out after each change of width. */
+const LAYOUT_MS = 300;
+
+// The page's sideways overflow, and the element reaching furthest right outside any scrolling
+// box. A string, like SCAN_SCRIPT: tsx injects helpers into functions the page lacks.
+const MEASURE_SCRIPT = `(() => {
+  const doc = document.documentElement;
+  const view = doc.clientWidth;
+  const overflowPx = Math.max(doc.scrollWidth, document.body ? document.body.scrollWidth : 0) - view;
+  if (overflowPx <= 2) return { overflowPx: 0, widest: '', exception: '' };
+  const inScroller = (el) => {
+    for (let p = el.parentElement; p && p !== document.body && p !== doc; p = p.parentElement) {
+      const x = getComputedStyle(p).overflowX;
+      if (x !== 'visible') return true;
+    }
+    return false;
+  };
+  // Past the right edge, or on a right-to-left page past the left one. (A left-to-right page
+  // can't scroll left, so menus parked off its left edge don't count.)
+  const rtl = getComputedStyle(document.body).direction === 'rtl';
+  let widest = null, furthest = 2;
+  for (const el of document.body.querySelectorAll('*')) {
+    const r = el.getBoundingClientRect();
+    const out = rtl ? -r.left : r.right - view;
+    if (r.width > 0 && out > furthest && !inScroller(el)) { widest = el; furthest = out; }
+  }
+  // WCAG lets content that needs two dimensions scroll: data tables, maps, diagrams, video, code.
+  const twoD = widest && widest.closest('table, [role="grid"], [role="table"], pre, canvas, video, iframe, svg, map');
+  const name = (el) => {
+    const classes = typeof el.className === 'string' ? el.className.trim().split(/\\s+/).filter(Boolean).slice(0, 2) : [];
+    const text = (el.innerText || '').replace(/\\s+/g, ' ').trim().slice(0, 50);
+    return el.tagName.toLowerCase() + (el.id ? '#' + el.id : '') + classes.map((c) => '.' + c).join('') + (text ? ' ("' + text + '")' : '');
+  };
+  return {
+    overflowPx: Math.round(overflowPx),
+    widest: widest ? name(widest) + ', ' + Math.round(widest.getBoundingClientRect().width) + ' px wide' : '',
+    exception: twoD ? 'it is inside ' + (twoD.tagName === 'TABLE' || /grid|table/.test(twoD.getAttribute('role') || '') ? 'a data table' : 'a ' + twoD.tagName.toLowerCase()) + ', which may scroll' : '',
+  };
+})()`;
+
+export class ReflowMeter {
+  private readonly seen = new Set<string>();
+  readonly checks: ReflowCheck[] = [];
+
+  /** Measures the page the agent is on, the first time it arrives at each address. */
+  async measure(page: Page): Promise<void> {
+    const url = page.url().split("#")[0];
+    const size = page.viewportSize();
+    if (!size || !/^https?:/.test(url) || this.seen.has(url) || this.seen.size >= MAX_PAGES) return;
+    this.seen.add(url);
+    try {
+      await page.setViewportSize({ width: REFLOW_WIDTH, height: size.height });
+      await page.waitForTimeout(LAYOUT_MS);
+      const measured = (await page.evaluate(MEASURE_SCRIPT)) as Omit<ReflowCheck, "url" | "widthPx">;
+      this.checks.push({ url, widthPx: REFLOW_WIDTH, ...measured });
+    } catch {
+      // The page changed under the measurement: this one goes without it.
+    } finally {
+      await page.setViewportSize(size).catch(() => {});
+      await page.waitForTimeout(LAYOUT_MS);
+    }
+  }
+}

package/src/browser/samples.ts

@@ -0,0 +1,119 @@
+// Files an agent can attach where an app asks for one: a photo (for a post, an avatar, a logo)
+// and a document (a CV, a brief). Made here rather than shipped, and made up: nothing real or
+// personal is ever uploaded. Written once per process to the system's temp folder.
+import { mkdir, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { deflateSync } from "node:zlib";
+
+export type SampleKind = "photo" | "document";
+
+/** A square photo, large enough for social posts that refuse small images (1080 px). */
+const PHOTO_PX = 1080;
+
+let written: Promise<Record<SampleKind, string>> | undefined;
+
+/** Paths to the sample files, written on first use. */
+export function sampleFiles(): Promise<Record<SampleKind, string>> {
+  written ??= (async () => {
+    const dir = join(tmpdir(), "debutly-samples");
+    await mkdir(dir, { recursive: true });
+    const photo = join(dir, "sample-photo.png");
+    const document = join(dir, "sample-cv.pdf");
+    await writeFile(photo, png(PHOTO_PX, PHOTO_PX));
+    await writeFile(document, pdf(SAMPLE_CV));
+    return { photo, document };
+  })();
+  return written;
+}
+
+/** Which sample an upload field wants, from its `accept` attribute; a photo when it doesn't say. */
+export function sampleFor(accept: string | null | undefined): SampleKind {
+  const a = (accept ?? "").toLowerCase();
+  if (!a || /image|\.png|\.jpe?g|\.gif|\.webp/.test(a)) return "photo";
+  if (/pdf|word|document|text|\.docx?|\.txt|\.rtf/.test(a)) return "document";
+  return "photo";
+}
+
+const SAMPLE_CV = [
+  "Alex Sample - Curriculum Vitae",
+  "alex.sample@example.com - London, UK",
+  "",
+  "Experience",
+  "Marketing Coordinator, Example Ltd (2022 - present)",
+  "- Ran weekly social campaigns and reported on their results",
+  "Marketing Assistant, Sample & Co (2020 - 2022)",
+  "- Wrote newsletters and kept the company blog up to date",
+  "",
+  "Education",
+  "BA Business Studies, Example University (2017 - 2020)",
+  "",
+  "Skills",
+  "Copywriting, Social media, Google Analytics, Canva, Excel",
+];
+
+// A gradient, so image checks that refuse blank pictures accept it.
+function png(width: number, height: number): Buffer {
+  const rows: Buffer[] = [];
+  for (let y = 0; y < height; y++) {
+    const row = Buffer.alloc(1 + width * 3);
+    for (let x = 0; x < width; x++) {
+      row[1 + x * 3] = Math.round((x / width) * 200) + 30;
+      row[2 + x * 3] = Math.round((y / height) * 160) + 60;
+      row[3 + x * 3] = 170;
+    }
+    rows.push(row);
+  }
+  const header = Buffer.alloc(13);
+  header.writeUInt32BE(width, 0);
+  header.writeUInt32BE(height, 4);
+  header.set([8, 2, 0, 0, 0], 8); // 8-bit RGB
+  return Buffer.concat([
+    Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]),
+    chunk("IHDR", header),
+    chunk("IDAT", deflateSync(Buffer.concat(rows))),
+    chunk("IEND", Buffer.alloc(0)),
+  ]);
+}
+
+function chunk(type: string, data: Buffer): Buffer {
+  const length = Buffer.alloc(4);
+  length.writeUInt32BE(data.length);
+  const body = Buffer.concat([Buffer.from(type, "ascii"), data]);
+  const crc = Buffer.alloc(4);
+  crc.writeUInt32BE(crc32(body));
+  return Buffer.concat([length, body, crc]);
+}
+
+function crc32(bytes: Buffer): number {
+  let c = ~0;
+  for (const b of bytes) {
+    c ^= b;
+    for (let k = 0; k < 8; k++) c = c & 1 ? (c >>> 1) ^ 0xedb88320 : c >>> 1;
+  }
+  return ~c >>> 0;
+}
+
+// A one-page PDF with real text, so apps that read a CV's text find some.
+function pdf(lines: string[]): Buffer {
+  const escape = (s: string) => s.replace(/[\\()]/g, (m) => `\\${m}`);
+  const text = ["BT", "/F1 12 Tf", "16 TL", "56 780 Td", ...lines.map((l) => `(${escape(l)}) '`), "ET"].join("\n");
+  const objects = [
+    "<< /Type /Catalog /Pages 2 0 R >>",
+    "<< /Type /Pages /Kids [3 0 R] /Count 1 >>",
+    "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 595 842] /Contents 4 0 R /Resources << /Font << /F1 5 0 R >> >> >>",
+    `<< /Length ${Buffer.byteLength(text)} >>\nstream\n${text}\nendstream`,
+    "<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>",
+  ];
+  let out = "%PDF-1.4\n";
+  const offsets: number[] = [];
+  objects.forEach((body, i) => {
+    offsets.push(Buffer.byteLength(out));
+    out += `${i + 1} 0 obj\n${body}\nendobj\n`;
+  });
+  const xref = Buffer.byteLength(out);
+  out += `xref\n0 ${objects.length + 1}\n0000000000 65535 f \n`;
+  out += offsets.map((o) => `${String(o).padStart(10, "0")} 00000 n \n`).join("");
+  out += `trailer\n<< /Size ${objects.length + 1} /Root 1 0 R >>\nstartxref\n${xref}\n%%EOF\n`;
+  return Buffer.from(out, "latin1");
+}

package/src/browser/scan-script.ts

@@ -0,0 +1,62 @@
+// Runs in the page. Passed as a string, not a function, because tsx/esbuild can inject
+// helpers (e.g. __name) into function bodies that don't exist inside the browser.
+// Includes ARIA widgets (custom selects, menus, tabs, toggles) that apps build from divs:
+// without them, agents can open a dropdown but never see its options.
+export const SCAN_SCRIPT = `(() => {
+  const selector = [
+    'a[href]', 'button', 'input', 'select', 'textarea', 'summary',
+    '[role="button"]', '[role="link"]', '[role="option"]', '[role="menuitem"]', '[role="menuitemcheckbox"]',
+    '[role="menuitemradio"]', '[role="tab"]', '[role="checkbox"]', '[role="radio"]', '[role="switch"]', '[role="combobox"]',
+  ].join(', ');
+  const visible = (el) => typeof el.checkVisibility === 'function'
+    ? el.checkVisibility({ visibilityProperty: true })
+    : (getComputedStyle(el).display !== 'none' && getComputedStyle(el).visibility !== 'hidden');
+  const textOf = (el) => (el && (el.innerText || el.textContent)) || '';
+  const out = [];
+  let next = 0;
+  // Ids from the last look stay on elements that have since gone out of sight; the same number
+  // given to another element then matches two ("strict mode violation"). Start clean each time.
+  for (const el of document.querySelectorAll('[data-debutly-id]')) el.removeAttribute('data-debutly-id');
+  for (const el of document.querySelectorAll(selector)) {
+    // Apps hide the real file field behind a styled button or drop zone; files can still be
+    // attached to it, so list it.
+    const fileField = el.tagName === 'INPUT' && el.type === 'file';
+    if (el.type === 'hidden' || (!visible(el) && !fileField)) continue;
+    const id = next++;
+    el.setAttribute('data-debutly-id', String(id));
+    const tag = el.tagName.toLowerCase();
+    const role = el.getAttribute('role') || (tag === 'a' ? 'link' : tag === 'input' ? 'input:' + (el.type || 'text') : tag);
+    const labelledBy = (el.getAttribute('aria-labelledby') || '').split(/\\s+/).filter(Boolean)
+      .map((ref) => textOf(document.getElementById(ref))).join(' ');
+    // A field's label points at it (for=) or wraps it; a wrapping label's name is its own text,
+    // without the field's (a select's options, a textarea's content).
+    const field = tag === 'input' || tag === 'select' || tag === 'textarea';
+    const labelEl = (field && el.labels && el.labels[0]) || (el.id ? document.querySelector('label[for="' + CSS.escape(el.id) + '"]') : null);
+    let label = null;
+    if (labelEl) {
+      label = labelEl.cloneNode(true);
+      label.querySelectorAll('input, select, textarea, button').forEach((n) => n.remove());
+    }
+    // title is the accessible name of last resort, as browsers and screen readers treat it.
+    // A native select's text is all its options run together: name it by its label, and list them apart.
+    const isSelect = tag === 'select';
+    const fileHint = fileField ? 'file upload' + (el.getAttribute('accept') ? ' (' + el.getAttribute('accept') + ')' : '') : '';
+    const labelled = el.getAttribute('aria-label') || labelledBy || (label && label.textContent.trim());
+    const placeholder = el.getAttribute('placeholder');
+    const raw = labelled || placeholder || (isSelect ? '' : textOf(el).trim()) || el.getAttribute('title') || fileHint;
+    const item = { id, role, name: raw.replace(/\\s+/g, ' ').trim().slice(0, 80), tag };
+    // A placeholder is an example, not a label: screen readers read it as the field's name (a
+    // password's dots as "13 bullet"), and it's gone once someone types.
+    if (!labelled && placeholder) item.nameFrom = 'placeholder';
+    // What a keyboard user can reach: a disabled control, or one out of the tab order, isn't.
+    if (el.disabled === true || el.getAttribute('aria-disabled') === 'true') item.disabled = true;
+    else if (el.tabIndex < 0 || el.closest('[inert]')) item.tabbable = false;
+    if (isSelect) {
+      const chosen = el.selectedOptions[0];
+      item.value = chosen ? chosen.label.slice(0, 80) : '';
+      item.options = Array.from(el.options).slice(0, 30).map((o) => o.label.slice(0, 60));
+    }
+    out.push(item);
+  }
+  return out;
+})()`;

package/src/browser/settle.ts

@@ -0,0 +1,32 @@
+import type { Page } from "playwright";
+
+export interface SettleOptions {
+  /** Longest wait for the network to go quiet (ms). */
+  idleMs?: number;
+  /** Longest extra wait while the page still says it is loading (ms). */
+  busyMs?: number;
+}
+
+/**
+ * Waits, as a person would, until the page has finished loading before it is read: the network
+ * goes quiet, and no "Loading…" placeholder or aria-busy region is left. Client-rendered apps show
+ * a spinner or an empty shell at DOMContentLoaded; on a slow connection agents otherwise judge (and
+ * navigate away from) pages that were only still loading. Bounded, so a page that never settles
+ * (polling, a stuck spinner) is read as it is.
+ */
+export async function settle(page: Page, o: SettleOptions = {}): Promise<void> {
+  const { idleMs = 5_000, busyMs = 10_000 } = o;
+  await page.waitForLoadState("networkidle", { timeout: idleMs }).catch(() => {});
+  const deadline = Date.now() + busyMs;
+  while (Date.now() < deadline) {
+    const busy = await page
+      .evaluate(() => {
+        if (document.querySelector('[aria-busy="true"]')) return true;
+        const text = (document.body?.innerText ?? "").trim();
+        return text.length < 300 && /\b(loading|please wait)\b/i.test(text);
+      })
+      .catch(() => false);
+    if (!busy) return;
+    await page.waitForTimeout(500);
+  }
+}

package/src/browser/signals.ts

@@ -0,0 +1,73 @@
+import type { Page } from "playwright";
+import type { Signal } from "../types";
+
+const BODY_CHARS = 500;
+/** How long a step waits for server-error bodies still being read before it is stored without them. */
+const BODY_WAIT_MS = 2000;
+
+export class SignalCollector {
+  private buffer: Signal[] = [];
+  private reading = new Set<Promise<void>>();
+
+  constructor(page: Page) {
+    page.on("console", (msg) => {
+      if (msg.type() === "error") this.buffer.push({ kind: "console_error", message: msg.text(), url: page.url() });
+    });
+    page.on("pageerror", (err) => {
+      this.buffer.push({ kind: "page_error", message: err.message, url: page.url() });
+    });
+    page.on("response", (res) => {
+      if (res.status() < 400) return;
+      // A failed navigation belongs to the page that was requested, not the one being left.
+      const url = res.request().isNavigationRequest() ? res.url() : page.url();
+      const startedAt = res.request().timing().startTime;
+      const signal: Signal = { kind: "http_error", status: res.status(), requestUrl: res.url(), url, startedAt: startedAt > 0 ? startedAt : undefined, at: Date.now() };
+      this.buffer.push(signal);
+      // A server error often says why (e.g. "No API key found"). Read the start of it; it lands on
+      // the same signal object, which findings read once the run is over.
+      if (res.status() >= 500) {
+        const read = res
+          .text()
+          .then((text) => {
+            if (signal.kind === "http_error") signal.body = text.slice(0, BODY_CHARS);
+          })
+          .catch(() => {})
+          .finally(() => this.reading.delete(read));
+        this.reading.add(read);
+      }
+    });
+    // Generating a report often only downloads a file; without this the agent saw "nothing
+    // visibly changed" and concluded the button was broken.
+    page.on("download", (download) => {
+      this.buffer.push({ kind: "download", filename: download.suggestedFilename(), url: page.url() });
+    });
+    // Playwright dismisses dialogs by default, so every confirm() would read as Cancel. Answer OK,
+    // as a person going along with the app would, and let the agent know what it asked.
+    page.on("dialog", (dialog) => {
+      this.buffer.push({ kind: "dialog", dialogType: dialog.type(), message: dialog.message(), url: page.url() });
+      dialog.accept().catch(() => {});
+    });
+  }
+
+  /** A signal the agent loop observed itself, such as a slow page. */
+  note(signal: Signal): void {
+    this.buffer.push(signal);
+  }
+
+  /**
+   * Waits, briefly, for the server-error bodies still being read. A step is stored as soon as it
+   * ends; a fast 502 was stored without the body that said why it failed (2026-10-17).
+   */
+  async settle(): Promise<void> {
+    if (!this.reading.size) return;
+    let timer: ReturnType<typeof setTimeout> | undefined;
+    await Promise.race([Promise.allSettled([...this.reading]), new Promise((r) => (timer = setTimeout(r, BODY_WAIT_MS)))]);
+    clearTimeout(timer);
+  }
+
+  drain(): Signal[] {
+    const out = this.buffer;
+    this.buffer = [];
+    return out;
+  }
+}

package/src/cancel.ts

@@ -0,0 +1,20 @@
+// The run kill switch from anywhere: npm run debutly:cancel -- <runId> --store postgres://…
+// Agents stop at their next step; the run still writes its (partial) report.
+import { parseArgs } from "node:util";
+import { openDb } from "./store/db";
+import { PgRunStore } from "./store/pg-run-store";
+
+const { values, positionals } = parseArgs({ allowPositionals: true, options: { store: { type: "string" } } });
+const [runId] = positionals;
+const url = values.store ?? process.env.DATABASE_URL;
+if (!runId || !url) {
+  console.error("Usage: npm run debutly:cancel -- <runId> --store postgres://… (or set DATABASE_URL)");
+  process.exit(1);
+}
+const { db, close } = await openDb(url);
+const store = new PgRunStore(db);
+await store.requestCancel(runId);
+const run = await store.getRun(runId);
+await close();
+console.log(run ? `Run ${runId}: ${run.status}` : `No run ${runId}`);
+process.exit(run ? 0 : 1);