debutly 0.0.0-stage → 0.2.0

package/src/types.ts

@@ -0,0 +1,201 @@
+/** Accessibility & inclusion needs every run includes by default. */
+export const INCLUSION_NEEDS = ["screen_reader", "low_vision", "slow_connection", "non_native_english"] as const;
+export type InclusionNeed = (typeof INCLUSION_NEEDS)[number];
+
+export interface Persona {
+  id: string;
+  name: string;
+  background: string;
+  goal: string;
+  patience: "low" | "medium" | "high";
+  techSavvy: "low" | "medium" | "high";
+  /** Job role inside a business crew, e.g. "Owner" or "Content writer". */
+  role?: string;
+  inclusion?: InclusionNeed;
+  /** First language, as a lang code ("ar"); English when unset. */
+  language?: string;
+}
+
+export interface Business {
+  name: string;
+  /** agency, SME, small business, freelancer, startup, enterprise team… */
+  archetype: string;
+  /** Role-played headcount; the crew itself is capped. */
+  size: number;
+  /** Monthly budget ceiling for tools like this, in USD. */
+  monthlyBudget: number;
+  goal: string;
+}
+
+/** Who a simulation run models: one person (B2C) or a business crew of owner + employees (B2B). */
+export type CustomerUnit =
+  | { kind: "individual"; id: string; person: Persona }
+  | { kind: "business"; id: string; business: Business; owner: Persona; employees: Persona[] };
+
+/** Keys a person uses to drive popups and forms; nothing that could close tabs or run shortcuts. */
+export const PRESS_KEYS = ["Escape", "Enter", "Space", "Tab", "ArrowDown", "ArrowUp"] as const;
+export type PressKey = (typeof PRESS_KEYS)[number];
+
+export type Action =
+  | { type: "click"; id: number }
+  | { type: "fill"; id: number; text: string }
+  // file: which sample to attach (a photo, or a document such as a CV); by default what the field accepts.
+  | { type: "upload"; id: number; file?: "photo" | "document" }
+  | { type: "goto"; path: string }
+  | { type: "check_email" }
+  | { type: "press"; key: PressKey }
+  | { type: "wait" }
+  | { type: "done"; success: boolean; note: string };
+
+export type Signal =
+  | { kind: "console_error"; message: string; url: string }
+  | { kind: "page_error"; message: string; url: string }
+  // startedAt/at (epoch ms) bracket the request, to match it against what a sandbox blocked meanwhile.
+  // body: the start of a server error's response, which can say why (e.g. a missing API key).
+  | { kind: "http_error"; status: number; requestUrl: string; url: string; startedAt?: number; at?: number; body?: string }
+  | { kind: "dialog"; dialogType: string; message: string; url: string }
+  // A page that took long to arrive. finished: false when the agent's wait ran out first.
+  // assets: the scripts and styles received by then, as sent, and how much of that came uncompressed.
+  | { kind: "slow_load"; url: string; ms: number; finished: boolean; slowNetwork: boolean; assets?: { files: number; bytes: number; uncompressedBytes: number } }
+  // A file the page handed over (a report, an export): the page itself often doesn't change.
+  | { kind: "download"; filename: string; url: string };
+
+/** What the browser tells assistive technology about an element. hidden: left out of the tree (e.g. aria-hidden). */
+export interface AccessibleName {
+  name: string;
+  role: string;
+  hidden: boolean;
+  /** What a screen reader reads after the role (aria-describedby, a tooltip); absent when there's none. */
+  description?: string;
+}
+
+export interface InteractiveElement {
+  id: number;
+  role: string;
+  name: string;
+  tag: string;
+  /** What a field holds: a native select's chosen option, or a text field's text (never a password's). */
+  value?: string;
+  /** A native select's options, by label. */
+  options?: string[];
+  /** A password field with something typed in it. */
+  filled?: true;
+  /** Shown but not editable. */
+  readOnly?: true;
+  /** A checkbox, radio button or switch: on or off. */
+  checked?: boolean;
+  /** A control that opens something (a menu, a dropdown): open or closed. */
+  expanded?: boolean;
+  /** The browser's own accessible name and role (Chromium), as a screen reader would announce them. */
+  ax?: AccessibleName;
+  /** Named only by its placeholder: no label, aria-label or aria-labelledby. */
+  nameFrom?: "placeholder";
+  /** Disabled, so Tab skips it. */
+  disabled?: true;
+  /** Out of the tab order (tabindex -1, or inside an inert part of the page). */
+  tabbable?: false;
+  /** A control with no text: the icon it shows, as its markup names it ("menu", "trash"). What a sighted person sees. */
+  icon?: string;
+}
+
+export interface Observation {
+  url: string;
+  title: string;
+  elements: InteractiveElement[];
+  text: string;
+  /** The page's lang attribute ("" when missing), its text direction, and the share of its letters that are Arabic. */
+  lang?: string;
+  dir?: string;
+  arabicShare?: number;
+  /** The page's viewport setting when it stops pinch-zoom (user-scalable=no, maximum-scale under 2), else "". */
+  zoomBlocked?: string;
+}
+
+export interface ActionResult {
+  blocked?: string;
+  error?: string;
+  /** Keyboard only: what entirely hid the control once it had focus. */
+  focusObscured?: string;
+  /** Keyboard only: Enter did nothing on the button, and Space pressed it. */
+  enterIgnored?: true;
+}
+
+export interface Step extends ActionResult {
+  index: number;
+  observation: Observation;
+  action: Action;
+  reasoning: string;
+  signals: Signal[];
+  screenshotPath?: string;
+  /** A goto to an address the agent typed itself, not a link from an email. */
+  guessedUrl?: boolean;
+}
+
+/** A page measured at a narrow width after the low-vision agent's run (WCAG 1.4.10 Reflow). */
+export interface ReflowCheck {
+  url: string;
+  widthPx: number;
+  /** How much wider than the window the page is; 0 when it fits. */
+  overflowPx: number;
+  /** The element reaching furthest to the right, outside any scrolling box. */
+  widest: string;
+  /** Why the overflow may be allowed (e.g. it is a data table), or "". */
+  exception: string;
+}
+
+export interface Session {
+  agentId: string;
+  personaName: string;
+  goal: string;
+  steps: Step[];
+  outcome: "success" | "gave_up" | "budget_exhausted" | "error" | "not_invited" | "cancelled";
+  outcomeNote: string;
+  unitId?: string;
+  role?: string;
+  inclusion?: InclusionNeed;
+  language?: string;
+  /** Tokens this session spent, and their cost when the model price is known. */
+  usage?: { inputTokens: number; outputTokens: number; costUsd?: number };
+  /** The low-vision agent's pages, measured again at 320 CSS pixels wide. */
+  reflow?: ReflowCheck[];
+}
+
+export type Severity = "blocker" | "major" | "minor";
+export type Confidence = "high" | "medium" | "low";
+export type FindingKind =
+  | "http_error"
+  | "page_error"
+  | "console_error"
+  | "a11y_unlabeled"
+  | "stuck"
+  | "slow_load"
+  | "covered"
+  | "keyboard"
+  | "focus_obscured"
+  | "reflow"
+  | "language"
+  | "zoom";
+
+export interface Finding {
+  signature: string;
+  kind: FindingKind;
+  severity: Severity;
+  title: string;
+  detail: string;
+  pagePath: string;
+  hitBy: string[];
+  visitors: number;
+  confidence: Confidence;
+  evidence: { agentId: string; stepIndex: number }[];
+  /**
+   * Why every occurrence of this failure is the test environment's doing: hosts it blocked at the
+   * same moment, `missing key: <what the app said>` when the app reported a key the test
+   * environment leaves out on purpose, and/or `made-up site: <host>` when the app couldn't reach
+   * the website an agent invented for its company.
+   */
+  sandboxCause?: string[];
+  /** WCAG 2.2 success criteria the finding relates to ("4.1.2"), for a person to check. */
+  wcag?: string[];
+  /** The control a screen-reader finding is about, so `debutly hear` can find it again. */
+  control?: { role: string; name?: string; problem: "unnamed" | "described" | "unreached" | "placeholder" | "keyboard" };
+}

package/src/util/chromium.ts

@@ -0,0 +1,20 @@
+import { spawnSync } from "node:child_process";
+import { existsSync } from "node:fs";
+import { createRequire } from "node:module";
+import { dirname, join } from "node:path";
+import { chromium } from "playwright";
+
+/** Is the browser the agents use downloaded yet? */
+export function chromiumInstalled(): boolean {
+  try {
+    return existsSync(chromium.executablePath());
+  } catch {
+    return false;
+  }
+}
+
+/** Downloads it with Playwright's own installer (about 150 MB, once per machine). */
+export function installChromium(): boolean {
+  const cli = join(dirname(createRequire(import.meta.url).resolve("playwright/package.json")), "cli.js");
+  return spawnSync(process.execPath, [cli, "install", "chromium"], { stdio: "inherit" }).status === 0;
+}

package/src/util/json.ts

@@ -0,0 +1,7 @@
+/** The JSON object in a model's reply, tolerating prose or code fences around it. */
+export function extractJsonObject(raw: string): unknown {
+  const start = raw.indexOf("{");
+  const end = raw.lastIndexOf("}");
+  if (start === -1 || end <= start) throw new Error("no JSON object in the reply");
+  return JSON.parse(raw.slice(start, end + 1));
+}

package/src/util/key-defaults.ts

@@ -0,0 +1,34 @@
+// One key is enough. Whatever is unset is filled in from the key itself: an OpenRouter key
+// (sk-or-…) gets the models that worked best in our comparisons, an Anthropic key (sk-ant-…)
+// the Anthropic provider. OPENROUTER_API_KEY and ANTHROPIC_API_KEY are read when
+// DEBUTLY_API_KEY isn't set, so a key already in the environment for other tools just works.
+// Anything set explicitly wins. Entry points import this right after legacy-env.
+import { isOpenRouter } from "../llm/openrouter";
+
+export const OPENROUTER_DEFAULTS = {
+  DEBUTLY_PROVIDER: "openai-compatible",
+  DEBUTLY_BASE_URL: "https://openrouter.ai/api/v1",
+  DEBUTLY_NAV_MODEL: "z-ai/glm-5.3-flash",
+  DEBUTLY_JUDGE_MODEL: "deepseek/deepseek-v4-flash",
+  DEBUTLY_REASONING_EFFORT: "low",
+} as const;
+
+export function applyKeyDefaults(env: NodeJS.ProcessEnv = process.env): NodeJS.ProcessEnv {
+  // A borrowed key only goes where it belongs: OpenRouter's to OpenRouter, Anthropic's to
+  // Anthropic, never to an address set elsewhere.
+  if (!env.DEBUTLY_API_KEY) {
+    const toOpenRouter = !env.DEBUTLY_BASE_URL || isOpenRouter(env.DEBUTLY_BASE_URL);
+    if (env.OPENROUTER_API_KEY && toOpenRouter && (!env.DEBUTLY_PROVIDER || env.DEBUTLY_PROVIDER === "openai-compatible")) env.DEBUTLY_API_KEY = env.OPENROUTER_API_KEY;
+    else if (env.ANTHROPIC_API_KEY && (!env.DEBUTLY_PROVIDER || env.DEBUTLY_PROVIDER === "anthropic")) env.DEBUTLY_API_KEY = env.ANTHROPIC_API_KEY;
+  }
+  const key = env.DEBUTLY_API_KEY ?? "";
+  if (!env.DEBUTLY_PROVIDER && key.startsWith("sk-ant-")) env.DEBUTLY_PROVIDER = "anthropic";
+  const openRouter = key.startsWith("sk-or-") && (!env.DEBUTLY_PROVIDER || env.DEBUTLY_PROVIDER === "openai-compatible") && (!env.DEBUTLY_BASE_URL || isOpenRouter(env.DEBUTLY_BASE_URL));
+  if (openRouter) {
+    // Set to empty is set: DEBUTLY_REASONING_EFFORT= turns reasoning control off.
+    for (const [name, value] of Object.entries(OPENROUTER_DEFAULTS)) if (env[name] === undefined) env[name] = value;
+  }
+  return env;
+}
+
+applyKeyDefaults();

package/src/util/legacy-env.ts

@@ -0,0 +1,14 @@
+// Settings are named DEBUTLY_*. Debutly was called Wind Tunnel until 2026-10-03, and the WT_*
+// names from then still work: each fills in its DEBUTLY_* name when that one is unset.
+// Entry points import this module first, so the copy happens before anything reads a setting.
+
+export function adoptLegacyEnv(env: NodeJS.ProcessEnv = process.env): NodeJS.ProcessEnv {
+  for (const [name, value] of Object.entries(env)) {
+    if (!name.startsWith("WT_")) continue;
+    const current = `DEBUTLY_${name.slice(3)}`;
+    if (env[current] === undefined) env[current] = value;
+  }
+  return env;
+}
+
+adoptLegacyEnv();

package/src/util/load-env.ts

@@ -0,0 +1,28 @@
+// The debutly command reads a .env file in the folder it runs in, as `node --env-file=.env` does.
+// Variables already set win. Imported first, before anything reads a setting.
+import { existsSync, readFileSync } from "node:fs";
+import { parseEnv } from "node:util";
+
+const KEYS = ["DEBUTLY_API_KEY", "OPENROUTER_API_KEY", "ANTHROPIC_API_KEY"];
+// Where the key is sent. A .env may choose that only along with the key itself: one in a repo
+// you cloned must not send the key in your shell to an address of its choosing.
+const ENDPOINTS = ["DEBUTLY_BASE_URL", "ANTHROPIC_BASE_URL"];
+
+/** Adds the file's variables to `env`; returns a warning for each one left out. */
+export function loadEnv(text: string, env: NodeJS.ProcessEnv = process.env): string[] {
+  const file = parseEnv(text);
+  const used = KEYS.find((k) => env[k] || file[k]);
+  const keyFromFile = used !== undefined && !env[used] && !!file[used];
+  const warnings: string[] = [];
+  for (const [name, value] of Object.entries(file)) {
+    if (env[name] !== undefined) continue;
+    if (ENDPOINTS.includes(name) && used && !keyFromFile) {
+      warnings.push(`.env here sets ${name}, but your key comes from your environment, so that address is ignored. Put the key in the same .env to use it.`);
+      continue;
+    }
+    env[name] = value;
+  }
+  return warnings;
+}
+
+if (existsSync(".env")) for (const warning of loadEnv(readFileSync(".env", "utf8"))) console.warn(warning);

package/src/util/pool.ts

@@ -0,0 +1,12 @@
+export async function mapWithConcurrency<T, R>(items: T[], limit: number, fn: (item: T, index: number) => Promise<R>): Promise<R[]> {
+  const results = new Array<R>(items.length);
+  let next = 0;
+  const worker = async () => {
+    while (next < items.length) {
+      const i = next++;
+      results[i] = await fn(items[i], i);
+    }
+  };
+  await Promise.all(Array.from({ length: Math.min(limit, items.length) }, worker));
+  return results;
+}

package/src/util/run-origin.ts

@@ -0,0 +1,27 @@
+// Which app a run tested, what started it and where its CI run lives, so the dashboard can
+// group runs by app and link back. Flags win; in GitHub Actions the rest comes from its own
+// variables, so a workflow needs no extra arguments.
+
+export interface RunOrigin {
+  app?: string;
+  trigger?: string;
+  ciUrl?: string;
+}
+
+const TRIGGERS: Record<string, string> = {
+  schedule: "nightly",
+  pull_request: "pr",
+  pull_request_target: "pr",
+  workflow_dispatch: "manual",
+  push: "push",
+};
+
+export function runOrigin(flags: { app?: string; trigger?: string; ciUrl?: string }, env: Record<string, string | undefined>): RunOrigin {
+  const repo = env.GITHUB_REPOSITORY;
+  const inCi = !!(repo && env.GITHUB_RUN_ID);
+  return {
+    app: flags.app ?? (repo ? repo.split("/").pop() : undefined),
+    trigger: flags.trigger ?? (inCi ? (TRIGGERS[env.GITHUB_EVENT_NAME ?? ""] ?? env.GITHUB_EVENT_NAME) : "local"),
+    ciUrl: flags.ciUrl ?? (inCi ? `${env.GITHUB_SERVER_URL ?? "https://github.com"}/${repo}/actions/runs/${env.GITHUB_RUN_ID}` : undefined),
+  };
+}

package/src/worker.ts

@@ -0,0 +1,53 @@
+// A queue worker: runs agent sessions that runs (CLI with --queue, or later the API) enqueue.
+// Scale out by starting more (docker compose up --scale worker=4).
+//
+//   REDIS_URL=redis://… DATABASE_URL=postgres://… DEBUTLY_API_KEY=… npm run debutly:worker
+//
+// Optional: DEBUTLY_WORKER_CONCURRENCY (default 2), DEBUTLY_S3_* for screenshots (else ./runs/worker/<runId>).
+import "./util/legacy-env";
+import "./util/key-defaults";
+import { join } from "node:path";
+import { Worker } from "bullmq";
+import { chromium } from "playwright";
+import { S3ArtifactStore } from "./artifacts/s3";
+import { LocalArtifactStore } from "./artifacts/store";
+import { MailpitInbox } from "./inbox/mailpit";
+import { createLlms } from "./llm/factory";
+import { AGENT_QUEUE, redisClient, type AgentJob } from "./queue/jobs";
+import { createAgentProcessor } from "./queue/processor";
+import { openDb } from "./store/db";
+import { PgRunStore } from "./store/pg-run-store";
+import type { Session } from "./types";
+
+const { REDIS_URL, DATABASE_URL } = process.env;
+if (!REDIS_URL || !DATABASE_URL) {
+  console.error("The worker needs REDIS_URL and DATABASE_URL (and the DEBUTLY_* model settings).");
+  process.exit(1);
+}
+
+const { db, close } = await openDb(DATABASE_URL);
+const browser = await chromium.launch();
+const process_ = createAgentProcessor({
+  browser,
+  llm: createLlms(process.env).nav,
+  store: new PgRunStore(db),
+  artifactsFor: (runId) => S3ArtifactStore.fromEnv(process.env, runId) ?? new LocalArtifactStore(join("runs", "worker", runId)),
+  inboxFor: (url) => new MailpitInbox(url),
+});
+
+const worker = new Worker<AgentJob, Session>(AGENT_QUEUE, (job) => process_(job), {
+  connection: redisClient(REDIS_URL),
+  concurrency: Number(process.env.DEBUTLY_WORKER_CONCURRENCY ?? 2),
+});
+worker.on("completed", (job, s) => console.log(`[worker] ${job.data.persona.name} (${job.data.runId}): ${s.outcome}`));
+worker.on("failed", (job, err) => console.error(`[worker] ${job?.data.persona.name ?? "?"}: ${err.message}`));
+console.log(`[worker] waiting for agent sessions on ${AGENT_QUEUE}`);
+
+const shutdown = async () => {
+  await worker.close();
+  await browser.close();
+  await close();
+  process.exit(0);
+};
+process.on("SIGTERM", shutdown);
+process.on("SIGINT", shutdown);

package/tsconfig.json

@@ -0,0 +1,13 @@
+{
+  "compilerOptions": {
+    "target": "ES2022",
+    "module": "ESNext",
+    "moduleResolution": "Bundler",
+    "strict": true,
+    "esModuleInterop": true,
+    "skipLibCheck": true,
+    "noEmit": true,
+    "types": ["node"]
+  },
+  "include": ["src", "fixture", "tests", "scripts", "vitest.config.ts"]
+}