debutly 0.0.0-stage → 0.2.0

package/src/perception/run-perception.ts

@@ -0,0 +1,178 @@
+import { z } from "zod";
+import { Meter, type ModelPrice } from "../llm/meter";
+import type { LlmClient } from "../llm/types";
+import { generatePersonas } from "../persona/generate";
+import type { NewPrediction } from "../store/predictions";
+import type { Finding, Persona } from "../types";
+import { extractJsonObject } from "../util/json";
+import { mapWithConcurrency } from "../util/pool";
+import { aggregatePairs, aggregateVerdicts, rankObjections, type PairAggregate, type VerdictAggregate } from "./aggregate";
+import { comparePair, type PairJudgment } from "./compare";
+import { evaluate, type Verdict } from "./evaluate";
+import type { Exhibit } from "./exhibits";
+import { marketBrief, type MarketContext } from "./market-context";
+
+const PERSONA_BATCH = 25;
+
+export interface PerceptionOptions {
+  /** Cheap model for the evaluators. */
+  llm: LlmClient;
+  /** Stronger model for evaluator personas and the summary. */
+  judge: LlmClient;
+  icp: string;
+  product: string;
+  /** One key ("main") for a perception run; two (e.g. A and B) for an experiment. */
+  exhibitsByVariant: Record<string, Exhibit[]>;
+  evaluators: number;
+  concurrency: number;
+  market?: MarketContext;
+  price?: ModelPrice | null;
+  judgePrice?: ModelPrice | null;
+  /** Functional findings from a depth run, for the Launch Readiness verdict. */
+  functional?: Finding[];
+  rng?: () => number;
+}
+
+export interface Summary {
+  verdict: string;
+  fixes: { fix: string; addresses: string }[];
+}
+
+export interface PerceptionResult {
+  mode: "single" | "experiment";
+  variants: string[];
+  evaluators: number;
+  unusable: number;
+  verdicts: Verdict[];
+  pairs: PairJudgment[];
+  /** Per variant. In an experiment only objections are filled (from the pairwise judgments). */
+  aggregates: Record<string, VerdictAggregate>;
+  pairAggregate?: PairAggregate;
+  summary: Summary;
+  predictions: NewPrediction[];
+  usage: { inputTokens: number; outputTokens: number; costUsd?: number };
+  market?: MarketContext;
+}
+
+const SummarySchema = z.object({ verdict: z.string().min(1), fixes: z.array(z.object({ fix: z.string(), addresses: z.string() })) });
+
+const SUMMARY_SYSTEM = `You write the verdict of a Launch Readiness report from simulated buyer evidence.
+Use only the numbers and quotes you are given; never invent statistics, percentages or conversion rates. Scores are on a 1–5 scale. This is simulated evidence, not real traction: say what the simulated buyers did, comparatively.
+Reply with one JSON object: {"verdict": 3-5 plain sentences, "fixes": [{"fix": a concrete change to the product or its pages, "addresses": the objection category or finding it answers}]} with at most 5 fixes, most important first.`;
+
+async function evaluatorPersonas(judge: LlmClient, icp: string, market: string | undefined, count: number): Promise<Persona[]> {
+  const out: Persona[] = [];
+  while (out.length < count) {
+    const n = Math.min(PERSONA_BATCH, count - out.length);
+    const batch = await generatePersonas(judge, market ? `${icp}\n\n${market}` : icp, n);
+    out.push(...batch.map((p, i) => ({ ...p, id: `e${out.length + i + 1}` })));
+  }
+  return out;
+}
+
+const emptyAggregate = (evaluators: number, objections: VerdictAggregate["objections"]): VerdictAggregate => ({
+  evaluators,
+  unusable: 0,
+  objections,
+  intent: { no: 0, maybe: 0, yes: 0 },
+  meanScores: { clarity: 0, credibility: 0, value: 0, urgency: 0 },
+  wouldPay: { answers: 0, medianUsd: null, plans: {} },
+  confusions: [],
+});
+
+function predictionsFor(r: Omit<PerceptionResult, "predictions" | "summary" | "usage">): NewPrediction[] {
+  const out: NewPrediction[] = [];
+  if (r.mode === "experiment" && r.pairAggregate) {
+    const p = r.pairAggregate;
+    const other = r.variants.find((v) => v !== p.leader) ?? "?";
+    out.push({
+      metric: "winner",
+      claim: `${p.leader} beats ${other} (${Math.round(p.winRate * 100)}% of buyers who chose; 95% interval ${Math.round(p.interval.low * 100)}–${Math.round(p.interval.high * 100)}%${p.decisive ? "" : ", not decisive"})`,
+      predicted: { winner: p.leader, winRate: p.winRate, interval: p.interval, decisive: p.decisive },
+    });
+    return out;
+  }
+  const a = r.aggregates[r.variants[0]];
+  const top = a.objections[0];
+  if (top) out.push({ metric: "top_objection", claim: `The top objection is ${top.category} (${top.buyers} of ${a.evaluators} buyers)`, predicted: { category: top.category, buyers: top.buyers, of: a.evaluators } });
+  const share = a.evaluators ? (a.intent.yes + a.intent.maybe) / a.evaluators : 0;
+  out.push({ metric: "intent_share", claim: `${Math.round(share * 100)}% of simulated buyers would consider it (not a conversion rate)`, predicted: { share, of: a.evaluators } });
+  return out;
+}
+
+/** Breadth evaluators judge the exhibits; an experiment compares two variants pairwise. */
+export async function runPerception(o: PerceptionOptions): Promise<PerceptionResult> {
+  const variants = Object.keys(o.exhibitsByVariant);
+  if (!variants.length || variants.length > 2) throw new Error("give one set of exhibits, or two variants to compare");
+  const nav = new Meter();
+  const judgeMeter = new Meter();
+  const brief = o.market ? marketBrief(o.market) : undefined;
+  const personas = await evaluatorPersonas({ complete: (req) => o.judge.complete({ ...req, onUsage: judgeMeter.add }) }, o.icp, brief, o.evaluators);
+
+  let base: Omit<PerceptionResult, "predictions" | "summary" | "usage">;
+  if (variants.length === 1) {
+    const [only] = variants;
+    const results = await mapWithConcurrency(personas, o.concurrency, (p) => evaluate(o.llm, p, o.exhibitsByVariant[only], { market: brief, onUsage: nav.add }));
+    const verdicts = results.filter((v): v is Verdict => v !== null);
+    const unusable = results.length - verdicts.length;
+    base = { mode: "single", variants, evaluators: personas.length, unusable, verdicts, pairs: [], aggregates: { [only]: aggregateVerdicts(verdicts, unusable) }, market: o.market };
+  } else {
+    const [a, b] = variants;
+    const rng = o.rng ?? Math.random;
+    const results = await mapWithConcurrency(personas, o.concurrency, (p) =>
+      comparePair(o.llm, p, o.exhibitsByVariant[a], o.exhibitsByVariant[b], { market: brief, onUsage: nav.add }, rng),
+    );
+    const pairs = results.filter((x): x is PairJudgment => x !== null);
+    base = {
+      mode: "experiment",
+      variants,
+      evaluators: personas.length,
+      unusable: results.length - pairs.length,
+      verdicts: [],
+      pairs,
+      aggregates: {
+        [a]: emptyAggregate(pairs.length, rankObjections(pairs.map((p) => p.objectionsA))),
+        [b]: emptyAggregate(pairs.length, rankObjections(pairs.map((p) => p.objectionsB))),
+      },
+      pairAggregate: aggregatePairs(pairs),
+      market: o.market,
+    };
+  }
+
+  const evidence = {
+    product: o.product,
+    buyers: o.icp,
+    mode: base.mode,
+    evaluators: base.evaluators,
+    unusable: base.unusable,
+    perVariant: base.aggregates,
+    experiment: base.pairAggregate,
+    functionalBlockers: (o.functional ?? []).filter((f) => f.severity !== "minor" && !f.sandboxCause).map((f) => `${f.severity}: ${f.title}`),
+    market: brief,
+  };
+  let summary: Summary = { verdict: "Summary unavailable.", fixes: [] };
+  try {
+    const raw = await o.judge.complete({
+      system: SUMMARY_SYSTEM,
+      messages: [{ role: "user", content: `Evidence:\n${JSON.stringify(evidence, null, 2)}` }],
+      maxTokens: 1500,
+      onUsage: judgeMeter.add,
+    });
+    summary = SummarySchema.parse(extractJsonObject(raw));
+  } catch (err) {
+    console.warn(`Launch Readiness summary unavailable: ${err instanceof Error ? err.message : err}`);
+  }
+
+  const navCost = nav.costUsd(o.price);
+  const judgeCost = judgeMeter.costUsd(o.judgePrice);
+  return {
+    ...base,
+    summary,
+    predictions: predictionsFor(base),
+    usage: {
+      inputTokens: nav.total.inputTokens + judgeMeter.total.inputTokens,
+      outputTokens: nav.total.outputTokens + judgeMeter.total.outputTokens,
+      costUsd: navCost !== undefined && judgeCost !== undefined ? navCost + judgeCost : undefined,
+    },
+  };
+}

package/src/persona/generate.ts

@@ -0,0 +1,92 @@
+import { z } from "zod";
+import type { LlmClient } from "../llm/types";
+import { languageRequest } from "./language";
+import type { CustomerUnit, Persona } from "../types";
+
+const PersonaSchema = z.object({
+  // A period would break "You are <name>." in the agent prompt.
+  name: z
+    .string()
+    .min(1)
+    .refine((n) => !n.includes("."), "name must not contain a period"),
+  background: z.string(),
+  goal: z.string(),
+  patience: z.enum(["low", "medium", "high"]),
+  techSavvy: z.enum(["low", "medium", "high"]),
+  // Asked for only with --languages; withLanguages() keeps it only if it's on the list.
+  language: z
+    .string()
+    .transform((l) => l.trim().toLowerCase())
+    .optional(),
+});
+
+const BusinessUnitSchema = z.object({
+  business: z.object({
+    name: z.string().min(1),
+    archetype: z.string().min(1),
+    size: z.number().int().positive(),
+    monthlyBudget: z.number().nonnegative(),
+    goal: z.string(),
+  }),
+  owner: PersonaSchema,
+  employees: z.array(PersonaSchema.extend({ role: z.string().min(1) })),
+});
+
+// Personas describe customers, not their beliefs: no politics or religion.
+const NEUTRAL = "Leave out politics and religion.";
+const SYSTEM = `You design realistic, diverse customer personas for product simulation. Vary role, company size, budget, urgency, patience and technical skill. Include skeptical and hard-to-please people, not just enthusiasts. ${NEUTRAL} Output only a JSON array.`;
+const INDIVIDUAL_SYSTEM = `You design realistic, diverse individual consumers for product simulation. Vary life and career stage, urgency, price sensitivity, region and technical skill. Include skeptical and hard-to-please people, not just enthusiasts. ${NEUTRAL} Output only a JSON array.`;
+const BUSINESS_SYSTEM = `You design realistic, diverse businesses for product simulation. Vary archetype (agency, SME, small business, freelancer, startup, enterprise team), industry, size, budget and urgency. Give employees distinct everyday roles. Include skeptical and hard-to-please people, not just enthusiasts. ${NEUTRAL} Output only a JSON array.`;
+const PERSONA_SHAPE = `{"name": first name only, "background": 1-2 sentences, "goal": what they want from this product, "patience": "low"|"medium"|"high", "techSavvy": "low"|"medium"|"high"}`;
+
+function extractJsonArray(raw: string): unknown {
+  const start = raw.indexOf("[");
+  const end = raw.lastIndexOf("]");
+  if (start === -1 || end <= start) throw new Error("no JSON array in persona reply");
+  return JSON.parse(raw.slice(start, end + 1));
+}
+
+async function ask(llm: LlmClient, system: string, content: string): Promise<unknown> {
+  return extractJsonArray(await llm.complete({ system, messages: [{ role: "user", content }], maxTokens: 16000 }));
+}
+
+export async function generatePersonas(llm: LlmClient, icp: string, count: number, system = SYSTEM, languages: string[] = []): Promise<Persona[]> {
+  const raw = await ask(llm, system, `Ideal customer profile:\n${icp}\n\nReturn a JSON array of ${count} distinct personas. Each item: ${PERSONA_SHAPE}.${withRequest(languages)}`);
+  const parsed = z.array(PersonaSchema).parse(raw);
+  return parsed.slice(0, count).map((persona, i) => ({ id: `p${i + 1}`, ...persona }));
+}
+
+export interface UnitOptions {
+  kind: CustomerUnit["kind"];
+  count: number;
+  /** Employees per business crew (0 = owner only). */
+  employees: number;
+  /** First languages to give the people (lang codes); none means English. */
+  languages?: string[];
+}
+
+const withRequest = (languages: string[]) => (languages.length ? ` ${languageRequest(languages)}` : "");
+
+/** Customer units from an ICP: individuals (B2C) or business crews of owner + employees (B2B). */
+export async function generateUnits(llm: LlmClient, icp: string, o: UnitOptions): Promise<CustomerUnit[]> {
+  if (o.kind === "individual") {
+    const people = await generatePersonas(llm, icp, o.count, INDIVIDUAL_SYSTEM, o.languages);
+    return people.map((person) => ({ kind: "individual", id: person.id, person }));
+  }
+  const raw = await ask(
+    llm,
+    BUSINESS_SYSTEM,
+    `Ideal customer profile:\n${icp}\n\nReturn a JSON array of ${o.count} distinct businesses. Each item: {"business": {"name": string, "archetype": string, "size": number of people, "monthlyBudget": number in USD they would spend on a tool like this, "goal": string}, "owner": the decision-maker, ${PERSONA_SHAPE}, "employees": ${o.employees} employees, each ${PERSONA_SHAPE.slice(0, -1)}, "role": their job title}}.${withRequest(o.languages ?? [])}`,
+  );
+  const parsed = z.array(BusinessUnitSchema).parse(raw);
+  return parsed.slice(0, o.count).map((u, i) => {
+    const id = `u${i + 1}`;
+    return {
+      kind: "business",
+      id,
+      business: u.business,
+      owner: { id: `${id}-o`, ...u.owner, role: "Owner" },
+      employees: u.employees.slice(0, o.employees).map((e, j) => ({ id: `${id}-e${j + 1}`, ...e })),
+    };
+  });
+}

package/src/persona/inclusion.ts

@@ -0,0 +1,53 @@
+import type { Browser, BrowserContext, Page } from "playwright";
+import { INCLUSION_NEEDS, type CustomerUnit, type InclusionNeed, type Persona } from "../types";
+
+const BRIEFS: Record<InclusionNeed, string> = {
+  screen_reader:
+    'You are blind and use a screen reader with your keyboard; you cannot use a mouse. You only know what it announces: element roles and names, and the page text. A button or link with an empty name is announced only as "button" or "link", so you cannot tell what it does; avoid it unless you have no other choice. You reach controls with the Tab key, so one the keyboard cannot reach is one you cannot use.',
+  low_vision:
+    "You have low vision and browse zoomed in to 200%, so pages look cramped and you often have to scroll. Small or low-contrast controls are hard for you to find.",
+  slow_connection:
+    "You are on a slow, unreliable mobile connection. Pages take several seconds to load; you wait a while before assuming something is broken, but not forever.",
+  non_native_english:
+    "English is not your first language. You read carefully and slowly, jargon and idioms confuse you, and you sometimes make small typing mistakes.",
+};
+
+export const inclusionBrief = (need: InclusionNeed) => BRIEFS[need];
+
+/** Assigns one need to each of the first agents of a run (owner before employees), starting at `offset` so needs rotate between runs. */
+export function withInclusion(units: CustomerUnit[], offset: number): CustomerUnit[] {
+  let next = 0;
+  const assign = (p: Persona): Persona => {
+    if (next >= INCLUSION_NEEDS.length) return p;
+    const inclusion = INCLUSION_NEEDS[(offset + next++) % INCLUSION_NEEDS.length];
+    return { ...p, inclusion };
+  };
+  return units.map((u) =>
+    u.kind === "individual" ? { ...u, person: assign(u.person) } : { ...u, owner: assign(u.owner), employees: u.employees.map(assign) },
+  );
+}
+
+// Chrome DevTools' "Slow 3G"-like: 400 ms round trips, 500 kbit/s each way.
+const SLOW_NETWORK = { offline: false, latency: 400, downloadThroughput: (500 * 1024) / 8, uploadThroughput: (500 * 1024) / 8 };
+const SLOW_NAVIGATION_MS = 90_000;
+
+/** A fresh browser context and page for one agent, shaped by its inclusion need and first language. */
+export async function openAgentPage(browser: Browser, need?: InclusionNeed, language?: string): Promise<{ context: BrowserContext; page: Page }> {
+  const context = await browser.newContext({
+    // 200% zoom on a 1280×800 laptop lays pages out like a 640×400 viewport.
+    ...(need === "low_vision" ? { viewport: { width: 640, height: 400 } } : {}),
+    // Someone whose first language isn't English usually has their browser set to it, so the
+    // site hears it in Accept-Language and navigator.language, as it would from them.
+    ...(language && language !== "en" ? { locale: language } : {}),
+  });
+  const page = await context.newPage();
+  if (need === "slow_connection") {
+    const cdp = await context.newCDPSession(page);
+    await cdp.send("Network.enable");
+    await cdp.send("Network.emulateNetworkConditions", SLOW_NETWORK);
+    // At 500 kbit/s a page can take longer than Playwright's 30 s default to arrive; a person on
+    // that connection waits, so the agent's navigations do too.
+    page.setDefaultNavigationTimeout(SLOW_NAVIGATION_MS);
+  }
+  return { context, page };
+}

package/src/persona/language.ts

@@ -0,0 +1,59 @@
+import type { CustomerUnit, Persona } from "../types";
+
+// Customers' first languages (--languages ar,en): agents take them in turn, and one whose first
+// language isn't English uses the site in it when it can. Arabic speakers also type as an Arabic
+// phone keyboard does, which is where Gulf sign-up forms often break.
+
+const NAMES: Record<string, string> = {
+  ar: "Arabic",
+  en: "English",
+  fr: "French",
+  de: "German",
+  es: "Spanish",
+  hi: "Hindi",
+  ur: "Urdu",
+  fa: "Persian",
+  tr: "Turkish",
+  ru: "Russian",
+  zh: "Chinese",
+};
+
+export const languageName = (code: string) => NAMES[code] ?? code;
+
+/** "ar, en" → ["ar", "en"]. Language codes as in a page's lang attribute. */
+export function parseLanguages(value: string | undefined): string[] {
+  if (!value?.trim()) return [];
+  const codes = value.split(",").map((c) => c.trim().toLowerCase()).filter(Boolean);
+  const bad = codes.find((c) => !/^[a-z]{2,3}$/.test(c));
+  if (bad) throw new Error(`--languages takes language codes such as ar,en (got "${bad}")`);
+  return [...new Set(codes)];
+}
+
+/** What the persona generator is told, so each person gets a first language and a name to match. */
+export function languageRequest(languages: string[]): string {
+  if (!languages.length) return "";
+  const list = languages.map((c) => `"${c}" (${languageName(c)})`).join(", ");
+  return `Give each person a "language": their first language, one of ${list}, mixing them across the people, and a name that fits it.`;
+}
+
+/** Each person gets one of the languages: the one the generator gave, if it's on the list, or the next in turn. */
+export function withLanguages(units: CustomerUnit[], languages: string[]): CustomerUnit[] {
+  if (!languages.length) return units;
+  let next = 0;
+  const assign = (p: Persona): Persona => {
+    const chosen = p.language && languages.includes(p.language) ? p.language : languages[next % languages.length];
+    next++;
+    return { ...p, language: chosen };
+  };
+  return units.map((u) =>
+    u.kind === "individual" ? { ...u, person: assign(u.person) } : { ...u, owner: assign(u.owner), employees: u.employees.map(assign) },
+  );
+}
+
+/** The agent's instructions for its first language; nothing for English. */
+export function languageBrief(code: string | undefined): string {
+  if (!code || code === "en") return "";
+  if (code === "ar")
+    return "Your first language is Arabic. If the site offers Arabic, switch to it. Write your name in Arabic script, and type numbers such as phone numbers the way your phone's Arabic keyboard does, with Arabic-Indic digits (٠١٢٣٤٥٦٧٨٩), unless the field shows another format.";
+  return `Your first language is ${languageName(code)}. If the site offers ${languageName(code)}, switch to it.`;
+}

package/src/persona/people.ts

@@ -0,0 +1,32 @@
+import { readFile } from "node:fs/promises";
+import type { CustomerUnit } from "../types";
+
+/** Each run writes its people here, next to the report, so a later run can meet the same ones. */
+export const PEOPLE_FILE = "people.json";
+
+/**
+ * The people of an earlier run, to run the same agents again: after a fix, whether it worked is
+ * only clear with the same people, needs and languages. Generated afresh, a run had a different
+ * crew every day (needs rotate daily), and two runs of one app could not be compared.
+ */
+export async function readPeople(path: string): Promise<CustomerUnit[]> {
+  let parsed: unknown;
+  try {
+    parsed = JSON.parse(await readFile(path, "utf8"));
+  } catch (err) {
+    throw new Error(`${path} can't be read: ${err instanceof Error ? err.message : err}`);
+  }
+  const ok = (u: unknown) => {
+    const unit = u as Partial<CustomerUnit> | null;
+    if (!unit || typeof unit !== "object" || typeof unit.id !== "string") return false;
+    if (unit.kind === "individual") return !!unit.person?.id;
+    return unit.kind === "business" && !!unit.owner?.id && Array.isArray(unit.employees);
+  };
+  if (!Array.isArray(parsed) || !parsed.length || !parsed.every(ok)) {
+    throw new Error(`${path} doesn't hold a run's people: use the ${PEOPLE_FILE} from an earlier run's report folder.`);
+  }
+  return parsed as CustomerUnit[];
+}
+
+/** How many agents the units make: everyone in a crew is one. */
+export const peopleCount = (units: CustomerUnit[]) => units.reduce((n, u) => n + (u.kind === "individual" ? 1 : 1 + u.employees.length), 0);

package/src/predictions.ts

@@ -0,0 +1,33 @@
+// The Calibration Loop from the command line:
+//   npm run debutly:predictions -- list [--unresolved] --store <url>
+//   npm run debutly:predictions -- record <predictionId> <actual> --store <url>   e.g. B, or price
+//   npm run debutly:predictions -- accuracy --store <url>
+import { parseArgs } from "node:util";
+import { openDb } from "./store/db";
+import { PgPredictionStore } from "./store/predictions";
+
+const { values, positionals } = parseArgs({
+  allowPositionals: true,
+  options: { store: { type: "string" }, unresolved: { type: "boolean", default: false } },
+});
+const [command, id, actual] = positionals;
+const url = values.store ?? process.env.DATABASE_URL;
+if (!url || !["list", "record", "accuracy"].includes(command ?? "") || (command === "record" && (!id || !actual))) {
+  console.error("Usage: npm run debutly:predictions -- list [--unresolved] | record <id> <actual> | accuracy   --store postgres://… (or DATABASE_URL)");
+  process.exit(1);
+}
+const { db, close } = await openDb(url);
+const store = new PgPredictionStore(db);
+if (command === "list") {
+  for (const p of await store.list({ unresolved: values.unresolved })) {
+    const outcome = p.checkedAt ? `actual ${JSON.stringify(p.actual)} → ${p.correct === null ? "not scored" : p.correct ? "correct" : "wrong"}` : "awaiting outcome";
+    console.log(`${p.id}  [${p.metric}] ${p.claim}  (${outcome})`);
+  }
+} else if (command === "record") {
+  const p = await store.record(id, actual);
+  console.log(`Recorded: ${p.claim} → actual ${actual}: ${p.correct === null ? "stored (not scored yet)" : p.correct ? "correct" : "wrong"}`);
+} else {
+  const a = await store.accuracy();
+  console.log(a.rate === null ? "No scored predictions yet." : `Accuracy: ${a.correct} of ${a.scored} correct (${Math.round(a.rate * 100)}%)`);
+}
+await close();

package/src/queue/dispatch.ts

@@ -0,0 +1,18 @@
+import type { Queue, QueueEvents } from "bullmq";
+import type { RunAgent } from "../crew/run-unit";
+import type { Session } from "../types";
+import type { AgentJob } from "./jobs";
+
+/**
+ * A RunAgent that queues the session for a worker and waits for its result. Crew order is
+ * unchanged: runUnit still waits for the owner before queuing the employees.
+ */
+export function queueDispatch(queue: Queue, events: QueueEvents, base: Omit<AgentJob, "persona" | "goal" | "brief">): RunAgent {
+  return async (persona, goal, brief) => {
+    const job = await queue.add("agent-session", { ...base, persona, goal, brief } satisfies AgentJob, {
+      removeOnComplete: 1000,
+      removeOnFail: 1000,
+    });
+    return (await job.waitUntilFinished(events)) as Session;
+  };
+}

package/src/queue/jobs.ts

@@ -0,0 +1,28 @@
+import { Redis } from "ioredis";
+import type { ModelPrice } from "../llm/meter";
+import type { Persona } from "../types";
+
+export const AGENT_QUEUE = "debutly-agent-sessions";
+
+/**
+ * One agent session to run on a worker. No secrets: workers take the LLM key, database and
+ * artifact credentials from their own environment.
+ */
+export interface AgentJob {
+  runId: string;
+  targetUrl: string;
+  persona: Persona;
+  goal: string;
+  brief?: string;
+  maxSteps: number;
+  caps?: { maxTokens?: number; maxMs?: number };
+  price?: ModelPrice | null;
+  /** The run's test inbox (Mailpit), for magic links and invites. */
+  mailpitUrl?: string;
+  emailDomain: string;
+  /** Keeps agent email addresses the same whichever worker runs them. */
+  runTag: string;
+}
+
+/** A Redis client for BullMQ (it needs one it can block on: no request retry limit). */
+export const redisClient = (url: string) => new Redis(url, { maxRetriesPerRequest: null });

package/src/queue/processor.ts

@@ -0,0 +1,42 @@
+import type { Job } from "bullmq";
+import type { Browser } from "playwright";
+import { runAgentSession } from "../agent/run-agent";
+import type { ArtifactStore } from "../artifacts/store";
+import type { Inbox } from "../inbox/types";
+import type { LlmClient } from "../llm/types";
+import { agentEmail } from "../run";
+import type { RunStore } from "../store/run-store";
+import type { Session } from "../types";
+import type { AgentJob } from "./jobs";
+
+export interface ProcessorDeps {
+  browser: Browser;
+  llm: LlmClient;
+  store: RunStore;
+  artifactsFor: (runId: string) => ArtifactStore;
+  inboxFor: (mailpitUrl: string) => Inbox;
+}
+
+/** Runs one queued agent session, recording it to the store, and returns the session. */
+export function createAgentProcessor(d: ProcessorDeps): (job: Pick<Job<AgentJob>, "data">) => Promise<Session> {
+  return async ({ data: j }) =>
+    runAgentSession(
+      {
+        browser: d.browser,
+        llm: d.llm,
+        origin: new URL(j.targetUrl).origin,
+        maxSteps: j.maxSteps,
+        artifacts: d.artifactsFor(j.runId),
+        emailOf: (p) => agentEmail(p, j.runTag, j.emailDomain),
+        inbox: j.mailpitUrl ? d.inboxFor(j.mailpitUrl) : undefined,
+        caps: j.caps,
+        price: j.price,
+        shouldStop: () => d.store.isCancelled(j.runId),
+        store: d.store,
+        runId: j.runId,
+      },
+      j.persona,
+      j.goal,
+      j.brief,
+    );
+}

package/src/report/fix-first.ts

@@ -0,0 +1,121 @@
+import { UNCOMPRESSED } from "../detect/findings";
+import type { Finding, Session } from "../types";
+
+/** At most this many items, so the section reads in a couple of minutes. */
+const MAX_ITEMS = 5;
+const MAX_QUOTE = 200;
+const RANK: Record<Finding["severity"], number> = { blocker: 0, major: 1, minor: 2 };
+
+// Agents that didn't reach the goal because of the app, or seemingly so. An engine error or a
+// cancelled run says nothing about the app; a run cut short for credit has its own section.
+const STOPPED: Partial<Record<Session["outcome"], string>> = {
+  gave_up: "gave up",
+  budget_exhausted: "ran out of steps",
+  not_invited: "never got an invite",
+};
+
+/** What to do about a finding, in a line. */
+export function nextStep(f: Finding): string {
+  switch (f.kind) {
+    case "http_error": {
+      const status = Number(/^HTTP (\d{3})/.exec(f.title)?.[1] ?? 0);
+      return status >= 500
+        ? "The server failed: its log for this request says why."
+        : "The server refused the request: compare what the page sent with what the server expects.";
+    }
+    case "page_error":
+      return "Fix the JavaScript error; the reproduction below shows the step that sets it off.";
+    case "console_error":
+      return "Read the error in the browser console on that page.";
+    case "a11y_unlabeled":
+      if (f.control?.problem === "placeholder") return "Give the field a visible <label>, and keep the placeholder as an example.";
+      if (f.control?.problem === "described") return "Name it: visible text or an aria-label, with the description kept as a hint.";
+      if (f.control && !/^(button|link|menuitem|tab|summary)/.test(f.control.role)) return "Give each one a <label> (or an aria-label) that says what goes in it.";
+      return 'Give each one visible text, or an aria-label that says what it does (aria-label="Open menu" on a menu icon).';
+    case "stuck":
+      return "Watch the reproduction below: the agent repeated an action that seemed to do nothing. Make the page show that it worked, or why it didn't.";
+    case "slow_load":
+      return f.detail.includes(UNCOMPRESSED)
+        ? "Turn on compression (gzip or Brotli) for scripts and styles, or check that the real site has it."
+        : "Find what holds the page up: the agents sat waiting for it.";
+    case "covered":
+      return "Move or close whatever lies over the control, so it can be clicked.";
+    case "keyboard":
+      if (f.signature.startsWith("enter:")) return "Let Enter press it too: look for a key handler that catches Enter and stops it (a page-wide shortcut, a form's own Enter handling).";
+      return 'Use a real <button> or <a>, or add tabindex="0" and handle Enter and Space.';
+    case "focus_obscured":
+      return "Keep the control with keyboard focus in view: move or shrink what covers it.";
+    case "reflow":
+      return "Let the page wrap at narrow widths instead of scrolling sideways: look for fixed widths.";
+    case "language":
+      if (f.signature.startsWith("dir:")) return 'Set dir="rtl" on Arabic pages (on <html>, or on the part that holds the Arabic).';
+      if (f.signature === "lang:missing") return 'Add a lang attribute to the <html> element, e.g. lang="en".';
+      return 'Set lang="ar" on Arabic pages.';
+    case "zoom":
+      return "Remove user-scalable=no and maximum-scale from the page's viewport meta tag.";
+  }
+}
+
+const lastPath = (s: Session) => {
+  try {
+    return new URL(s.steps.at(-1)?.observation.url ?? "").pathname;
+  } catch {
+    return "";
+  }
+};
+
+const quote = (s: string) => `"${s.length > MAX_QUOTE ? `${s.slice(0, MAX_QUOTE)}…` : s}"`;
+
+/** A screenshot as the report's reader finds it: next to the report, not on the machine that ran it. */
+export function shotPath(path: string): string {
+  if (/^https?:/.test(path)) return path;
+  const at = path.replace(/\\/g, "/").lastIndexOf("/screenshots/");
+  return at >= 0 ? path.replace(/\\/g, "/").slice(at + 1) : path;
+}
+
+function stoppedItem(s: Session, needs: string, environment: boolean): string {
+  const who = [s.role, needs !== "—" ? needs : ""].filter(Boolean).join("; ");
+  const where = lastPath(s) ? ` on ${lastPath(s)}` : "";
+  const shot = [...s.steps].reverse().find((st) => st.screenshotPath)?.screenshotPath;
+  // Out of steps, the note only says so; the agent's last thought says where it was.
+  const said = s.outcome === "budget_exhausted" && s.steps.at(-1)?.reasoning ? `Its last thought: ${quote(s.steps.at(-1)!.reasoning)}` : `In the agent's words: ${quote(s.outcomeNote)}`;
+  return [
+    `**${s.personaName}${who ? ` (${who})` : ""} ${STOPPED[s.outcome]}${where}.** ${said}`,
+    environment ? "Part of it may be the test environment: see the end of the report." : "",
+    shot ? `Start with the last screenshot, ${shotPath(shot)}.` : "",
+  ]
+    .filter(Boolean)
+    .join(" ");
+}
+
+/**
+ * The few things to look at first: agents the app seemingly stopped, in their own words, then the
+ * blocker and major findings, those that stopped an agent first, each with what to do.
+ */
+export function renderFixFirst(sessions: Session[], findings: Finding[], needLabel: (s: Session) => string): string[] {
+  const stopped = sessions.filter((s) => STOPPED[s.outcome]);
+  const failed = new Set(stopped.map((s) => s.agentId));
+  const caused = findings.filter((f) => f.sandboxCause);
+  const serious = findings
+    .filter((f) => !f.sandboxCause && f.confidence !== "low" && f.severity !== "minor")
+    .map((f, order) => ({ f, order, stoppedSome: f.hitBy.some((id) => failed.has(id)) }))
+    // Most severe first; within that, one an agent didn't get past.
+    .sort((a, b) => RANK[a.f.severity] - RANK[b.f.severity] || Number(b.stoppedSome) - Number(a.stoppedSome) || a.order - b.order)
+    .map(({ f }) => f);
+  const reached = sessions.filter((s) => s.outcome === "success").length;
+  const stops = stopped.slice(0, MAX_ITEMS).map((s) => stoppedItem(s, needLabel(s), caused.some((f) => f.hitBy.includes(s.agentId))));
+  const fixes = serious.slice(0, MAX_ITEMS - stops.length).map((f) => `**${f.title}** (${f.severity}). ${nextStep(f)}`);
+  const shown = [...stops, ...fixes];
+  const rest = findings.filter((f) => !f.sandboxCause).length - fixes.length;
+  return [
+    `## Fix these first`,
+    "",
+    `${reached} of ${sessions.length} agents reached the goal.`,
+    "",
+    ...(shown.length
+      ? shown.map((item, i) => `${i + 1}. ${item}`)
+      : ["Nothing stopped an agent, and nothing serious was found."]),
+    ...(rest > 0 ? ["", `${rest === 1 ? "One" : rest}${fixes.length ? " more" : ""} ${rest === 1 ? "finding is" : "findings are"} below.`] : []),
+    "",
+  ];
+}