debutly 0.0.0-stage → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +662 -0
- package/README.md +192 -2
- package/bin/debutly.mjs +8 -0
- package/drizzle/0000_init.sql +107 -0
- package/drizzle/0001_predictions.sql +14 -0
- package/drizzle/0002_dashboard.sql +15 -0
- package/drizzle/meta/0000_snapshot.json +722 -0
- package/drizzle/meta/0001_snapshot.json +821 -0
- package/drizzle/meta/0002_snapshot.json +942 -0
- package/drizzle/meta/_journal.json +27 -0
- package/package.json +83 -4
- package/src/agent/describe.ts +29 -0
- package/src/agent/effect.ts +37 -0
- package/src/agent/loop.ts +232 -0
- package/src/agent/parse-reply.ts +76 -0
- package/src/agent/prompt.ts +76 -0
- package/src/agent/run-agent.ts +68 -0
- package/src/artifacts/db.ts +55 -0
- package/src/artifacts/s3.ts +36 -0
- package/src/artifacts/store.ts +26 -0
- package/src/bin.ts +140 -0
- package/src/browser/ax.ts +77 -0
- package/src/browser/execute.ts +338 -0
- package/src/browser/observe.ts +56 -0
- package/src/browser/reflow.ts +73 -0
- package/src/browser/samples.ts +119 -0
- package/src/browser/scan-script.ts +62 -0
- package/src/browser/settle.ts +32 -0
- package/src/browser/signals.ts +73 -0
- package/src/cancel.ts +20 -0
- package/src/cli.ts +265 -0
- package/src/config.ts +62 -0
- package/src/cost/estimate.ts +32 -0
- package/src/cost/prices.ts +32 -0
- package/src/cost/tiers.ts +27 -0
- package/src/costs.ts +96 -0
- package/src/crew/run-unit.ts +62 -0
- package/src/detect/findings.ts +617 -0
- package/src/detect/wcag.ts +38 -0
- package/src/doctor.ts +145 -0
- package/src/hear-cli.ts +137 -0
- package/src/hear.ts +309 -0
- package/src/import.ts +108 -0
- package/src/inbox/mailpit.ts +46 -0
- package/src/inbox/types.ts +10 -0
- package/src/init.ts +183 -0
- package/src/llm/anthropic.ts +41 -0
- package/src/llm/credits.ts +35 -0
- package/src/llm/factory.ts +65 -0
- package/src/llm/meter.ts +39 -0
- package/src/llm/openai-compatible.ts +90 -0
- package/src/llm/openrouter.ts +9 -0
- package/src/llm/scripted.ts +18 -0
- package/src/llm/types.ts +36 -0
- package/src/perceive.ts +150 -0
- package/src/perception/aggregate.ts +111 -0
- package/src/perception/compare.ts +77 -0
- package/src/perception/evaluate.ts +83 -0
- package/src/perception/exhibits.ts +146 -0
- package/src/perception/market-context.ts +50 -0
- package/src/perception/run-perception.ts +178 -0
- package/src/persona/generate.ts +92 -0
- package/src/persona/inclusion.ts +53 -0
- package/src/persona/language.ts +59 -0
- package/src/predictions.ts +33 -0
- package/src/queue/dispatch.ts +18 -0
- package/src/queue/jobs.ts +28 -0
- package/src/queue/processor.ts +42 -0
- package/src/report/launch-readiness.ts +106 -0
- package/src/report/markdown.ts +200 -0
- package/src/run.ts +139 -0
- package/src/sandbox/outbox.ts +65 -0
- package/src/store/db.ts +40 -0
- package/src/store/import-run.ts +116 -0
- package/src/store/pg-run-store.ts +163 -0
- package/src/store/predictions.ts +70 -0
- package/src/store/run-store.ts +78 -0
- package/src/store/schema.ts +193 -0
- package/src/store/url.ts +15 -0
- package/src/types.ts +187 -0
- package/src/util/chromium.ts +20 -0
- package/src/util/json.ts +7 -0
- package/src/util/key-defaults.ts +34 -0
- package/src/util/legacy-env.ts +14 -0
- package/src/util/load-env.ts +28 -0
- package/src/util/pool.ts +12 -0
- package/src/util/run-origin.ts +27 -0
- package/src/worker.ts +53 -0
- package/tsconfig.json +13 -0
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
{
|
|
2
|
+
"version": "7",
|
|
3
|
+
"dialect": "postgresql",
|
|
4
|
+
"entries": [
|
|
5
|
+
{
|
|
6
|
+
"idx": 0,
|
|
7
|
+
"version": "7",
|
|
8
|
+
"when": 1790627348170,
|
|
9
|
+
"tag": "0000_init",
|
|
10
|
+
"breakpoints": true
|
|
11
|
+
},
|
|
12
|
+
{
|
|
13
|
+
"idx": 1,
|
|
14
|
+
"version": "7",
|
|
15
|
+
"when": 1790631168176,
|
|
16
|
+
"tag": "0001_predictions",
|
|
17
|
+
"breakpoints": true
|
|
18
|
+
},
|
|
19
|
+
{
|
|
20
|
+
"idx": 2,
|
|
21
|
+
"version": "7",
|
|
22
|
+
"when": 1791025761130,
|
|
23
|
+
"tag": "0002_dashboard",
|
|
24
|
+
"breakpoints": true
|
|
25
|
+
}
|
|
26
|
+
]
|
|
27
|
+
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,85 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "debutly",
|
|
3
|
-
"version": "0.
|
|
4
|
-
"
|
|
5
|
-
"
|
|
6
|
-
|
|
3
|
+
"version": "0.1.0",
|
|
4
|
+
"description": "Rehearse your launch: AI agents use your web app like your first customers and report what breaks.",
|
|
5
|
+
"type": "module",
|
|
6
|
+
"license": "AGPL-3.0-only",
|
|
7
|
+
"author": "Chetan Sroay (https://github.com/sroay)",
|
|
8
|
+
"homepage": "https://debutly.dev",
|
|
9
|
+
"repository": {
|
|
10
|
+
"type": "git",
|
|
11
|
+
"url": "git+https://github.com/sroay/debutly.git"
|
|
12
|
+
},
|
|
13
|
+
"bugs": {
|
|
14
|
+
"url": "https://github.com/sroay/debutly/issues"
|
|
15
|
+
},
|
|
16
|
+
"keywords": [
|
|
17
|
+
"testing",
|
|
18
|
+
"e2e",
|
|
19
|
+
"ai-agents",
|
|
20
|
+
"playwright",
|
|
21
|
+
"llm",
|
|
22
|
+
"synthetic-users",
|
|
23
|
+
"qa",
|
|
24
|
+
"launch"
|
|
25
|
+
],
|
|
26
|
+
"engines": {
|
|
27
|
+
"node": ">=20.12"
|
|
28
|
+
},
|
|
29
|
+
"bin": {
|
|
30
|
+
"debutly": "bin/debutly.mjs"
|
|
31
|
+
},
|
|
32
|
+
"files": [
|
|
33
|
+
"bin/",
|
|
34
|
+
"src/",
|
|
35
|
+
"drizzle/",
|
|
36
|
+
"tsconfig.json",
|
|
37
|
+
"LICENSE",
|
|
38
|
+
"README.md"
|
|
39
|
+
],
|
|
40
|
+
"workspaces": [
|
|
41
|
+
"dashboard"
|
|
42
|
+
],
|
|
43
|
+
"scripts": {
|
|
44
|
+
"test": "vitest run",
|
|
45
|
+
"typecheck": "tsc --noEmit",
|
|
46
|
+
"fixture": "tsx fixture/main.ts",
|
|
47
|
+
"debutly": "tsx src/cli.ts",
|
|
48
|
+
"debutly:cancel": "tsx src/cancel.ts",
|
|
49
|
+
"debutly:worker": "tsx src/worker.ts",
|
|
50
|
+
"debutly:predictions": "tsx src/predictions.ts",
|
|
51
|
+
"debutly:perceive": "tsx src/perceive.ts",
|
|
52
|
+
"debutly:costs": "tsx src/costs.ts",
|
|
53
|
+
"debutly:import": "tsx src/import.ts",
|
|
54
|
+
"dashboard": "npm run dev -w dashboard",
|
|
55
|
+
"dashboard:build": "npm run build -w dashboard",
|
|
56
|
+
"docs": "tsx scripts/build-docs.ts"
|
|
57
|
+
},
|
|
58
|
+
"dependencies": {
|
|
59
|
+
"@anthropic-ai/sdk": "^0.128.0",
|
|
60
|
+
"@aws-sdk/client-s3": "^3.1142.0",
|
|
61
|
+
"@electric-sql/pglite": "^0.5.8",
|
|
62
|
+
"bullmq": "^6.3.9",
|
|
63
|
+
"drizzle-orm": "^0.45.3",
|
|
64
|
+
"express": "^5.2.1",
|
|
65
|
+
"ioredis": "^5.11.1",
|
|
66
|
+
"playwright": "^1.63.0",
|
|
67
|
+
"postgres": "^3.4.9",
|
|
68
|
+
"tsx": "^4.23.15",
|
|
69
|
+
"zod": "^4.6.5"
|
|
70
|
+
},
|
|
71
|
+
"devDependencies": {
|
|
72
|
+
"@types/express": "^5.0.6",
|
|
73
|
+
"@types/node": "^26.6.3",
|
|
74
|
+
"drizzle-kit": "^0.31.11",
|
|
75
|
+
"marked": "18.0.14",
|
|
76
|
+
"typescript": "^7.0.2",
|
|
77
|
+
"vitest": "^5.0.2"
|
|
78
|
+
},
|
|
79
|
+
"publishConfig": {
|
|
80
|
+
"access": "public"
|
|
81
|
+
},
|
|
82
|
+
"optionalDependencies": {
|
|
83
|
+
"@guidepup/guidepup": "0.36.4"
|
|
84
|
+
}
|
|
85
|
+
}
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
import type { Action, Observation } from "../types";
|
|
2
|
+
|
|
3
|
+
export function describeAction(action: Action, obs: Observation): string {
|
|
4
|
+
const target = (id: number) => {
|
|
5
|
+
const el = obs.elements.find((e) => e.id === id);
|
|
6
|
+
return el ? `${el.role} "${el.name}"` : `element #${id}`;
|
|
7
|
+
};
|
|
8
|
+
switch (action.type) {
|
|
9
|
+
case "click":
|
|
10
|
+
return `clicked ${target(action.id)}`;
|
|
11
|
+
case "fill": {
|
|
12
|
+
const el = obs.elements.find((e) => e.id === action.id);
|
|
13
|
+
const picks = el && (el.tag === "select" || (el.role === "combobox" && el.tag !== "input" && el.tag !== "textarea"));
|
|
14
|
+
return picks ? `chose "${action.text}" in ${target(action.id)}` : `typed "${action.text}" into ${target(action.id)}`;
|
|
15
|
+
}
|
|
16
|
+
case "upload":
|
|
17
|
+
return `uploaded the sample ${action.file ?? "file"} with ${target(action.id)}`;
|
|
18
|
+
case "goto":
|
|
19
|
+
return `went to ${action.path}`;
|
|
20
|
+
case "check_email":
|
|
21
|
+
return "checked email";
|
|
22
|
+
case "press":
|
|
23
|
+
return `pressed ${action.key}`;
|
|
24
|
+
case "wait":
|
|
25
|
+
return "waited for the page to load";
|
|
26
|
+
case "done":
|
|
27
|
+
return `finished (${action.success ? "success" : "gave up"}): ${action.note}`;
|
|
28
|
+
}
|
|
29
|
+
}
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
import type { Action, ActionResult, Observation, Signal } from "../types";
|
|
2
|
+
|
|
3
|
+
const pathOf = (url: string) => {
|
|
4
|
+
try {
|
|
5
|
+
const u = new URL(url);
|
|
6
|
+
return u.pathname;
|
|
7
|
+
} catch {
|
|
8
|
+
return url;
|
|
9
|
+
}
|
|
10
|
+
};
|
|
11
|
+
|
|
12
|
+
// What a person would notice after acting: where they are now, whether anything changed,
|
|
13
|
+
// and whether something visibly broke. Without this, agents retry dead buttons forever.
|
|
14
|
+
export function describeEffect(action: Action, before: Observation, after: Observation, signals: Signal[], result: ActionResult): string {
|
|
15
|
+
if (result.blocked) return `blocked: ${result.blocked}`;
|
|
16
|
+
if (result.error) return `failed: ${result.error}`;
|
|
17
|
+
// check_email reports its own result; done ends the session.
|
|
18
|
+
if (action.type === "done" || action.type === "check_email") return "";
|
|
19
|
+
|
|
20
|
+
const parts: string[] = [];
|
|
21
|
+
if (pathOf(after.url) !== pathOf(before.url)) parts.push(`now on ${pathOf(after.url)}`);
|
|
22
|
+
else if (after.text !== before.text) parts.push("the page changed");
|
|
23
|
+
else if (action.type !== "fill" && !signals.some((s) => s.kind === "download")) parts.push("nothing visibly changed");
|
|
24
|
+
|
|
25
|
+
for (const s of signals) if (s.kind === "download") parts.push(`a file downloaded: ${s.filename}`);
|
|
26
|
+
|
|
27
|
+
for (const s of signals) if (s.kind === "dialog") parts.push(`a ${s.dialogType} dialog said "${s.message}" (answered OK)`);
|
|
28
|
+
if (signals.some((s) => s.kind === "page_error")) parts.push("a JavaScript error occurred");
|
|
29
|
+
const failed = [
|
|
30
|
+
...new Set(
|
|
31
|
+
signals.filter((s) => s.kind === "http_error" && pathOf(s.requestUrl) !== "/favicon.ico").map((s) => (s.kind === "http_error" ? s.status : 0)),
|
|
32
|
+
),
|
|
33
|
+
];
|
|
34
|
+
for (const status of failed) parts.push(`a request failed (HTTP ${status})`);
|
|
35
|
+
|
|
36
|
+
return parts.join("; ");
|
|
37
|
+
}
|
|
@@ -0,0 +1,232 @@
|
|
|
1
|
+
import type { Page } from "playwright";
|
|
2
|
+
import type { ArtifactStore } from "../artifacts/store";
|
|
3
|
+
import { execute } from "../browser/execute";
|
|
4
|
+
import { observe } from "../browser/observe";
|
|
5
|
+
import { ReflowMeter } from "../browser/reflow";
|
|
6
|
+
import { settle, type SettleOptions } from "../browser/settle";
|
|
7
|
+
import { SignalCollector } from "../browser/signals";
|
|
8
|
+
import type { EmailMessage, Inbox } from "../inbox/types";
|
|
9
|
+
import { Meter, type ModelPrice } from "../llm/meter";
|
|
10
|
+
import type { LlmClient, LlmMessage } from "../llm/types";
|
|
11
|
+
import type { ActionResult, Persona, Session, Step } from "../types";
|
|
12
|
+
import { describeAction } from "./describe";
|
|
13
|
+
import { describeEffect } from "./effect";
|
|
14
|
+
import { parseAgentReply } from "./parse-reply";
|
|
15
|
+
import { buildStepMessage, buildSystemPrompt, type AgentIdentity } from "./prompt";
|
|
16
|
+
|
|
17
|
+
const MAX_REPLY_ATTEMPTS = 3;
|
|
18
|
+
const LINKS_PER_EMAIL = 3;
|
|
19
|
+
/** How long the first page may take to arrive; a slow-connection agent needs more than 30 s. */
|
|
20
|
+
const FIRST_LOAD_MS = 90_000;
|
|
21
|
+
/** A first page slower than this is reported. */
|
|
22
|
+
const SLOW_LOAD_MS = 10_000;
|
|
23
|
+
/** How long a step's screenshot may take before the step goes without one. */
|
|
24
|
+
const SCREENSHOT_MS = 15_000;
|
|
25
|
+
|
|
26
|
+
export interface RunSessionOptions {
|
|
27
|
+
page: Page;
|
|
28
|
+
llm: LlmClient;
|
|
29
|
+
persona: Persona;
|
|
30
|
+
goal: string;
|
|
31
|
+
origin: string;
|
|
32
|
+
maxSteps: number;
|
|
33
|
+
/** Where step screenshots go (a local folder, or S3); none means no screenshots. */
|
|
34
|
+
artifacts?: ArtifactStore;
|
|
35
|
+
identity?: AgentIdentity;
|
|
36
|
+
inbox?: Inbox;
|
|
37
|
+
/** Shared context for the system prompt, e.g. a crew member's company. */
|
|
38
|
+
brief?: string;
|
|
39
|
+
/** Per-agent limits on top of maxSteps. */
|
|
40
|
+
caps?: { maxTokens?: number; maxMs?: number };
|
|
41
|
+
/** The run's kill switch, checked before every step. */
|
|
42
|
+
shouldStop?: () => Promise<boolean> | boolean;
|
|
43
|
+
/** Prices the session's usage; omitted or null leaves the cost unknown. */
|
|
44
|
+
price?: ModelPrice | null;
|
|
45
|
+
/** Called with each step as it completes, e.g. to record it for live views. */
|
|
46
|
+
onStep?: (step: Step) => Promise<void> | void;
|
|
47
|
+
/** How long to let each page finish loading before the agent reads it. */
|
|
48
|
+
settle?: SettleOptions;
|
|
49
|
+
/** How long the first page may take, and from when it counts as slow (defaults: 90 s, 10 s). */
|
|
50
|
+
firstLoad?: { timeoutMs?: number; slowMs?: number };
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
function describeInbox(messages: EmailMessage[]): string {
|
|
54
|
+
if (!messages.length) return "checked email → no emails yet";
|
|
55
|
+
const items = messages.map((m) => `"${m.subject}"${m.links.length ? ` links: ${m.links.slice(0, LINKS_PER_EMAIL).join(" ")}` : ""}`);
|
|
56
|
+
return `checked email → ${messages.length} email${messages.length === 1 ? "" : "s"}: ${items.join("; ")}`;
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
/**
|
|
60
|
+
* Open the app the way a person does: wait for the page, but never fail the session because it
|
|
61
|
+
* is slow. Waiting for every image and font ("load") within Playwright's 30 s ended the
|
|
62
|
+
* slow-connection agents on a test app's home page before their first step, two nights running.
|
|
63
|
+
* Now the agent starts once the document is in (settle() waits for the rest), with up to
|
|
64
|
+
* FIRST_LOAD_MS for that; whatever has arrived by then is what it sees, and it can wait or give up.
|
|
65
|
+
* A slow start is itself a finding.
|
|
66
|
+
*/
|
|
67
|
+
async function openFirstPage(
|
|
68
|
+
page: Page,
|
|
69
|
+
origin: string,
|
|
70
|
+
signals: SignalCollector,
|
|
71
|
+
slowNetwork: boolean,
|
|
72
|
+
{ timeoutMs = FIRST_LOAD_MS, slowMs = SLOW_LOAD_MS }: RunSessionOptions["firstLoad"] = {},
|
|
73
|
+
): Promise<void> {
|
|
74
|
+
const startedAt = Date.now();
|
|
75
|
+
let finished = true;
|
|
76
|
+
try {
|
|
77
|
+
await page.goto(origin, { waitUntil: "domcontentloaded", timeout: timeoutMs });
|
|
78
|
+
} catch (err) {
|
|
79
|
+
if (!(err instanceof Error && err.name === "TimeoutError")) throw err;
|
|
80
|
+
finished = false;
|
|
81
|
+
}
|
|
82
|
+
const ms = Date.now() - startedAt;
|
|
83
|
+
if (!finished || ms >= slowMs) signals.note({ kind: "slow_load", url: origin, ms, finished, slowNetwork });
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
export async function runSession(o: RunSessionOptions): Promise<Session> {
|
|
87
|
+
const signals = new SignalCollector(o.page);
|
|
88
|
+
const system = buildSystemPrompt(o.persona, o.goal, o.inbox ? o.identity : undefined, o.brief);
|
|
89
|
+
const steps: Step[] = [];
|
|
90
|
+
const history: string[] = [];
|
|
91
|
+
const emailedLinks = new Set<string>();
|
|
92
|
+
const meter = new Meter();
|
|
93
|
+
const startedAt = Date.now();
|
|
94
|
+
const base = { agentId: o.persona.id, personaName: o.persona.name, goal: o.goal, steps };
|
|
95
|
+
// The low-vision agent's pages, each measured at 320 CSS pixels wide as it arrives (WCAG 1.4.10).
|
|
96
|
+
const reflow = o.persona.inclusion === "low_vision" ? new ReflowMeter() : undefined;
|
|
97
|
+
// Every way out of the session carries what it spent, and what it measured.
|
|
98
|
+
const end = (outcome: Session["outcome"], outcomeNote: string): Session => ({
|
|
99
|
+
...base,
|
|
100
|
+
outcome,
|
|
101
|
+
outcomeNote,
|
|
102
|
+
usage: { ...meter.total, costUsd: meter.costUsd(o.price) },
|
|
103
|
+
...(reflow?.checks.length ? { reflow: [...reflow.checks] } : {}),
|
|
104
|
+
});
|
|
105
|
+
|
|
106
|
+
await openFirstPage(o.page, o.origin, signals, o.persona.inclusion === "slow_connection", o.firstLoad);
|
|
107
|
+
|
|
108
|
+
// A failure part-way (the provider out of credits, the browser gone) still ends the session
|
|
109
|
+
// with the steps it took: they hold findings, and they were paid for.
|
|
110
|
+
try {
|
|
111
|
+
return await takeSteps(o, { signals, system, steps, history, emailedLinks, meter, startedAt, end, reflow });
|
|
112
|
+
} catch (err) {
|
|
113
|
+
return end("error", err instanceof Error ? err.message : String(err));
|
|
114
|
+
}
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
interface SessionState {
|
|
118
|
+
signals: SignalCollector;
|
|
119
|
+
system: string;
|
|
120
|
+
steps: Step[];
|
|
121
|
+
history: string[];
|
|
122
|
+
emailedLinks: Set<string>;
|
|
123
|
+
meter: Meter;
|
|
124
|
+
startedAt: number;
|
|
125
|
+
end: (outcome: Session["outcome"], outcomeNote: string) => Session;
|
|
126
|
+
reflow?: ReflowMeter;
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
async function takeSteps(o: RunSessionOptions, { signals, system, steps, history, emailedLinks, meter, startedAt, end, reflow }: SessionState): Promise<Session> {
|
|
130
|
+
// The screen-reader agent hears what the browser tells assistive technology and works by keyboard.
|
|
131
|
+
const screenReader = o.persona.inclusion === "screen_reader";
|
|
132
|
+
for (let index = 0; index < o.maxSteps; index++) {
|
|
133
|
+
if (o.shouldStop && (await o.shouldStop())) return end("cancelled", "run cancelled");
|
|
134
|
+
const { inputTokens, outputTokens } = meter.total;
|
|
135
|
+
if (o.caps?.maxTokens !== undefined && inputTokens + outputTokens >= o.caps.maxTokens) {
|
|
136
|
+
return end("budget_exhausted", `stopped at the token cap (${o.caps.maxTokens} tokens)`);
|
|
137
|
+
}
|
|
138
|
+
if (o.caps?.maxMs !== undefined && Date.now() - startedAt >= o.caps.maxMs) {
|
|
139
|
+
return end("budget_exhausted", `stopped at the time cap (${Math.round(o.caps.maxMs / 1000)}s)`);
|
|
140
|
+
}
|
|
141
|
+
await settle(o.page, o.settle);
|
|
142
|
+
await reflow?.measure(o.page);
|
|
143
|
+
const observation = await observe(o.page);
|
|
144
|
+
// Now that we can see the page after the previous action, tell the agent what it did.
|
|
145
|
+
const prev = steps.at(-1);
|
|
146
|
+
if (prev) {
|
|
147
|
+
const effect = describeEffect(prev.action, prev.observation, observation, prev.signals, prev);
|
|
148
|
+
if (effect) history[history.length - 1] += ` → ${effect}`;
|
|
149
|
+
}
|
|
150
|
+
const messages: LlmMessage[] = [{ role: "user", content: buildStepMessage(observation, history, { screenReader }) }];
|
|
151
|
+
let reply: ReturnType<typeof parseAgentReply> | undefined;
|
|
152
|
+
let raw = "";
|
|
153
|
+
let problem = "";
|
|
154
|
+
// Real models occasionally send prose, an empty reply or malformed JSON; say what was wrong and ask again.
|
|
155
|
+
for (let attempt = 0; attempt < MAX_REPLY_ATTEMPTS && !reply; attempt++) {
|
|
156
|
+
raw = await o.llm.complete({ system, messages: [...messages], maxTokens: 4096, onUsage: meter.add });
|
|
157
|
+
try {
|
|
158
|
+
reply = parseAgentReply(raw);
|
|
159
|
+
} catch (err) {
|
|
160
|
+
problem = err instanceof Error ? err.message : String(err);
|
|
161
|
+
messages.push(
|
|
162
|
+
{ role: "assistant", content: raw || "(empty reply)" },
|
|
163
|
+
{
|
|
164
|
+
role: "user",
|
|
165
|
+
content: `Your last reply was not valid: ${problem.slice(0, 300)}\nReply with only one JSON object: {"reasoning": string, "action": Action}.`,
|
|
166
|
+
},
|
|
167
|
+
);
|
|
168
|
+
}
|
|
169
|
+
}
|
|
170
|
+
if (!reply) {
|
|
171
|
+
return end("error", `${problem} | last reply: ${raw.slice(0, 300) || "(empty)"}`);
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
let result: ActionResult;
|
|
175
|
+
let historyLine = describeAction(reply.action, observation);
|
|
176
|
+
if (reply.action.type === "check_email") {
|
|
177
|
+
if (!o.inbox || !o.identity) {
|
|
178
|
+
result = { error: "this agent has no inbox" };
|
|
179
|
+
} else {
|
|
180
|
+
try {
|
|
181
|
+
const mail = await o.inbox.latest(o.identity.email);
|
|
182
|
+
for (const m of mail) for (const link of m.links) emailedLinks.add(link);
|
|
183
|
+
historyLine = describeInbox(mail);
|
|
184
|
+
result = {};
|
|
185
|
+
} catch (err) {
|
|
186
|
+
result = { error: err instanceof Error ? err.message : String(err) };
|
|
187
|
+
}
|
|
188
|
+
}
|
|
189
|
+
} else {
|
|
190
|
+
result = await execute(o.page, reply.action, observation, o.origin, (href) => emailedLinks.has(href), { keyboard: screenReader });
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
let screenshotPath: string | undefined;
|
|
194
|
+
if (o.artifacts) {
|
|
195
|
+
// JPEG where the store asks for it (the database keeps them small); PNG otherwise. A
|
|
196
|
+
// screenshot that fails (one timed out on a slow page) costs this step
|
|
197
|
+
// its picture, not the agent its session.
|
|
198
|
+
const format = o.artifacts.screenshotFormat;
|
|
199
|
+
try {
|
|
200
|
+
const shot = await o.page.screenshot({ ...(format ?? {}), timeout: SCREENSHOT_MS });
|
|
201
|
+
const name = `${o.persona.id}-${String(index).padStart(3, "0")}`;
|
|
202
|
+
screenshotPath = format
|
|
203
|
+
? await o.artifacts.put(`${name}.jpg`, shot, "image/jpeg")
|
|
204
|
+
: await o.artifacts.put(`${name}.png`, shot, "image/png");
|
|
205
|
+
} catch (err) {
|
|
206
|
+
console.warn(`[${o.persona.name}] no screenshot for step ${index}: ${err instanceof Error ? err.message.split("\n")[0] : err}`);
|
|
207
|
+
}
|
|
208
|
+
}
|
|
209
|
+
|
|
210
|
+
const guessedUrl = reply.action.type === "goto" && !emailedLinks.has(reply.action.path);
|
|
211
|
+
await signals.settle();
|
|
212
|
+
const step: Step = {
|
|
213
|
+
index,
|
|
214
|
+
observation,
|
|
215
|
+
action: reply.action,
|
|
216
|
+
reasoning: reply.reasoning,
|
|
217
|
+
signals: signals.drain(),
|
|
218
|
+
screenshotPath,
|
|
219
|
+
...result,
|
|
220
|
+
...(guessedUrl ? { guessedUrl } : {}),
|
|
221
|
+
};
|
|
222
|
+
steps.push(step);
|
|
223
|
+
await o.onStep?.(step);
|
|
224
|
+
history.push(historyLine);
|
|
225
|
+
|
|
226
|
+
if (reply.action.type === "done") {
|
|
227
|
+
return end(reply.action.success ? "success" : "gave_up", reply.action.note);
|
|
228
|
+
}
|
|
229
|
+
}
|
|
230
|
+
|
|
231
|
+
return end("budget_exhausted", `stopped after ${o.maxSteps} steps`);
|
|
232
|
+
}
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
import { z } from "zod";
|
|
2
|
+
import { PRESS_KEYS, type Action } from "../types";
|
|
3
|
+
|
|
4
|
+
const ActionSchema = z.discriminatedUnion("type", [
|
|
5
|
+
z.object({ type: z.literal("click"), id: z.number().int() }),
|
|
6
|
+
z.object({ type: z.literal("fill"), id: z.number().int(), text: z.string() }),
|
|
7
|
+
z.object({ type: z.literal("upload"), id: z.number().int(), file: z.enum(["photo", "document"]).optional() }),
|
|
8
|
+
z.object({ type: z.literal("goto"), path: z.string() }),
|
|
9
|
+
z.object({ type: z.literal("check_email") }),
|
|
10
|
+
z.object({ type: z.literal("press"), key: z.enum(PRESS_KEYS) }),
|
|
11
|
+
z.object({ type: z.literal("wait") }),
|
|
12
|
+
z.object({ type: z.literal("done"), success: z.boolean(), note: z.string() }),
|
|
13
|
+
]);
|
|
14
|
+
|
|
15
|
+
// Real models sometimes drop the reasoning; the action is what matters.
|
|
16
|
+
const ReplySchema = z.object({ reasoning: z.string().default(""), action: ActionSchema });
|
|
17
|
+
|
|
18
|
+
export function parseAgentReply(raw: string): { reasoning: string; action: Action } {
|
|
19
|
+
const start = raw.indexOf("{");
|
|
20
|
+
const end = raw.lastIndexOf("}");
|
|
21
|
+
if (start === -1 || end <= start) throw new Error("no JSON object in agent reply");
|
|
22
|
+
return ReplySchema.parse(actionAsObject(parseQuotedAction(raw.slice(start, end + 1))));
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
// Some models (GLM 5.3 Flash) send the action as a string. Unescaped,
|
|
26
|
+
// {"reasoning":"…","action":"{"type":"fill","id":1,"text":"x"}"} is not JSON at all, and asking
|
|
27
|
+
// again got the same reply three times, ending the agent. An action is a flat object, so its
|
|
28
|
+
// quotes can be dropped and the rest parsed as meant. Sometimes the closing quote and brace never
|
|
29
|
+
// come at all; then the reasoning and the action are read out separately.
|
|
30
|
+
function parseQuotedAction(text: string): unknown {
|
|
31
|
+
try {
|
|
32
|
+
return JSON.parse(text);
|
|
33
|
+
} catch (err) {
|
|
34
|
+
const unquoted = text.replace(/("action"\s*:\s*)"(\{[^{}]*\})"/, "$1$2");
|
|
35
|
+
if (unquoted !== text) {
|
|
36
|
+
try {
|
|
37
|
+
return JSON.parse(unquoted);
|
|
38
|
+
} catch {
|
|
39
|
+
// fall through to reading the parts
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
const parts = readParts(text);
|
|
43
|
+
if (!parts) throw err;
|
|
44
|
+
return parts;
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
function readParts(text: string): { reasoning: string; action: unknown } | undefined {
|
|
49
|
+
const action = text.match(/"action"\s*:\s*"?\s*(\{[^{}]*\})/)?.[1];
|
|
50
|
+
if (!action) return undefined;
|
|
51
|
+
const reasoning = text.match(/"reasoning"\s*:\s*"((?:[^"\\]|\\.)*)"/)?.[1];
|
|
52
|
+
try {
|
|
53
|
+
return { reasoning: reasoning ? JSON.parse(`"${reasoning}"`) : "", action: JSON.parse(action.replace(/\\"/g, '"')) };
|
|
54
|
+
} catch {
|
|
55
|
+
return undefined;
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
const ACTION_TYPES = new Set(["click", "fill", "upload", "goto", "check_email", "press", "wait", "done"]);
|
|
60
|
+
const ACTION_FIELDS = ["id", "text", "file", "path", "key", "success", "note"] as const;
|
|
61
|
+
|
|
62
|
+
// Escaped, the same mistake parses, with the action as a string holding its JSON. Or the action is
|
|
63
|
+
// flattened into the reply, {"reasoning":"…","action":"click","id":40}: its type as a string, its
|
|
64
|
+
// fields beside it (2026-10-18; asking again got the same reply).
|
|
65
|
+
function actionAsObject(reply: unknown): unknown {
|
|
66
|
+
if (!reply || typeof reply !== "object" || typeof (reply as { action?: unknown }).action !== "string") return reply;
|
|
67
|
+
const r = reply as Record<string, unknown> & { action: string };
|
|
68
|
+
try {
|
|
69
|
+
return { ...r, action: JSON.parse(r.action) };
|
|
70
|
+
} catch {
|
|
71
|
+
if (!ACTION_TYPES.has(r.action)) return reply; // the schema says what is wrong with it
|
|
72
|
+
const action: Record<string, unknown> = { type: r.action };
|
|
73
|
+
for (const field of ACTION_FIELDS) if (field in r) action[field] = r[field];
|
|
74
|
+
return { reasoning: r.reasoning, action };
|
|
75
|
+
}
|
|
76
|
+
}
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
import { inclusionBrief } from "../persona/inclusion";
|
|
2
|
+
import { languageBrief } from "../persona/language";
|
|
3
|
+
import type { InteractiveElement, Observation, Persona } from "../types";
|
|
4
|
+
|
|
5
|
+
// Stays constant for the whole session so providers can cache it; everything that changes goes in the user message.
|
|
6
|
+
export interface AgentIdentity {
|
|
7
|
+
email: string;
|
|
8
|
+
}
|
|
9
|
+
|
|
10
|
+
// `brief` carries shared context, e.g. the company a crew member works at and what their owner reported.
|
|
11
|
+
export function buildSystemPrompt(persona: Persona, goal: string, identity?: AgentIdentity, brief?: string): string {
|
|
12
|
+
const email = identity
|
|
13
|
+
? `Your email address is ${identity.email}. Use it whenever the site asks for your email.
|
|
14
|
+
To read your inbox (for example a login link or a confirmation), reply {"type":"check_email"}. To open a link from an email, use {"type":"goto","path":"<the full link>"}.`
|
|
15
|
+
: `Use emails at example.com.`;
|
|
16
|
+
const about = [
|
|
17
|
+
`You are ${persona.name}. ${persona.background}`,
|
|
18
|
+
`Patience: ${persona.patience}. Tech savvy: ${persona.techSavvy}.`,
|
|
19
|
+
persona.role ? `Your role: ${persona.role}.` : "",
|
|
20
|
+
brief ?? "",
|
|
21
|
+
persona.inclusion ? inclusionBrief(persona.inclusion) : "",
|
|
22
|
+
languageBrief(persona.language),
|
|
23
|
+
]
|
|
24
|
+
.filter(Boolean)
|
|
25
|
+
.join("\n");
|
|
26
|
+
return `${about}
|
|
27
|
+
|
|
28
|
+
You are a real prospective customer using this web product for the first time. Act exactly as this person would: read what is on screen, decide what they would do next, and react honestly to confusion or breakage. If something fails, try a reasonable alternative like a real person. Give up (done with success false) at the point this person would give up, and say why.
|
|
29
|
+
|
|
30
|
+
Your goal: ${goal}
|
|
31
|
+
|
|
32
|
+
${email}
|
|
33
|
+
Use test data only: card number 4242 4242 4242 4242. Never enter real personal or payment data.
|
|
34
|
+
|
|
35
|
+
Reply with exactly one JSON object: {"reasoning": string, "action": Action}
|
|
36
|
+
Action is one of:
|
|
37
|
+
{"type":"click","id":number}
|
|
38
|
+
{"type":"fill","id":number,"text":string} (on a select or dropdown, text is the option to choose)
|
|
39
|
+
{"type":"upload","id":number,"file":"photo"|"document"} (attach a file through an upload button or file field: you have a sample photo and a sample CV/document)
|
|
40
|
+
{"type":"goto","path":string}
|
|
41
|
+
{"type":"press","key":"Escape"|"Enter"|"Tab"|"ArrowDown"|"ArrowUp"} (e.g. Escape closes a popup that blocks the page)
|
|
42
|
+
{"type":"wait"} (the page is still loading, e.g. a spinner or a form that has not appeared yet: wait a few seconds rather than leave)${identity ? `\n{"type":"check_email"}` : ""}
|
|
43
|
+
{"type":"done","success":boolean,"note":string}`;
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
function describeElement(e: InteractiveElement, screenReader = false): string {
|
|
47
|
+
// A screen reader announces the role and name the browser gives it.
|
|
48
|
+
const heard = screenReader && e.ax ? e.ax : e;
|
|
49
|
+
const line = `[${e.id}] ${heard.role} "${heard.name}"`;
|
|
50
|
+
if (!e.options) return line;
|
|
51
|
+
return `${line} set to "${e.value ?? ""}", options: ${e.options.map((o) => `"${o}"`).join(" | ")}`;
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
export interface StepView {
|
|
55
|
+
/** As a screen reader presents the page: the browser's names and roles, without what is hidden from it. */
|
|
56
|
+
screenReader?: boolean;
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
export function buildStepMessage(obs: Observation, history: string[], view: StepView = {}): string {
|
|
60
|
+
const shown = view.screenReader ? obs.elements.filter((e) => !e.ax?.hidden) : obs.elements;
|
|
61
|
+
const elements = shown.map((e) => describeElement(e, view.screenReader)).join("\n");
|
|
62
|
+
const past = history.length ? history.map((h, i) => `${i + 1}. ${h}`).join("\n") : "(none yet)";
|
|
63
|
+
return `URL: ${obs.url}
|
|
64
|
+
Title: ${obs.title}
|
|
65
|
+
|
|
66
|
+
Interactive elements:
|
|
67
|
+
${elements}
|
|
68
|
+
|
|
69
|
+
Visible text:
|
|
70
|
+
${obs.text}
|
|
71
|
+
|
|
72
|
+
Your actions so far:
|
|
73
|
+
${past}
|
|
74
|
+
|
|
75
|
+
What do you do next?`;
|
|
76
|
+
}
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
import { mkdir } from "node:fs/promises";
|
|
2
|
+
import { join } from "node:path";
|
|
3
|
+
import type { Browser } from "playwright";
|
|
4
|
+
import type { ArtifactStore } from "../artifacts/store";
|
|
5
|
+
import type { Inbox } from "../inbox/types";
|
|
6
|
+
import type { ModelPrice } from "../llm/meter";
|
|
7
|
+
import type { LlmClient } from "../llm/types";
|
|
8
|
+
import { openAgentPage } from "../persona/inclusion";
|
|
9
|
+
import type { RunStore } from "../store/run-store";
|
|
10
|
+
import type { Persona, Session } from "../types";
|
|
11
|
+
import { runSession } from "./loop";
|
|
12
|
+
|
|
13
|
+
export interface AgentRunDeps {
|
|
14
|
+
browser: Browser;
|
|
15
|
+
llm: LlmClient;
|
|
16
|
+
origin: string;
|
|
17
|
+
maxSteps: number;
|
|
18
|
+
artifacts: ArtifactStore;
|
|
19
|
+
emailOf: (persona: Persona) => string;
|
|
20
|
+
inbox?: Inbox;
|
|
21
|
+
caps?: { maxTokens?: number; maxMs?: number };
|
|
22
|
+
price?: ModelPrice | null;
|
|
23
|
+
shouldStop?: () => boolean | Promise<boolean>;
|
|
24
|
+
/** With a store and run id, the session and each step are recorded as they happen. */
|
|
25
|
+
store?: RunStore;
|
|
26
|
+
runId?: string;
|
|
27
|
+
/** Keeps the agent's browser state here as <agent id>.json when it finishes, for `debutly hear`. */
|
|
28
|
+
browserStateDir?: string;
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
/**
|
|
32
|
+
* One agent, start to finish, in its own browser context (no shared cookies or login, even
|
|
33
|
+
* inside a crew). Used in-process by runSimulation and by queue workers alike.
|
|
34
|
+
*/
|
|
35
|
+
export async function runAgentSession(d: AgentRunDeps, persona: Persona, goal: string, brief: string | undefined): Promise<Session> {
|
|
36
|
+
const record = d.store && d.runId ? { store: d.store, runId: d.runId } : undefined;
|
|
37
|
+
const { context, page } = await openAgentPage(d.browser, persona.inclusion, persona.language);
|
|
38
|
+
if (record) await record.store.startSession(record.runId, persona.id, goal);
|
|
39
|
+
let session: Session;
|
|
40
|
+
try {
|
|
41
|
+
session = await runSession({
|
|
42
|
+
page,
|
|
43
|
+
llm: d.llm,
|
|
44
|
+
persona,
|
|
45
|
+
goal,
|
|
46
|
+
origin: d.origin,
|
|
47
|
+
maxSteps: d.maxSteps,
|
|
48
|
+
artifacts: d.artifacts,
|
|
49
|
+
identity: d.inbox ? { email: d.emailOf(persona) } : undefined,
|
|
50
|
+
inbox: d.inbox,
|
|
51
|
+
brief,
|
|
52
|
+
caps: d.caps,
|
|
53
|
+
price: d.price,
|
|
54
|
+
shouldStop: d.shouldStop,
|
|
55
|
+
onStep: record ? (step) => record.store.appendStep(record.runId, persona.id, step) : undefined,
|
|
56
|
+
});
|
|
57
|
+
} catch (err) {
|
|
58
|
+
session = { agentId: persona.id, personaName: persona.name, goal, steps: [], outcome: "error", outcomeNote: err instanceof Error ? err.message : String(err) };
|
|
59
|
+
} finally {
|
|
60
|
+
if (d.browserStateDir) {
|
|
61
|
+
await mkdir(d.browserStateDir, { recursive: true });
|
|
62
|
+
await context.storageState({ path: join(d.browserStateDir, `${persona.id}.json`) }).catch(() => {});
|
|
63
|
+
}
|
|
64
|
+
await context.close();
|
|
65
|
+
}
|
|
66
|
+
if (record) await record.store.finishSession(record.runId, session);
|
|
67
|
+
return session;
|
|
68
|
+
}
|