@ap3x/browser-agent 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (74) hide show
  1. package/LICENSE +21 -0
  2. package/README.md +34 -0
  3. package/dist/agent/agent.d.ts +160 -0
  4. package/dist/agent/agent.d.ts.map +1 -0
  5. package/dist/agent/backend.d.ts +38 -0
  6. package/dist/agent/backend.d.ts.map +1 -0
  7. package/dist/agent/captcha-watchdog.d.ts +31 -0
  8. package/dist/agent/captcha-watchdog.d.ts.map +1 -0
  9. package/dist/agent/context-builder.d.ts +24 -0
  10. package/dist/agent/context-builder.d.ts.map +1 -0
  11. package/dist/agent/event-stream.d.ts +104 -0
  12. package/dist/agent/event-stream.d.ts.map +1 -0
  13. package/dist/agent/executor.d.ts +37 -0
  14. package/dist/agent/executor.d.ts.map +1 -0
  15. package/dist/agent/gif.d.ts +23 -0
  16. package/dist/agent/gif.d.ts.map +1 -0
  17. package/dist/agent/history.d.ts +89 -0
  18. package/dist/agent/history.d.ts.map +1 -0
  19. package/dist/agent/judge.d.ts +36 -0
  20. package/dist/agent/judge.d.ts.map +1 -0
  21. package/dist/agent/message-manager.d.ts +94 -0
  22. package/dist/agent/message-manager.d.ts.map +1 -0
  23. package/dist/agent/output-schema.d.ts +26 -0
  24. package/dist/agent/output-schema.d.ts.map +1 -0
  25. package/dist/agent/png-decode.d.ts +9 -0
  26. package/dist/agent/png-decode.d.ts.map +1 -0
  27. package/dist/agent/rematch.d.ts +26 -0
  28. package/dist/agent/rematch.d.ts.map +1 -0
  29. package/dist/agent/replay.d.ts +155 -0
  30. package/dist/agent/replay.d.ts.map +1 -0
  31. package/dist/agent/step-loop.d.ts +124 -0
  32. package/dist/agent/step-loop.d.ts.map +1 -0
  33. package/dist/agent/variable-detector.d.ts +19 -0
  34. package/dist/agent/variable-detector.d.ts.map +1 -0
  35. package/dist/agent/video.d.ts +12 -0
  36. package/dist/agent/video.d.ts.map +1 -0
  37. package/dist/chunk-32HNRAQS.js +82 -0
  38. package/dist/chunk-UZALZUEE.js +2907 -0
  39. package/dist/filesystem/external-read.d.ts +18 -0
  40. package/dist/filesystem/external-read.d.ts.map +1 -0
  41. package/dist/filesystem/file-system.d.ts +61 -0
  42. package/dist/filesystem/file-system.d.ts.map +1 -0
  43. package/dist/index.d.ts +37 -0
  44. package/dist/index.d.ts.map +1 -0
  45. package/dist/index.js +1070 -0
  46. package/dist/mcp/bin.d.ts +3 -0
  47. package/dist/mcp/bin.d.ts.map +1 -0
  48. package/dist/mcp/bin.js +19 -0
  49. package/dist/mcp/client.d.ts +45 -0
  50. package/dist/mcp/client.d.ts.map +1 -0
  51. package/dist/mcp/index.d.ts +3 -0
  52. package/dist/mcp/index.d.ts.map +1 -0
  53. package/dist/mcp/log.d.ts +6 -0
  54. package/dist/mcp/log.d.ts.map +1 -0
  55. package/dist/mcp/server.d.ts +36 -0
  56. package/dist/mcp/server.d.ts.map +1 -0
  57. package/dist/observability/logger.d.ts +25 -0
  58. package/dist/observability/logger.d.ts.map +1 -0
  59. package/dist/testing.d.ts +37 -0
  60. package/dist/testing.d.ts.map +1 -0
  61. package/dist/testing.js +18 -0
  62. package/dist/tools/actions.d.ts +9 -0
  63. package/dist/tools/actions.d.ts.map +1 -0
  64. package/dist/tools/registry.d.ts +125 -0
  65. package/dist/tools/registry.d.ts.map +1 -0
  66. package/dist/tools/sensitive.d.ts +59 -0
  67. package/dist/tools/sensitive.d.ts.map +1 -0
  68. package/dist/tools/types.d.ts +117 -0
  69. package/dist/tools/types.d.ts.map +1 -0
  70. package/dist/types.d.ts +2 -0
  71. package/dist/types.d.ts.map +1 -0
  72. package/dist/usage/ledger.d.ts +27 -0
  73. package/dist/usage/ledger.d.ts.map +1 -0
  74. package/package.json +50 -0
@@ -0,0 +1,2907 @@
1
+ // src/tools/types.ts
2
+ function stepResult(fields = {}) {
3
+ return { ...fields, isDone: false };
4
+ }
5
+ function doneResult(success, fields = {}) {
6
+ return { ...fields, isDone: true, success };
7
+ }
8
+ function isDoneResult(result) {
9
+ return result.isDone === true;
10
+ }
11
+
12
+ // src/tools/sensitive.ts
13
+ import { createHmac } from "crypto";
14
+ import { matchUrlWithDomainPattern } from "@ap3x/browser";
15
+ function domainApplies(pattern, url) {
16
+ if (pattern === "*") return true;
17
+ return matchUrlWithDomainPattern(url, pattern);
18
+ }
19
+ function isPresent(value) {
20
+ if (typeof value === "string") return value.length > 0;
21
+ return typeof value.totpSeed === "string" && value.totpSeed.length > 0;
22
+ }
23
+ function resolveSecretsForUrl(data, url) {
24
+ const out = {};
25
+ for (const [pattern, entries] of Object.entries(data)) {
26
+ if (!domainApplies(pattern, url)) continue;
27
+ for (const [key, value] of Object.entries(entries)) {
28
+ if (isPresent(value)) out[key] = value;
29
+ }
30
+ }
31
+ return out;
32
+ }
33
+ function applicablePlaceholders(data, url) {
34
+ return Object.keys(resolveSecretsForUrl(data, url)).sort();
35
+ }
36
+ function sensitiveDataDescription(data, url) {
37
+ const keys = applicablePlaceholders(data, url);
38
+ if (keys.length === 0) return void 0;
39
+ const list = keys.map((k) => ` - ${k}`).join("\n");
40
+ return `The following placeholders hold sensitive values for this page. To use one, write it as <secret>key</secret> where you want the value typed; the system substitutes the real value at execution time. Never write the raw value.
41
+ ${list}`;
42
+ }
43
+ function maskOutbound(text, data) {
44
+ const pairs = [];
45
+ for (const entries of Object.values(data)) {
46
+ for (const [key, value] of Object.entries(entries)) {
47
+ if (typeof value === "string" && value.length > 0) pairs.push({ key, value });
48
+ }
49
+ }
50
+ pairs.sort((a, b) => b.value.length - a.value.length);
51
+ let out = text;
52
+ for (const { key, value } of pairs) {
53
+ out = out.split(value).join(`<secret>${key}</secret>`);
54
+ }
55
+ return out;
56
+ }
57
+ function maskSecretMap(text, secrets) {
58
+ const pairs = [];
59
+ for (const [key, value] of Object.entries(secrets)) {
60
+ if (typeof value === "string" && value.length > 0) pairs.push({ key, value });
61
+ }
62
+ pairs.sort((a, b) => b.value.length - a.value.length);
63
+ let out = text;
64
+ for (const { key, value } of pairs) {
65
+ out = out.split(value).join(`<secret>${key}</secret>`);
66
+ }
67
+ return out;
68
+ }
69
+ var SECRET_TAG = /<secret>([^<]+)<\/secret>/g;
70
+ function realValue(value) {
71
+ return typeof value === "string" ? value : totp(value.totpSeed);
72
+ }
73
+ function substituteSecrets(params, secrets) {
74
+ const unresolved = /* @__PURE__ */ new Set();
75
+ const walk = (node) => {
76
+ if (typeof node === "string") return substituteString(node);
77
+ if (Array.isArray(node)) return node.map(walk);
78
+ if (node && typeof node === "object") {
79
+ const out = {};
80
+ for (const [k, v] of Object.entries(node)) out[k] = walk(v);
81
+ return out;
82
+ }
83
+ return node;
84
+ };
85
+ const substituteString = (text) => {
86
+ if (Object.hasOwn(secrets, text)) return realValue(secrets[text]);
87
+ return text.replace(SECRET_TAG, (match, key) => {
88
+ if (Object.hasOwn(secrets, key)) return realValue(secrets[key]);
89
+ unresolved.add(key);
90
+ return match;
91
+ });
92
+ };
93
+ return { value: walk(params), unresolved: [...unresolved] };
94
+ }
95
+ var BASE32_ALPHABET = "ABCDEFGHIJKLMNOPQRSTUVWXYZ234567";
96
+ function base32Decode(input) {
97
+ const clean = input.replace(/[\s=]/g, "").toUpperCase();
98
+ const bytes = [];
99
+ let bits = 0;
100
+ let acc = 0;
101
+ for (const char of clean) {
102
+ const index = BASE32_ALPHABET.indexOf(char);
103
+ if (index < 0) throw new Error(`invalid base32 character: ${char}`);
104
+ acc = acc << 5 | index;
105
+ bits += 5;
106
+ if (bits >= 8) {
107
+ bits -= 8;
108
+ bytes.push(acc >> bits & 255);
109
+ }
110
+ }
111
+ return Buffer.from(bytes);
112
+ }
113
+ function totp(seed, options = {}) {
114
+ const step = options.step ?? 30;
115
+ const digits = options.digits ?? 6;
116
+ const time = options.time ?? Math.floor(Date.now() / 1e3);
117
+ const counter = Math.floor(time / step);
118
+ const buf = Buffer.alloc(8);
119
+ buf.writeBigUInt64BE(BigInt(counter));
120
+ const hmac = createHmac("sha1", base32Decode(seed)).update(buf).digest();
121
+ const offset = hmac[hmac.length - 1] & 15;
122
+ const binary = (hmac[offset] & 127) << 24 | (hmac[offset + 1] & 255) << 16 | (hmac[offset + 2] & 255) << 8 | hmac[offset + 3] & 255;
123
+ return (binary % 10 ** digits).toString().padStart(digits, "0");
124
+ }
125
+
126
+ // src/tools/registry.ts
127
+ import { matchUrlWithDomainPattern as matchUrlWithDomainPattern2 } from "@ap3x/browser";
128
+ import { Type } from "@sinclair/typebox";
129
+ import { Value } from "@sinclair/typebox/value";
130
+ function defineAction(def) {
131
+ return def;
132
+ }
133
+ function isAvailable(action, pageUrl) {
134
+ if (!action.domains || action.domains.length === 0) return true;
135
+ if (pageUrl === void 0) return false;
136
+ return action.domains.some((pattern) => matchUrlWithDomainPattern2(pageUrl, pattern));
137
+ }
138
+ function firstError(schema, value) {
139
+ const [error] = [...Value.Errors(schema, value)];
140
+ return error ? `${error.path || "/"}: ${error.message}` : "does not match schema";
141
+ }
142
+ var Registry = class _Registry {
143
+ actions = /* @__PURE__ */ new Map();
144
+ excluded = /* @__PURE__ */ new Set();
145
+ unionCache = /* @__PURE__ */ new Map();
146
+ /** Register an action. Later registration of an excluded name is a no-op. */
147
+ register(def) {
148
+ if (this.excluded.has(def.name)) return;
149
+ if (this.actions.has(def.name)) {
150
+ throw new Error(`Action "${def.name}" is already registered`);
151
+ }
152
+ this.actions.set(def.name, {
153
+ name: def.name,
154
+ description: def.description,
155
+ schema: def.schema,
156
+ domains: def.domains,
157
+ terminatesSequence: def.terminatesSequence ?? false,
158
+ execute: def.execute
159
+ });
160
+ this.unionCache.clear();
161
+ }
162
+ /**
163
+ * A shallow copy sharing the same action definitions. The step loop clones a
164
+ * caller's registry before applying a vision filter so the caller's object is
165
+ * never mutated (E7). Excluded names carry over.
166
+ */
167
+ clone() {
168
+ const copy = new _Registry();
169
+ for (const [name, action] of this.actions) copy.actions.set(name, action);
170
+ for (const name of this.excluded) copy.excluded.add(name);
171
+ return copy;
172
+ }
173
+ /** Remove an action and block it from being re-registered. */
174
+ exclude(name) {
175
+ this.excluded.add(name);
176
+ if (this.actions.delete(name)) this.unionCache.clear();
177
+ }
178
+ get(name) {
179
+ return this.actions.get(name);
180
+ }
181
+ /** Actions available on `pageUrl` (unfiltered + domain-matching), name-sorted. */
182
+ available(pageUrl) {
183
+ return [...this.actions.values()].filter((a) => isAvailable(a, pageUrl)).sort((a, b) => a.name.localeCompare(b.name));
184
+ }
185
+ /**
186
+ * The per-page union of single-key action objects for structured output (A8).
187
+ * Cached by the set of admitted action names (so equivalent URLs share one).
188
+ */
189
+ actionUnion(pageUrl) {
190
+ const admitted = this.available(pageUrl);
191
+ const key = admitted.map((a) => a.name).join("|");
192
+ const cached = this.unionCache.get(key);
193
+ if (cached) return cached;
194
+ const members = admitted.map(
195
+ (a) => Type.Object({ [a.name]: a.schema }, { additionalProperties: false, title: a.name })
196
+ );
197
+ const union = members.length === 0 ? Type.Never() : members.length === 1 ? members[0] : Type.Union(members);
198
+ this.unionCache.set(key, union);
199
+ return union;
200
+ }
201
+ /** `name: description (param=type, ...)` lines for the prompt. */
202
+ promptDescriptions(pageUrl) {
203
+ return this.available(pageUrl).map((a) => {
204
+ const props = a.schema.properties ?? {};
205
+ const params = Object.keys(props).join(", ");
206
+ return `${a.name}: ${a.description}${params ? ` (${params})` : ""}`;
207
+ }).join("\n");
208
+ }
209
+ /**
210
+ * Execute one action. `<secret>key</secret>` tags are substituted (enforcement
211
+ * point 3) and — ALWAYS, even with no secrets configured — scanned for
212
+ * survivors: any unresolved `<secret>…</secret>` short-circuits to an
213
+ * `ActionResult.error` and the action never runs, so a literal tag can never be
214
+ * typed (C5). Params are then validated against the action's schema before
215
+ * `execute`; afterwards every result string is masked against the resolved
216
+ * secrets so a real substituted value can never land in history.
217
+ *
218
+ * `allSensitiveData` (M8 follow-up), when provided, masks results against
219
+ * EVERY domain's secret values (mirroring `maskOutbound`'s all-domains
220
+ * approach), not just `secrets` (the current-URL-scoped subset used for
221
+ * substitution). Page content is not domain-sandboxed the way secret
222
+ * substitution is — a domain-B secret echoed while acting on domain A must
223
+ * still be masked on this method's outbound text (narration / event stream /
224
+ * ToolResultMessage), the same way the LLM-prompt path already masks via
225
+ * `maskOutbound` in the message manager. Without it, masking falls back to
226
+ * `secrets` alone (the pre-M11 behavior — still correct for the single-domain
227
+ * case, just narrower).
228
+ */
229
+ async execute(name, rawParams, ctx, secrets, allSensitiveData) {
230
+ const action = this.actions.get(name);
231
+ if (!action) return stepResult({ error: `Action "${name}" not found` });
232
+ const resolved = secrets ?? {};
233
+ const { value: params, unresolved } = substituteSecrets(rawParams, resolved);
234
+ if (unresolved.length > 0) {
235
+ return stepResult({
236
+ error: `Unresolved secret placeholder(s): ${unresolved.join(", ")}. No such key in sensitive data for this page \u2014 the value was not typed.`
237
+ });
238
+ }
239
+ if (!Value.Check(action.schema, params)) {
240
+ return stepResult({
241
+ error: `Invalid parameters for "${name}": ${firstError(action.schema, params)}`
242
+ });
243
+ }
244
+ try {
245
+ const result = await action.execute(params, ctx);
246
+ return maskResultSecrets(result, resolved, allSensitiveData);
247
+ } catch (error) {
248
+ const message = error instanceof Error ? error.message : String(error);
249
+ return maskResultSecrets(stepResult({ error: message }), resolved, allSensitiveData);
250
+ }
251
+ }
252
+ };
253
+ function maskResultSecrets(result, secrets, allSensitiveData) {
254
+ const mask = (text) => {
255
+ if (text === void 0) return void 0;
256
+ return allSensitiveData ? maskOutbound(text, allSensitiveData) : maskSecretMap(text, secrets);
257
+ };
258
+ const masked = { ...result };
259
+ if (masked.longTermMemory !== void 0) masked.longTermMemory = mask(masked.longTermMemory);
260
+ if (masked.extractedContent !== void 0)
261
+ masked.extractedContent = mask(masked.extractedContent);
262
+ if (masked.error !== void 0) masked.error = mask(masked.error);
263
+ return masked;
264
+ }
265
+
266
+ // src/filesystem/external-read.ts
267
+ import { readFile as fsReadFile } from "fs/promises";
268
+ import { strFromU8, unzipSync } from "fflate";
269
+ import { getDocument } from "pdfjs-dist/legacy/build/pdf.mjs";
270
+ var MAX_CHARS = 6e4;
271
+ function truncate(text) {
272
+ return text.length <= MAX_CHARS ? text : `${text.slice(0, MAX_CHARS)}
273
+ [...truncated]`;
274
+ }
275
+ var MAX_DOCX_INFLATED_BYTES = 32 * 1024 * 1024;
276
+ var FileParseError = class extends Error {
277
+ constructor(path, message, options) {
278
+ super(message, options);
279
+ this.path = path;
280
+ }
281
+ path;
282
+ name = "FileParseError";
283
+ };
284
+ function errorMessage(err) {
285
+ return err instanceof Error ? err.message : String(err);
286
+ }
287
+ async function parsePdf(buf, path) {
288
+ let doc;
289
+ try {
290
+ doc = await getDocument({
291
+ data: new Uint8Array(buf),
292
+ isEvalSupported: false,
293
+ useSystemFonts: true,
294
+ verbosity: 0
295
+ // ERRORS only — silence pdf.js's recovery-mode console noise
296
+ }).promise;
297
+ } catch (err) {
298
+ throw new FileParseError(path, `Could not parse PDF '${path}': ${errorMessage(err)}`, {
299
+ cause: err
300
+ });
301
+ }
302
+ try {
303
+ const pages = [];
304
+ for (let i = 1; i <= doc.numPages; i++) {
305
+ const page = await doc.getPage(i);
306
+ const content = await page.getTextContent();
307
+ const text = content.items.map((item) => "str" in item ? item.str : "").join(" ").trim();
308
+ pages.push(text);
309
+ }
310
+ return pages.join("\n\n");
311
+ } catch (err) {
312
+ throw new FileParseError(
313
+ path,
314
+ `Could not read PDF page text in '${path}': ${errorMessage(err)}`,
315
+ { cause: err }
316
+ );
317
+ } finally {
318
+ await doc.destroy().catch(() => {
319
+ });
320
+ }
321
+ }
322
+ var DOCX_PARAGRAPH = /<w:p[ >][\s\S]*?<\/w:p>/g;
323
+ var DOCX_TEXT_RUN = /<w:t(?:\s[^>]*)?>([\s\S]*?)<\/w:t>/g;
324
+ var DOCX_LINE_BREAK = /<w:(?:br|tab|cr)\s*\/>/g;
325
+ function decodeXmlEntities(text) {
326
+ return text.replace(/&lt;/g, "<").replace(/&gt;/g, ">").replace(/&quot;/g, '"').replace(/&apos;/g, "'").replace(/&amp;/g, "&");
327
+ }
328
+ function parseDocx(buf, path) {
329
+ let xml;
330
+ try {
331
+ const entries = unzipSync(new Uint8Array(buf), {
332
+ // Checked against the local header's declared uncompressed size BEFORE
333
+ // inflation — a bomb entry never gets decompressed at all.
334
+ filter: (info) => info.name === "word/document.xml" && info.originalSize <= MAX_DOCX_INFLATED_BYTES
335
+ });
336
+ const bytes = entries["word/document.xml"];
337
+ if (!bytes) {
338
+ throw new Error(
339
+ `missing word/document.xml, or it exceeds the ${MAX_DOCX_INFLATED_BYTES}-byte inflate cap \u2014 not a valid (or safe) .docx file`
340
+ );
341
+ }
342
+ xml = strFromU8(bytes);
343
+ } catch (err) {
344
+ throw new FileParseError(path, `Could not parse DOCX '${path}': ${errorMessage(err)}`, {
345
+ cause: err
346
+ });
347
+ }
348
+ const paragraphs = xml.match(DOCX_PARAGRAPH) ?? [];
349
+ const lines = paragraphs.map((paragraph) => {
350
+ const withBreaks = paragraph.replace(DOCX_LINE_BREAK, "\n");
351
+ const runs = [...withBreaks.matchAll(DOCX_TEXT_RUN)].map((m) => decodeXmlEntities(m[1] ?? ""));
352
+ return runs.join("");
353
+ });
354
+ return lines.join("\n").trim();
355
+ }
356
+ async function readExternalFile(path) {
357
+ const ext = path.slice(path.lastIndexOf(".") + 1).toLowerCase();
358
+ if (ext === "pdf") return truncate(await parsePdf(await fsReadFile(path), path));
359
+ if (ext === "docx") return truncate(parseDocx(await fsReadFile(path), path));
360
+ return truncate(await fsReadFile(path, "utf8"));
361
+ }
362
+
363
+ // src/tools/actions.ts
364
+ import { basename, join } from "path";
365
+ import { InteractionError, PAGE_SCROLL_PX, asElementIndex, isPathContained } from "@ap3x/browser";
366
+ import { Type as Type2 } from "@sinclair/typebox";
367
+ var sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
368
+ function nodeAt(ctx, index) {
369
+ const node = ctx.session.getElementByIndex(index);
370
+ if (!node) {
371
+ return {
372
+ ok: false,
373
+ result: stepResult({
374
+ error: `Element ${index} not found \u2014 the page may have changed. Re-read the page state to re-index.`
375
+ })
376
+ };
377
+ }
378
+ return { ok: true, node };
379
+ }
380
+ var navigate = defineAction({
381
+ name: "navigate",
382
+ description: "Navigate the current tab to a URL (opens a new tab when newTab is true).",
383
+ schema: Type2.Object({ url: Type2.String(), newTab: Type2.Optional(Type2.Boolean()) }),
384
+ terminatesSequence: true,
385
+ async execute(params, ctx) {
386
+ if (params.newTab) {
387
+ await ctx.session.openTab(params.url);
388
+ return stepResult({ longTermMemory: `Opened new tab at ${params.url}` });
389
+ }
390
+ await ctx.driver.navigate(params.url, { signal: ctx.signal });
391
+ return stepResult({ longTermMemory: `Navigated to ${params.url}` });
392
+ }
393
+ });
394
+ var goBack = defineAction({
395
+ name: "go_back",
396
+ description: "Go back one entry in the browser history.",
397
+ schema: Type2.Object({}),
398
+ terminatesSequence: true,
399
+ async execute(_params, ctx) {
400
+ await ctx.session.goBack();
401
+ return stepResult({ longTermMemory: "Navigated back" });
402
+ }
403
+ });
404
+ var click = defineAction({
405
+ name: "click",
406
+ description: "Click an interactive element by its [index].",
407
+ schema: Type2.Object({ index: Type2.Integer({ minimum: 1 }) }),
408
+ async execute(params, ctx) {
409
+ const resolved = nodeAt(ctx, params.index);
410
+ if (!resolved.ok) return resolved.result;
411
+ await ctx.driver.click(asElementIndex(params.index), { signal: ctx.signal });
412
+ return stepResult({ longTermMemory: `Clicked element ${params.index}` });
413
+ }
414
+ });
415
+ var inputText = defineAction({
416
+ name: "input_text",
417
+ description: "Type text into an element by its [index]. clear=false appends; text='' clears.",
418
+ schema: Type2.Object({
419
+ index: Type2.Integer({ minimum: 0 }),
420
+ text: Type2.String(),
421
+ clear: Type2.Optional(Type2.Boolean())
422
+ }),
423
+ async execute(params, ctx) {
424
+ const resolved = nodeAt(ctx, params.index);
425
+ if (!resolved.ok) return resolved.result;
426
+ await ctx.driver.type(asElementIndex(params.index), params.text, {
427
+ clear: params.clear ?? true,
428
+ // Suppress value read-back when a secret is in play so it never round-trips out.
429
+ sensitive: ctx.hasSensitiveData,
430
+ signal: ctx.signal
431
+ });
432
+ const shown = ctx.hasSensitiveData ? "<sensitive>" : params.text;
433
+ return stepResult({ longTermMemory: `Typed "${shown}" into element ${params.index}` });
434
+ }
435
+ });
436
+ var scroll = defineAction({
437
+ name: "scroll",
438
+ description: "Scroll the page (or an element by [index]) up/down by a number of pages.",
439
+ schema: Type2.Object({
440
+ down: Type2.Optional(Type2.Boolean()),
441
+ pages: Type2.Optional(Type2.Number()),
442
+ index: Type2.Optional(Type2.Integer({ minimum: 1 }))
443
+ }),
444
+ async execute(params, ctx) {
445
+ const pages = params.pages ?? 1;
446
+ const amount = Math.round(pages * PAGE_SCROLL_PX) * (params.down === false ? -1 : 1);
447
+ if (params.index !== void 0) {
448
+ const resolved = nodeAt(ctx, params.index);
449
+ if (!resolved.ok) return resolved.result;
450
+ await ctx.session.input.scroll({ amount, node: resolved.node, signal: ctx.signal });
451
+ } else {
452
+ await ctx.driver.scroll(amount, { signal: ctx.signal });
453
+ }
454
+ return stepResult({
455
+ longTermMemory: `Scrolled ${params.down === false ? "up" : "down"} ${pages} page(s)`
456
+ });
457
+ }
458
+ });
459
+ var scrollToText = defineAction({
460
+ name: "scroll_to_text",
461
+ description: "Scroll to the first occurrence of the given text.",
462
+ schema: Type2.Object({ text: Type2.String() }),
463
+ async execute(params, ctx) {
464
+ try {
465
+ await ctx.session.input.scrollToText(params.text, { signal: ctx.signal });
466
+ return stepResult({ longTermMemory: `Scrolled to text "${params.text}"` });
467
+ } catch (error) {
468
+ if (error instanceof InteractionError && error.reason === "text_not_found") {
469
+ return stepResult({ extractedContent: `Text "${params.text}" not found on the page.` });
470
+ }
471
+ throw error;
472
+ }
473
+ }
474
+ });
475
+ var sendKeys = defineAction({
476
+ name: "send_keys",
477
+ description: "Dispatch raw keyboard keys or a shortcut (e.g. 'Enter', 'Control+a').",
478
+ schema: Type2.Object({ keys: Type2.String() }),
479
+ async execute(params, ctx) {
480
+ await ctx.session.input.sendKeys(params.keys, { signal: ctx.signal });
481
+ return stepResult({ longTermMemory: `Sent keys ${params.keys}` });
482
+ }
483
+ });
484
+ var selectDropdown = defineAction({
485
+ name: "select_dropdown",
486
+ description: "Select a dropdown option by its exact text, on the element at [index].",
487
+ schema: Type2.Object({ index: Type2.Integer({ minimum: 1 }), text: Type2.String() }),
488
+ async execute(params, ctx) {
489
+ const resolved = nodeAt(ctx, params.index);
490
+ if (!resolved.ok) return resolved.result;
491
+ const result = await ctx.session.input.selectDropdownOption(resolved.node, params.text, {
492
+ signal: ctx.signal
493
+ });
494
+ if (!result.success) {
495
+ return stepResult({ error: result.message ?? `Could not select "${params.text}"` });
496
+ }
497
+ return stepResult({
498
+ longTermMemory: `Selected "${params.text}" (value=${result.value ?? ""})`
499
+ });
500
+ }
501
+ });
502
+ var getDropdownOptions = defineAction({
503
+ name: "get_dropdown_options",
504
+ description: "List the available options of the dropdown element at [index].",
505
+ schema: Type2.Object({ index: Type2.Integer({ minimum: 1 }) }),
506
+ async execute(params, ctx) {
507
+ const resolved = nodeAt(ctx, params.index);
508
+ if (!resolved.ok) return resolved.result;
509
+ const options = await ctx.session.input.getDropdownOptions(resolved.node, {
510
+ signal: ctx.signal
511
+ });
512
+ return stepResult({
513
+ extractedContent: `Options: ${options.join(", ")}. Use the exact text string to select.`,
514
+ includeExtractedContentOnlyOnce: true
515
+ });
516
+ }
517
+ });
518
+ var switchTab = defineAction({
519
+ name: "switch_tab",
520
+ description: "Focus another tab by its 4-char id.",
521
+ schema: Type2.Object({ tabId: Type2.String() }),
522
+ terminatesSequence: true,
523
+ async execute(params, ctx) {
524
+ await ctx.driver.switchTab(params.tabId);
525
+ return stepResult({ longTermMemory: `Switched to tab ${params.tabId}` });
526
+ }
527
+ });
528
+ var openTab = defineAction({
529
+ name: "open_tab",
530
+ description: "Open a new tab at a URL.",
531
+ schema: Type2.Object({ url: Type2.String() }),
532
+ terminatesSequence: true,
533
+ async execute(params, ctx) {
534
+ const tabId = await ctx.session.openTab(params.url);
535
+ return stepResult({ longTermMemory: `Opened tab ${tabId} at ${params.url}` });
536
+ }
537
+ });
538
+ var closeTab = defineAction({
539
+ name: "close_tab",
540
+ description: "Close a tab by its 4-char id.",
541
+ schema: Type2.Object({ tabId: Type2.String() }),
542
+ async execute(params, ctx) {
543
+ await ctx.driver.closeTab(params.tabId);
544
+ return stepResult({ longTermMemory: `Closed tab ${params.tabId}` });
545
+ }
546
+ });
547
+ var uploadFile = defineAction({
548
+ name: "upload_file",
549
+ description: "Set a file on the file-input element at [index]. Path must be available.",
550
+ schema: Type2.Object({ index: Type2.Integer({ minimum: 0 }), path: Type2.String() }),
551
+ async execute(params, ctx) {
552
+ let uploadPath;
553
+ const base = basename(params.path);
554
+ if (ctx.fileSystem.listFiles().includes(base)) {
555
+ uploadPath = join(ctx.fileSystem.getDir(), base);
556
+ if (!await isPathContained(uploadPath, ctx.fileSystem.getDir())) {
557
+ return stepResult({ error: `Upload of "${params.path}" escapes the workspace; refusing.` });
558
+ }
559
+ } else if (ctx.availableFilePaths.includes(params.path)) {
560
+ uploadPath = params.path;
561
+ } else {
562
+ return stepResult({ error: `Path "${params.path}" is not available for upload.` });
563
+ }
564
+ const resolved = nodeAt(ctx, params.index);
565
+ if (!resolved.ok) return resolved.result;
566
+ await ctx.session.input.uploadFile(resolved.node, uploadPath, { signal: ctx.signal });
567
+ return stepResult({ longTermMemory: `Uploaded ${uploadPath} to element ${params.index}` });
568
+ }
569
+ });
570
+ var wait = defineAction({
571
+ name: "wait",
572
+ description: "Wait for a number of seconds (capped at 30).",
573
+ schema: Type2.Object({ seconds: Type2.Optional(Type2.Number()) }),
574
+ async execute(params) {
575
+ const requested = params.seconds ?? 3;
576
+ await sleep(Math.min(Math.max(requested - 1, 0), 30) * 1e3);
577
+ return stepResult({ longTermMemory: `Waited for ${requested} seconds` });
578
+ }
579
+ });
580
+ var extractContent = defineAction({
581
+ name: "extract_content",
582
+ description: "Extract the page's readable content, guided by a query.",
583
+ schema: Type2.Object({
584
+ query: Type2.String()
585
+ }),
586
+ async execute(params, ctx) {
587
+ const text = await ctx.driver.extract(params.query, { signal: ctx.signal });
588
+ const content = ctx.extractionLlm ? await ctx.extractionLlm.extract(`Query: ${params.query}
589
+
590
+ ${text}`, { signal: ctx.signal }) : text;
591
+ const once = content.length > 1e4;
592
+ return stepResult({
593
+ extractedContent: once ? content : `<query>${params.query}</query>
594
+ ${content}`,
595
+ includeExtractedContentOnlyOnce: once
596
+ });
597
+ }
598
+ });
599
+ var writeFile = defineAction({
600
+ name: "write_file",
601
+ description: "Write (replacing) a file in the agent workspace.",
602
+ schema: Type2.Object({
603
+ fileName: Type2.String(),
604
+ content: Type2.String(),
605
+ append: Type2.Optional(Type2.Boolean())
606
+ }),
607
+ async execute(params, ctx) {
608
+ const message = params.append ? await ctx.fileSystem.appendFile(params.fileName, params.content) : await ctx.fileSystem.writeFile(params.fileName, params.content);
609
+ return stepResult({ extractedContent: message, longTermMemory: message });
610
+ }
611
+ });
612
+ var appendFile = defineAction({
613
+ name: "append_file",
614
+ description: "Append content to a file in the agent workspace.",
615
+ schema: Type2.Object({ fileName: Type2.String(), content: Type2.String() }),
616
+ async execute(params, ctx) {
617
+ const message = await ctx.fileSystem.appendFile(params.fileName, params.content);
618
+ return stepResult({ extractedContent: message, longTermMemory: message });
619
+ }
620
+ });
621
+ var replaceFile = defineAction({
622
+ name: "replace_file",
623
+ description: "Replace a substring in a workspace file (the todo.md checkbox primitive).",
624
+ schema: Type2.Object({ fileName: Type2.String(), oldStr: Type2.String(), newStr: Type2.String() }),
625
+ async execute(params, ctx) {
626
+ const message = await ctx.fileSystem.replaceFileStr(
627
+ params.fileName,
628
+ params.oldStr,
629
+ params.newStr
630
+ );
631
+ return stepResult({ extractedContent: message, longTermMemory: message });
632
+ }
633
+ });
634
+ var readFile = defineAction({
635
+ name: "read_file",
636
+ description: "Read a workspace file, or a whitelisted external file (text, .pdf, or .docx).",
637
+ schema: Type2.Object({ fileName: Type2.String() }),
638
+ async execute(params, ctx) {
639
+ const managed = ctx.fileSystem.getFile(params.fileName);
640
+ if (managed !== void 0) {
641
+ return stepResult({ extractedContent: ctx.fileSystem.readFile(params.fileName) });
642
+ }
643
+ const external = ctx.availableFilePaths.find((p) => basename(p) === basename(params.fileName));
644
+ if (external) {
645
+ try {
646
+ const content = await readExternalFile(external);
647
+ return stepResult({ extractedContent: `Read from file ${params.fileName}.
648
+ ${content}` });
649
+ } catch (error) {
650
+ const message = error instanceof Error ? error.message : String(error);
651
+ return stepResult({ error: `Could not read external file ${params.fileName}: ${message}` });
652
+ }
653
+ }
654
+ return stepResult({ error: `File '${params.fileName}' not found.` });
655
+ }
656
+ });
657
+ var done = defineAction({
658
+ name: "done",
659
+ description: "Finish the task. success=true ONLY when fully complete. text must report only observed facts.",
660
+ schema: Type2.Object({
661
+ text: Type2.String(),
662
+ success: Type2.Boolean(),
663
+ filesToDisplay: Type2.Optional(Type2.Array(Type2.String()))
664
+ }),
665
+ execute(params, ctx) {
666
+ const attachments = (params.filesToDisplay ?? []).map((name) => ctx.fileSystem.displayFile(name) !== void 0 ? name : void 0).filter((n) => n !== void 0);
667
+ return doneResult(params.success, {
668
+ extractedContent: params.text,
669
+ attachments: attachments.length > 0 ? attachments : void 0
670
+ });
671
+ }
672
+ });
673
+ var BUILTIN_ACTIONS = [
674
+ navigate,
675
+ goBack,
676
+ click,
677
+ inputText,
678
+ scroll,
679
+ scrollToText,
680
+ sendKeys,
681
+ selectDropdown,
682
+ getDropdownOptions,
683
+ switchTab,
684
+ openTab,
685
+ closeTab,
686
+ uploadFile,
687
+ wait,
688
+ extractContent,
689
+ writeFile,
690
+ appendFile,
691
+ replaceFile,
692
+ readFile,
693
+ done
694
+ ];
695
+ function registerBuiltins(registry) {
696
+ for (const action of BUILTIN_ACTIONS) registry.register(action);
697
+ }
698
+ function createDefaultRegistry() {
699
+ const registry = new Registry();
700
+ registerBuiltins(registry);
701
+ return registry;
702
+ }
703
+
704
+ // src/filesystem/file-system.ts
705
+ import { mkdir, rm, writeFile as writeFile2 } from "fs/promises";
706
+ import { basename as basename2, join as join2 } from "path";
707
+ var SUPPORTED_EXTENSIONS = ["md", "txt", "json", "jsonl", "csv", "html", "xml"];
708
+ var STUBBED_EXTENSIONS = ["pdf", "docx"];
709
+ var SUPPORTED = new Set(SUPPORTED_EXTENSIONS);
710
+ var STUBBED = new Set(STUBBED_EXTENSIONS);
711
+ var DATA_DIR_NAME = "ap3x_agent_data";
712
+ var DEFAULT_FILE = "todo.md";
713
+ var DESCRIBE_WHOLE_LIMIT = 600;
714
+ var DESCRIBE_PREVIEW_CHARS = 200;
715
+ function splitName(raw) {
716
+ const base = basename2(raw);
717
+ const dot = base.lastIndexOf(".");
718
+ if (dot <= 0) return null;
719
+ return { stem: base.slice(0, dot), ext: base.slice(dot + 1).toLowerCase() };
720
+ }
721
+ var WINDOWS_RESERVED = /^(con|prn|aux|nul|com[1-9]|lpt[1-9])$/i;
722
+ function sanitizeStem(stem) {
723
+ const cleaned = stem.replace(/\s+/g, "-").replace(/[^\p{L}\p{N}_().-]/gu, "").replace(/-+/g, "-");
724
+ if (cleaned === "") return "file";
725
+ return WINDOWS_RESERVED.test(cleaned) ? `_${cleaned}` : cleaned;
726
+ }
727
+ function sanitizeFilename(name) {
728
+ const split = splitName(name);
729
+ if (!split) return name;
730
+ return `${sanitizeStem(split.stem)}.${split.ext}`;
731
+ }
732
+ function parseCsv(text) {
733
+ const rows = [];
734
+ let row = [];
735
+ let field = "";
736
+ let inQuotes = false;
737
+ for (let i = 0; i < text.length; i++) {
738
+ const c = text.charAt(i);
739
+ if (inQuotes) {
740
+ if (c === '"') {
741
+ if (text.charAt(i + 1) === '"') {
742
+ field += '"';
743
+ i++;
744
+ } else {
745
+ inQuotes = false;
746
+ }
747
+ } else {
748
+ field += c;
749
+ }
750
+ continue;
751
+ }
752
+ if (c === '"') inQuotes = true;
753
+ else if (c === ",") {
754
+ row.push(field);
755
+ field = "";
756
+ } else if (c === "\n") {
757
+ row.push(field);
758
+ rows.push(row);
759
+ row = [];
760
+ field = "";
761
+ } else if (c !== "\r") field += c;
762
+ }
763
+ row.push(field);
764
+ rows.push(row);
765
+ return rows;
766
+ }
767
+ function serializeCsvField(field) {
768
+ return /[",\n]/.test(field) ? `"${field.replace(/"/g, '""')}"` : field;
769
+ }
770
+ function normalizeCsv(content) {
771
+ let text = content;
772
+ if (!text.includes("\n") && text.includes("\\n")) {
773
+ text = text.replace(/\\"/g, '"').replace(/\\n/g, "\n");
774
+ }
775
+ const rows = parseCsv(text);
776
+ while (rows.length > 0 && rows[0].length === 1 && rows[0][0] === "") rows.shift();
777
+ while (rows.length > 0 && rows[rows.length - 1].length === 1 && rows[rows.length - 1][0] === "")
778
+ rows.pop();
779
+ return rows.map((r) => r.map(serializeCsvField).join(",")).join("\n");
780
+ }
781
+ function normalizeContent(ext, content) {
782
+ return ext === "csv" ? normalizeCsv(content) : content;
783
+ }
784
+ var FileSystem = class _FileSystem {
785
+ files = /* @__PURE__ */ new Map();
786
+ extractedContentCount = 0;
787
+ baseDir;
788
+ dataDir;
789
+ constructor(baseDir) {
790
+ this.baseDir = baseDir;
791
+ this.dataDir = join2(baseDir, DATA_DIR_NAME);
792
+ }
793
+ /** Create a fresh filesystem. The data dir is wiped and recreated. */
794
+ static async create(baseDir, options = {}) {
795
+ const fs = new _FileSystem(baseDir);
796
+ await rm(fs.dataDir, { recursive: true, force: true });
797
+ await mkdir(fs.dataDir, { recursive: true });
798
+ if (options.createDefaultFiles ?? true) {
799
+ fs.files.set(DEFAULT_FILE, { stem: "todo", ext: "md", content: "" });
800
+ await fs.syncFile(DEFAULT_FILE);
801
+ }
802
+ return fs;
803
+ }
804
+ /** Restore from a checkpoint at the same base dir; unknown types skipped. */
805
+ static async fromState(state) {
806
+ const fs = new _FileSystem(state.baseDir);
807
+ await rm(fs.dataDir, { recursive: true, force: true });
808
+ await mkdir(fs.dataDir, { recursive: true });
809
+ fs.extractedContentCount = state.extractedContentCount;
810
+ for (const [full, { type, data }] of Object.entries(state.files)) {
811
+ if (!SUPPORTED.has(type)) continue;
812
+ const split = splitName(full);
813
+ if (!split) continue;
814
+ fs.files.set(full, { stem: split.stem, ext: type, content: data });
815
+ await fs.syncFile(full);
816
+ }
817
+ return fs;
818
+ }
819
+ /** Serialize for checkpoint. */
820
+ getState() {
821
+ const files = {};
822
+ for (const [full, entry] of this.files) files[full] = { type: entry.ext, data: entry.content };
823
+ return { files, baseDir: this.baseDir, extractedContentCount: this.extractedContentCount };
824
+ }
825
+ getAllowedExtensions() {
826
+ return SUPPORTED_EXTENSIONS;
827
+ }
828
+ getDir() {
829
+ return this.dataDir;
830
+ }
831
+ listFiles() {
832
+ return [...this.files.keys()];
833
+ }
834
+ /** Raw content of a managed file, or undefined. */
835
+ getFile(name) {
836
+ return this.files.get(name)?.content;
837
+ }
838
+ /** The always-present todo.md content. */
839
+ getTodoContents() {
840
+ return this.files.get(DEFAULT_FILE)?.content ?? "";
841
+ }
842
+ /** Write (replacing) a managed file. Returns an LLM-facing message. */
843
+ async writeFile(name, content) {
844
+ const resolved = this.resolve(name);
845
+ if ("error" in resolved) return resolved.error;
846
+ const { full, stem, ext, note } = resolved;
847
+ this.files.set(full, { stem, ext, content: normalizeContent(ext, content) });
848
+ await this.syncFile(full);
849
+ return `Data written to file ${full} successfully.${note}`;
850
+ }
851
+ /** Append to a managed file (creating it if new). */
852
+ async appendFile(name, content) {
853
+ const resolved = this.resolve(name);
854
+ if ("error" in resolved) return resolved.error;
855
+ const { full, stem, ext, note } = resolved;
856
+ const existing = this.files.get(full)?.content ?? "";
857
+ this.files.set(full, { stem, ext, content: normalizeContent(ext, existing + content) });
858
+ await this.syncFile(full);
859
+ return `Data appended to file ${full} successfully.${note}`;
860
+ }
861
+ /** Targeted string replace — the todo.md checkbox-update primitive. */
862
+ async replaceFileStr(name, oldStr, newStr) {
863
+ if (oldStr === "") return "Cannot replace with an empty search string.";
864
+ const full = sanitizeFilename(name);
865
+ const entry = this.files.get(full);
866
+ if (!entry) return `File '${full}' not found.`;
867
+ entry.content = normalizeContent(entry.ext, entry.content.split(oldStr).join(newStr));
868
+ await this.syncFile(full);
869
+ return `Successfully replaced text in ${full}.`;
870
+ }
871
+ /** Read a managed file; formatted message. */
872
+ readFile(name) {
873
+ const full = sanitizeFilename(name);
874
+ const entry = this.files.get(full);
875
+ if (!entry) return `File '${full}' not found.`;
876
+ return `Read from file ${full}.
877
+ <content>
878
+ ${entry.content}
879
+ </content>`;
880
+ }
881
+ /** Raw content for done attachments. */
882
+ displayFile(name) {
883
+ return this.files.get(sanitizeFilename(name))?.content;
884
+ }
885
+ /** Save extracted content to an auto-numbered file; returns the filename. */
886
+ async saveExtractedContent(content) {
887
+ const full = `extracted_content_${this.extractedContentCount}.md`;
888
+ this.extractedContentCount++;
889
+ this.files.set(full, { stem: full.slice(0, -3), ext: "md", content });
890
+ await this.syncFile(full);
891
+ return full;
892
+ }
893
+ /** Remove the data dir from disk (in-memory state untouched). */
894
+ async nuke() {
895
+ await rm(this.dataDir, { recursive: true, force: true });
896
+ }
897
+ /**
898
+ * Prompt-friendly listing of every file EXCEPT todo.md (which is surfaced whole
899
+ * elsewhere). Small files shown whole; large files previewed head+tail.
900
+ */
901
+ describe() {
902
+ const blocks = [];
903
+ for (const [full, entry] of this.files) {
904
+ if (full === DEFAULT_FILE) continue;
905
+ const lines = entry.content === "" ? 0 : entry.content.split("\n").length;
906
+ const body = entry.content.length < DESCRIBE_WHOLE_LIMIT ? entry.content : this.preview(entry.content, lines);
907
+ blocks.push(`<file>
908
+ ${full} - ${lines} lines
909
+ ${body}
910
+ </file>`);
911
+ }
912
+ return blocks.join("\n");
913
+ }
914
+ preview(content, totalLines) {
915
+ const head = content.slice(0, DESCRIBE_PREVIEW_CHARS);
916
+ const tail = content.slice(-DESCRIBE_PREVIEW_CHARS);
917
+ const shownLines = head.split("\n").length + tail.split("\n").length;
918
+ const omitted = Math.max(0, totalLines - shownLines);
919
+ return `${head}
920
+ ... ${omitted} more lines ...
921
+ ${tail}`;
922
+ }
923
+ /** Validate + sanitize a target name, or return an LLM-facing error. */
924
+ resolve(name) {
925
+ const split = splitName(name);
926
+ if (!split) {
927
+ return { error: `File '${basename2(name)}' has no extension. Try a name like 'notes.md'.` };
928
+ }
929
+ if (isBinaryExtension(split.ext)) {
930
+ return {
931
+ error: `Cannot write '${split.ext}' \u2014 that is a binary/image extension. Only text files are supported (screenshots are captured automatically).`
932
+ };
933
+ }
934
+ if (STUBBED.has(split.ext)) {
935
+ return { error: `Writing '.${split.ext}' files is not supported in this version yet.` };
936
+ }
937
+ if (!SUPPORTED.has(split.ext)) {
938
+ return {
939
+ error: `Unsupported file extension '.${split.ext}'. Allowed: ${SUPPORTED_EXTENSIONS.join(", ")}.`
940
+ };
941
+ }
942
+ const full = sanitizeFilename(name);
943
+ const changed = full !== name;
944
+ const note = changed ? ` (Note: filename auto-corrected to ${full})` : "";
945
+ return {
946
+ full,
947
+ stem: full.slice(0, full.lastIndexOf(".")),
948
+ ext: split.ext,
949
+ note
950
+ };
951
+ }
952
+ async syncFile(full) {
953
+ const entry = this.files.get(full);
954
+ if (!entry) return;
955
+ await writeFile2(join2(this.dataDir, full), entry.content, "utf8");
956
+ }
957
+ };
958
+ var BINARY_EXTENSIONS = /* @__PURE__ */ new Set([
959
+ "png",
960
+ "jpg",
961
+ "jpeg",
962
+ "gif",
963
+ "webp",
964
+ "bmp",
965
+ "ico",
966
+ "svg",
967
+ "zip",
968
+ "gz",
969
+ "tar",
970
+ "exe",
971
+ "dll",
972
+ "mp3",
973
+ "mp4",
974
+ "wav",
975
+ "mov"
976
+ ]);
977
+ function isBinaryExtension(ext) {
978
+ return BINARY_EXTENSIONS.has(ext);
979
+ }
980
+
981
+ // src/agent/output-schema.ts
982
+ import { Type as Type3 } from "@sinclair/typebox";
983
+ function buildAgentOutputSchema(actionUnion, variant) {
984
+ const action = Type3.Array(actionUnion, { minItems: 1 });
985
+ if (variant === "flash") {
986
+ return Type3.Object(
987
+ { memory: Type3.String(), action },
988
+ { additionalProperties: false, title: "AgentOutput" }
989
+ );
990
+ }
991
+ const fields = {
992
+ evaluation_previous_goal: Type3.String(),
993
+ memory: Type3.String(),
994
+ next_goal: Type3.String(),
995
+ action
996
+ };
997
+ if (variant === "thinking") {
998
+ return Type3.Object(
999
+ { thinking: Type3.Optional(Type3.String()), ...fields },
1000
+ { additionalProperties: false, title: "AgentOutput" }
1001
+ );
1002
+ }
1003
+ return Type3.Object(fields, { additionalProperties: false, title: "AgentOutput" });
1004
+ }
1005
+ function parseAction(entry) {
1006
+ const keys = Object.keys(entry);
1007
+ if (keys.length !== 1) return void 0;
1008
+ const name = keys[0];
1009
+ const params = entry[name];
1010
+ return { name, params: params ?? {} };
1011
+ }
1012
+
1013
+ // src/agent/message-manager.ts
1014
+ var COMPACT_EVERY_N_STEPS = 25;
1015
+ var COMPACT_TRIGGER_CHARS = 4e4;
1016
+ var KEEP_LAST_ITEMS = 6;
1017
+ var SUMMARY_MAX_CHARS = 6e3;
1018
+ var ERROR_TRUNCATE_CHARS = 200;
1019
+ var MAX_READ_STATE_CHARS = 6e4;
1020
+ function truncateContent(text) {
1021
+ return text.length <= MAX_READ_STATE_CHARS ? text : `${text.slice(0, MAX_READ_STATE_CHARS)}
1022
+ ... [Content truncated at 60k characters]`;
1023
+ }
1024
+ function truncateError(error) {
1025
+ if (error.length <= ERROR_TRUNCATE_CHARS) return error;
1026
+ const half = ERROR_TRUNCATE_CHARS / 2;
1027
+ return `${error.slice(0, half)}\u2026\u2026${error.slice(-half)}`;
1028
+ }
1029
+ function renderItem(item) {
1030
+ const tag = item.stepNumber ?? 0;
1031
+ if (item.systemMessage) return `<step_${tag}>
1032
+ <sys>${item.systemMessage}</sys>
1033
+ </step_${tag}>`;
1034
+ const lines = [];
1035
+ if (item.evaluationPreviousGoal) lines.push(`Evaluation: ${item.evaluationPreviousGoal}`);
1036
+ if (item.memory) lines.push(`Memory: ${item.memory}`);
1037
+ if (item.nextGoal) lines.push(`Next goal: ${item.nextGoal}`);
1038
+ for (const result of item.actionResults ?? []) lines.push(`Result: ${result}`);
1039
+ if (item.error) lines.push(`Error: ${truncateError(item.error)}`);
1040
+ return `<step_${tag}>
1041
+ ${lines.join("\n")}
1042
+ </step_${tag}>`;
1043
+ }
1044
+ function persistentLine(result) {
1045
+ if (result.error !== void 0) return `Error: ${truncateError(result.error)}`;
1046
+ if (result.longTermMemory !== void 0) return result.longTermMemory;
1047
+ if (result.extractedContent !== void 0 && !result.includeExtractedContentOnlyOnce) {
1048
+ return result.extractedContent;
1049
+ }
1050
+ return void 0;
1051
+ }
1052
+ var MessageManager = class {
1053
+ systemPrompt;
1054
+ task;
1055
+ sensitiveData;
1056
+ fileSystemDescribe;
1057
+ todoContents;
1058
+ availableFilePaths;
1059
+ items = [];
1060
+ readState = "";
1061
+ compactedMemory;
1062
+ compactionCountValue = 0;
1063
+ lastCompactionStep = 0;
1064
+ /**
1065
+ * One-shot: set when a step's result asked for the screenshot to be shown
1066
+ * next (the on-demand `screenshot` action's `metadata.includeScreenshot`,
1067
+ * design/04b §4.3 — "the message manager attaches the *next* state
1068
+ * screenshot even in use_vision='auto'"). Consumed by the following
1069
+ * `buildMessages` call regardless of the caller's own `attachScreenshot`.
1070
+ */
1071
+ pendingScreenshotRequest = false;
1072
+ /** The raw state text built for the most recent step (kept for history debugging). */
1073
+ lastStateText = "";
1074
+ constructor(options) {
1075
+ this.systemPrompt = options.systemPrompt;
1076
+ this.task = options.task;
1077
+ this.sensitiveData = options.sensitiveData;
1078
+ this.fileSystemDescribe = options.fileSystemDescribe;
1079
+ this.todoContents = options.todoContents;
1080
+ this.availableFilePaths = options.availableFilePaths ?? [];
1081
+ this.items.push({ stepNumber: 0, systemMessage: "Agent initialized" });
1082
+ }
1083
+ get compactionCount() {
1084
+ return this.compactionCountValue;
1085
+ }
1086
+ get stateMessageText() {
1087
+ return this.lastStateText;
1088
+ }
1089
+ /**
1090
+ * Fold a completed step's output + results into the running history (the RECORD
1091
+ * half of the loop as the LLM sees it). Once-only content becomes the next
1092
+ * prompt's one-shot <read_state>; everything else becomes a persistent line.
1093
+ */
1094
+ recordStep(stepNumber, output, results) {
1095
+ const actionResults = [];
1096
+ const readStateParts = [];
1097
+ for (const result of results) {
1098
+ const line = persistentLine(result);
1099
+ if (line !== void 0) actionResults.push(line);
1100
+ if (result.includeExtractedContentOnlyOnce && result.extractedContent !== void 0) {
1101
+ readStateParts.push(result.extractedContent);
1102
+ }
1103
+ if (result.metadata?.includeScreenshot === true) this.pendingScreenshotRequest = true;
1104
+ }
1105
+ this.readState = truncateContent(readStateParts.join("\n\n"));
1106
+ this.items.push({
1107
+ stepNumber,
1108
+ evaluationPreviousGoal: output?.evaluation_previous_goal,
1109
+ memory: output?.memory,
1110
+ nextGoal: output?.next_goal,
1111
+ actionResults: actionResults.length > 0 ? actionResults : void 0
1112
+ });
1113
+ }
1114
+ /** Record a synthetic system note (e.g. "Failed to complete task in max steps"). */
1115
+ recordSystemNote(stepNumber, message) {
1116
+ this.items.push({ stepNumber, systemMessage: message });
1117
+ }
1118
+ /** Full rendered history text (used for both the prompt and the compaction gate). */
1119
+ renderHistory() {
1120
+ const blocks = [];
1121
+ if (this.compactedMemory) {
1122
+ blocks.push(
1123
+ `<compacted_memory>
1124
+ (Treat as unverified \u2014 do not report as completed unless re-verified this session.)
1125
+ ${this.compactedMemory}
1126
+ </compacted_memory>`
1127
+ );
1128
+ }
1129
+ for (const item of this.items) blocks.push(renderItem(item));
1130
+ return blocks.join("\n");
1131
+ }
1132
+ /**
1133
+ * Compaction (design/04b §4.2). Fires only when BOTH gates hold:
1134
+ * `stepNumber - lastCompactionStep >= 25` AND rendered history `>= 40000` chars.
1135
+ * Summarizes the dropped items into `compactedMemory`, keeps the first item and
1136
+ * the last 6, and bumps the compaction count. Returns whether it fired.
1137
+ *
1138
+ * ponytail: deterministic join-summary rather than a summarizer LLM call — the
1139
+ * gate + keep-window is the load-bearing behavior; upgrade path is to pass a
1140
+ * compaction model and summarize `dropped` instead of joining their memories.
1141
+ */
1142
+ maybeCompact(stepNumber) {
1143
+ if (stepNumber - this.lastCompactionStep < COMPACT_EVERY_N_STEPS) return false;
1144
+ if (this.renderHistory().length < COMPACT_TRIGGER_CHARS) return false;
1145
+ if (this.items.length <= KEEP_LAST_ITEMS + 1) return false;
1146
+ const first = this.items[0];
1147
+ const dropped = this.items.slice(1, this.items.length - KEEP_LAST_ITEMS);
1148
+ const kept = this.items.slice(this.items.length - KEEP_LAST_ITEMS);
1149
+ const summary = dropped.map((item) => item.memory ?? item.systemMessage ?? (item.actionResults ?? []).join("; ")).filter((text) => text.length > 0).join("\n").slice(0, SUMMARY_MAX_CHARS);
1150
+ this.compactedMemory = [this.compactedMemory, summary].filter(Boolean).join("\n").slice(0, SUMMARY_MAX_CHARS);
1151
+ this.items.length = 0;
1152
+ this.items.push(first, ...kept);
1153
+ this.compactionCountValue += 1;
1154
+ this.lastCompactionStep = stepNumber;
1155
+ return true;
1156
+ }
1157
+ /** Assemble `[system, state, ...context]` for one step. */
1158
+ buildMessages(state, options) {
1159
+ const stateText = this.buildStateText(state, options);
1160
+ this.lastStateText = stateText;
1161
+ const attachScreenshot = (options.attachScreenshot ?? false) || this.pendingScreenshotRequest;
1162
+ this.pendingScreenshotRequest = false;
1163
+ const systemMessage = userText(this.systemPrompt);
1164
+ const stateMessage = this.buildStateMessage(stateText, state, attachScreenshot);
1165
+ const contextMessages = (options.nudges ?? []).map((nudge) => userText(nudge));
1166
+ return [systemMessage, stateMessage, ...contextMessages];
1167
+ }
1168
+ buildStateMessage(stateText, state, attachScreenshot) {
1169
+ if (attachScreenshot && state.screenshot) {
1170
+ const parts = [
1171
+ { type: "text", text: stateText },
1172
+ { type: "text", text: "Current screenshot:" },
1173
+ { type: "image", data: state.screenshot, mimeType: "image/png" }
1174
+ ];
1175
+ return { role: "user", content: parts, timestamp: Date.now() };
1176
+ }
1177
+ return { role: "user", content: stateText, timestamp: Date.now() };
1178
+ }
1179
+ buildStateText(state, options) {
1180
+ const sections = [];
1181
+ sections.push(`<user_request>
1182
+ ${this.task}
1183
+ </user_request>`);
1184
+ sections.push(`<agent_history>
1185
+ ${this.renderHistory()}
1186
+ </agent_history>`);
1187
+ sections.push(this.buildAgentState(state.url));
1188
+ sections.push(this.buildBrowserState(state));
1189
+ if (this.readState) sections.push(`<read_state>
1190
+ ${this.readState}
1191
+ </read_state>`);
1192
+ sections.push(
1193
+ `<step_info>
1194
+ Step ${options.stepNumber} of at most ${options.maxSteps}.
1195
+ </step_info>`
1196
+ );
1197
+ const text = sections.join("\n\n");
1198
+ return this.sensitiveData ? maskOutbound(text, this.sensitiveData) : text;
1199
+ }
1200
+ buildAgentState(url) {
1201
+ const parts = [];
1202
+ if (this.fileSystemDescribe)
1203
+ parts.push(`<file_system>
1204
+ ${this.fileSystemDescribe()}
1205
+ </file_system>`);
1206
+ if (this.todoContents) {
1207
+ const todo = this.todoContents();
1208
+ parts.push(
1209
+ `<todo_contents>
1210
+ ${todo || "[empty todo.md \u2014 fill it when useful]"}
1211
+ </todo_contents>`
1212
+ );
1213
+ }
1214
+ if (this.sensitiveData) {
1215
+ const block = sensitiveDataDescription(this.sensitiveData, url);
1216
+ if (block) parts.push(`<sensitive_data>
1217
+ ${block}
1218
+ </sensitive_data>`);
1219
+ }
1220
+ if (this.availableFilePaths.length > 0) {
1221
+ parts.push(
1222
+ `<available_file_paths>
1223
+ ${this.availableFilePaths.join("\n")}
1224
+ Use absolute paths.
1225
+ </available_file_paths>`
1226
+ );
1227
+ }
1228
+ return `<agent_state>
1229
+ ${parts.join("\n")}
1230
+ </agent_state>`;
1231
+ }
1232
+ buildBrowserState(state) {
1233
+ const tabs = state.tabs.map((tab) => ` [${tab.id}] ${tab.url} \u2014 ${tab.title.slice(0, 30)}`).join("\n");
1234
+ return [
1235
+ "<browser_state>",
1236
+ `Current URL: ${state.url}`,
1237
+ `Title: ${state.title}`,
1238
+ `Open tabs:
1239
+ ${tabs || " (none)"}`,
1240
+ "Interactive elements (only [index]ed elements are interactive):",
1241
+ state.serializedText || "[empty page]",
1242
+ "</browser_state>"
1243
+ ].join("\n");
1244
+ }
1245
+ };
1246
+ function userText(text) {
1247
+ return { role: "user", content: text, timestamp: Date.now() };
1248
+ }
1249
+ function resultsAreDone(results) {
1250
+ const last = results[results.length - 1];
1251
+ return last !== void 0 && isDoneResult(last);
1252
+ }
1253
+
1254
+ // src/agent/context-builder.ts
1255
+ var BUDGET_WARN_FRACTION = 0.75;
1256
+ var REPETITION_THRESHOLD = 5;
1257
+ var STAGNATION_THRESHOLD = 5;
1258
+ var ACTION_WINDOW = 20;
1259
+ var FINGERPRINT_WINDOW = 5;
1260
+ function perceive(driver, vision) {
1261
+ return driver.getState({ screenshot: vision !== "off" });
1262
+ }
1263
+ function budgetNudge(stepNumber, maxSteps) {
1264
+ if (maxSteps <= 0) return void 0;
1265
+ if (stepNumber / maxSteps < BUDGET_WARN_FRACTION) return void 0;
1266
+ return `Budget warning: you are on step ${stepNumber} of ${maxSteps}. Consolidate what you have found and call done soon.`;
1267
+ }
1268
+ function actionSignature(action) {
1269
+ const [name] = Object.keys(action);
1270
+ return `${name}|${JSON.stringify(action[name] ?? {})}`;
1271
+ }
1272
+ function pageFingerprint(state) {
1273
+ const text = state.serializedText;
1274
+ return `${state.url}|${text.length}|${text.slice(0, 64)}`;
1275
+ }
1276
+ var LoopDetector = class {
1277
+ actionHashes = [];
1278
+ fingerprints = [];
1279
+ stagnation = 0;
1280
+ /** Record a step's executed actions and the page state it was taken against. */
1281
+ record(output, state) {
1282
+ for (const action of output?.action ?? []) {
1283
+ this.actionHashes.push(actionSignature(action));
1284
+ }
1285
+ while (this.actionHashes.length > ACTION_WINDOW) this.actionHashes.shift();
1286
+ const fingerprint = pageFingerprint(state);
1287
+ const previous = this.fingerprints[this.fingerprints.length - 1];
1288
+ this.stagnation = previous === fingerprint ? this.stagnation + 1 : 0;
1289
+ this.fingerprints.push(fingerprint);
1290
+ while (this.fingerprints.length > FINGERPRINT_WINDOW) this.fingerprints.shift();
1291
+ }
1292
+ /** Advisory nudges to inject this step (may be empty). */
1293
+ nudges() {
1294
+ const out = [];
1295
+ const repetition = this.maxRepetition();
1296
+ if (repetition >= REPETITION_THRESHOLD) {
1297
+ out.push(
1298
+ `You have repeated the same action ${repetition} times. If you are not making progress, try a different approach or call done.`
1299
+ );
1300
+ }
1301
+ if (this.stagnation >= STAGNATION_THRESHOLD) {
1302
+ out.push(
1303
+ `The page has not changed for ${this.stagnation} steps. Re-read the state and consider a different action.`
1304
+ );
1305
+ }
1306
+ return out;
1307
+ }
1308
+ maxRepetition() {
1309
+ const counts = /* @__PURE__ */ new Map();
1310
+ let max = 0;
1311
+ for (const hash of this.actionHashes) {
1312
+ const next = (counts.get(hash) ?? 0) + 1;
1313
+ counts.set(hash, next);
1314
+ if (next > max) max = next;
1315
+ }
1316
+ return max;
1317
+ }
1318
+ };
1319
+
1320
+ // src/agent/event-stream.ts
1321
+ function isTerminal(event) {
1322
+ return event.type === "done" || event.type === "error";
1323
+ }
1324
+ var AgentEventStream = class {
1325
+ typed = /* @__PURE__ */ new Map();
1326
+ any = /* @__PURE__ */ new Set();
1327
+ /** Fan an event out to all listeners. */
1328
+ emit(event) {
1329
+ for (const cb of this.typed.get(event.type) ?? []) cb(event);
1330
+ for (const cb of this.any) cb(event);
1331
+ }
1332
+ /** Subscribe to one event type. Returns an unsubscribe function. */
1333
+ on(type, cb) {
1334
+ let set = this.typed.get(type);
1335
+ if (!set) {
1336
+ set = /* @__PURE__ */ new Set();
1337
+ this.typed.set(type, set);
1338
+ }
1339
+ set.add(cb);
1340
+ return () => {
1341
+ set?.delete(cb);
1342
+ };
1343
+ }
1344
+ /** Subscribe to every event. Returns an unsubscribe function. */
1345
+ onEvent(cb) {
1346
+ this.any.add(cb);
1347
+ return () => {
1348
+ this.any.delete(cb);
1349
+ };
1350
+ }
1351
+ /**
1352
+ * An async iterable of every event, ending after the terminal (done/error).
1353
+ * Subscribe before/while a run produces events; iterating after a run has
1354
+ * already finished (with no further run) parks on the first `next()`.
1355
+ */
1356
+ events() {
1357
+ const buffer = [];
1358
+ let pending = null;
1359
+ let finished = false;
1360
+ const unsub = this.onEvent((event) => {
1361
+ if (pending) {
1362
+ const resolve = pending;
1363
+ pending = null;
1364
+ resolve({ value: event, done: false });
1365
+ } else {
1366
+ buffer.push(event);
1367
+ }
1368
+ if (isTerminal(event)) finished = true;
1369
+ });
1370
+ const iterator = {
1371
+ next() {
1372
+ const head = buffer.shift();
1373
+ if (head !== void 0) return Promise.resolve({ value: head, done: false });
1374
+ if (finished) {
1375
+ unsub();
1376
+ return Promise.resolve({ value: void 0, done: true });
1377
+ }
1378
+ return new Promise((resolve) => {
1379
+ pending = resolve;
1380
+ });
1381
+ },
1382
+ return() {
1383
+ unsub();
1384
+ return Promise.resolve({ value: void 0, done: true });
1385
+ },
1386
+ [Symbol.asyncIterator]() {
1387
+ return this;
1388
+ }
1389
+ };
1390
+ return iterator;
1391
+ }
1392
+ };
1393
+ function toTypedAction(name, params) {
1394
+ return { kind: name, ...params };
1395
+ }
1396
+ function summarizeActions(output) {
1397
+ const names = (output?.action ?? []).map((entry) => Object.keys(entry)[0]).filter((name) => name !== void 0);
1398
+ return names.length > 0 ? `Running ${names.join(", ")}.` : "";
1399
+ }
1400
+ function assembleNarration(output, results) {
1401
+ const done2 = results.find(isDoneResult);
1402
+ if (done2) {
1403
+ const text = (done2.extractedContent ?? "").trim();
1404
+ if (text) return text;
1405
+ return done2.success ? "Task complete." : "Stopping \u2014 the task could not be completed.";
1406
+ }
1407
+ const parts = [];
1408
+ const evaluation = output?.evaluation_previous_goal?.trim();
1409
+ const nextGoal = output?.next_goal?.trim();
1410
+ if (evaluation) parts.push(evaluation);
1411
+ if (nextGoal) parts.push(nextGoal);
1412
+ if (parts.length === 0) {
1413
+ const summary = summarizeActions(output);
1414
+ if (summary) parts.push(summary);
1415
+ }
1416
+ if (parts.length === 0) {
1417
+ const errored = results.find((result) => result.error !== void 0);
1418
+ if (errored?.error) parts.push(errored.error);
1419
+ }
1420
+ return parts.join(" ").trim();
1421
+ }
1422
+
1423
+ // src/agent/executor.ts
1424
+ function snapshot(session) {
1425
+ const focus = session.registry.focusedTargetId;
1426
+ if (!focus) return {};
1427
+ return { url: session.registry.getCurrentUrl(focus), focus };
1428
+ }
1429
+ async function multiAct(input) {
1430
+ const {
1431
+ actions,
1432
+ registry,
1433
+ ctx,
1434
+ session,
1435
+ secrets,
1436
+ allSensitiveData,
1437
+ signal,
1438
+ shouldContinue,
1439
+ emit
1440
+ } = input;
1441
+ const step = input.step ?? 0;
1442
+ const results = [];
1443
+ const limit = Math.min(actions.length, input.maxActions ?? actions.length);
1444
+ for (let i = 0; i < limit; i++) {
1445
+ if (signal?.aborted) throw new DOMException("Aborted", "AbortError");
1446
+ if (shouldContinue && !shouldContinue()) break;
1447
+ const parsed = parseAction(actions[i]);
1448
+ if (!parsed) {
1449
+ results.push(
1450
+ stepResult({ error: `Malformed action at position ${i} (expected a single-key object)` })
1451
+ );
1452
+ break;
1453
+ }
1454
+ if (i > 0 && parsed.name === "done") break;
1455
+ const before = snapshot(session);
1456
+ emit?.({
1457
+ type: "action-started",
1458
+ step,
1459
+ index: i,
1460
+ name: parsed.name,
1461
+ action: toTypedAction(parsed.name, parsed.params),
1462
+ params: parsed.params
1463
+ });
1464
+ const elementIndex = typeof parsed.params.index === "number" ? parsed.params.index : void 0;
1465
+ if (input.demoMode && elementIndex !== void 0) {
1466
+ await ctx.driver.highlightElement(elementIndex, { signal });
1467
+ }
1468
+ const result = await registry.execute(
1469
+ parsed.name,
1470
+ parsed.params,
1471
+ ctx,
1472
+ secrets,
1473
+ allSensitiveData
1474
+ );
1475
+ if (input.demoMode && elementIndex !== void 0) {
1476
+ await ctx.driver.clearHighlight({ signal });
1477
+ }
1478
+ results.push(result);
1479
+ emit?.({ type: "action-finished", step, index: i, name: parsed.name, result });
1480
+ const isLast = i === limit - 1;
1481
+ if (isDoneResult(result) || result.error !== void 0 || isLast) break;
1482
+ if (registry.get(parsed.name)?.terminatesSequence) break;
1483
+ const after = snapshot(session);
1484
+ if (after.url !== before.url || after.focus !== before.focus) break;
1485
+ }
1486
+ return results;
1487
+ }
1488
+
1489
+ // src/agent/history.ts
1490
+ function stepDurationSeconds(metadata) {
1491
+ return (metadata.stepEndTime - metadata.stepStartTime) / 1e3;
1492
+ }
1493
+ function firstError2(results) {
1494
+ for (const result of results) {
1495
+ if (result.error !== void 0) return result.error;
1496
+ }
1497
+ return void 0;
1498
+ }
1499
+ var AgentHistoryList = class _AgentHistoryList {
1500
+ history = [];
1501
+ usage;
1502
+ /** Set once by the loop when it exits; drives StopReason mapping (no regex). */
1503
+ terminationReason;
1504
+ /** Append a step. Items are never mutated or reordered afterwards. */
1505
+ add(item) {
1506
+ this.history.push(item);
1507
+ }
1508
+ get length() {
1509
+ return this.history.length;
1510
+ }
1511
+ numberOfSteps() {
1512
+ return this.history.length;
1513
+ }
1514
+ lastResult() {
1515
+ const last = this.history[this.history.length - 1];
1516
+ if (!last) return void 0;
1517
+ return last.result[last.result.length - 1];
1518
+ }
1519
+ /** The `extractedContent` of the last result of the last step. */
1520
+ finalResult() {
1521
+ return this.lastResult()?.extractedContent;
1522
+ }
1523
+ /** True once a terminal `done` result has been recorded. */
1524
+ isDone() {
1525
+ const last = this.lastResult();
1526
+ return last !== void 0 && isDoneResult(last);
1527
+ }
1528
+ /** The agent's self-reported success — only meaningful when done, else undefined. */
1529
+ isSuccessful() {
1530
+ const last = this.lastResult();
1531
+ if (!last || !isDoneResult(last)) return void 0;
1532
+ return last.success;
1533
+ }
1534
+ /** One error (the first) per step; undefined where a step had none. */
1535
+ errors() {
1536
+ return this.history.map((step) => firstError2(step.result));
1537
+ }
1538
+ hasErrors() {
1539
+ return this.errors().some((error) => error !== void 0);
1540
+ }
1541
+ /** Per-step page URLs. */
1542
+ urls() {
1543
+ return this.history.map((step) => step.state.url);
1544
+ }
1545
+ /** Flattened action names across all steps (one per single-key action object). */
1546
+ actionNames() {
1547
+ const names = [];
1548
+ for (const step of this.history) {
1549
+ for (const action of step.modelOutput?.action ?? []) {
1550
+ const [name] = Object.keys(action);
1551
+ if (name) names.push(name);
1552
+ }
1553
+ }
1554
+ return names;
1555
+ }
1556
+ /** Sum of per-step durations (seconds). */
1557
+ totalDurationSeconds() {
1558
+ return this.history.reduce(
1559
+ (sum, step) => sum + (step.metadata ? stepDurationSeconds(step.metadata) : 0),
1560
+ 0
1561
+ );
1562
+ }
1563
+ /** Plain-data snapshot for persistence (design/04b §4.1 `AgentHistoryList.save_to_file`). */
1564
+ toJSON() {
1565
+ return { version: 1, history: this.history, terminationReason: this.terminationReason };
1566
+ }
1567
+ /**
1568
+ * Rebuild a list from a plain-data snapshot. The CALLER is responsible for
1569
+ * schema-validating untrusted input before calling this (replay.ts's
1570
+ * `loadHistoryFromFile` is the trust boundary — this constructor trusts its
1571
+ * argument, matching every other in-process history mutation in this class).
1572
+ *
1573
+ * `interactedElements` entries round-trip through JSON as `null` (undefined
1574
+ * array elements serialize that way — `Type.Array` has no other way to carry
1575
+ * a "no element" slot); normalize them back to `undefined` here so in-memory
1576
+ * histories built directly (never touching JSON) and loaded ones agree.
1577
+ */
1578
+ static fromJSON(data) {
1579
+ const list = new _AgentHistoryList();
1580
+ for (const item of data.history) {
1581
+ const interactedElements = item.state.interactedElements?.map(
1582
+ (el) => el ?? void 0
1583
+ );
1584
+ list.add(
1585
+ interactedElements ? { ...item, state: { ...item.state, interactedElements } } : item
1586
+ );
1587
+ }
1588
+ list.terminationReason = data.terminationReason;
1589
+ return list;
1590
+ }
1591
+ };
1592
+
1593
+ // src/observability/logger.ts
1594
+ var RANK = { debug: 10, info: 20, warn: 30, error: 40 };
1595
+ function renderValue(value) {
1596
+ if (typeof value === "string") return value;
1597
+ if (value === null || value === void 0) return String(value);
1598
+ if (typeof value === "object") return JSON.stringify(value);
1599
+ return String(value);
1600
+ }
1601
+ function format(level, message, fields) {
1602
+ const keys = Object.keys(fields);
1603
+ if (keys.length === 0) return `[${level}] ${message}`;
1604
+ const rendered = keys.map((key) => `${key}=${renderValue(fields[key])}`).join(" ");
1605
+ return `[${level}] ${message} ${rendered}`;
1606
+ }
1607
+ var defaultSink = (level, line) => {
1608
+ const method = level === "debug" ? console.debug : level === "warn" ? console.warn : level === "error" ? console.error : console.info;
1609
+ method(line);
1610
+ };
1611
+ function createLogger(options = {}) {
1612
+ const level = options.level ?? "info";
1613
+ const base = options.fields ?? {};
1614
+ const sink = options.sink ?? defaultSink;
1615
+ const min = RANK[level];
1616
+ const write = (at, message, fields) => {
1617
+ if (RANK[at] < min) return;
1618
+ sink(at, format(at, message, fields ? { ...base, ...fields } : base));
1619
+ };
1620
+ return {
1621
+ debug: (message, fields) => write("debug", message, fields),
1622
+ info: (message, fields) => write("info", message, fields),
1623
+ warn: (message, fields) => write("warn", message, fields),
1624
+ error: (message, fields) => write("error", message, fields),
1625
+ child: (fields) => createLogger({ level, fields: { ...base, ...fields }, sink })
1626
+ };
1627
+ }
1628
+ var silentLogger = {
1629
+ debug() {
1630
+ },
1631
+ info() {
1632
+ },
1633
+ warn() {
1634
+ },
1635
+ error() {
1636
+ },
1637
+ child() {
1638
+ return silentLogger;
1639
+ }
1640
+ };
1641
+
1642
+ // src/agent/captcha-watchdog.ts
1643
+ var VENDOR_PATTERNS = [
1644
+ { vendor: "recaptcha", pattern: /\brecaptcha\b/i },
1645
+ { vendor: "hcaptcha", pattern: /\bhcaptcha\b/i },
1646
+ {
1647
+ vendor: "cloudflare",
1648
+ pattern: /checking (?:if )?(?:the )?(?:connection|site connection) is secure|cf-turnstile|cloudflare/i
1649
+ },
1650
+ { vendor: "perimeterx", pattern: /perimeterx|_px-captcha|press\s*(?:&|and)\s*hold/i },
1651
+ { vendor: "datadome", pattern: /datadome/i },
1652
+ { vendor: "akamai", pattern: /akamai bot manager|ak_bmsc/i }
1653
+ ];
1654
+ var GENERIC_PATTERN = /\b(are you a (?:human|robot)|i.?m not a robot|prove you.?re (?:not a )?robot|verify you are human|verify you are a human|unusual traffic|complete the security check|access denied.{0,20}bot|suspected automated (?:traffic|access))\b/i;
1655
+ function detectCaptcha(state) {
1656
+ const haystack = `${state.title}
1657
+ ${state.serializedText}`;
1658
+ for (const { vendor, pattern } of VENDOR_PATTERNS) {
1659
+ const match = haystack.match(pattern);
1660
+ if (match) return { vendor, url: state.url, title: state.title, matchedText: match[0] };
1661
+ }
1662
+ const generic = haystack.match(GENERIC_PATTERN);
1663
+ if (generic) {
1664
+ return { vendor: "generic", url: state.url, title: state.title, matchedText: generic[0] };
1665
+ }
1666
+ return void 0;
1667
+ }
1668
+ var CaptchaWatchdog = class {
1669
+ last;
1670
+ /** Check state; returns a signal only the FIRST time it's seen for this vendor+url. */
1671
+ check(state) {
1672
+ const signal = detectCaptcha(state);
1673
+ if (!signal) {
1674
+ this.last = void 0;
1675
+ return void 0;
1676
+ }
1677
+ const key = `${signal.vendor}|${signal.url}`;
1678
+ if (key === this.last) return void 0;
1679
+ this.last = key;
1680
+ return signal;
1681
+ }
1682
+ };
1683
+
1684
+ // src/agent/step-loop.ts
1685
+ import {
1686
+ StructuredOutputError,
1687
+ generateObject
1688
+ } from "@ap3x/ai";
1689
+ import {
1690
+ buildInteractedElement
1691
+ } from "@ap3x/browser";
1692
+ import { Type as Type4 } from "@sinclair/typebox";
1693
+ function anySignal(...signals) {
1694
+ const present = signals.filter((s) => s !== void 0);
1695
+ if (present.length === 0) return void 0;
1696
+ if (present.length === 1) return present[0];
1697
+ return AbortSignal.any(present);
1698
+ }
1699
+ var StepLoop = class {
1700
+ deps;
1701
+ log;
1702
+ loopDetector = new LoopDetector();
1703
+ captchaWatchdog = new CaptchaWatchdog();
1704
+ nSteps = 1;
1705
+ consecutiveFailures = 0;
1706
+ lastStepDurationSeconds;
1707
+ pendingNudges = [];
1708
+ done = false;
1709
+ stopped = false;
1710
+ paused = false;
1711
+ pauseGate;
1712
+ pauseRelease;
1713
+ maxSteps;
1714
+ runSignal;
1715
+ runDeadline;
1716
+ constructor(deps) {
1717
+ this.deps = deps;
1718
+ this.log = deps.logger ?? silentLogger;
1719
+ this.maxSteps = deps.limits.maxSteps;
1720
+ }
1721
+ /** Fan an event out to the O2 stream (no-op when no sink is wired). */
1722
+ emit(event) {
1723
+ this.deps.emit?.(event);
1724
+ }
1725
+ // --- control surface -------------------------------------------------------
1726
+ pause() {
1727
+ if (this.paused) return;
1728
+ this.paused = true;
1729
+ this.pauseGate = new Promise((resolve) => {
1730
+ this.pauseRelease = resolve;
1731
+ });
1732
+ this.emit({ type: "paused" });
1733
+ }
1734
+ resume() {
1735
+ const wasPaused = this.paused;
1736
+ this.paused = false;
1737
+ this.pauseRelease?.();
1738
+ this.pauseGate = void 0;
1739
+ this.pauseRelease = void 0;
1740
+ if (wasPaused) this.emit({ type: "resumed" });
1741
+ }
1742
+ stop() {
1743
+ this.stopped = true;
1744
+ this.resume();
1745
+ }
1746
+ get isDone() {
1747
+ return this.done;
1748
+ }
1749
+ /** Current consecutive-failure count (observability / tests). */
1750
+ get failures() {
1751
+ return this.consecutiveFailures;
1752
+ }
1753
+ /** The 1-based number of the step about to run (observability / tests). */
1754
+ get currentStep() {
1755
+ return this.nSteps;
1756
+ }
1757
+ // --- run / step ------------------------------------------------------------
1758
+ async run(options = {}) {
1759
+ this.maxSteps = options.maxSteps ?? this.deps.limits.maxSteps;
1760
+ this.runSignal = options.signal;
1761
+ this.runDeadline = this.deps.limits.runTimeoutMs ? AbortSignal.timeout(this.deps.limits.runTimeoutMs) : void 0;
1762
+ this.emit({
1763
+ type: "run-started",
1764
+ task: this.deps.task ?? "",
1765
+ maxSteps: this.maxSteps,
1766
+ startedAt: Date.now()
1767
+ });
1768
+ this.log.info("run started", { maxSteps: this.maxSteps });
1769
+ try {
1770
+ while (this.nSteps <= this.maxSteps) {
1771
+ await this.waitWhilePaused();
1772
+ if (this.stopped || this.runSignal?.aborted) {
1773
+ this.deps.history.terminationReason = this.runSignal?.aborted ? "aborted" : "stopped";
1774
+ break;
1775
+ }
1776
+ if (this.runDeadline?.aborted) {
1777
+ this.deps.history.terminationReason = "run_timeout";
1778
+ this.recordSyntheticFailure(
1779
+ `Stopped: the run exceeded its ${this.deps.limits.runTimeoutMs}ms time budget.`
1780
+ );
1781
+ break;
1782
+ }
1783
+ const ceiling = this.deps.limits.maxFailures + (this.deps.limits.finalResponseAfterFailure ? 1 : 0);
1784
+ if (this.consecutiveFailures >= ceiling) {
1785
+ this.deps.messageManager.recordSystemNote(
1786
+ this.nSteps,
1787
+ `Stopped after ${this.consecutiveFailures} consecutive failures.`
1788
+ );
1789
+ this.deps.history.terminationReason = "max_failures";
1790
+ break;
1791
+ }
1792
+ const isDone = await this.step();
1793
+ if (isDone) {
1794
+ this.deps.history.terminationReason = "done";
1795
+ break;
1796
+ }
1797
+ if (this.stopped) {
1798
+ this.deps.history.terminationReason = this.runSignal?.aborted ? "aborted" : "stopped";
1799
+ break;
1800
+ }
1801
+ }
1802
+ if (!this.done && this.nSteps > this.maxSteps) {
1803
+ this.deps.history.terminationReason = "max_steps";
1804
+ this.recordSyntheticFailure(
1805
+ "Failed to complete the task within the maximum number of steps."
1806
+ );
1807
+ }
1808
+ this.deps.history.usage = this.deps.ledger.summary();
1809
+ await this.deps.hooks?.onDone?.(this.deps.history);
1810
+ this.emit({
1811
+ type: "done",
1812
+ success: this.deps.history.isSuccessful() ?? false,
1813
+ finalText: this.deps.history.finalResult() ?? "",
1814
+ steps: this.deps.history.length
1815
+ });
1816
+ this.log.info("run finished", {
1817
+ steps: this.deps.history.length,
1818
+ done: this.deps.history.isDone()
1819
+ });
1820
+ return this.deps.history;
1821
+ } catch (error) {
1822
+ this.emit({ type: "error", message: errorText(error), step: this.nSteps });
1823
+ throw error;
1824
+ }
1825
+ }
1826
+ /**
1827
+ * One perceive→decide→act→record cycle. Returns whether the task is now done.
1828
+ * A no-op (returns false) when paused/stopped/already-done.
1829
+ */
1830
+ async step() {
1831
+ if (this.paused || this.stopped || this.done) return false;
1832
+ const stepStart = Date.now();
1833
+ const stepSignal = anySignal(
1834
+ this.runSignal,
1835
+ this.runDeadline,
1836
+ AbortSignal.timeout(this.deps.limits.stepTimeoutMs)
1837
+ );
1838
+ let state;
1839
+ try {
1840
+ state = await perceive(this.deps.driver, this.deps.vision);
1841
+ } catch (error) {
1842
+ this.emit({ type: "step-started", step: this.nSteps, url: "", title: "" });
1843
+ this.log.warn("perceive failed", { step: this.nSteps, error: errorText(error) });
1844
+ this.recordWholeStepFailure(errorText(error), placeholderState(), stepStart);
1845
+ return false;
1846
+ }
1847
+ const histState = historyState(state, this.deps.sensitiveData);
1848
+ this.emit({
1849
+ type: "step-started",
1850
+ step: this.nSteps,
1851
+ url: histState.url,
1852
+ title: histState.title
1853
+ });
1854
+ this.log.info("step", { step: this.nSteps, url: histState.url });
1855
+ const captcha = this.captchaWatchdog.check(state);
1856
+ if (captcha) {
1857
+ this.emit({ type: "captcha-detected", step: this.nSteps, ...captcha });
1858
+ this.log.warn("captcha detected", {
1859
+ step: this.nSteps,
1860
+ vendor: captcha.vendor,
1861
+ url: state.url
1862
+ });
1863
+ }
1864
+ const nudges = [...this.pendingNudges];
1865
+ this.pendingNudges = [];
1866
+ const budget = budgetNudge(this.nSteps, this.maxSteps);
1867
+ if (budget) nudges.push(budget);
1868
+ nudges.push(...this.loopDetector.nudges());
1869
+ this.deps.messageManager.maybeCompact(this.nSteps);
1870
+ const forced = this.nSteps >= this.maxSteps || this.deps.limits.finalResponseAfterFailure && this.consecutiveFailures >= this.deps.limits.maxFailures;
1871
+ if (forced) {
1872
+ nudges.push(
1873
+ "This is your final step. Your only available tool is done \u2014 set success honestly and include everything you have found."
1874
+ );
1875
+ }
1876
+ const schema = buildAgentOutputSchema(
1877
+ forced ? this.doneOnlyUnion() : this.deps.registry.actionUnion(state.url),
1878
+ this.deps.variant
1879
+ );
1880
+ const messages = this.deps.messageManager.buildMessages(state, {
1881
+ stepNumber: this.nSteps,
1882
+ maxSteps: this.maxSteps,
1883
+ nudges,
1884
+ attachScreenshot: this.deps.vision === "everyStep"
1885
+ });
1886
+ let output;
1887
+ try {
1888
+ const llmSignal = anySignal(stepSignal, AbortSignal.timeout(this.deps.limits.llmTimeoutMs));
1889
+ const { object, usage } = await generateObject(this.deps.model, messages, schema, {
1890
+ ...this.deps.llmStreamOptions,
1891
+ signal: llmSignal
1892
+ });
1893
+ this.deps.ledger.record(this.deps.model.id, usage);
1894
+ output = object;
1895
+ } catch (error) {
1896
+ return this.handleLlmError(error, histState, stepStart);
1897
+ }
1898
+ histState.interactedElements = output.action.map((action) => {
1899
+ const parsed = parseAction(action);
1900
+ const index = parsed && typeof parsed.params.index === "number" ? parsed.params.index : void 0;
1901
+ if (index === void 0) return void 0;
1902
+ const node = state.selectorMap.get(index);
1903
+ return node ? buildInteractedElement(node) : void 0;
1904
+ });
1905
+ this.emit({
1906
+ type: "model-thought",
1907
+ step: this.nSteps,
1908
+ thinking: output.thinking,
1909
+ evaluationPreviousGoal: output.evaluation_previous_goal,
1910
+ memory: output.memory,
1911
+ nextGoal: output.next_goal
1912
+ });
1913
+ await this.deps.hooks?.onStep?.(histState, output, this.nSteps);
1914
+ const secrets = this.deps.sensitiveData ? resolveSecretsForUrl(this.deps.sensitiveData, state.url) : {};
1915
+ const ctx = this.buildContext(state.url, Object.keys(secrets).length > 0, stepSignal);
1916
+ let results;
1917
+ try {
1918
+ results = await multiAct({
1919
+ actions: output.action,
1920
+ registry: this.deps.registry,
1921
+ ctx,
1922
+ session: this.deps.session,
1923
+ secrets,
1924
+ allSensitiveData: this.deps.sensitiveData,
1925
+ signal: stepSignal,
1926
+ shouldContinue: () => !this.stopped && !this.paused,
1927
+ maxActions: this.deps.limits.maxActionsPerStep,
1928
+ emit: (event) => this.emit(event),
1929
+ step: this.nSteps,
1930
+ demoMode: this.deps.demoMode
1931
+ });
1932
+ } catch (error) {
1933
+ if (this.stopped || this.runSignal?.aborted) {
1934
+ this.stopped = true;
1935
+ return false;
1936
+ }
1937
+ results = [stepResult({ error: errorText(error) })];
1938
+ }
1939
+ if (this.runSignal?.aborted) {
1940
+ this.stopped = true;
1941
+ } else {
1942
+ this.accountFailures(results);
1943
+ }
1944
+ this.loopDetector.record(output, state);
1945
+ this.recordStep(output, results, histState, stepStart);
1946
+ const isDone = resultsAreDone(results);
1947
+ if (isDone) this.done = true;
1948
+ this.nSteps += 1;
1949
+ return isDone;
1950
+ }
1951
+ // --- internals -------------------------------------------------------------
1952
+ async waitWhilePaused() {
1953
+ while (this.paused && !this.stopped) {
1954
+ if (!this.pauseGate) {
1955
+ this.pauseGate = new Promise((resolve) => {
1956
+ this.pauseRelease = resolve;
1957
+ });
1958
+ }
1959
+ await this.pauseGate;
1960
+ }
1961
+ }
1962
+ /** Consecutive-failure accounting (design/07 §7.2.3). */
1963
+ accountFailures(results) {
1964
+ const hasError = results.some((r) => r.error !== void 0);
1965
+ if (!hasError) {
1966
+ this.consecutiveFailures = 0;
1967
+ } else if (results.length === 1) {
1968
+ this.consecutiveFailures += 1;
1969
+ }
1970
+ }
1971
+ handleLlmError(error, state, stepStart) {
1972
+ if (this.stopped || this.runSignal?.aborted) {
1973
+ this.stopped = true;
1974
+ return false;
1975
+ }
1976
+ if (error instanceof StructuredOutputError && error.reason === "empty") {
1977
+ const results = [
1978
+ doneResult(false, { extractedContent: "No next action returned by the model." })
1979
+ ];
1980
+ this.recordStep(void 0, results, state, stepStart);
1981
+ this.done = true;
1982
+ this.nSteps += 1;
1983
+ return true;
1984
+ }
1985
+ if (error instanceof StructuredOutputError && (error.reason === "invalid" || error.reason === "truncated")) {
1986
+ this.pendingNudges.push(
1987
+ "Your previous output did not match the required schema. Respond with a single valid object exactly matching the schema, including a non-empty action list."
1988
+ );
1989
+ this.recordWholeStepFailure(`Invalid model output: ${error.message}`, state, stepStart);
1990
+ return false;
1991
+ }
1992
+ this.recordWholeStepFailure(errorText(error), state, stepStart);
1993
+ return false;
1994
+ }
1995
+ /** A whole-step failure: increment the counter and record an error result. */
1996
+ recordWholeStepFailure(message, state, stepStart) {
1997
+ this.consecutiveFailures += 1;
1998
+ const results = [stepResult({ error: message })];
1999
+ this.recordStep(void 0, results, state, stepStart);
2000
+ this.nSteps += 1;
2001
+ }
2002
+ /** Append a synthetic terminal failure entry (max steps / consecutive failures). */
2003
+ recordSyntheticFailure(message) {
2004
+ this.deps.messageManager.recordSystemNote(this.nSteps, message);
2005
+ this.deps.history.add({
2006
+ result: [doneResult(false, { extractedContent: message })],
2007
+ state: placeholderState(),
2008
+ metadata: this.metadata(Date.now())
2009
+ });
2010
+ this.done = true;
2011
+ }
2012
+ /** Fold a step into both the message-manager history and the AgentHistoryList. */
2013
+ recordStep(output, results, state, stepStart) {
2014
+ this.deps.messageManager.recordStep(this.nSteps, output, results);
2015
+ this.deps.history.add({
2016
+ modelOutput: output,
2017
+ result: results,
2018
+ state,
2019
+ metadata: this.metadata(stepStart),
2020
+ stateMessage: this.deps.messageManager.stateMessageText
2021
+ });
2022
+ const end = Date.now();
2023
+ this.lastStepDurationSeconds = (end - stepStart) / 1e3;
2024
+ this.emit({
2025
+ type: "step-finished",
2026
+ step: this.nSteps,
2027
+ narration: assembleNarration(output, results),
2028
+ output,
2029
+ results
2030
+ });
2031
+ }
2032
+ metadata(stepStart) {
2033
+ return {
2034
+ stepStartTime: stepStart,
2035
+ stepEndTime: Date.now(),
2036
+ stepNumber: this.nSteps,
2037
+ stepInterval: this.lastStepDurationSeconds
2038
+ };
2039
+ }
2040
+ buildContext(pageUrl, hasSensitiveData, signal) {
2041
+ return {
2042
+ session: this.deps.session,
2043
+ driver: this.deps.driver,
2044
+ pageUrl,
2045
+ fileSystem: this.deps.fileSystem,
2046
+ extractionLlm: this.deps.extractionLlm,
2047
+ availableFilePaths: this.deps.availableFilePaths ?? [],
2048
+ hasSensitiveData,
2049
+ signal
2050
+ };
2051
+ }
2052
+ /** A single-key union carrying only `done` (forced-termination schema). */
2053
+ doneOnlyUnion() {
2054
+ const done2 = this.deps.registry.get("done");
2055
+ const schema = done2?.schema ?? Type4.Object({ text: Type4.String(), success: Type4.Boolean() });
2056
+ return Type4.Object({ done: schema }, { additionalProperties: false, title: "done" });
2057
+ }
2058
+ };
2059
+ function historyState(state, sensitiveData) {
2060
+ const mask = (text) => sensitiveData ? maskOutbound(text, sensitiveData) : text;
2061
+ return {
2062
+ url: mask(state.url),
2063
+ title: mask(state.title),
2064
+ tabs: state.tabs.map((tab) => ({ ...tab, url: mask(tab.url), title: mask(tab.title) })),
2065
+ screenshot: state.screenshot
2066
+ };
2067
+ }
2068
+ function placeholderState() {
2069
+ return { url: "", title: "", tabs: [] };
2070
+ }
2071
+ function errorText(error) {
2072
+ return error instanceof Error ? error.message : String(error);
2073
+ }
2074
+
2075
+ // src/usage/ledger.ts
2076
+ var UsageLedger = class {
2077
+ byModel = /* @__PURE__ */ new Map();
2078
+ /** Fold one LLM call's usage into the running totals for `model`. */
2079
+ record(model, usage) {
2080
+ let stats = this.byModel.get(model);
2081
+ if (!stats) {
2082
+ stats = {
2083
+ model,
2084
+ invocations: 0,
2085
+ input: 0,
2086
+ output: 0,
2087
+ cacheRead: 0,
2088
+ cacheWrite: 0,
2089
+ totalTokens: 0,
2090
+ cost: 0
2091
+ };
2092
+ this.byModel.set(model, stats);
2093
+ }
2094
+ stats.invocations += 1;
2095
+ stats.input += usage.input;
2096
+ stats.output += usage.output;
2097
+ stats.cacheRead += usage.cacheRead;
2098
+ stats.cacheWrite += usage.cacheWrite;
2099
+ stats.totalTokens += usage.totalTokens;
2100
+ stats.cost += usage.cost.total;
2101
+ }
2102
+ /** The end-of-run summary (per-model + grand totals). */
2103
+ summary() {
2104
+ const perModel = [...this.byModel.values()];
2105
+ return {
2106
+ perModel,
2107
+ totalTokens: perModel.reduce((sum, m) => sum + m.totalTokens, 0),
2108
+ totalCost: perModel.reduce((sum, m) => sum + m.cost, 0),
2109
+ totalInvocations: perModel.reduce((sum, m) => sum + m.invocations, 0)
2110
+ };
2111
+ }
2112
+ };
2113
+
2114
+ // src/agent/gif.ts
2115
+ import { createRequire } from "module";
2116
+ import { PLACEHOLDER_SCREENSHOT } from "@ap3x/browser";
2117
+
2118
+ // src/agent/png-decode.ts
2119
+ import { inflateSync } from "zlib";
2120
+ var PNG_SIGNATURE = Buffer.from([137, 80, 78, 71, 13, 10, 26, 10]);
2121
+ function paeth(a, b, c) {
2122
+ const p = a + b - c;
2123
+ const pa = Math.abs(p - a);
2124
+ const pb = Math.abs(p - b);
2125
+ const pc = Math.abs(p - c);
2126
+ if (pa <= pb && pa <= pc) return a;
2127
+ if (pb <= pc) return b;
2128
+ return c;
2129
+ }
2130
+ function decodePngBase64(base64) {
2131
+ try {
2132
+ return decodePng(Buffer.from(base64, "base64"));
2133
+ } catch {
2134
+ return void 0;
2135
+ }
2136
+ }
2137
+ function decodePng(buf) {
2138
+ if (buf.length < 8 || !buf.subarray(0, 8).equals(PNG_SIGNATURE)) return void 0;
2139
+ let width;
2140
+ let height;
2141
+ let bitDepth;
2142
+ let colorType;
2143
+ let interlace;
2144
+ const idat = [];
2145
+ let offset = 8;
2146
+ while (offset + 8 <= buf.length) {
2147
+ const length = buf.readUInt32BE(offset);
2148
+ const type = buf.toString("ascii", offset + 4, offset + 8);
2149
+ const dataStart = offset + 8;
2150
+ const data = buf.subarray(dataStart, dataStart + length);
2151
+ if (type === "IHDR") {
2152
+ width = data.readUInt32BE(0);
2153
+ height = data.readUInt32BE(4);
2154
+ bitDepth = data.readUInt8(8);
2155
+ colorType = data.readUInt8(9);
2156
+ interlace = data.readUInt8(12);
2157
+ } else if (type === "IDAT") {
2158
+ idat.push(data);
2159
+ } else if (type === "IEND") {
2160
+ break;
2161
+ }
2162
+ offset = dataStart + length + 4;
2163
+ }
2164
+ if (width === void 0 || height === void 0 || bitDepth !== 8 || interlace !== 0 || colorType !== 2 && colorType !== 6 || idat.length === 0) {
2165
+ return void 0;
2166
+ }
2167
+ const channels = colorType === 6 ? 4 : 3;
2168
+ const raw = inflateSync(Buffer.concat(idat));
2169
+ const stride = width * channels;
2170
+ const rgba = new Uint8Array(width * height * 4);
2171
+ let prevLine = new Uint8Array(stride);
2172
+ let pos = 0;
2173
+ for (let y = 0; y < height; y++) {
2174
+ if (pos >= raw.length) return void 0;
2175
+ const filterType = raw[pos];
2176
+ pos += 1;
2177
+ if (pos + stride > raw.length) return void 0;
2178
+ const line = raw.subarray(pos, pos + stride);
2179
+ pos += stride;
2180
+ const out = new Uint8Array(stride);
2181
+ for (let x = 0; x < stride; x++) {
2182
+ const a = x >= channels ? out[x - channels] : 0;
2183
+ const b = prevLine[x];
2184
+ const c = x >= channels ? prevLine[x - channels] : 0;
2185
+ const raw8 = line[x];
2186
+ let value;
2187
+ switch (filterType) {
2188
+ case 0:
2189
+ value = raw8;
2190
+ break;
2191
+ case 1:
2192
+ value = raw8 + a;
2193
+ break;
2194
+ case 2:
2195
+ value = raw8 + b;
2196
+ break;
2197
+ case 3:
2198
+ value = raw8 + Math.floor((a + b) / 2);
2199
+ break;
2200
+ case 4:
2201
+ value = raw8 + paeth(a, b, c);
2202
+ break;
2203
+ default:
2204
+ return void 0;
2205
+ }
2206
+ out[x] = value & 255;
2207
+ }
2208
+ for (let x = 0; x < width; x++) {
2209
+ const si = x * channels;
2210
+ const di = (y * width + x) * 4;
2211
+ rgba[di] = out[si];
2212
+ rgba[di + 1] = out[si + 1];
2213
+ rgba[di + 2] = out[si + 2];
2214
+ rgba[di + 3] = channels === 4 ? out[si + 3] : 255;
2215
+ }
2216
+ prevLine = out;
2217
+ }
2218
+ return { width, height, rgba };
2219
+ }
2220
+
2221
+ // src/agent/gif.ts
2222
+ var { GIFEncoder, applyPalette, quantize } = createRequire(import.meta.url)(
2223
+ "gifenc"
2224
+ );
2225
+ var DEFAULT_FRAME_DURATION_MS = 3e3;
2226
+ var DEFAULT_MAX_COLORS = 256;
2227
+ function collectFrames(history) {
2228
+ const frames = [];
2229
+ for (const step of history.history) {
2230
+ const shot = step.state.screenshot;
2231
+ if (!shot || shot === PLACEHOLDER_SCREENSHOT) continue;
2232
+ const decoded = decodePngBase64(shot);
2233
+ if (decoded && (frames.length === 0 || sameSize(decoded, frames[0]))) {
2234
+ frames.push(decoded);
2235
+ }
2236
+ }
2237
+ return frames;
2238
+ }
2239
+ function sameSize(a, b) {
2240
+ return a.width === b.width && a.height === b.height;
2241
+ }
2242
+ function encodeFramesToGif(frames, options = {}) {
2243
+ if (frames.length === 0) return void 0;
2244
+ const delay = options.frameDurationMs ?? DEFAULT_FRAME_DURATION_MS;
2245
+ const maxColors = options.maxColors ?? DEFAULT_MAX_COLORS;
2246
+ const repeat = options.repeat ?? 0;
2247
+ const { width, height } = frames[0];
2248
+ const gif = GIFEncoder();
2249
+ for (const frame of frames) {
2250
+ const palette = quantize(frame.rgba, maxColors);
2251
+ const indexed = applyPalette(frame.rgba, palette);
2252
+ gif.writeFrame(indexed, width, height, { palette, delay, repeat });
2253
+ }
2254
+ gif.finish();
2255
+ return gif.bytes();
2256
+ }
2257
+ function generateRunGif(history, options = {}) {
2258
+ return encodeFramesToGif(collectFrames(history), options);
2259
+ }
2260
+
2261
+ // src/agent/judge.ts
2262
+ import {
2263
+ generateObject as generateObject2
2264
+ } from "@ap3x/ai";
2265
+ import { PLACEHOLDER_SCREENSHOT as PLACEHOLDER_SCREENSHOT2 } from "@ap3x/browser";
2266
+ import { Type as Type5 } from "@sinclair/typebox";
2267
+ var JudgementResultSchema = Type5.Object({
2268
+ reasoning: Type5.String(),
2269
+ verdict: Type5.Boolean(),
2270
+ failureReason: Type5.String(),
2271
+ impossibleTask: Type5.Boolean(),
2272
+ reachedCaptcha: Type5.Boolean()
2273
+ });
2274
+ var MAX_TEXT_CHARS = 4e4;
2275
+ var DEFAULT_MAX_IMAGES = 10;
2276
+ function truncate2(text) {
2277
+ return text.length <= MAX_TEXT_CHARS ? text : `${text.slice(0, MAX_TEXT_CHARS)}...[truncated]`;
2278
+ }
2279
+ function formatSteps(history) {
2280
+ return history.history.map((step, i) => {
2281
+ const lines = [`Step ${i + 1}:`];
2282
+ const actions = step.modelOutput?.action ?? [];
2283
+ if (actions.length > 0) lines.push(`Actions: ${JSON.stringify(actions)}`);
2284
+ step.result.forEach((result, j) => {
2285
+ if (result.extractedContent) lines.push(`Result ${j + 1}: ${result.extractedContent}`);
2286
+ if (result.error) lines.push(`Error ${j + 1}: ${result.error}`);
2287
+ });
2288
+ return lines.join("\n");
2289
+ }).join("\n\n");
2290
+ }
2291
+ function selectScreenshots(history, maxImages) {
2292
+ const shots = history.history.map((step) => step.state.screenshot).filter((shot) => !!shot && shot !== PLACEHOLDER_SCREENSHOT2);
2293
+ return maxImages > 0 ? shots.slice(-maxImages) : [];
2294
+ }
2295
+ function buildSystemPrompt(hasGroundTruth) {
2296
+ return [
2297
+ "You are an expert judge evaluating a browser-automation agent's completed run.",
2298
+ hasGroundTruth ? "A <ground_truth> block is provided below and takes ABSOLUTE precedence: if the run does not satisfy it, verdict MUST be false." : void 0,
2299
+ "Judge, in order of importance: (1) whether the task was actually accomplished, matching the user's exact request \u2014 not a partial or approximate match; (2) whether the final output is complete and in the right format; (3) whether the browser interactions actually succeeded (don't just trust the agent's self-report \u2014 cross-check it against the trajectory and screenshots); (4) the quality of the agent's reasoning and recovery from errors.",
2300
+ "Automatically fail the run if: it was blocked by a captcha or login wall, the output format is wrong or missing, the agent looped or crashed, a required step was skipped, the agent fabricated content not present in the page/screenshots, or `done` was called before the task was actually complete.",
2301
+ "Set impossibleTask=true only when the task itself could not have been completed by any agent (a genuinely broken/inaccessible site, missing required credentials, a nonsensical request) \u2014 not merely because this agent performed poorly.",
2302
+ "Set reachedCaptcha=true if any screenshot or error indicates a captcha/bot-detection challenge was encountered.",
2303
+ "Be skeptical of the agent's own success claim; verify it against the trajectory. Respond with the exact JSON shape requested by the schema."
2304
+ ].filter((line) => !!line).join("\n");
2305
+ }
2306
+ function buildUserPrompt(task, finalResult, stepsText, groundTruth, imageCount) {
2307
+ const sections = [
2308
+ `<task>
2309
+ ${truncate2(task) || "No task provided"}
2310
+ </task>`,
2311
+ groundTruth ? `<ground_truth>
2312
+ ${truncate2(groundTruth)}
2313
+ </ground_truth>` : void 0,
2314
+ `<agent_trajectory>
2315
+ ${truncate2(stepsText) || "No agent trajectory provided"}
2316
+ </agent_trajectory>`,
2317
+ `<final_result>
2318
+ ${truncate2(finalResult) || "No final result provided"}
2319
+ </final_result>`,
2320
+ `${imageCount} screenshot(s) from execution are attached.`,
2321
+ "Evaluate this run against the criteria above and respond with the required JSON structure."
2322
+ ];
2323
+ return sections.filter((line) => !!line).join("\n\n");
2324
+ }
2325
+ async function judgeRun(model, task, history, options = {}) {
2326
+ const useVision = options.vision ?? true;
2327
+ const maxImages = options.maxImages ?? DEFAULT_MAX_IMAGES;
2328
+ const screenshots = useVision ? selectScreenshots(history, maxImages) : [];
2329
+ const system = buildSystemPrompt(!!options.groundTruth);
2330
+ const user = buildUserPrompt(
2331
+ task,
2332
+ history.finalResult() ?? "",
2333
+ formatSteps(history),
2334
+ options.groundTruth,
2335
+ screenshots.length
2336
+ );
2337
+ const userParts = [{ type: "text", text: user }];
2338
+ for (const shot of screenshots)
2339
+ userParts.push({ type: "image", data: shot, mimeType: "image/png" });
2340
+ const messages = [
2341
+ { role: "user", content: system, timestamp: Date.now() },
2342
+ { role: "user", content: userParts, timestamp: Date.now() }
2343
+ ];
2344
+ const { object, usage } = await generateObject2(model, messages, JudgementResultSchema, {
2345
+ ...options.streamOptions,
2346
+ signal: options.signal
2347
+ });
2348
+ return { judgement: object, usage };
2349
+ }
2350
+
2351
+ // src/agent/video.ts
2352
+ function sameSize2(a, b) {
2353
+ return a.width === b.width && a.height === b.height;
2354
+ }
2355
+ function decodeFrames(frames) {
2356
+ const decoded = [];
2357
+ for (const frame of frames) {
2358
+ const image = decodePngBase64(frame.data);
2359
+ if (image && (decoded.length === 0 || sameSize2(image, decoded[0]))) {
2360
+ decoded.push(image);
2361
+ }
2362
+ }
2363
+ return decoded;
2364
+ }
2365
+ function generateRunVideoGif(frames, options = {}) {
2366
+ const DEFAULT_VIDEO_FRAME_MS = 200;
2367
+ return encodeFramesToGif(decodeFrames(frames), {
2368
+ frameDurationMs: DEFAULT_VIDEO_FRAME_MS,
2369
+ ...options
2370
+ });
2371
+ }
2372
+
2373
+ // src/agent/agent.ts
2374
+ import { randomUUID } from "crypto";
2375
+ import { tmpdir } from "os";
2376
+ import { join as join3 } from "path";
2377
+ import { BrowserDriver } from "@ap3x/browser";
2378
+ import { Type as Type6 } from "@sinclair/typebox";
2379
+ var screenshotAction = defineAction({
2380
+ name: "screenshot",
2381
+ description: "Capture a screenshot of the current page for visual inspection.",
2382
+ schema: Type6.Object({}),
2383
+ async execute(_params, ctx) {
2384
+ const data = await ctx.driver.screenshot({ signal: ctx.signal });
2385
+ return stepResult({
2386
+ images: [{ name: "screenshot", data }],
2387
+ metadata: { includeScreenshot: true }
2388
+ });
2389
+ }
2390
+ });
2391
+ var DEFAULT_LIMITS = {
2392
+ maxSteps: 100,
2393
+ maxFailures: 5,
2394
+ maxActionsPerStep: 5,
2395
+ finalResponseAfterFailure: true,
2396
+ stepTimeoutMs: 18e4,
2397
+ llmTimeoutMs: 9e4
2398
+ };
2399
+ var DEFAULT_SYSTEM_PROMPT = [
2400
+ "You are an autonomous browser agent. You run in a loop: read the page state, decide the next action(s), execute them, then observe the result.",
2401
+ "",
2402
+ "Each turn you receive a state message with these sections:",
2403
+ "- <user_request>: the task. Follow specific instructions literally; plan open-ended tasks yourself.",
2404
+ "- <agent_history>: your previous steps as <step_N> blocks (evaluation, memory, next goal, results).",
2405
+ "- <agent_state>: your file system, todo list, and any sensitive-data placeholders.",
2406
+ "- <browser_state>: the current URL, tabs, and the numbered interactive elements. ONLY elements shown with an [index] are interactive.",
2407
+ "- <read_state>: one-shot content from the previous step (shown once).",
2408
+ "",
2409
+ "Rules:",
2410
+ "- Act only on element indexes present in <browser_state>. A stale index means the page changed \u2014 re-read it.",
2411
+ "- Put page-changing actions (navigate, go_back, switch/open tab) last: anything queued after them is skipped and must be retried next step.",
2412
+ "- When sensitive data applies, write <secret>key</secret> where the value should go; never write the raw value.",
2413
+ "- Call done when the task is complete, impossible, or on the final step. Set success=true ONLY when fully complete, and report only facts you observed this session.",
2414
+ "",
2415
+ "Respond with a single object matching the required schema, including a non-empty action list."
2416
+ ].join("\n");
2417
+ var Agent = class {
2418
+ task;
2419
+ options;
2420
+ model;
2421
+ session;
2422
+ driver;
2423
+ registry;
2424
+ vision;
2425
+ variant;
2426
+ limits;
2427
+ ledger = new UsageLedger();
2428
+ historyList = new AgentHistoryList();
2429
+ messageManager;
2430
+ fileSystemDir;
2431
+ stream = new AgentEventStream();
2432
+ logger;
2433
+ taskId = randomUUID();
2434
+ fileSystem;
2435
+ stepLoop;
2436
+ pendingPaused = false;
2437
+ pendingStopped = false;
2438
+ /** Set once, after `run()` completes, when `output.generateGif` is on. */
2439
+ lastGif;
2440
+ /** Set once, after `run()` completes, when `output.recordHar` is on. */
2441
+ lastHar;
2442
+ /** Set once, after `run()` completes, when `output.recordVideo` is on. */
2443
+ lastVideo;
2444
+ /** Set once, after `run()` completes, when `judge` is on and the judge succeeded. */
2445
+ lastJudgement;
2446
+ constructor(task, options) {
2447
+ this.task = task;
2448
+ this.options = options;
2449
+ this.model = options.llm.model;
2450
+ this.logger = (options.logger ?? silentLogger).child({ taskId: this.taskId });
2451
+ this.session = options.browser.session;
2452
+ this.driver = options.browser.driver ?? new BrowserDriver(this.session);
2453
+ this.vision = options.vision ?? "everyStep";
2454
+ this.variant = options.output?.flash ? "flash" : options.output?.thinking === false ? "noThinking" : "thinking";
2455
+ this.limits = { ...DEFAULT_LIMITS, ...options.limits };
2456
+ this.fileSystemDir = options.fileSystemDir ?? join3(tmpdir(), `ap3x_agent_${randomUUID()}`);
2457
+ this.registry = this.buildRegistry();
2458
+ this.messageManager = new MessageManager({
2459
+ systemPrompt: options.systemPrompt ?? DEFAULT_SYSTEM_PROMPT,
2460
+ task,
2461
+ sensitiveData: options.security?.sensitiveData,
2462
+ fileSystemDescribe: () => this.fileSystem?.describe() ?? "",
2463
+ todoContents: () => this.fileSystem?.getTodoContents() ?? "",
2464
+ availableFilePaths: options.security?.availableFilePaths ?? []
2465
+ });
2466
+ }
2467
+ /** The run history (populated as the loop runs). */
2468
+ get history() {
2469
+ return this.historyList;
2470
+ }
2471
+ /** The effective (cloned, vision-filtered) registry the agent drives. */
2472
+ get effectiveRegistry() {
2473
+ return this.registry;
2474
+ }
2475
+ /** The model this agent reasons with. */
2476
+ get currentModel() {
2477
+ return this.model;
2478
+ }
2479
+ /** The GIF run-summary built after `run()`, when `output.generateGif` is on. */
2480
+ get gif() {
2481
+ return this.lastGif;
2482
+ }
2483
+ /** The HAR 1.2 network-traffic artifact built after `run()`, when `output.recordHar` is on. */
2484
+ get har() {
2485
+ return this.lastHar;
2486
+ }
2487
+ /** The video-as-GIF artifact built after `run()`, when `output.recordVideo` is on. */
2488
+ get video() {
2489
+ return this.lastVideo;
2490
+ }
2491
+ /**
2492
+ * The post-run judge's verdict, when `judge` is on. Informational — it never
2493
+ * overrides the agent's own reported success. `undefined` if the judge was off
2494
+ * or the evaluation failed.
2495
+ */
2496
+ get judgement() {
2497
+ return this.lastJudgement;
2498
+ }
2499
+ /** The curated O2 event stream as an async iterable (`for await`). */
2500
+ events() {
2501
+ return this.stream.events();
2502
+ }
2503
+ /** Subscribe to one event type (callback sugar over {@link events}). */
2504
+ on(type, cb) {
2505
+ return this.stream.on(type, cb);
2506
+ }
2507
+ /** Subscribe to every event. Returns an unsubscribe function. */
2508
+ onEvent(cb) {
2509
+ return this.stream.onEvent(cb);
2510
+ }
2511
+ async run(options = {}) {
2512
+ const loop = await this.ensureReady();
2513
+ if (this.options.output?.recordHar && !this.session.har.isRecording) this.session.har.start();
2514
+ if (this.options.output?.recordVideo && !this.session.screencast.isRecording) {
2515
+ this.session.screencast.start();
2516
+ }
2517
+ return loop.run(options);
2518
+ }
2519
+ /** Drive one step (respects pause/stop). Returns whether the task is now done. */
2520
+ async step() {
2521
+ const loop = await this.ensureReady();
2522
+ return loop.step();
2523
+ }
2524
+ pause() {
2525
+ if (this.stepLoop) this.stepLoop.pause();
2526
+ else this.pendingPaused = true;
2527
+ }
2528
+ resume() {
2529
+ if (this.stepLoop) this.stepLoop.resume();
2530
+ else this.pendingPaused = false;
2531
+ }
2532
+ stop() {
2533
+ if (this.stepLoop) this.stepLoop.stop();
2534
+ else this.pendingStopped = true;
2535
+ }
2536
+ // --- internals -------------------------------------------------------------
2537
+ /** Clone the caller's (or default) registry and apply the vision filter (E7). */
2538
+ buildRegistry() {
2539
+ const base = this.options.tools ?? createDefaultRegistry();
2540
+ const registry = base.clone();
2541
+ if ((this.options.vision ?? "everyStep") === "off") {
2542
+ registry.exclude("screenshot");
2543
+ } else if (!registry.get("screenshot")) {
2544
+ registry.register(screenshotAction);
2545
+ }
2546
+ return registry;
2547
+ }
2548
+ /**
2549
+ * Compose the caller's `onDone` (if any) with the post-run artifact steps:
2550
+ * the caller's hook always runs first, then — for whichever of GIF/HAR/video
2551
+ * is gated on — the run's screenshots/network trace/screencast are turned
2552
+ * into {@link Agent.gif} / {@link Agent.har} / {@link Agent.video}. Each
2553
+ * artifact step is independently logged-and-swallowed on failure — a pure
2554
+ * output artifact must never fail the run.
2555
+ */
2556
+ buildHooks() {
2557
+ const userHooks = this.options.hooks;
2558
+ const output = this.options.output ?? {};
2559
+ if (!output.generateGif && !output.recordHar && !output.recordVideo && !this.options.judge) {
2560
+ return userHooks ?? {};
2561
+ }
2562
+ const gifOptions = typeof output.generateGif === "object" ? output.generateGif : {};
2563
+ const videoOptions = typeof output.recordVideo === "object" ? output.recordVideo : {};
2564
+ return {
2565
+ ...userHooks,
2566
+ onDone: async (history) => {
2567
+ await userHooks?.onDone?.(history);
2568
+ if (output.generateGif) {
2569
+ try {
2570
+ this.lastGif = generateRunGif(history, gifOptions);
2571
+ } catch (error) {
2572
+ this.logger.warn("gif generation failed", {
2573
+ error: error instanceof Error ? error.message : String(error)
2574
+ });
2575
+ }
2576
+ }
2577
+ if (output.recordHar && this.session.har.isRecording) {
2578
+ try {
2579
+ this.lastHar = this.session.har.stop();
2580
+ } catch (error) {
2581
+ this.logger.warn("har recording failed", {
2582
+ error: error instanceof Error ? error.message : String(error)
2583
+ });
2584
+ }
2585
+ }
2586
+ if (output.recordVideo && this.session.screencast.isRecording) {
2587
+ try {
2588
+ const frames = this.session.screencast.stop();
2589
+ this.lastVideo = generateRunVideoGif(frames, videoOptions);
2590
+ } catch (error) {
2591
+ this.logger.warn("video generation failed", {
2592
+ error: error instanceof Error ? error.message : String(error)
2593
+ });
2594
+ }
2595
+ }
2596
+ if (this.options.judge) {
2597
+ try {
2598
+ const cfg = typeof this.options.judge === "object" ? this.options.judge : {};
2599
+ const judgeModel = cfg.model ?? this.model;
2600
+ const { judgement, usage } = await judgeRun(judgeModel, this.task, history, {
2601
+ groundTruth: cfg.groundTruth,
2602
+ maxImages: cfg.maxImages,
2603
+ vision: cfg.vision,
2604
+ // ponytail: reuse the agent's per-call LLM overrides (auth) — correct
2605
+ // for the default same-model judge; add a judge-specific override
2606
+ // only if a split-provider judge ever needs different credentials.
2607
+ streamOptions: this.options.llm.streamOptions
2608
+ });
2609
+ this.lastJudgement = judgement;
2610
+ this.ledger.record(judgeModel.id, usage);
2611
+ history.usage = this.ledger.summary();
2612
+ } catch (error) {
2613
+ this.logger.warn("judge evaluation failed", {
2614
+ error: error instanceof Error ? error.message : String(error)
2615
+ });
2616
+ }
2617
+ }
2618
+ }
2619
+ };
2620
+ }
2621
+ /** Create the FileSystem + StepLoop once; apply any buffered pause/stop. */
2622
+ async ensureReady() {
2623
+ if (this.stepLoop) return this.stepLoop;
2624
+ this.fileSystem = await FileSystem.create(this.fileSystemDir);
2625
+ this.stepLoop = new StepLoop({
2626
+ model: this.model,
2627
+ driver: this.driver,
2628
+ session: this.session,
2629
+ registry: this.registry,
2630
+ fileSystem: this.fileSystem,
2631
+ messageManager: this.messageManager,
2632
+ history: this.historyList,
2633
+ ledger: this.ledger,
2634
+ vision: this.vision,
2635
+ variant: this.variant,
2636
+ limits: this.limits,
2637
+ sensitiveData: this.options.security?.sensitiveData,
2638
+ availableFilePaths: this.options.security?.availableFilePaths ?? [],
2639
+ extractionLlm: this.options.llm.extraction,
2640
+ llmStreamOptions: this.options.llm.streamOptions,
2641
+ demoMode: this.options.demoMode,
2642
+ hooks: this.buildHooks(),
2643
+ task: this.task,
2644
+ emit: (event) => this.stream.emit(event),
2645
+ logger: this.logger
2646
+ });
2647
+ if (this.pendingStopped) this.stepLoop.stop();
2648
+ else if (this.pendingPaused) this.stepLoop.pause();
2649
+ return this.stepLoop;
2650
+ }
2651
+ };
2652
+
2653
+ // src/mcp/log.ts
2654
+ var mcpLogger = createLogger({
2655
+ level: "info",
2656
+ fields: { component: "mcp" },
2657
+ sink: (_level, line) => process.stderr.write(`${line}
2658
+ `)
2659
+ });
2660
+ function logStderr(message) {
2661
+ mcpLogger.info(message);
2662
+ }
2663
+
2664
+ // src/mcp/server.ts
2665
+ import { randomUUID as randomUUID2 } from "crypto";
2666
+ import { tmpdir as tmpdir2 } from "os";
2667
+ import { join as join4 } from "path";
2668
+ import { BrowserDriver as BrowserDriver2 } from "@ap3x/browser";
2669
+ import { Server } from "@modelcontextprotocol/sdk/server/index.js";
2670
+ import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js";
2671
+ import {
2672
+ CallToolRequestSchema,
2673
+ ListToolsRequestSchema
2674
+ } from "@modelcontextprotocol/sdk/types.js";
2675
+ import { Type as Type7 } from "@sinclair/typebox";
2676
+ var ACT_TOOLS = [
2677
+ { tool: "browser_navigate", action: "navigate" },
2678
+ { tool: "browser_click", action: "click" },
2679
+ { tool: "browser_type", action: "input_text" },
2680
+ { tool: "browser_scroll", action: "scroll" },
2681
+ { tool: "browser_extract", action: "extract_content" },
2682
+ { tool: "browser_go_back", action: "go_back" },
2683
+ { tool: "browser_switch_tab", action: "switch_tab" },
2684
+ { tool: "browser_close_tab", action: "close_tab" }
2685
+ ];
2686
+ var GET_STATE_SCHEMA = Type7.Object({ include_screenshot: Type7.Optional(Type7.Boolean()) });
2687
+ var SCREENSHOT_SCHEMA = Type7.Object({ full_page: Type7.Optional(Type7.Boolean()) });
2688
+ var LIST_TABS_SCHEMA = Type7.Object({});
2689
+ var RUN_AGENT_SCHEMA = Type7.Object({
2690
+ task: Type7.String(),
2691
+ max_steps: Type7.Optional(Type7.Integer({ minimum: 1 }))
2692
+ });
2693
+ function textResult(text) {
2694
+ return { content: [{ type: "text", text }] };
2695
+ }
2696
+ function errorResult(message) {
2697
+ return { content: [{ type: "text", text: message }], isError: true };
2698
+ }
2699
+ function actionResultToContent(result) {
2700
+ if (result.error !== void 0) return errorResult(result.error);
2701
+ const text = result.longTermMemory ?? result.extractedContent ?? "OK";
2702
+ const content = [{ type: "text", text }];
2703
+ for (const image of result.images ?? []) {
2704
+ content.push({ type: "image", data: image.data, mimeType: "image/png" });
2705
+ }
2706
+ return { content };
2707
+ }
2708
+ async function getStateResult(driver, includeScreenshot) {
2709
+ const state = await driver.getState({ screenshot: includeScreenshot });
2710
+ const content = [
2711
+ {
2712
+ type: "text",
2713
+ text: JSON.stringify({
2714
+ url: state.url,
2715
+ title: state.title,
2716
+ tabs: state.tabs,
2717
+ elements: state.serializedText
2718
+ })
2719
+ }
2720
+ ];
2721
+ if (state.screenshot)
2722
+ content.push({ type: "image", data: state.screenshot, mimeType: "image/png" });
2723
+ return { content };
2724
+ }
2725
+ async function screenshotResult(driver, fullPage, signal) {
2726
+ const data = await driver.screenshot({ fullPage, signal });
2727
+ return { content: [{ type: "image", data, mimeType: "image/png" }] };
2728
+ }
2729
+ function createBrowserMcpServer(options) {
2730
+ const { session } = options;
2731
+ const driver = options.driver ?? new BrowserDriver2(session);
2732
+ const registry = options.registry ?? createDefaultRegistry();
2733
+ function catalog() {
2734
+ const tools = [];
2735
+ for (const { tool, action } of ACT_TOOLS) {
2736
+ const registered = registry.get(action);
2737
+ if (!registered) continue;
2738
+ tools.push({
2739
+ name: tool,
2740
+ description: registered.description,
2741
+ inputSchema: registered.schema
2742
+ });
2743
+ }
2744
+ tools.push({
2745
+ name: "browser_get_state",
2746
+ description: "Read the current page: url, title, tabs, and the numbered interactive elements the agent sees.",
2747
+ inputSchema: GET_STATE_SCHEMA
2748
+ });
2749
+ tools.push({
2750
+ name: "browser_screenshot",
2751
+ description: "Capture a screenshot of the current page.",
2752
+ inputSchema: SCREENSHOT_SCHEMA
2753
+ });
2754
+ tools.push({
2755
+ name: "browser_list_tabs",
2756
+ description: "List the open tabs with their 4-char ids.",
2757
+ inputSchema: LIST_TABS_SCHEMA
2758
+ });
2759
+ if (options.llm) {
2760
+ tools.push({
2761
+ name: "browser_run_agent",
2762
+ description: "Run an autonomous browser agent to complete a task end-to-end.",
2763
+ inputSchema: RUN_AGENT_SCHEMA
2764
+ });
2765
+ }
2766
+ return tools;
2767
+ }
2768
+ let fsPromise;
2769
+ const fileSystem = () => {
2770
+ if (!fsPromise) {
2771
+ fsPromise = FileSystem.create(
2772
+ options.fileSystemDir ?? join4(tmpdir2(), `ap3x_mcp_${randomUUID2()}`)
2773
+ );
2774
+ }
2775
+ return fsPromise;
2776
+ };
2777
+ async function buildCtx(signal) {
2778
+ const focus = session.registry.focusedTargetId;
2779
+ return {
2780
+ session,
2781
+ driver,
2782
+ pageUrl: focus ? session.registry.getCurrentUrl(focus) ?? "" : "",
2783
+ fileSystem: await fileSystem(),
2784
+ extractionLlm: options.llm?.extraction,
2785
+ availableFilePaths: options.availableFilePaths ?? [],
2786
+ hasSensitiveData: false,
2787
+ signal
2788
+ };
2789
+ }
2790
+ async function runAgent(args, signal) {
2791
+ if (!options.llm) return errorResult("browser_run_agent is not available: no LLM configured.");
2792
+ const task = typeof args.task === "string" ? args.task : "";
2793
+ if (!task) return errorResult("browser_run_agent requires a 'task' string.");
2794
+ const agent = new Agent(task, {
2795
+ llm: options.llm,
2796
+ browser: { session, driver },
2797
+ limits: typeof args.max_steps === "number" ? { maxSteps: args.max_steps } : void 0
2798
+ });
2799
+ const history = await agent.run({ signal });
2800
+ return textResult(
2801
+ JSON.stringify({
2802
+ done: history.isDone(),
2803
+ success: history.isSuccessful(),
2804
+ steps: history.length,
2805
+ finalResult: history.finalResult(),
2806
+ urls: history.urls(),
2807
+ errors: history.errors().filter((error) => error !== void 0)
2808
+ })
2809
+ );
2810
+ }
2811
+ async function dispatch(name, args, signal) {
2812
+ const act = ACT_TOOLS.find((mapping) => mapping.tool === name);
2813
+ if (act) {
2814
+ const ctx = await buildCtx(signal);
2815
+ return actionResultToContent(await registry.execute(act.action, args, ctx));
2816
+ }
2817
+ switch (name) {
2818
+ case "browser_get_state":
2819
+ return getStateResult(driver, args.include_screenshot === true);
2820
+ case "browser_screenshot":
2821
+ return screenshotResult(driver, args.full_page === true, signal);
2822
+ case "browser_list_tabs":
2823
+ return textResult(JSON.stringify(driver.tabs()));
2824
+ case "browser_run_agent":
2825
+ return runAgent(args, signal);
2826
+ default:
2827
+ return errorResult(`Unknown tool "${name}"`);
2828
+ }
2829
+ }
2830
+ const server = new Server(
2831
+ {
2832
+ name: options.serverInfo?.name ?? "ap3x-browser",
2833
+ version: options.serverInfo?.version ?? "0.1.0"
2834
+ },
2835
+ { capabilities: { tools: {} } }
2836
+ );
2837
+ server.setRequestHandler(ListToolsRequestSchema, async () => ({ tools: catalog() }));
2838
+ server.setRequestHandler(CallToolRequestSchema, async (request, extra) => {
2839
+ const name = request.params.name;
2840
+ const args = request.params.arguments ?? {};
2841
+ try {
2842
+ return await dispatch(name, args, extra.signal);
2843
+ } catch (error) {
2844
+ return errorResult(error instanceof Error ? error.message : String(error));
2845
+ }
2846
+ });
2847
+ return server;
2848
+ }
2849
+ async function runBrowserMcpServer(options) {
2850
+ const server = createBrowserMcpServer(options);
2851
+ await server.connect(new StdioServerTransport());
2852
+ logStderr("browser MCP server ready on stdio");
2853
+ return server;
2854
+ }
2855
+
2856
+ export {
2857
+ stepResult,
2858
+ doneResult,
2859
+ isDoneResult,
2860
+ resolveSecretsForUrl,
2861
+ applicablePlaceholders,
2862
+ sensitiveDataDescription,
2863
+ maskOutbound,
2864
+ maskSecretMap,
2865
+ substituteSecrets,
2866
+ totp,
2867
+ defineAction,
2868
+ Registry,
2869
+ FileParseError,
2870
+ readExternalFile,
2871
+ BUILTIN_ACTIONS,
2872
+ registerBuiltins,
2873
+ createDefaultRegistry,
2874
+ sanitizeFilename,
2875
+ FileSystem,
2876
+ buildAgentOutputSchema,
2877
+ parseAction,
2878
+ COMPACT_EVERY_N_STEPS,
2879
+ COMPACT_TRIGGER_CHARS,
2880
+ MessageManager,
2881
+ resultsAreDone,
2882
+ perceive,
2883
+ budgetNudge,
2884
+ LoopDetector,
2885
+ AgentEventStream,
2886
+ toTypedAction,
2887
+ assembleNarration,
2888
+ multiAct,
2889
+ stepDurationSeconds,
2890
+ AgentHistoryList,
2891
+ createLogger,
2892
+ silentLogger,
2893
+ detectCaptcha,
2894
+ CaptchaWatchdog,
2895
+ anySignal,
2896
+ StepLoop,
2897
+ UsageLedger,
2898
+ encodeFramesToGif,
2899
+ generateRunGif,
2900
+ JudgementResultSchema,
2901
+ judgeRun,
2902
+ generateRunVideoGif,
2903
+ Agent,
2904
+ logStderr,
2905
+ createBrowserMcpServer,
2906
+ runBrowserMcpServer
2907
+ };