@parcha/pi-browser 0.1.0-beta.27 → 0.1.0-beta.29

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (57) hide show
  1. package/CHANGELOG.md +19 -0
  2. package/README.md +29 -8
  3. package/dist/coding-agent-local.d.ts +3 -7
  4. package/dist/coding-agent-local.js +12 -80
  5. package/dist/coding-agent.d.ts +10 -1
  6. package/dist/coding-agent.js +45 -10
  7. package/dist/core/contract.d.ts +2 -0
  8. package/dist/core/contract.js +11 -4
  9. package/dist/core/decisions.d.ts +3 -1
  10. package/dist/core/effects.d.ts +8 -0
  11. package/dist/core/effects.js +54 -5
  12. package/dist/core/entities.d.ts +3 -0
  13. package/dist/core/entities.js +68 -0
  14. package/dist/core/guard.d.ts +3 -0
  15. package/dist/core/guard.js +5 -0
  16. package/dist/core/host.d.ts +24 -6
  17. package/dist/core/page-classifier.d.ts +77 -0
  18. package/dist/core/page-classifier.js +92 -0
  19. package/dist/core/page-markdown.d.ts +5 -0
  20. package/dist/core/page-markdown.js +151 -0
  21. package/dist/core/page-scan.d.ts +95 -0
  22. package/dist/core/page-scan.js +542 -0
  23. package/dist/core/public-fetch.d.ts +32 -0
  24. package/dist/core/public-fetch.js +134 -0
  25. package/dist/core/read-page-worker.d.ts +8 -0
  26. package/dist/core/read-page-worker.js +32 -0
  27. package/dist/core/read-page.d.ts +46 -0
  28. package/dist/core/read-page.js +96 -0
  29. package/dist/core/tools.d.ts +9 -2
  30. package/dist/core/tools.js +56 -31
  31. package/dist/core/vault.d.ts +68 -0
  32. package/dist/core/vault.js +79 -0
  33. package/dist/core/web.d.ts +33 -0
  34. package/dist/core/web.js +155 -16
  35. package/dist/driver/stagehand.js +2 -1
  36. package/dist/durable/web.d.ts +7 -1
  37. package/dist/durable/web.js +4 -1
  38. package/dist/durable.d.ts +2 -1
  39. package/dist/durable.js +58 -10
  40. package/dist/index.d.ts +4 -0
  41. package/dist/index.js +4 -0
  42. package/dist/providers/browserbase.d.ts +1 -1
  43. package/dist/providers/browserbase.js +11 -1
  44. package/dist/providers/cdp-session.d.ts +28 -0
  45. package/dist/providers/cdp-session.js +153 -0
  46. package/dist/providers/cdp.d.ts +6 -0
  47. package/dist/providers/cdp.js +52 -2
  48. package/dist/providers/chrome-args.d.ts +3 -1
  49. package/dist/providers/chrome-args.js +3 -2
  50. package/dist/providers/kernel.js +6 -0
  51. package/dist/providers/profiles.d.ts +4 -0
  52. package/dist/providers/profiles.js +10 -0
  53. package/dist/providers/worker-extension.d.ts +14 -0
  54. package/dist/providers/worker-extension.js +63 -0
  55. package/dist/vault-session.d.ts +58 -0
  56. package/dist/vault-session.js +219 -0
  57. package/package.json +5 -1
@@ -0,0 +1,134 @@
1
+ // A page from a public host only, so a page-steered agent never reaches the owner's private network: every hop's name
2
+ // is resolved once, each address it resolves to is checked, and the connection is dialled at the address checked.
3
+ // http(s) on ports 80 and 443 only, no cookies or credentials, a byte cap on the body, and the caller's signal bounds
4
+ // every hop, its name lookup included.
5
+ import dns from "node:dns/promises";
6
+ import net from "node:net";
7
+ import { Agent, fetch as undiciFetch } from "undici";
8
+ import { refusedError } from "./custody.js";
9
+ /** 0.0.0.0/8 this network, 10/8, 100.64/10 carrier-grade, 127/8, 169.254/16 link-local, 172.16/12, 192.0.0.0/24
10
+ * protocol assignments, 192.168/16, 198.18/15 benchmarking, 224/4 multicast and 240/4 reserved (the broadcast address
11
+ * included). */
12
+ const V4_PRIVATE = [[0x00000000, 8], [0x0a000000, 8], [0x64400000, 10], [0x7f000000, 8], [0xa9fe0000, 16], [0xac100000, 12], [0xc0000000, 24], [0xc0a80000, 16], [0xc6120000, 15], [0xe0000000, 4], [0xf0000000, 4]];
13
+ /** ::/8 reserved by the IETF (the unspecified and loopback addresses, IPv4-compatible ones, and the NAT64 prefixes
14
+ * 64:ff9b::/96 and 64:ff9b:1::/48), fc00::/7 unique-local, fe80::/10 link-local, fec0::/10 site-local and ff00::/8
15
+ * multicast. An IPv4-mapped address (::ffff:0:0/96) is checked as the IPv4 address it carries. */
16
+ const V6_PRIVATE = [["::", 8], ["fc00::", 7], ["fe80::", 10], ["fec0::", 10], ["ff00::", 8]];
17
+ const v4 = (ip) => ip.split(".").reduce((n, octet) => n * 256 + Number(octet), 0);
18
+ /** An IPv6 address's eight 16-bit groups, read through the URL parser's canonical form; null when it does not parse
19
+ * (a zone index among them). */
20
+ function v6Groups(ip) {
21
+ let canonical;
22
+ try {
23
+ canonical = new URL(`http://[${ip}]/`).hostname.slice(1, -1);
24
+ }
25
+ catch {
26
+ return null;
27
+ }
28
+ const groups = (part) => (part ? part.split(":").map((group) => parseInt(group, 16)) : []);
29
+ if (!canonical.includes("::"))
30
+ return groups(canonical);
31
+ const [head, tail] = canonical.split("::").map(groups);
32
+ return [...head, ...new Array(8 - head.length - tail.length).fill(0), ...tail];
33
+ }
34
+ const inV6 = (groups, base, bits) => groups.every((group, i) => { const span = Math.max(0, Math.min(16, bits - 16 * i)); const mask = (0xffff << (16 - span)) & 0xffff; return (group & mask) === (base[i] & mask); });
35
+ /** Any address but a public unicast one, v4 and v6 (see V4_PRIVATE and V6_PRIVATE); what does not parse as one too. */
36
+ export function isPrivateAddress(ip) {
37
+ if (net.isIPv4(ip))
38
+ return V4_PRIVATE.some(([base, bits]) => Math.floor(v4(ip) / 2 ** (32 - bits)) === Math.floor(base / 2 ** (32 - bits)));
39
+ const groups = net.isIPv6(ip) ? v6Groups(ip) : null;
40
+ if (!groups)
41
+ return true;
42
+ if (groups.slice(0, 5).every((group) => group === 0) && groups[5] === 0xffff)
43
+ return isPrivateAddress([groups[6] >> 8, groups[6] & 0xff, groups[7] >> 8, groups[7] & 0xff].join("."));
44
+ return V6_PRIVATE.some(([base, bits]) => inV6(groups, v6Groups(base), bits));
45
+ }
46
+ /** Answers `address` for any name: the address a request was checked against is the one it connects to. */
47
+ export const pinnedLookup = (address) => (_host, options, callback) => {
48
+ const family = net.isIPv6(address) ? 6 : 4;
49
+ if (options?.all)
50
+ callback(null, [{ address, family }]);
51
+ else
52
+ callback(null, address, family);
53
+ };
54
+ export const PUBLIC_MAX_BYTES = 5 * 1024 * 1024;
55
+ const MAX_REDIRECTS = 5;
56
+ const PORTS = new Set(["", "80", "443"]);
57
+ /** `work()`'s answer, or the signal's reason as soon as it aborts: a resolver that never answers cannot outlast the
58
+ * caller. A signal already aborted starts no work. */
59
+ function untilAborted(work, signal) {
60
+ if (!signal)
61
+ return work();
62
+ signal.throwIfAborted();
63
+ return new Promise((resolve, reject) => {
64
+ const abort = () => reject(signal.reason);
65
+ signal.addEventListener("abort", abort, { once: true });
66
+ work().then(resolve, reject).finally(() => signal.removeEventListener("abort", abort));
67
+ });
68
+ }
69
+ /**
70
+ * Fetches `url` from a public host, following at most 5 redirects, each hop resolved, checked and dialled at the
71
+ * address checked. A 2xx answer of a text type is read whole, up to `maxBytes`; another status comes back without its
72
+ * body. Refused (`BrowserFailureError` `refused`), on the first hop or a redirect: a scheme other than http(s); a host
73
+ * that is local by name or resolves to any private address; a port other than 80 or 443; too many redirects; a 2xx body
74
+ * that is not text or is over the cap. `signal` ends the fetch at any point, a hop's name lookup included (it rejects
75
+ * with the signal's reason), and an aborted hop is never dialled.
76
+ */
77
+ export async function publicFetch(url, options = {}) {
78
+ const lookup = options.lookup ?? (async (host) => (await dns.lookup(host, { all: true })).map((a) => a.address));
79
+ const request = options.fetchImpl ?? ((target, init) => undiciFetch(target, init));
80
+ const maxBytes = options.maxBytes ?? PUBLIC_MAX_BYTES;
81
+ if (!Number.isSafeInteger(maxBytes) || maxBytes < 1)
82
+ throw new RangeError(`maxBytes must be a whole number of bytes above 0, not ${maxBytes}`);
83
+ let target = new URL(url);
84
+ if (!/^https?:$/.test(target.protocol))
85
+ throw refusedError(`refused: ${target.protocol} is not http(s)`);
86
+ for (let hop = 0;; hop += 1) {
87
+ const host = target.hostname.replace(/^\[|\]$/g, "");
88
+ if (host === "localhost" || host.endsWith(".localhost") || host.endsWith(".local") || host.endsWith(".internal"))
89
+ throw refusedError(`refused: ${host} is not a public host`);
90
+ const addresses = net.isIP(host) ? [host] : await untilAborted(() => lookup(host), options.signal);
91
+ if (!addresses.length || addresses.some(isPrivateAddress))
92
+ throw refusedError(`refused: ${host} is not a public host`);
93
+ if (!PORTS.has(target.port))
94
+ throw refusedError(`refused: port ${target.port} is not 80 or 443`);
95
+ options.signal?.throwIfAborted();
96
+ const dispatcher = new Agent({ connect: { lookup: pinnedLookup(addresses[0]) } });
97
+ let res;
98
+ try {
99
+ res = await request(target, { redirect: "manual", signal: options.signal, dispatcher, headers: { accept: "text/html,text/plain;q=0.9,*/*;q=0.1", "user-agent": "pi-browser" } });
100
+ if (res.status >= 300 && res.status < 400 && res.headers.get("location")) {
101
+ if (hop >= MAX_REDIRECTS)
102
+ throw refusedError("refused: too many redirects");
103
+ target = new URL(res.headers.get("location"), target);
104
+ if (!/^https?:$/.test(target.protocol))
105
+ throw refusedError("refused: a redirect left http(s)");
106
+ continue;
107
+ }
108
+ const contentType = res.headers.get("content-type") ?? "";
109
+ if (!res.ok)
110
+ return { url: target.href, status: res.status, contentType, body: null };
111
+ if (!/^(text\/|application\/(xhtml\+xml|json|xml))/i.test(contentType))
112
+ throw refusedError(`refused: ${contentType || "an unknown content type"} is not text`);
113
+ const chunks = [];
114
+ let size = 0;
115
+ for (const reader = res.body.getReader();;) {
116
+ const { done, value } = await reader.read();
117
+ if (done)
118
+ break;
119
+ if ((size += value.length) > maxBytes) {
120
+ await reader.cancel();
121
+ throw refusedError(`refused: the page is over ${maxBytes === PUBLIC_MAX_BYTES ? "5 MiB" : `${maxBytes} bytes`}`);
122
+ }
123
+ chunks.push(value);
124
+ }
125
+ return { url: target.href, status: res.status, contentType, body: Buffer.concat(chunks).toString("utf8") };
126
+ }
127
+ finally {
128
+ // A reply refused unread (not text, an error status, a redirect) still holds its connection: close() waits
129
+ // for it.
130
+ await res?.body?.cancel().catch(() => undefined);
131
+ await dispatcher.close().catch(() => undefined);
132
+ }
133
+ }
134
+ }
@@ -0,0 +1,8 @@
1
+ /** `html` as given (readPage drops and cuts it first), or, with `max`, dropped (its first article, main or body only,
2
+ * with `main`) and cut at `max` here; `url`: the page's title and links are read too. */
3
+ export type Job = {
4
+ html: string;
5
+ url?: string;
6
+ main?: boolean;
7
+ max?: number;
8
+ };
@@ -0,0 +1,32 @@
1
+ // read-page.ts's worker. For `workerData`, a Job: the page's markdown, posted once as { text } (with { title, links }
2
+ // when the job has a `url`, and { truncated, html_chars } when the worker cut the page), or as { error } when the
3
+ // conversion threw (a page nested past the stack, say).
4
+ import { parentPort, workerData } from "node:worker_threads";
5
+ import { TextNode } from "node-html-parser";
6
+ import { markdownOf } from "./page-markdown.js";
7
+ import { cutHtml, dropInert, mainOf, PageScan } from "./page-scan.js";
8
+ if (parentPort) {
9
+ const job = workerData;
10
+ try {
11
+ let html = job.html;
12
+ let cut = {};
13
+ if (job.max !== undefined) {
14
+ const kept = dropInert(job.html);
15
+ const picked = job.main ? mainOf(kept) : kept;
16
+ html = cutHtml(picked, job.max);
17
+ if (html.length < picked.length)
18
+ cut = { truncated: true, html_chars: job.html.length };
19
+ }
20
+ let read = {};
21
+ if (job.url !== undefined) {
22
+ const scan = new PageScan(html, job.url, (text) => new TextNode(text).text, false);
23
+ scan.step(Number.POSITIVE_INFINITY);
24
+ const { title, links } = scan.result();
25
+ read = { title, links };
26
+ }
27
+ parentPort.postMessage({ ...read, ...cut, text: markdownOf(html) });
28
+ }
29
+ catch (error) {
30
+ parentPort.postMessage({ error: String(error?.message ?? error) });
31
+ }
32
+ }
@@ -0,0 +1,46 @@
1
+ /** The most of a page's HTML that is read, counted once its script, style, svg, noscript and template blocks and its
2
+ * comments are dropped. A million characters of go.dev/dl converts in about 0.8 s on a busy 48-core host and gives
3
+ * 590,000 characters of markdown, ten times the 60,000 one web_fetch answer carries. */
4
+ export declare const READ_MAX_CHARS = 1000000;
5
+ /** How long the worker has for the markdown by default: six times what READ_MAX_CHARS of tables took. */
6
+ export declare const READ_DEADLINE_MS = 5000;
7
+ /** `n` with its digits in groups of three ("3,154,060"): Intl's first use costs tens of milliseconds on the loop. */
8
+ export declare const thousands: (n: number) => string;
9
+ /** `converted`: "markdown", or "text" when the conversion did not finish in time (or failed) and the text is the page's
10
+ * plain text. `truncated` and `html_chars` (the page's whole length, in characters) when the page was cut at
11
+ * READ_MAX_CHARS before it was read: the text, the title and the links are then the cut page's. */
12
+ export type PageRead = {
13
+ title: string;
14
+ text: string;
15
+ links: string[];
16
+ converted: "markdown" | "text";
17
+ truncated?: true;
18
+ html_chars?: number;
19
+ };
20
+ /** `deadlineMs`: how long the conversion to markdown has (default READ_DEADLINE_MS). `signal` ends the read, the
21
+ * conversion stopped, with the signal's reason. */
22
+ export type ReadPageOptions = {
23
+ deadlineMs?: number;
24
+ signal?: AbortSignal;
25
+ };
26
+ /**
27
+ * An HTML page's title, its text as markdown, and the absolute http(s) URLs it links to (resolved against its
28
+ * `<base href>` or `url`), each once, in page order. The event loop is never held for longer than one linear step of a
29
+ * bounded part of the page (see the file's head and PageRead). Rejects only with `signal`'s reason.
30
+ */
31
+ export declare function readPage(html: string, url: string, { deadlineMs, signal }?: ReadPageOptions): Promise<PageRead>;
32
+ /**
33
+ * `html`'s markdown, made in the worker within `deadlineMs` (default READ_DEADLINE_MS): of its first article, else its
34
+ * main, else its body element when `main` (mainOf), once its inert blocks are dropped, cut at READ_MAX_CHARS (then
35
+ * `truncated`, and `html_chars` is the page's whole length). Null when the worker has not answered in time, failed, or
36
+ * could not start. The event loop does no more than hand the worker the page. Rejects only with `signal`'s reason.
37
+ */
38
+ export declare function htmlMarkdown(html: string, { main, deadlineMs, signal }?: {
39
+ main?: boolean;
40
+ } & ReadPageOptions): Promise<{
41
+ text: string;
42
+ truncated?: true;
43
+ html_chars?: number;
44
+ } | null>;
45
+ /** `html` without its inert blocks (InertDrop), STEP_CHARS at a time with the loop yielded between steps. */
46
+ export declare function dropOnLoop(html: string, signal?: AbortSignal): Promise<string>;
@@ -0,0 +1,96 @@
1
+ // A fetched HTML page read as its title, its text and its links, without ever holding the event loop. The host's tools
2
+ // run in the run's own process, so a read that holds the loop holds the run's heartbeat with it, and a run whose
3
+ // heartbeat lapses is fenced: go.dev/dl, 3.15 MB of tables, held it past 240 s in one markdown conversion. So one linear
4
+ // pass drops the blocks that hold no readable text (page-scan.ts); what is left is cut at READ_MAX_CHARS (the page is
5
+ // then `truncated`, and `html_chars` is its whole length); and a worker thread converts that to markdown, in pieces
6
+ // small enough that the converter stays linear (page-markdown.ts), within a deadline. A worker that has not answered by
7
+ // then, or that fails, is stopped, and the text is the page's plain text, read on the loop in steps that yield it
8
+ // (`converted: "text"`). browser_read's markdown (htmlMarkdown) goes through the same worker, which drops, picks and
9
+ // cuts the live page's HTML itself, since that HTML has no bound; its fallback is the browser's own text.
10
+ import { setImmediate as yieldToLoop } from "node:timers/promises";
11
+ import { Worker } from "node:worker_threads";
12
+ import { decodeEntities } from "./entities.js";
13
+ import { cutHtml, InertDrop, PageScan } from "./page-scan.js";
14
+ /** The most of a page's HTML that is read, counted once its script, style, svg, noscript and template blocks and its
15
+ * comments are dropped. A million characters of go.dev/dl converts in about 0.8 s on a busy 48-core host and gives
16
+ * 590,000 characters of markdown, ten times the 60,000 one web_fetch answer carries. */
17
+ export const READ_MAX_CHARS = 1_000_000;
18
+ /** How long the worker has for the markdown by default: six times what READ_MAX_CHARS of tables took. */
19
+ export const READ_DEADLINE_MS = 5_000;
20
+ /** `n` with its digits in groups of three ("3,154,060"): Intl's first use costs tens of milliseconds on the loop. */
21
+ export const thousands = (n) => String(n).replace(/\B(?=(\d{3})+$)/g, ",");
22
+ /** How much of the page one step of the plain-text scan reads before it yields the loop. */
23
+ const STEP_CHARS = 32_768;
24
+ /**
25
+ * An HTML page's title, its text as markdown, and the absolute http(s) URLs it links to (resolved against its
26
+ * `<base href>` or `url`), each once, in page order. The event loop is never held for longer than one linear step of a
27
+ * bounded part of the page (see the file's head and PageRead). Rejects only with `signal`'s reason.
28
+ */
29
+ export async function readPage(html, url, { deadlineMs = READ_DEADLINE_MS, signal } = {}) {
30
+ signal?.throwIfAborted();
31
+ const kept = await dropOnLoop(html, signal);
32
+ const read = cutHtml(kept, READ_MAX_CHARS);
33
+ const cut = read.length < kept.length ? { truncated: true, html_chars: html.length } : {};
34
+ await yieldToLoop();
35
+ signal?.throwIfAborted();
36
+ const markdown = await inWorker({ html: read, url }, deadlineMs, signal);
37
+ if (markdown)
38
+ return { title: markdown.title ?? "", text: markdown.text, links: markdown.links ?? [], converted: "markdown", ...cut };
39
+ return { ...(await plainRead(read, url, signal)), converted: "text", ...cut };
40
+ }
41
+ /**
42
+ * `html`'s markdown, made in the worker within `deadlineMs` (default READ_DEADLINE_MS): of its first article, else its
43
+ * main, else its body element when `main` (mainOf), once its inert blocks are dropped, cut at READ_MAX_CHARS (then
44
+ * `truncated`, and `html_chars` is the page's whole length). Null when the worker has not answered in time, failed, or
45
+ * could not start. The event loop does no more than hand the worker the page. Rejects only with `signal`'s reason.
46
+ */
47
+ export async function htmlMarkdown(html, { main = false, deadlineMs = READ_DEADLINE_MS, signal } = {}) {
48
+ signal?.throwIfAborted();
49
+ const answer = await inWorker({ html, main, max: READ_MAX_CHARS }, deadlineMs, signal);
50
+ return answer && { text: answer.text, ...(answer.truncated ? { truncated: true, html_chars: answer.html_chars } : {}) };
51
+ }
52
+ /** The worker's answer to `job`, or null when it has not answered within `deadlineMs`, failed, or could not start. */
53
+ function inWorker(job, deadlineMs, signal) {
54
+ return new Promise((resolve, reject) => {
55
+ let worker;
56
+ // execArgv: none of the host's own flags (a test runner's --import, say) reach the worker.
57
+ try {
58
+ worker = new Worker(new URL("./read-page-worker.js", import.meta.url), { workerData: job, execArgv: [], resourceLimits: { maxOldGenerationSizeMb: 1024 } });
59
+ }
60
+ catch {
61
+ resolve(null);
62
+ return;
63
+ }
64
+ const settle = (finish) => {
65
+ clearTimeout(timer);
66
+ signal?.removeEventListener("abort", onAbort);
67
+ void worker.terminate().catch(() => undefined);
68
+ finish();
69
+ };
70
+ const onAbort = () => settle(() => reject(signal.reason));
71
+ const timer = setTimeout(() => settle(() => resolve(null)), deadlineMs);
72
+ signal?.addEventListener("abort", onAbort, { once: true });
73
+ worker.once("message", (answer) => settle(() => resolve(typeof answer?.text === "string" ? answer : null)));
74
+ worker.once("error", () => settle(() => resolve(null)));
75
+ worker.once("exit", () => settle(() => resolve(null)));
76
+ });
77
+ }
78
+ /** `html` without its inert blocks (InertDrop), STEP_CHARS at a time with the loop yielded between steps. */
79
+ export async function dropOnLoop(html, signal) {
80
+ const drop = new InertDrop(html);
81
+ while (!drop.step(STEP_CHARS)) {
82
+ await yieldToLoop();
83
+ signal?.throwIfAborted();
84
+ }
85
+ return drop.result();
86
+ }
87
+ /** `html`'s plain text, title and links, scanned STEP_CHARS at a time with the loop yielded between steps. */
88
+ async function plainRead(html, url, signal) {
89
+ // Decoded here without node-html-parser, whose first load would hold the loop (entities.ts).
90
+ const scan = new PageScan(html, url, decodeEntities, true);
91
+ while (!scan.step(STEP_CHARS)) {
92
+ await yieldToLoop();
93
+ signal?.throwIfAborted();
94
+ }
95
+ return scan.result();
96
+ }
@@ -7,7 +7,12 @@ import { type Locate } from "./clip.js";
7
7
  import { type ScreenshotOptions } from "./images.js";
8
8
  import { type Redact } from "./redact.js";
9
9
  import type { Geo } from "./host.js";
10
- /** One action of a run's batch, as the contract's schema admits it. */
10
+ /** `run` code is the body of an async function with `page`, `context` and `browser` in scope. Code written instead as one
11
+ * function expression (`async () => { ... }`) would only define that function and return nothing, so such code is
12
+ * called: a function with no parameters is awaited and its value returned, and one that declares parameters fails
13
+ * with the body form spelled out. The code counts as one function expression when it begins like one and the parser
14
+ * compiles the whole of it as a single expression (compiled, never run). Any other code reaches the driver unchanged. */
15
+ export declare function runnableCode(code: string): string;
11
16
  export type RunAction = {
12
17
  op: "click";
13
18
  id: string;
@@ -129,10 +134,12 @@ type ReconnectTimings = {
129
134
  tryMs: number;
130
135
  backoffMs: readonly number[];
131
136
  };
132
- export declare function pageTools({ guards, reconnect: timings }?: {
137
+ export declare function pageTools({ guards, reconnect: timings, readDeadlineMs }?: {
133
138
  /** Each conversation's repeat guard; a host that owns the conversation's lifecycle passes it to evict one. */
134
139
  guards?: Map<number, RepeatGuard>;
135
140
  /** The reconnect timings, for tests; hosts keep the defaults. */
136
141
  reconnect?: ReconnectTimings;
142
+ /** How long browser_read's markdown has before the browser's own text stands in (default READ_DEADLINE_MS), for tests. */
143
+ readDeadlineMs?: number;
137
144
  }): Pick<BrowserToolImpls<PageDriver>, "snapshot" | "run" | "screenshot" | "browser_read" | "browser_upload" | "browser_downloads">;
138
145
  export {};
@@ -1,3 +1,4 @@
1
+ import { setImmediate as yieldToLoop } from "node:timers/promises";
1
2
  import { BrowserFailureError, FenceRefusal, refusedError, loadedNoPage, passedThrough } from "./custody.js";
2
3
  import { defaultInline } from "./evidence.js";
3
4
  import { fetchDownloads } from "./downloads.js";
@@ -5,10 +6,40 @@ import { browserFailure, classifyBrowserError, transportFactOf } from "./failure
5
6
  import { createRepeatGuard } from "./guard.js";
6
7
  import { regionOf } from "./clip.js";
7
8
  import { captureWithinBudget } from "./images.js";
9
+ import { decodeEntities } from "./entities.js";
10
+ import { FrameScan } from "./page-scan.js";
11
+ import { dropOnLoop, htmlMarkdown, READ_DEADLINE_MS, READ_MAX_CHARS, thousands } from "./read-page.js";
8
12
  import { orderFacts, receiptFacts, sha256Hex } from "./receipts.js";
9
13
  import { scrubPageUrl } from "./redact.js";
10
14
  import { UNANSWERED_CALL } from "./unanswered.js";
11
15
  import { pageNotice, wallFailure, wallOf } from "./web.js";
16
+ /** One action of a run's batch, as the contract's schema admits it. */
17
+ const AsyncFunction = Object.getPrototypeOf(async function () { }).constructor;
18
+ /** How a function expression begins: `function`, `async function`, an arrow's parameter list or single parameter. */
19
+ const FUNCTION_START = /^(?:async\s+)?(?:function\b|\([^)]*\)\s*=>|[A-Za-z_$][\w$]*\s*=>)/;
20
+ /** `run` code is the body of an async function with `page`, `context` and `browser` in scope. Code written instead as one
21
+ * function expression (`async () => { ... }`) would only define that function and return nothing, so such code is
22
+ * called: a function with no parameters is awaited and its value returned, and one that declares parameters fails
23
+ * with the body form spelled out. The code counts as one function expression when it begins like one and the parser
24
+ * compiles the whole of it as a single expression (compiled, never run). Any other code reaches the driver unchanged. */
25
+ export function runnableCode(code) {
26
+ const expression = code.trim().replace(/;+$/, "").trimEnd();
27
+ if (!FUNCTION_START.test(expression))
28
+ return code;
29
+ try {
30
+ new AsyncFunction(`return (\n${expression}\n);`);
31
+ }
32
+ catch {
33
+ return code;
34
+ }
35
+ return [
36
+ `const __runCode = (\n${expression}\n);`,
37
+ "if (typeof __runCode === \"function\") {",
38
+ " if (__runCode.length > 0) throw new TypeError(\"run code is a function that takes parameters. Write the steps as the code itself (the body of a function): page, context and browser are already in scope, and `return` gives a value back.\");",
39
+ " return await __runCode();",
40
+ "}",
41
+ ].join("\n");
42
+ }
12
43
  const WHATS = ["markdown", "text", "html", "url", "title"];
13
44
  const DROPPED_RUN = "The connection to your browser session dropped; the session is still open, and your next browser call reconnects to it with its pages and cookies. This run may have acted before the drop: take a snapshot to see the page before running again.";
14
45
  const DROPPED_FOR_GOOD = "The connection to your browser session dropped and could not be re-established, though the session is still open. Call browser_relaunch for a fresh session (its pages and cookies are gone), or use web_fetch.";
@@ -38,17 +69,19 @@ const MAX_FRAMES = 5;
38
69
  const FRAME_TEXT_CHARS = 200;
39
70
  const FRAME_TEXT_READ_CHARS = 20_000;
40
71
  const FRAME_SCAN_CHARS = 2_000_000;
72
+ /** How much of the page one step of the frame scan reads before it yields the event loop. */
73
+ const FRAME_STEP_CHARS = 32_768;
41
74
  /** What the judge is shown of the page's frames, in document order: a frame's content is not in the page's text, so a page whose content sits in a challenge frame reads as empty without it. `frames` are the distinct origins (origin only: a
42
75
  * frame's path and query can carry a token; http(s) only; a relative source resolves against the page's `<base href>` when it has one, as the browser does) and `frameText` what the page puts inside the frame elements (the fallback a browser
43
76
  * does not render, where a block page keeps its message), collapsed to one line and cut at 200 characters. A frame is an element: markup inside a script, a comment, a noscript or a template is not one, so it neither counts nor takes one of the five.
44
- * A page that cannot be parsed gives none. */
77
+ * The first FRAME_SCAN_CHARS of the page are read in one linear scan (FrameScan), FRAME_STEP_CHARS at a time with the event loop yielded between steps, so a big page never holds it. A page that cannot be read gives none. */
45
78
  async function framesOf(html, pageUrl, redact) {
46
79
  try {
47
- const { parse } = await import("node-html-parser");
48
- // HTML's own void elements (the parser's default list lacks `frame`, so an unclosed <frame> would swallow its sibling).
49
- const root = parse(html.slice(0, FRAME_SCAN_CHARS), { voidTag: { tags: ["area", "base", "br", "col", "embed", "frame", "hr", "img", "input", "keygen", "link", "meta", "param", "source", "track", "wbr"] } });
80
+ const scan = new FrameScan(await dropOnLoop(html.slice(0, FRAME_SCAN_CHARS)), decodeEntities, FRAME_TEXT_READ_CHARS);
81
+ while (!scan.step(FRAME_STEP_CHARS))
82
+ await yieldToLoop();
83
+ const { frames, base: baseHref } = scan.result();
50
84
  let base = pageUrl;
51
- const baseHref = root.querySelector("base[href]")?.getAttribute("href");
52
85
  if (baseHref) {
53
86
  try {
54
87
  base = new URL(baseHref, pageUrl).href;
@@ -57,18 +90,10 @@ async function framesOf(html, pageUrl, redact) {
57
90
  }
58
91
  const origins = new Set();
59
92
  const texts = new Set();
60
- for (const frame of root.querySelectorAll("iframe, frame")) {
93
+ for (const frame of frames) {
61
94
  if (origins.size >= MAX_FRAMES && texts.size >= MAX_FRAMES)
62
95
  break;
63
- let inert = false;
64
- for (let up = frame.parentNode; up; up = up.parentNode)
65
- if (up.rawTagName?.toLowerCase() === "template") {
66
- inert = true;
67
- break;
68
- }
69
- if (inert)
70
- continue;
71
- const source = frame.getAttribute("src")?.trim();
96
+ const source = frame.src?.trim();
72
97
  if (source && origins.size < MAX_FRAMES) {
73
98
  try {
74
99
  const url = new URL(source, base);
@@ -127,19 +152,17 @@ const factsOf = (p, now) => ({
127
152
  callId: p.callId, urlAtDispatch: p.url, urlNow: now, observed: p.observed === true,
128
153
  sent: p.watched ? (p.sent ?? []).map((r) => ({ method: r.method, url: `${r.origin}${r.path}`, held: r.held })) : null,
129
154
  });
130
- async function markdown(html, fallback) {
131
- try {
132
- const [{ parse }, { NodeHtmlMarkdown }] = await Promise.all([import("node-html-parser"), import("node-html-markdown")]);
133
- const root = parse(html);
134
- const main = root.querySelector("article") ?? root.querySelector("main") ?? root.querySelector("body") ?? root;
135
- const md = NodeHtmlMarkdown.translate(main.toString(), { keepDataImages: false });
136
- if (md.trim())
137
- return { body: md, extractor: "readability-md", ext: "md" };
138
- }
139
- catch { }
140
- return { body: fallback, extractor: "inner-text", ext: "txt" };
155
+ /** The page's main content as markdown (its first article, else main, else body: htmlMarkdown, off the event loop); the
156
+ * browser's own text when the conversion gives nothing, fails or runs past `deadlineMs`. A page cut before it was read
157
+ * says so on its last line. Rejects only with `signal`'s reason. */
158
+ async function markdown(html, fallback, signal, deadlineMs) {
159
+ const read = await htmlMarkdown(html, { main: true, deadlineMs, ...(signal ? { signal } : {}) });
160
+ if (!read?.text.trim())
161
+ return { body: fallback, extractor: "inner-text", ext: "txt" };
162
+ const cut = read.truncated ? `\n\n(Cut: the page is ${thousands(read.html_chars)} characters of HTML, and only the first ${thousands(READ_MAX_CHARS)} of its main content, scripts and styles aside, were read as markdown. browser_read with what: "text" gives all of its text.)` : "";
163
+ return { body: read.text + cut, extractor: "readability-md", ext: "md" };
141
164
  }
142
- export function pageTools({ guards = new Map(), reconnect: timings = RECONNECT } = {}) {
165
+ export function pageTools({ guards = new Map(), reconnect: timings = RECONNECT, readDeadlineMs = READ_DEADLINE_MS } = {}) {
143
166
  const guardOf = (conversationId, redact) => guards.get(conversationId) ?? guards.set(conversationId, createRepeatGuard(redact)).get(conversationId);
144
167
  async function failureOf(error, effectful, port, call, action) {
145
168
  if (error instanceof BrowserFailureError)
@@ -342,7 +365,7 @@ export function pageTools({ guards = new Map(), reconnect: timings = RECONNECT }
342
365
  const stop = s.effects?.record((row) => { sent.push({ method: row.method, url: call.redact(`${row.origin}${row.path}`), held: row.held }); return port.journal(row); });
343
366
  let value;
344
367
  try {
345
- value = await s.driver.run(args.code !== undefined ? { code: args.code } : { actions: args.actions });
368
+ value = await s.driver.run(args.code !== undefined ? { code: runnableCode(args.code) } : { actions: args.actions });
346
369
  }
347
370
  finally {
348
371
  await s.effects?.flush(call.signal).catch(() => undefined);
@@ -350,10 +373,12 @@ export function pageTools({ guards = new Map(), reconnect: timings = RECONNECT }
350
373
  }
351
374
  const url = await s.driver.url().catch(() => null);
352
375
  // Compared as custody keeps it, so a page with a query value never looks new on every run.
353
- if (url && scrubPageUrl(url, call.redact) !== s.record.lastUrl)
376
+ if (url && scrubPageUrl(url, call.redact) !== s.record.lastUrl) {
377
+ guardOf(call.conversationId, call.redact).moved();
354
378
  await port.navigated(url);
379
+ }
355
380
  const reported = args.actions !== undefined && value && typeof value === "object" && typeof value.url === "string" ? { ...value, url: pageUrl(call, value.url) } : value;
356
- const shown = typeof reported === "string" ? reported : reported === undefined ? "(run completed; no return value)" : JSON.stringify(reported, null, 2) ?? String(reported);
381
+ const shown = typeof reported === "string" ? reported : reported === undefined ? "(run completed; the code returned no value: end it with `return <value>` to get one back)" : JSON.stringify(reported, null, 2) ?? String(reported);
357
382
  return { ...text(withNews(news, shown)), ...(sent.length ? { details: { sent } } : {}) };
358
383
  });
359
384
  },
@@ -381,7 +406,7 @@ export function pageTools({ guards = new Map(), reconnect: timings = RECONNECT }
381
406
  // so, and where to go. A script that wrote a title or text into about:blank made a page, which is read.
382
407
  if (page.url === "about:blank" && loadedNoPage(s.record) && !page.title.trim() && !page.text.trim())
383
408
  throw new BrowserFailureError(freshBrowser(s.replaced ?? null));
384
- const read = what === "markdown" ? await markdown(page.html, page.text)
409
+ const read = what === "markdown" ? await markdown(page.html, page.text, call.signal, readDeadlineMs)
385
410
  : what === "html" ? { body: page.html, extractor: "raw", ext: "html" }
386
411
  : what === "text" ? { body: page.text, extractor: "inner-text", ext: "txt" }
387
412
  : { body: what === "url" ? pageUrl(call, page.url) : page.title, ext: "txt" };
@@ -0,0 +1,68 @@
1
+ /** The most a session cookie may be restored after its capture: 24 hours. A host may set a lower age, never a higher one. */
2
+ export declare const MAX_SESSION_COOKIE_AGE_MS: number;
3
+ /** The age cap a restore applies to session cookies. Absent, 24 h; above 24 h, clamped to 24 h; zero, negative or not a number, refused. */
4
+ export declare function clampSessionCookieAgeMs(requested?: number): number;
5
+ export type VaultCookie = {
6
+ name: string;
7
+ value: string;
8
+ domain: string;
9
+ path: string;
10
+ /** Seconds since the epoch; absent means a SESSION cookie. */
11
+ expires?: number;
12
+ httpOnly: boolean;
13
+ secure: boolean;
14
+ sameSite?: "Strict" | "Lax" | "None";
15
+ };
16
+ export type CapturedState = {
17
+ kind: "cdp";
18
+ capturedAt: string;
19
+ /** Only cookies the allowed origins would be sent (see `cookieKept`). Partitioned cookies are not kept. */
20
+ cookies: VaultCookie[];
21
+ localStorage: Record<string, Record<string, string>>;
22
+ }
23
+ /** `origins`: the origins the context was captured for. A context holds a whole browser profile, so it is restored only for a request that covers EVERY one of them (`coversEveryOrigin`); one that records none is refused. */
24
+ | {
25
+ kind: "provider-context";
26
+ capturedAt: string;
27
+ provider: "browserbase" | "kernel";
28
+ contextId: string;
29
+ origins?: string[];
30
+ };
31
+ /** Whether `requested` includes every origin in `saved` (as scheme://host[:port], any order): a provider context is all-or-nothing, so a request for some of its origins would bring back the others' sign-ins. */
32
+ export declare function coversEveryOrigin(requested: readonly string[], saved: readonly string[] | undefined): boolean;
33
+ export interface SessionVaultPort {
34
+ /** At release and at the end of a handoff, and, for a context a new session made, right after the create (so a restart still finds it). The host supplied the origins; nothing outside them was captured.
35
+ * A capture has a deadline (20 s) after which nothing more is read or written, but a write already handed to this port cannot be recalled: ignore one whose `capturedAt` is older than the one you hold. */
36
+ capture(input: {
37
+ sessionKey: string;
38
+ origins: string[];
39
+ state: CapturedState;
40
+ }): Promise<void>;
41
+ /** At create, before the driver attaches. null: nothing kept, or all of it expired. */
42
+ restore(input: {
43
+ sessionKey: string;
44
+ origins: string[];
45
+ }): Promise<CapturedState | null>;
46
+ /** Crypto-shred: the item's key is destroyed. The package never calls this itself; the host does, on a revoke. */
47
+ forget(sessionKey: string): Promise<void>;
48
+ }
49
+ /** The host's choice, per conversation: which saved sign-in and which origins. Never the model's. Not a secret. */
50
+ export type VaultTarget = {
51
+ sessionKey: string;
52
+ origins: string[];
53
+ };
54
+ /** The host's allowed origins, each `scheme://host[:port]` of http or https, normalised and de-duplicated; anything else is refused. */
55
+ export declare function vaultOrigins(origins: readonly string[]): string[];
56
+ export declare function vaultTarget(target: VaultTarget): {
57
+ sessionKey: string;
58
+ origins: string[];
59
+ };
60
+ /** Whether a browser cookie belongs to the allowed origins: it would be SENT to at least one of them, by domain and secure flag (any path
61
+ * on an origin is the origin's). A host-only cookie matches its own host; a domain cookie (leading dot) matches the host and every host
62
+ * under it; a domain with no dot inside it (a public suffix) matches only a host equal to it; a secure cookie needs an https origin (a
63
+ * loopback host counts); a partitioned cookie is not kept. */
64
+ export declare function cookieKept(cookie: {
65
+ domain: string;
66
+ secure?: boolean;
67
+ partitionKey?: unknown;
68
+ }, origins: readonly string[]): boolean;