@parcha/pi-browser 0.1.0-beta.27 → 0.1.0-beta.29
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +19 -0
- package/README.md +29 -8
- package/dist/coding-agent-local.d.ts +3 -7
- package/dist/coding-agent-local.js +12 -80
- package/dist/coding-agent.d.ts +10 -1
- package/dist/coding-agent.js +45 -10
- package/dist/core/contract.d.ts +2 -0
- package/dist/core/contract.js +11 -4
- package/dist/core/decisions.d.ts +3 -1
- package/dist/core/effects.d.ts +8 -0
- package/dist/core/effects.js +54 -5
- package/dist/core/entities.d.ts +3 -0
- package/dist/core/entities.js +68 -0
- package/dist/core/guard.d.ts +3 -0
- package/dist/core/guard.js +5 -0
- package/dist/core/host.d.ts +24 -6
- package/dist/core/page-classifier.d.ts +77 -0
- package/dist/core/page-classifier.js +92 -0
- package/dist/core/page-markdown.d.ts +5 -0
- package/dist/core/page-markdown.js +151 -0
- package/dist/core/page-scan.d.ts +95 -0
- package/dist/core/page-scan.js +542 -0
- package/dist/core/public-fetch.d.ts +32 -0
- package/dist/core/public-fetch.js +134 -0
- package/dist/core/read-page-worker.d.ts +8 -0
- package/dist/core/read-page-worker.js +32 -0
- package/dist/core/read-page.d.ts +46 -0
- package/dist/core/read-page.js +96 -0
- package/dist/core/tools.d.ts +9 -2
- package/dist/core/tools.js +56 -31
- package/dist/core/vault.d.ts +68 -0
- package/dist/core/vault.js +79 -0
- package/dist/core/web.d.ts +33 -0
- package/dist/core/web.js +155 -16
- package/dist/driver/stagehand.js +2 -1
- package/dist/durable/web.d.ts +7 -1
- package/dist/durable/web.js +4 -1
- package/dist/durable.d.ts +2 -1
- package/dist/durable.js +58 -10
- package/dist/index.d.ts +4 -0
- package/dist/index.js +4 -0
- package/dist/providers/browserbase.d.ts +1 -1
- package/dist/providers/browserbase.js +11 -1
- package/dist/providers/cdp-session.d.ts +28 -0
- package/dist/providers/cdp-session.js +153 -0
- package/dist/providers/cdp.d.ts +6 -0
- package/dist/providers/cdp.js +52 -2
- package/dist/providers/chrome-args.d.ts +3 -1
- package/dist/providers/chrome-args.js +3 -2
- package/dist/providers/kernel.js +6 -0
- package/dist/providers/profiles.d.ts +4 -0
- package/dist/providers/profiles.js +10 -0
- package/dist/providers/worker-extension.d.ts +14 -0
- package/dist/providers/worker-extension.js +63 -0
- package/dist/vault-session.d.ts +58 -0
- package/dist/vault-session.js +219 -0
- package/package.json +5 -1
|
@@ -0,0 +1,134 @@
|
|
|
1
|
+
// A page from a public host only, so a page-steered agent never reaches the owner's private network: every hop's name
|
|
2
|
+
// is resolved once, each address it resolves to is checked, and the connection is dialled at the address checked.
|
|
3
|
+
// http(s) on ports 80 and 443 only, no cookies or credentials, a byte cap on the body, and the caller's signal bounds
|
|
4
|
+
// every hop, its name lookup included.
|
|
5
|
+
import dns from "node:dns/promises";
|
|
6
|
+
import net from "node:net";
|
|
7
|
+
import { Agent, fetch as undiciFetch } from "undici";
|
|
8
|
+
import { refusedError } from "./custody.js";
|
|
9
|
+
/** 0.0.0.0/8 this network, 10/8, 100.64/10 carrier-grade, 127/8, 169.254/16 link-local, 172.16/12, 192.0.0.0/24
|
|
10
|
+
* protocol assignments, 192.168/16, 198.18/15 benchmarking, 224/4 multicast and 240/4 reserved (the broadcast address
|
|
11
|
+
* included). */
|
|
12
|
+
const V4_PRIVATE = [[0x00000000, 8], [0x0a000000, 8], [0x64400000, 10], [0x7f000000, 8], [0xa9fe0000, 16], [0xac100000, 12], [0xc0000000, 24], [0xc0a80000, 16], [0xc6120000, 15], [0xe0000000, 4], [0xf0000000, 4]];
|
|
13
|
+
/** ::/8 reserved by the IETF (the unspecified and loopback addresses, IPv4-compatible ones, and the NAT64 prefixes
|
|
14
|
+
* 64:ff9b::/96 and 64:ff9b:1::/48), fc00::/7 unique-local, fe80::/10 link-local, fec0::/10 site-local and ff00::/8
|
|
15
|
+
* multicast. An IPv4-mapped address (::ffff:0:0/96) is checked as the IPv4 address it carries. */
|
|
16
|
+
const V6_PRIVATE = [["::", 8], ["fc00::", 7], ["fe80::", 10], ["fec0::", 10], ["ff00::", 8]];
|
|
17
|
+
const v4 = (ip) => ip.split(".").reduce((n, octet) => n * 256 + Number(octet), 0);
|
|
18
|
+
/** An IPv6 address's eight 16-bit groups, read through the URL parser's canonical form; null when it does not parse
|
|
19
|
+
* (a zone index among them). */
|
|
20
|
+
function v6Groups(ip) {
|
|
21
|
+
let canonical;
|
|
22
|
+
try {
|
|
23
|
+
canonical = new URL(`http://[${ip}]/`).hostname.slice(1, -1);
|
|
24
|
+
}
|
|
25
|
+
catch {
|
|
26
|
+
return null;
|
|
27
|
+
}
|
|
28
|
+
const groups = (part) => (part ? part.split(":").map((group) => parseInt(group, 16)) : []);
|
|
29
|
+
if (!canonical.includes("::"))
|
|
30
|
+
return groups(canonical);
|
|
31
|
+
const [head, tail] = canonical.split("::").map(groups);
|
|
32
|
+
return [...head, ...new Array(8 - head.length - tail.length).fill(0), ...tail];
|
|
33
|
+
}
|
|
34
|
+
const inV6 = (groups, base, bits) => groups.every((group, i) => { const span = Math.max(0, Math.min(16, bits - 16 * i)); const mask = (0xffff << (16 - span)) & 0xffff; return (group & mask) === (base[i] & mask); });
|
|
35
|
+
/** Any address but a public unicast one, v4 and v6 (see V4_PRIVATE and V6_PRIVATE); what does not parse as one too. */
|
|
36
|
+
export function isPrivateAddress(ip) {
|
|
37
|
+
if (net.isIPv4(ip))
|
|
38
|
+
return V4_PRIVATE.some(([base, bits]) => Math.floor(v4(ip) / 2 ** (32 - bits)) === Math.floor(base / 2 ** (32 - bits)));
|
|
39
|
+
const groups = net.isIPv6(ip) ? v6Groups(ip) : null;
|
|
40
|
+
if (!groups)
|
|
41
|
+
return true;
|
|
42
|
+
if (groups.slice(0, 5).every((group) => group === 0) && groups[5] === 0xffff)
|
|
43
|
+
return isPrivateAddress([groups[6] >> 8, groups[6] & 0xff, groups[7] >> 8, groups[7] & 0xff].join("."));
|
|
44
|
+
return V6_PRIVATE.some(([base, bits]) => inV6(groups, v6Groups(base), bits));
|
|
45
|
+
}
|
|
46
|
+
/** Answers `address` for any name: the address a request was checked against is the one it connects to. */
|
|
47
|
+
export const pinnedLookup = (address) => (_host, options, callback) => {
|
|
48
|
+
const family = net.isIPv6(address) ? 6 : 4;
|
|
49
|
+
if (options?.all)
|
|
50
|
+
callback(null, [{ address, family }]);
|
|
51
|
+
else
|
|
52
|
+
callback(null, address, family);
|
|
53
|
+
};
|
|
54
|
+
export const PUBLIC_MAX_BYTES = 5 * 1024 * 1024;
|
|
55
|
+
const MAX_REDIRECTS = 5;
|
|
56
|
+
const PORTS = new Set(["", "80", "443"]);
|
|
57
|
+
/** `work()`'s answer, or the signal's reason as soon as it aborts: a resolver that never answers cannot outlast the
|
|
58
|
+
* caller. A signal already aborted starts no work. */
|
|
59
|
+
function untilAborted(work, signal) {
|
|
60
|
+
if (!signal)
|
|
61
|
+
return work();
|
|
62
|
+
signal.throwIfAborted();
|
|
63
|
+
return new Promise((resolve, reject) => {
|
|
64
|
+
const abort = () => reject(signal.reason);
|
|
65
|
+
signal.addEventListener("abort", abort, { once: true });
|
|
66
|
+
work().then(resolve, reject).finally(() => signal.removeEventListener("abort", abort));
|
|
67
|
+
});
|
|
68
|
+
}
|
|
69
|
+
/**
|
|
70
|
+
* Fetches `url` from a public host, following at most 5 redirects, each hop resolved, checked and dialled at the
|
|
71
|
+
* address checked. A 2xx answer of a text type is read whole, up to `maxBytes`; another status comes back without its
|
|
72
|
+
* body. Refused (`BrowserFailureError` `refused`), on the first hop or a redirect: a scheme other than http(s); a host
|
|
73
|
+
* that is local by name or resolves to any private address; a port other than 80 or 443; too many redirects; a 2xx body
|
|
74
|
+
* that is not text or is over the cap. `signal` ends the fetch at any point, a hop's name lookup included (it rejects
|
|
75
|
+
* with the signal's reason), and an aborted hop is never dialled.
|
|
76
|
+
*/
|
|
77
|
+
export async function publicFetch(url, options = {}) {
|
|
78
|
+
const lookup = options.lookup ?? (async (host) => (await dns.lookup(host, { all: true })).map((a) => a.address));
|
|
79
|
+
const request = options.fetchImpl ?? ((target, init) => undiciFetch(target, init));
|
|
80
|
+
const maxBytes = options.maxBytes ?? PUBLIC_MAX_BYTES;
|
|
81
|
+
if (!Number.isSafeInteger(maxBytes) || maxBytes < 1)
|
|
82
|
+
throw new RangeError(`maxBytes must be a whole number of bytes above 0, not ${maxBytes}`);
|
|
83
|
+
let target = new URL(url);
|
|
84
|
+
if (!/^https?:$/.test(target.protocol))
|
|
85
|
+
throw refusedError(`refused: ${target.protocol} is not http(s)`);
|
|
86
|
+
for (let hop = 0;; hop += 1) {
|
|
87
|
+
const host = target.hostname.replace(/^\[|\]$/g, "");
|
|
88
|
+
if (host === "localhost" || host.endsWith(".localhost") || host.endsWith(".local") || host.endsWith(".internal"))
|
|
89
|
+
throw refusedError(`refused: ${host} is not a public host`);
|
|
90
|
+
const addresses = net.isIP(host) ? [host] : await untilAborted(() => lookup(host), options.signal);
|
|
91
|
+
if (!addresses.length || addresses.some(isPrivateAddress))
|
|
92
|
+
throw refusedError(`refused: ${host} is not a public host`);
|
|
93
|
+
if (!PORTS.has(target.port))
|
|
94
|
+
throw refusedError(`refused: port ${target.port} is not 80 or 443`);
|
|
95
|
+
options.signal?.throwIfAborted();
|
|
96
|
+
const dispatcher = new Agent({ connect: { lookup: pinnedLookup(addresses[0]) } });
|
|
97
|
+
let res;
|
|
98
|
+
try {
|
|
99
|
+
res = await request(target, { redirect: "manual", signal: options.signal, dispatcher, headers: { accept: "text/html,text/plain;q=0.9,*/*;q=0.1", "user-agent": "pi-browser" } });
|
|
100
|
+
if (res.status >= 300 && res.status < 400 && res.headers.get("location")) {
|
|
101
|
+
if (hop >= MAX_REDIRECTS)
|
|
102
|
+
throw refusedError("refused: too many redirects");
|
|
103
|
+
target = new URL(res.headers.get("location"), target);
|
|
104
|
+
if (!/^https?:$/.test(target.protocol))
|
|
105
|
+
throw refusedError("refused: a redirect left http(s)");
|
|
106
|
+
continue;
|
|
107
|
+
}
|
|
108
|
+
const contentType = res.headers.get("content-type") ?? "";
|
|
109
|
+
if (!res.ok)
|
|
110
|
+
return { url: target.href, status: res.status, contentType, body: null };
|
|
111
|
+
if (!/^(text\/|application\/(xhtml\+xml|json|xml))/i.test(contentType))
|
|
112
|
+
throw refusedError(`refused: ${contentType || "an unknown content type"} is not text`);
|
|
113
|
+
const chunks = [];
|
|
114
|
+
let size = 0;
|
|
115
|
+
for (const reader = res.body.getReader();;) {
|
|
116
|
+
const { done, value } = await reader.read();
|
|
117
|
+
if (done)
|
|
118
|
+
break;
|
|
119
|
+
if ((size += value.length) > maxBytes) {
|
|
120
|
+
await reader.cancel();
|
|
121
|
+
throw refusedError(`refused: the page is over ${maxBytes === PUBLIC_MAX_BYTES ? "5 MiB" : `${maxBytes} bytes`}`);
|
|
122
|
+
}
|
|
123
|
+
chunks.push(value);
|
|
124
|
+
}
|
|
125
|
+
return { url: target.href, status: res.status, contentType, body: Buffer.concat(chunks).toString("utf8") };
|
|
126
|
+
}
|
|
127
|
+
finally {
|
|
128
|
+
// A reply refused unread (not text, an error status, a redirect) still holds its connection: close() waits
|
|
129
|
+
// for it.
|
|
130
|
+
await res?.body?.cancel().catch(() => undefined);
|
|
131
|
+
await dispatcher.close().catch(() => undefined);
|
|
132
|
+
}
|
|
133
|
+
}
|
|
134
|
+
}
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
/** `html` as given (readPage drops and cuts it first), or, with `max`, dropped (its first article, main or body only,
|
|
2
|
+
* with `main`) and cut at `max` here; `url`: the page's title and links are read too. */
|
|
3
|
+
export type Job = {
|
|
4
|
+
html: string;
|
|
5
|
+
url?: string;
|
|
6
|
+
main?: boolean;
|
|
7
|
+
max?: number;
|
|
8
|
+
};
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
// read-page.ts's worker. For `workerData`, a Job: the page's markdown, posted once as { text } (with { title, links }
|
|
2
|
+
// when the job has a `url`, and { truncated, html_chars } when the worker cut the page), or as { error } when the
|
|
3
|
+
// conversion threw (a page nested past the stack, say).
|
|
4
|
+
import { parentPort, workerData } from "node:worker_threads";
|
|
5
|
+
import { TextNode } from "node-html-parser";
|
|
6
|
+
import { markdownOf } from "./page-markdown.js";
|
|
7
|
+
import { cutHtml, dropInert, mainOf, PageScan } from "./page-scan.js";
|
|
8
|
+
if (parentPort) {
|
|
9
|
+
const job = workerData;
|
|
10
|
+
try {
|
|
11
|
+
let html = job.html;
|
|
12
|
+
let cut = {};
|
|
13
|
+
if (job.max !== undefined) {
|
|
14
|
+
const kept = dropInert(job.html);
|
|
15
|
+
const picked = job.main ? mainOf(kept) : kept;
|
|
16
|
+
html = cutHtml(picked, job.max);
|
|
17
|
+
if (html.length < picked.length)
|
|
18
|
+
cut = { truncated: true, html_chars: job.html.length };
|
|
19
|
+
}
|
|
20
|
+
let read = {};
|
|
21
|
+
if (job.url !== undefined) {
|
|
22
|
+
const scan = new PageScan(html, job.url, (text) => new TextNode(text).text, false);
|
|
23
|
+
scan.step(Number.POSITIVE_INFINITY);
|
|
24
|
+
const { title, links } = scan.result();
|
|
25
|
+
read = { title, links };
|
|
26
|
+
}
|
|
27
|
+
parentPort.postMessage({ ...read, ...cut, text: markdownOf(html) });
|
|
28
|
+
}
|
|
29
|
+
catch (error) {
|
|
30
|
+
parentPort.postMessage({ error: String(error?.message ?? error) });
|
|
31
|
+
}
|
|
32
|
+
}
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
/** The most of a page's HTML that is read, counted once its script, style, svg, noscript and template blocks and its
|
|
2
|
+
* comments are dropped. A million characters of go.dev/dl converts in about 0.8 s on a busy 48-core host and gives
|
|
3
|
+
* 590,000 characters of markdown, ten times the 60,000 one web_fetch answer carries. */
|
|
4
|
+
export declare const READ_MAX_CHARS = 1000000;
|
|
5
|
+
/** How long the worker has for the markdown by default: six times what READ_MAX_CHARS of tables took. */
|
|
6
|
+
export declare const READ_DEADLINE_MS = 5000;
|
|
7
|
+
/** `n` with its digits in groups of three ("3,154,060"): Intl's first use costs tens of milliseconds on the loop. */
|
|
8
|
+
export declare const thousands: (n: number) => string;
|
|
9
|
+
/** `converted`: "markdown", or "text" when the conversion did not finish in time (or failed) and the text is the page's
|
|
10
|
+
* plain text. `truncated` and `html_chars` (the page's whole length, in characters) when the page was cut at
|
|
11
|
+
* READ_MAX_CHARS before it was read: the text, the title and the links are then the cut page's. */
|
|
12
|
+
export type PageRead = {
|
|
13
|
+
title: string;
|
|
14
|
+
text: string;
|
|
15
|
+
links: string[];
|
|
16
|
+
converted: "markdown" | "text";
|
|
17
|
+
truncated?: true;
|
|
18
|
+
html_chars?: number;
|
|
19
|
+
};
|
|
20
|
+
/** `deadlineMs`: how long the conversion to markdown has (default READ_DEADLINE_MS). `signal` ends the read, the
|
|
21
|
+
* conversion stopped, with the signal's reason. */
|
|
22
|
+
export type ReadPageOptions = {
|
|
23
|
+
deadlineMs?: number;
|
|
24
|
+
signal?: AbortSignal;
|
|
25
|
+
};
|
|
26
|
+
/**
|
|
27
|
+
* An HTML page's title, its text as markdown, and the absolute http(s) URLs it links to (resolved against its
|
|
28
|
+
* `<base href>` or `url`), each once, in page order. The event loop is never held for longer than one linear step of a
|
|
29
|
+
* bounded part of the page (see the file's head and PageRead). Rejects only with `signal`'s reason.
|
|
30
|
+
*/
|
|
31
|
+
export declare function readPage(html: string, url: string, { deadlineMs, signal }?: ReadPageOptions): Promise<PageRead>;
|
|
32
|
+
/**
|
|
33
|
+
* `html`'s markdown, made in the worker within `deadlineMs` (default READ_DEADLINE_MS): of its first article, else its
|
|
34
|
+
* main, else its body element when `main` (mainOf), once its inert blocks are dropped, cut at READ_MAX_CHARS (then
|
|
35
|
+
* `truncated`, and `html_chars` is the page's whole length). Null when the worker has not answered in time, failed, or
|
|
36
|
+
* could not start. The event loop does no more than hand the worker the page. Rejects only with `signal`'s reason.
|
|
37
|
+
*/
|
|
38
|
+
export declare function htmlMarkdown(html: string, { main, deadlineMs, signal }?: {
|
|
39
|
+
main?: boolean;
|
|
40
|
+
} & ReadPageOptions): Promise<{
|
|
41
|
+
text: string;
|
|
42
|
+
truncated?: true;
|
|
43
|
+
html_chars?: number;
|
|
44
|
+
} | null>;
|
|
45
|
+
/** `html` without its inert blocks (InertDrop), STEP_CHARS at a time with the loop yielded between steps. */
|
|
46
|
+
export declare function dropOnLoop(html: string, signal?: AbortSignal): Promise<string>;
|
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
// A fetched HTML page read as its title, its text and its links, without ever holding the event loop. The host's tools
|
|
2
|
+
// run in the run's own process, so a read that holds the loop holds the run's heartbeat with it, and a run whose
|
|
3
|
+
// heartbeat lapses is fenced: go.dev/dl, 3.15 MB of tables, held it past 240 s in one markdown conversion. So one linear
|
|
4
|
+
// pass drops the blocks that hold no readable text (page-scan.ts); what is left is cut at READ_MAX_CHARS (the page is
|
|
5
|
+
// then `truncated`, and `html_chars` is its whole length); and a worker thread converts that to markdown, in pieces
|
|
6
|
+
// small enough that the converter stays linear (page-markdown.ts), within a deadline. A worker that has not answered by
|
|
7
|
+
// then, or that fails, is stopped, and the text is the page's plain text, read on the loop in steps that yield it
|
|
8
|
+
// (`converted: "text"`). browser_read's markdown (htmlMarkdown) goes through the same worker, which drops, picks and
|
|
9
|
+
// cuts the live page's HTML itself, since that HTML has no bound; its fallback is the browser's own text.
|
|
10
|
+
import { setImmediate as yieldToLoop } from "node:timers/promises";
|
|
11
|
+
import { Worker } from "node:worker_threads";
|
|
12
|
+
import { decodeEntities } from "./entities.js";
|
|
13
|
+
import { cutHtml, InertDrop, PageScan } from "./page-scan.js";
|
|
14
|
+
/** The most of a page's HTML that is read, counted once its script, style, svg, noscript and template blocks and its
|
|
15
|
+
* comments are dropped. A million characters of go.dev/dl converts in about 0.8 s on a busy 48-core host and gives
|
|
16
|
+
* 590,000 characters of markdown, ten times the 60,000 one web_fetch answer carries. */
|
|
17
|
+
export const READ_MAX_CHARS = 1_000_000;
|
|
18
|
+
/** How long the worker has for the markdown by default: six times what READ_MAX_CHARS of tables took. */
|
|
19
|
+
export const READ_DEADLINE_MS = 5_000;
|
|
20
|
+
/** `n` with its digits in groups of three ("3,154,060"): Intl's first use costs tens of milliseconds on the loop. */
|
|
21
|
+
export const thousands = (n) => String(n).replace(/\B(?=(\d{3})+$)/g, ",");
|
|
22
|
+
/** How much of the page one step of the plain-text scan reads before it yields the loop. */
|
|
23
|
+
const STEP_CHARS = 32_768;
|
|
24
|
+
/**
|
|
25
|
+
* An HTML page's title, its text as markdown, and the absolute http(s) URLs it links to (resolved against its
|
|
26
|
+
* `<base href>` or `url`), each once, in page order. The event loop is never held for longer than one linear step of a
|
|
27
|
+
* bounded part of the page (see the file's head and PageRead). Rejects only with `signal`'s reason.
|
|
28
|
+
*/
|
|
29
|
+
export async function readPage(html, url, { deadlineMs = READ_DEADLINE_MS, signal } = {}) {
|
|
30
|
+
signal?.throwIfAborted();
|
|
31
|
+
const kept = await dropOnLoop(html, signal);
|
|
32
|
+
const read = cutHtml(kept, READ_MAX_CHARS);
|
|
33
|
+
const cut = read.length < kept.length ? { truncated: true, html_chars: html.length } : {};
|
|
34
|
+
await yieldToLoop();
|
|
35
|
+
signal?.throwIfAborted();
|
|
36
|
+
const markdown = await inWorker({ html: read, url }, deadlineMs, signal);
|
|
37
|
+
if (markdown)
|
|
38
|
+
return { title: markdown.title ?? "", text: markdown.text, links: markdown.links ?? [], converted: "markdown", ...cut };
|
|
39
|
+
return { ...(await plainRead(read, url, signal)), converted: "text", ...cut };
|
|
40
|
+
}
|
|
41
|
+
/**
|
|
42
|
+
* `html`'s markdown, made in the worker within `deadlineMs` (default READ_DEADLINE_MS): of its first article, else its
|
|
43
|
+
* main, else its body element when `main` (mainOf), once its inert blocks are dropped, cut at READ_MAX_CHARS (then
|
|
44
|
+
* `truncated`, and `html_chars` is the page's whole length). Null when the worker has not answered in time, failed, or
|
|
45
|
+
* could not start. The event loop does no more than hand the worker the page. Rejects only with `signal`'s reason.
|
|
46
|
+
*/
|
|
47
|
+
export async function htmlMarkdown(html, { main = false, deadlineMs = READ_DEADLINE_MS, signal } = {}) {
|
|
48
|
+
signal?.throwIfAborted();
|
|
49
|
+
const answer = await inWorker({ html, main, max: READ_MAX_CHARS }, deadlineMs, signal);
|
|
50
|
+
return answer && { text: answer.text, ...(answer.truncated ? { truncated: true, html_chars: answer.html_chars } : {}) };
|
|
51
|
+
}
|
|
52
|
+
/** The worker's answer to `job`, or null when it has not answered within `deadlineMs`, failed, or could not start. */
|
|
53
|
+
function inWorker(job, deadlineMs, signal) {
|
|
54
|
+
return new Promise((resolve, reject) => {
|
|
55
|
+
let worker;
|
|
56
|
+
// execArgv: none of the host's own flags (a test runner's --import, say) reach the worker.
|
|
57
|
+
try {
|
|
58
|
+
worker = new Worker(new URL("./read-page-worker.js", import.meta.url), { workerData: job, execArgv: [], resourceLimits: { maxOldGenerationSizeMb: 1024 } });
|
|
59
|
+
}
|
|
60
|
+
catch {
|
|
61
|
+
resolve(null);
|
|
62
|
+
return;
|
|
63
|
+
}
|
|
64
|
+
const settle = (finish) => {
|
|
65
|
+
clearTimeout(timer);
|
|
66
|
+
signal?.removeEventListener("abort", onAbort);
|
|
67
|
+
void worker.terminate().catch(() => undefined);
|
|
68
|
+
finish();
|
|
69
|
+
};
|
|
70
|
+
const onAbort = () => settle(() => reject(signal.reason));
|
|
71
|
+
const timer = setTimeout(() => settle(() => resolve(null)), deadlineMs);
|
|
72
|
+
signal?.addEventListener("abort", onAbort, { once: true });
|
|
73
|
+
worker.once("message", (answer) => settle(() => resolve(typeof answer?.text === "string" ? answer : null)));
|
|
74
|
+
worker.once("error", () => settle(() => resolve(null)));
|
|
75
|
+
worker.once("exit", () => settle(() => resolve(null)));
|
|
76
|
+
});
|
|
77
|
+
}
|
|
78
|
+
/** `html` without its inert blocks (InertDrop), STEP_CHARS at a time with the loop yielded between steps. */
|
|
79
|
+
export async function dropOnLoop(html, signal) {
|
|
80
|
+
const drop = new InertDrop(html);
|
|
81
|
+
while (!drop.step(STEP_CHARS)) {
|
|
82
|
+
await yieldToLoop();
|
|
83
|
+
signal?.throwIfAborted();
|
|
84
|
+
}
|
|
85
|
+
return drop.result();
|
|
86
|
+
}
|
|
87
|
+
/** `html`'s plain text, title and links, scanned STEP_CHARS at a time with the loop yielded between steps. */
|
|
88
|
+
async function plainRead(html, url, signal) {
|
|
89
|
+
// Decoded here without node-html-parser, whose first load would hold the loop (entities.ts).
|
|
90
|
+
const scan = new PageScan(html, url, decodeEntities, true);
|
|
91
|
+
while (!scan.step(STEP_CHARS)) {
|
|
92
|
+
await yieldToLoop();
|
|
93
|
+
signal?.throwIfAborted();
|
|
94
|
+
}
|
|
95
|
+
return scan.result();
|
|
96
|
+
}
|
package/dist/core/tools.d.ts
CHANGED
|
@@ -7,7 +7,12 @@ import { type Locate } from "./clip.js";
|
|
|
7
7
|
import { type ScreenshotOptions } from "./images.js";
|
|
8
8
|
import { type Redact } from "./redact.js";
|
|
9
9
|
import type { Geo } from "./host.js";
|
|
10
|
-
/**
|
|
10
|
+
/** `run` code is the body of an async function with `page`, `context` and `browser` in scope. Code written instead as one
|
|
11
|
+
* function expression (`async () => { ... }`) would only define that function and return nothing, so such code is
|
|
12
|
+
* called: a function with no parameters is awaited and its value returned, and one that declares parameters fails
|
|
13
|
+
* with the body form spelled out. The code counts as one function expression when it begins like one and the parser
|
|
14
|
+
* compiles the whole of it as a single expression (compiled, never run). Any other code reaches the driver unchanged. */
|
|
15
|
+
export declare function runnableCode(code: string): string;
|
|
11
16
|
export type RunAction = {
|
|
12
17
|
op: "click";
|
|
13
18
|
id: string;
|
|
@@ -129,10 +134,12 @@ type ReconnectTimings = {
|
|
|
129
134
|
tryMs: number;
|
|
130
135
|
backoffMs: readonly number[];
|
|
131
136
|
};
|
|
132
|
-
export declare function pageTools({ guards, reconnect: timings }?: {
|
|
137
|
+
export declare function pageTools({ guards, reconnect: timings, readDeadlineMs }?: {
|
|
133
138
|
/** Each conversation's repeat guard; a host that owns the conversation's lifecycle passes it to evict one. */
|
|
134
139
|
guards?: Map<number, RepeatGuard>;
|
|
135
140
|
/** The reconnect timings, for tests; hosts keep the defaults. */
|
|
136
141
|
reconnect?: ReconnectTimings;
|
|
142
|
+
/** How long browser_read's markdown has before the browser's own text stands in (default READ_DEADLINE_MS), for tests. */
|
|
143
|
+
readDeadlineMs?: number;
|
|
137
144
|
}): Pick<BrowserToolImpls<PageDriver>, "snapshot" | "run" | "screenshot" | "browser_read" | "browser_upload" | "browser_downloads">;
|
|
138
145
|
export {};
|
package/dist/core/tools.js
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { setImmediate as yieldToLoop } from "node:timers/promises";
|
|
1
2
|
import { BrowserFailureError, FenceRefusal, refusedError, loadedNoPage, passedThrough } from "./custody.js";
|
|
2
3
|
import { defaultInline } from "./evidence.js";
|
|
3
4
|
import { fetchDownloads } from "./downloads.js";
|
|
@@ -5,10 +6,40 @@ import { browserFailure, classifyBrowserError, transportFactOf } from "./failure
|
|
|
5
6
|
import { createRepeatGuard } from "./guard.js";
|
|
6
7
|
import { regionOf } from "./clip.js";
|
|
7
8
|
import { captureWithinBudget } from "./images.js";
|
|
9
|
+
import { decodeEntities } from "./entities.js";
|
|
10
|
+
import { FrameScan } from "./page-scan.js";
|
|
11
|
+
import { dropOnLoop, htmlMarkdown, READ_DEADLINE_MS, READ_MAX_CHARS, thousands } from "./read-page.js";
|
|
8
12
|
import { orderFacts, receiptFacts, sha256Hex } from "./receipts.js";
|
|
9
13
|
import { scrubPageUrl } from "./redact.js";
|
|
10
14
|
import { UNANSWERED_CALL } from "./unanswered.js";
|
|
11
15
|
import { pageNotice, wallFailure, wallOf } from "./web.js";
|
|
16
|
+
/** One action of a run's batch, as the contract's schema admits it. */
|
|
17
|
+
const AsyncFunction = Object.getPrototypeOf(async function () { }).constructor;
|
|
18
|
+
/** How a function expression begins: `function`, `async function`, an arrow's parameter list or single parameter. */
|
|
19
|
+
const FUNCTION_START = /^(?:async\s+)?(?:function\b|\([^)]*\)\s*=>|[A-Za-z_$][\w$]*\s*=>)/;
|
|
20
|
+
/** `run` code is the body of an async function with `page`, `context` and `browser` in scope. Code written instead as one
|
|
21
|
+
* function expression (`async () => { ... }`) would only define that function and return nothing, so such code is
|
|
22
|
+
* called: a function with no parameters is awaited and its value returned, and one that declares parameters fails
|
|
23
|
+
* with the body form spelled out. The code counts as one function expression when it begins like one and the parser
|
|
24
|
+
* compiles the whole of it as a single expression (compiled, never run). Any other code reaches the driver unchanged. */
|
|
25
|
+
export function runnableCode(code) {
|
|
26
|
+
const expression = code.trim().replace(/;+$/, "").trimEnd();
|
|
27
|
+
if (!FUNCTION_START.test(expression))
|
|
28
|
+
return code;
|
|
29
|
+
try {
|
|
30
|
+
new AsyncFunction(`return (\n${expression}\n);`);
|
|
31
|
+
}
|
|
32
|
+
catch {
|
|
33
|
+
return code;
|
|
34
|
+
}
|
|
35
|
+
return [
|
|
36
|
+
`const __runCode = (\n${expression}\n);`,
|
|
37
|
+
"if (typeof __runCode === \"function\") {",
|
|
38
|
+
" if (__runCode.length > 0) throw new TypeError(\"run code is a function that takes parameters. Write the steps as the code itself (the body of a function): page, context and browser are already in scope, and `return` gives a value back.\");",
|
|
39
|
+
" return await __runCode();",
|
|
40
|
+
"}",
|
|
41
|
+
].join("\n");
|
|
42
|
+
}
|
|
12
43
|
const WHATS = ["markdown", "text", "html", "url", "title"];
|
|
13
44
|
const DROPPED_RUN = "The connection to your browser session dropped; the session is still open, and your next browser call reconnects to it with its pages and cookies. This run may have acted before the drop: take a snapshot to see the page before running again.";
|
|
14
45
|
const DROPPED_FOR_GOOD = "The connection to your browser session dropped and could not be re-established, though the session is still open. Call browser_relaunch for a fresh session (its pages and cookies are gone), or use web_fetch.";
|
|
@@ -38,17 +69,19 @@ const MAX_FRAMES = 5;
|
|
|
38
69
|
const FRAME_TEXT_CHARS = 200;
|
|
39
70
|
const FRAME_TEXT_READ_CHARS = 20_000;
|
|
40
71
|
const FRAME_SCAN_CHARS = 2_000_000;
|
|
72
|
+
/** How much of the page one step of the frame scan reads before it yields the event loop. */
|
|
73
|
+
const FRAME_STEP_CHARS = 32_768;
|
|
41
74
|
/** What the judge is shown of the page's frames, in document order: a frame's content is not in the page's text, so a page whose content sits in a challenge frame reads as empty without it. `frames` are the distinct origins (origin only: a
|
|
42
75
|
* frame's path and query can carry a token; http(s) only; a relative source resolves against the page's `<base href>` when it has one, as the browser does) and `frameText` what the page puts inside the frame elements (the fallback a browser
|
|
43
76
|
* does not render, where a block page keeps its message), collapsed to one line and cut at 200 characters. A frame is an element: markup inside a script, a comment, a noscript or a template is not one, so it neither counts nor takes one of the five.
|
|
44
|
-
* A page that cannot be
|
|
77
|
+
* The first FRAME_SCAN_CHARS of the page are read in one linear scan (FrameScan), FRAME_STEP_CHARS at a time with the event loop yielded between steps, so a big page never holds it. A page that cannot be read gives none. */
|
|
45
78
|
async function framesOf(html, pageUrl, redact) {
|
|
46
79
|
try {
|
|
47
|
-
const
|
|
48
|
-
|
|
49
|
-
|
|
80
|
+
const scan = new FrameScan(await dropOnLoop(html.slice(0, FRAME_SCAN_CHARS)), decodeEntities, FRAME_TEXT_READ_CHARS);
|
|
81
|
+
while (!scan.step(FRAME_STEP_CHARS))
|
|
82
|
+
await yieldToLoop();
|
|
83
|
+
const { frames, base: baseHref } = scan.result();
|
|
50
84
|
let base = pageUrl;
|
|
51
|
-
const baseHref = root.querySelector("base[href]")?.getAttribute("href");
|
|
52
85
|
if (baseHref) {
|
|
53
86
|
try {
|
|
54
87
|
base = new URL(baseHref, pageUrl).href;
|
|
@@ -57,18 +90,10 @@ async function framesOf(html, pageUrl, redact) {
|
|
|
57
90
|
}
|
|
58
91
|
const origins = new Set();
|
|
59
92
|
const texts = new Set();
|
|
60
|
-
for (const frame of
|
|
93
|
+
for (const frame of frames) {
|
|
61
94
|
if (origins.size >= MAX_FRAMES && texts.size >= MAX_FRAMES)
|
|
62
95
|
break;
|
|
63
|
-
|
|
64
|
-
for (let up = frame.parentNode; up; up = up.parentNode)
|
|
65
|
-
if (up.rawTagName?.toLowerCase() === "template") {
|
|
66
|
-
inert = true;
|
|
67
|
-
break;
|
|
68
|
-
}
|
|
69
|
-
if (inert)
|
|
70
|
-
continue;
|
|
71
|
-
const source = frame.getAttribute("src")?.trim();
|
|
96
|
+
const source = frame.src?.trim();
|
|
72
97
|
if (source && origins.size < MAX_FRAMES) {
|
|
73
98
|
try {
|
|
74
99
|
const url = new URL(source, base);
|
|
@@ -127,19 +152,17 @@ const factsOf = (p, now) => ({
|
|
|
127
152
|
callId: p.callId, urlAtDispatch: p.url, urlNow: now, observed: p.observed === true,
|
|
128
153
|
sent: p.watched ? (p.sent ?? []).map((r) => ({ method: r.method, url: `${r.origin}${r.path}`, held: r.held })) : null,
|
|
129
154
|
});
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
}
|
|
139
|
-
catch { }
|
|
140
|
-
return { body: fallback, extractor: "inner-text", ext: "txt" };
|
|
155
|
+
/** The page's main content as markdown (its first article, else main, else body: htmlMarkdown, off the event loop); the
|
|
156
|
+
* browser's own text when the conversion gives nothing, fails or runs past `deadlineMs`. A page cut before it was read
|
|
157
|
+
* says so on its last line. Rejects only with `signal`'s reason. */
|
|
158
|
+
async function markdown(html, fallback, signal, deadlineMs) {
|
|
159
|
+
const read = await htmlMarkdown(html, { main: true, deadlineMs, ...(signal ? { signal } : {}) });
|
|
160
|
+
if (!read?.text.trim())
|
|
161
|
+
return { body: fallback, extractor: "inner-text", ext: "txt" };
|
|
162
|
+
const cut = read.truncated ? `\n\n(Cut: the page is ${thousands(read.html_chars)} characters of HTML, and only the first ${thousands(READ_MAX_CHARS)} of its main content, scripts and styles aside, were read as markdown. browser_read with what: "text" gives all of its text.)` : "";
|
|
163
|
+
return { body: read.text + cut, extractor: "readability-md", ext: "md" };
|
|
141
164
|
}
|
|
142
|
-
export function pageTools({ guards = new Map(), reconnect: timings = RECONNECT } = {}) {
|
|
165
|
+
export function pageTools({ guards = new Map(), reconnect: timings = RECONNECT, readDeadlineMs = READ_DEADLINE_MS } = {}) {
|
|
143
166
|
const guardOf = (conversationId, redact) => guards.get(conversationId) ?? guards.set(conversationId, createRepeatGuard(redact)).get(conversationId);
|
|
144
167
|
async function failureOf(error, effectful, port, call, action) {
|
|
145
168
|
if (error instanceof BrowserFailureError)
|
|
@@ -342,7 +365,7 @@ export function pageTools({ guards = new Map(), reconnect: timings = RECONNECT }
|
|
|
342
365
|
const stop = s.effects?.record((row) => { sent.push({ method: row.method, url: call.redact(`${row.origin}${row.path}`), held: row.held }); return port.journal(row); });
|
|
343
366
|
let value;
|
|
344
367
|
try {
|
|
345
|
-
value = await s.driver.run(args.code !== undefined ? { code: args.code } : { actions: args.actions });
|
|
368
|
+
value = await s.driver.run(args.code !== undefined ? { code: runnableCode(args.code) } : { actions: args.actions });
|
|
346
369
|
}
|
|
347
370
|
finally {
|
|
348
371
|
await s.effects?.flush(call.signal).catch(() => undefined);
|
|
@@ -350,10 +373,12 @@ export function pageTools({ guards = new Map(), reconnect: timings = RECONNECT }
|
|
|
350
373
|
}
|
|
351
374
|
const url = await s.driver.url().catch(() => null);
|
|
352
375
|
// Compared as custody keeps it, so a page with a query value never looks new on every run.
|
|
353
|
-
if (url && scrubPageUrl(url, call.redact) !== s.record.lastUrl)
|
|
376
|
+
if (url && scrubPageUrl(url, call.redact) !== s.record.lastUrl) {
|
|
377
|
+
guardOf(call.conversationId, call.redact).moved();
|
|
354
378
|
await port.navigated(url);
|
|
379
|
+
}
|
|
355
380
|
const reported = args.actions !== undefined && value && typeof value === "object" && typeof value.url === "string" ? { ...value, url: pageUrl(call, value.url) } : value;
|
|
356
|
-
const shown = typeof reported === "string" ? reported : reported === undefined ? "(run completed; no return value)" : JSON.stringify(reported, null, 2) ?? String(reported);
|
|
381
|
+
const shown = typeof reported === "string" ? reported : reported === undefined ? "(run completed; the code returned no value: end it with `return <value>` to get one back)" : JSON.stringify(reported, null, 2) ?? String(reported);
|
|
357
382
|
return { ...text(withNews(news, shown)), ...(sent.length ? { details: { sent } } : {}) };
|
|
358
383
|
});
|
|
359
384
|
},
|
|
@@ -381,7 +406,7 @@ export function pageTools({ guards = new Map(), reconnect: timings = RECONNECT }
|
|
|
381
406
|
// so, and where to go. A script that wrote a title or text into about:blank made a page, which is read.
|
|
382
407
|
if (page.url === "about:blank" && loadedNoPage(s.record) && !page.title.trim() && !page.text.trim())
|
|
383
408
|
throw new BrowserFailureError(freshBrowser(s.replaced ?? null));
|
|
384
|
-
const read = what === "markdown" ? await markdown(page.html, page.text)
|
|
409
|
+
const read = what === "markdown" ? await markdown(page.html, page.text, call.signal, readDeadlineMs)
|
|
385
410
|
: what === "html" ? { body: page.html, extractor: "raw", ext: "html" }
|
|
386
411
|
: what === "text" ? { body: page.text, extractor: "inner-text", ext: "txt" }
|
|
387
412
|
: { body: what === "url" ? pageUrl(call, page.url) : page.title, ext: "txt" };
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
/** The most a session cookie may be restored after its capture: 24 hours. A host may set a lower age, never a higher one. */
|
|
2
|
+
export declare const MAX_SESSION_COOKIE_AGE_MS: number;
|
|
3
|
+
/** The age cap a restore applies to session cookies. Absent, 24 h; above 24 h, clamped to 24 h; zero, negative or not a number, refused. */
|
|
4
|
+
export declare function clampSessionCookieAgeMs(requested?: number): number;
|
|
5
|
+
export type VaultCookie = {
|
|
6
|
+
name: string;
|
|
7
|
+
value: string;
|
|
8
|
+
domain: string;
|
|
9
|
+
path: string;
|
|
10
|
+
/** Seconds since the epoch; absent means a SESSION cookie. */
|
|
11
|
+
expires?: number;
|
|
12
|
+
httpOnly: boolean;
|
|
13
|
+
secure: boolean;
|
|
14
|
+
sameSite?: "Strict" | "Lax" | "None";
|
|
15
|
+
};
|
|
16
|
+
export type CapturedState = {
|
|
17
|
+
kind: "cdp";
|
|
18
|
+
capturedAt: string;
|
|
19
|
+
/** Only cookies the allowed origins would be sent (see `cookieKept`). Partitioned cookies are not kept. */
|
|
20
|
+
cookies: VaultCookie[];
|
|
21
|
+
localStorage: Record<string, Record<string, string>>;
|
|
22
|
+
}
|
|
23
|
+
/** `origins`: the origins the context was captured for. A context holds a whole browser profile, so it is restored only for a request that covers EVERY one of them (`coversEveryOrigin`); one that records none is refused. */
|
|
24
|
+
| {
|
|
25
|
+
kind: "provider-context";
|
|
26
|
+
capturedAt: string;
|
|
27
|
+
provider: "browserbase" | "kernel";
|
|
28
|
+
contextId: string;
|
|
29
|
+
origins?: string[];
|
|
30
|
+
};
|
|
31
|
+
/** Whether `requested` includes every origin in `saved` (as scheme://host[:port], any order): a provider context is all-or-nothing, so a request for some of its origins would bring back the others' sign-ins. */
|
|
32
|
+
export declare function coversEveryOrigin(requested: readonly string[], saved: readonly string[] | undefined): boolean;
|
|
33
|
+
export interface SessionVaultPort {
|
|
34
|
+
/** At release and at the end of a handoff, and, for a context a new session made, right after the create (so a restart still finds it). The host supplied the origins; nothing outside them was captured.
|
|
35
|
+
* A capture has a deadline (20 s) after which nothing more is read or written, but a write already handed to this port cannot be recalled: ignore one whose `capturedAt` is older than the one you hold. */
|
|
36
|
+
capture(input: {
|
|
37
|
+
sessionKey: string;
|
|
38
|
+
origins: string[];
|
|
39
|
+
state: CapturedState;
|
|
40
|
+
}): Promise<void>;
|
|
41
|
+
/** At create, before the driver attaches. null: nothing kept, or all of it expired. */
|
|
42
|
+
restore(input: {
|
|
43
|
+
sessionKey: string;
|
|
44
|
+
origins: string[];
|
|
45
|
+
}): Promise<CapturedState | null>;
|
|
46
|
+
/** Crypto-shred: the item's key is destroyed. The package never calls this itself; the host does, on a revoke. */
|
|
47
|
+
forget(sessionKey: string): Promise<void>;
|
|
48
|
+
}
|
|
49
|
+
/** The host's choice, per conversation: which saved sign-in and which origins. Never the model's. Not a secret. */
|
|
50
|
+
export type VaultTarget = {
|
|
51
|
+
sessionKey: string;
|
|
52
|
+
origins: string[];
|
|
53
|
+
};
|
|
54
|
+
/** The host's allowed origins, each `scheme://host[:port]` of http or https, normalised and de-duplicated; anything else is refused. */
|
|
55
|
+
export declare function vaultOrigins(origins: readonly string[]): string[];
|
|
56
|
+
export declare function vaultTarget(target: VaultTarget): {
|
|
57
|
+
sessionKey: string;
|
|
58
|
+
origins: string[];
|
|
59
|
+
};
|
|
60
|
+
/** Whether a browser cookie belongs to the allowed origins: it would be SENT to at least one of them, by domain and secure flag (any path
|
|
61
|
+
* on an origin is the origin's). A host-only cookie matches its own host; a domain cookie (leading dot) matches the host and every host
|
|
62
|
+
* under it; a domain with no dot inside it (a public suffix) matches only a host equal to it; a secure cookie needs an https origin (a
|
|
63
|
+
* loopback host counts); a partitioned cookie is not kept. */
|
|
64
|
+
export declare function cookieKept(cookie: {
|
|
65
|
+
domain: string;
|
|
66
|
+
secure?: boolean;
|
|
67
|
+
partitionKey?: unknown;
|
|
68
|
+
}, origins: readonly string[]): boolean;
|