@sixb/agent 0.2.0 → 0.2.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -1
- package/dist/agent-prompt.d.ts +13 -0
- package/dist/agent-prompt.d.ts.map +1 -1
- package/dist/agent-runtime/preflight.d.ts.map +1 -1
- package/dist/artifacts/layout-check/layout-check.d.ts.map +1 -1
- package/dist/background-commands.d.ts +95 -0
- package/dist/background-commands.d.ts.map +1 -0
- package/dist/background-notices.d.ts +31 -0
- package/dist/background-notices.d.ts.map +1 -0
- package/dist/computer-runner.d.ts +63 -2
- package/dist/computer-runner.d.ts.map +1 -1
- package/dist/context-compaction.d.ts +3 -0
- package/dist/context-compaction.d.ts.map +1 -1
- package/dist/execution-plan.d.ts +12 -2
- package/dist/execution-plan.d.ts.map +1 -1
- package/dist/index.d.ts +2 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +2287 -230
- package/dist/index.js.map +28 -18
- package/dist/login-shell.d.ts +16 -0
- package/dist/login-shell.d.ts.map +1 -0
- package/dist/run-child.d.ts +3 -0
- package/dist/run-child.d.ts.map +1 -1
- package/dist/run-conversation-turn.d.ts +6 -2
- package/dist/run-conversation-turn.d.ts.map +1 -1
- package/dist/run-environment.d.ts +12 -2
- package/dist/run-environment.d.ts.map +1 -1
- package/dist/run-sandbox.d.ts +23 -0
- package/dist/run-sandbox.d.ts.map +1 -1
- package/dist/sandbox-files.d.ts.map +1 -1
- package/dist/tools/background.d.ts +50 -0
- package/dist/tools/background.d.ts.map +1 -0
- package/dist/tools/bash.d.ts +12 -4
- package/dist/tools/bash.d.ts.map +1 -1
- package/dist/tools/model-spec.d.ts +9 -2
- package/dist/tools/model-spec.d.ts.map +1 -1
- package/dist/tools/tool-outputs.d.ts +12 -2
- package/dist/tools/tool-outputs.d.ts.map +1 -1
- package/dist/tools/web-fetch/cache.d.ts +33 -0
- package/dist/tools/web-fetch/cache.d.ts.map +1 -0
- package/dist/tools/web-fetch/html-markdown.d.ts +25 -0
- package/dist/tools/web-fetch/html-markdown.d.ts.map +1 -0
- package/dist/tools/web-fetch/network.d.ts +55 -0
- package/dist/tools/web-fetch/network.d.ts.map +1 -0
- package/dist/tools/web-fetch/tool.d.ts +96 -0
- package/dist/tools/web-fetch/tool.d.ts.map +1 -0
- package/dist/tools/web-search.d.ts +40 -0
- package/dist/tools/web-search.d.ts.map +1 -0
- package/dist/workspace-instructions.d.ts +15 -0
- package/dist/workspace-instructions.d.ts.map +1 -0
- package/package.json +13 -4
- package/src/agent-prompt.ts +60 -1
- package/src/agent-runtime/preflight.ts +2 -1
- package/src/artifacts/layout-check/layout-check.ts +4 -4
- package/src/background-commands.ts +573 -0
- package/src/background-notices.ts +178 -0
- package/src/computer-runner.ts +211 -9
- package/src/context-compaction.ts +4 -1
- package/src/execution-plan.ts +15 -2
- package/src/index.ts +3 -0
- package/src/login-shell.ts +18 -0
- package/src/run-child.ts +5 -0
- package/src/run-conversation-turn.ts +23 -4
- package/src/run-environment.ts +92 -12
- package/src/run-sandbox.ts +41 -0
- package/src/sandbox-files.ts +2 -1
- package/src/tools/background.ts +127 -0
- package/src/tools/bash.ts +80 -11
- package/src/tools/model-spec.ts +25 -3
- package/src/tools/tool-outputs.ts +46 -6
- package/src/tools/web-fetch/cache.ts +91 -0
- package/src/tools/web-fetch/html-markdown.ts +479 -0
- package/src/tools/web-fetch/network.ts +452 -0
- package/src/tools/web-fetch/tool.ts +619 -0
- package/src/tools/web-search.ts +139 -0
- package/src/workspace-instructions.ts +163 -0
|
@@ -0,0 +1,452 @@
|
|
|
1
|
+
import { promises as dns } from "node:dns"
|
|
2
|
+
import { BlockList, isIP } from "node:net"
|
|
3
|
+
import { AgentToolPublicError } from "@sixb/core"
|
|
4
|
+
import { waitForAbort } from "../../abort"
|
|
5
|
+
|
|
6
|
+
/**
|
|
7
|
+
* Where a fetch leaves from: Sixb's servers, or the user's computer through the `sixb` CLI. Both
|
|
8
|
+
* reach only public addresses; a computer may also read its own loopback, so a developer can point
|
|
9
|
+
* the agent at their dev server.
|
|
10
|
+
*/
|
|
11
|
+
export type WebFetchNetwork = "server" | "computer"
|
|
12
|
+
|
|
13
|
+
export interface GuardedFetchInput {
|
|
14
|
+
readonly network: WebFetchNetwork
|
|
15
|
+
readonly signal: AbortSignal
|
|
16
|
+
/** The body is read up to this many bytes, then cut (`truncated`). */
|
|
17
|
+
readonly maxBytes: number
|
|
18
|
+
/** For the whole fetch: every redirect hop and the body. */
|
|
19
|
+
readonly timeoutMs: number
|
|
20
|
+
/** Throws to refuse a URL before it is requested: the first, and each redirect followed. */
|
|
21
|
+
readonly admit?: (url: URL) => void
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
export interface GuardedPage {
|
|
25
|
+
/** The final URL, after the same-host redirects that were followed. */
|
|
26
|
+
readonly url: string
|
|
27
|
+
readonly status: number
|
|
28
|
+
/** The media type, lowercased and without parameters. */
|
|
29
|
+
readonly contentType?: string
|
|
30
|
+
readonly charset?: string
|
|
31
|
+
readonly body: Uint8Array
|
|
32
|
+
readonly truncated: boolean
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
/** A redirect to another host. It is not followed: the agent decides whether to fetch it. */
|
|
36
|
+
export interface GuardedRedirect {
|
|
37
|
+
/** The URL that answered with the redirect. */
|
|
38
|
+
readonly url: string
|
|
39
|
+
readonly redirect: { readonly url: string; readonly status: number }
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
export type GuardedResponse = GuardedPage | GuardedRedirect
|
|
43
|
+
|
|
44
|
+
export type WebFetchErrorCode =
|
|
45
|
+
| "unsupported-url"
|
|
46
|
+
| "blocked"
|
|
47
|
+
| "unresolved"
|
|
48
|
+
| "unreachable"
|
|
49
|
+
| "too-many-redirects"
|
|
50
|
+
| "timeout"
|
|
51
|
+
|
|
52
|
+
/** Why a page could not be fetched, worded for the model. */
|
|
53
|
+
export class WebFetchError extends AgentToolPublicError {
|
|
54
|
+
override readonly name = "WebFetchError"
|
|
55
|
+
|
|
56
|
+
constructor(
|
|
57
|
+
readonly code: WebFetchErrorCode,
|
|
58
|
+
message: string,
|
|
59
|
+
options?: ErrorOptions
|
|
60
|
+
) {
|
|
61
|
+
super(`[SixbAgent] ${message}`, options)
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
const MAX_REDIRECTS = 5
|
|
66
|
+
|
|
67
|
+
const REDIRECT_STATUSES = new Set([301, 302, 303, 307, 308])
|
|
68
|
+
|
|
69
|
+
// A fetch-protocol version, not the package's: `tsconfig.build.json` keeps `package.json` outside
|
|
70
|
+
// the program, and nothing else in the framework reads its own version.
|
|
71
|
+
const USER_AGENT = "Sixb-Agent/1 (+https://sixb.ai)"
|
|
72
|
+
|
|
73
|
+
const ACCEPT = "text/markdown, text/html;q=0.9, */*;q=0.8"
|
|
74
|
+
|
|
75
|
+
// Cloud metadata services also answer on these names, wherever DNS sends them.
|
|
76
|
+
const METADATA_HOSTNAMES = new Set(["metadata", "metadata.google.internal"])
|
|
77
|
+
|
|
78
|
+
/**
|
|
79
|
+
* Never reachable: "this network", private and shared networks, link-local (cloud metadata at
|
|
80
|
+
* 169.254.169.254, and fd00:ec2::254 inside fc00::/7), documentation, benchmarking, multicast and
|
|
81
|
+
* reserved ranges. `::/96` is `::` and the deprecated IPv4-compatible block; `::1` is loopback.
|
|
82
|
+
* `64:ff9b:1::/48` is NAT64 for a local network, and `fec0::/10` the deprecated site-local block.
|
|
83
|
+
*/
|
|
84
|
+
const PRIVATE = blockList([
|
|
85
|
+
"0.0.0.0/8",
|
|
86
|
+
"10.0.0.0/8",
|
|
87
|
+
"100.64.0.0/10",
|
|
88
|
+
"169.254.0.0/16",
|
|
89
|
+
"172.16.0.0/12",
|
|
90
|
+
"192.0.0.0/24",
|
|
91
|
+
"192.0.2.0/24",
|
|
92
|
+
"192.168.0.0/16",
|
|
93
|
+
"198.18.0.0/15",
|
|
94
|
+
"198.51.100.0/24",
|
|
95
|
+
"203.0.113.0/24",
|
|
96
|
+
"224.0.0.0/4",
|
|
97
|
+
"240.0.0.0/4",
|
|
98
|
+
"::/96",
|
|
99
|
+
"fc00::/7",
|
|
100
|
+
"fe80::/10",
|
|
101
|
+
"fec0::/10",
|
|
102
|
+
"64:ff9b:1::/48",
|
|
103
|
+
"ff00::/8",
|
|
104
|
+
"2001:db8::/32",
|
|
105
|
+
"100::/64",
|
|
106
|
+
])
|
|
107
|
+
|
|
108
|
+
/** Reachable from a computer only, written as such or mapped (`::ffff:127.0.0.1`). */
|
|
109
|
+
const LOOPBACK = blockList(["127.0.0.0/8", "::1/128"])
|
|
110
|
+
|
|
111
|
+
/**
|
|
112
|
+
* Fetch a page for the agent without letting it reach anything but the public web.
|
|
113
|
+
*
|
|
114
|
+
* Every address DNS returns for the host must be public, and the connection goes to the one that
|
|
115
|
+
* was checked: the URL carries the address, `Host` and the TLS server name carry the host, so DNS
|
|
116
|
+
* is never asked again and cannot answer differently in between. Same-host redirects are followed,
|
|
117
|
+
* each hop resolved and checked again; a redirect to another host is returned, unless its target
|
|
118
|
+
* is refused before DNS is even asked.
|
|
119
|
+
*/
|
|
120
|
+
export async function guardedFetch(url: URL, input: GuardedFetchInput): Promise<GuardedResponse> {
|
|
121
|
+
input.signal.throwIfAborted()
|
|
122
|
+
|
|
123
|
+
const deadline = new AbortController()
|
|
124
|
+
const timer = setTimeout(() => deadline.abort(), input.timeoutMs)
|
|
125
|
+
const signal = AbortSignal.any([input.signal, deadline.signal])
|
|
126
|
+
|
|
127
|
+
try {
|
|
128
|
+
let current = url
|
|
129
|
+
|
|
130
|
+
for (let hop = 0; ; hop += 1) {
|
|
131
|
+
input.admit?.(current)
|
|
132
|
+
const response = await fetchPinned(current, input.network, signal)
|
|
133
|
+
|
|
134
|
+
const location = redirectLocation(response, current)
|
|
135
|
+
if (!location) return await readPage(current, response, input.maxBytes)
|
|
136
|
+
|
|
137
|
+
await response.body?.cancel()
|
|
138
|
+
|
|
139
|
+
if (!isSameHost(current, location)) {
|
|
140
|
+
// The agent may fetch it next: a target refused before DNS is said now, with why.
|
|
141
|
+
const refused = refusalBeforeDns(location, input.network)
|
|
142
|
+
if (refused) {
|
|
143
|
+
throw new WebFetchError(
|
|
144
|
+
"blocked",
|
|
145
|
+
`Blocked: ${current.href} redirects to ${location.href}, which cannot be read: ${refused.reason}`
|
|
146
|
+
)
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
return { url: current.href, redirect: { url: location.href, status: response.status } }
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
if (hop === MAX_REDIRECTS) {
|
|
153
|
+
throw new WebFetchError(
|
|
154
|
+
"too-many-redirects",
|
|
155
|
+
`Stopped after ${MAX_REDIRECTS} redirects at ${current.href}.`
|
|
156
|
+
)
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
current = location
|
|
160
|
+
}
|
|
161
|
+
} catch (error) {
|
|
162
|
+
if (input.signal.aborted) throw input.signal.reason
|
|
163
|
+
|
|
164
|
+
if (deadline.signal.aborted) {
|
|
165
|
+
throw new WebFetchError(
|
|
166
|
+
"timeout",
|
|
167
|
+
`Timed out after ${input.timeoutMs / 1000} s fetching ${url.href}.`
|
|
168
|
+
)
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
throw error
|
|
172
|
+
} finally {
|
|
173
|
+
clearTimeout(timer)
|
|
174
|
+
}
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
async function fetchPinned(
|
|
178
|
+
url: URL,
|
|
179
|
+
network: WebFetchNetwork,
|
|
180
|
+
signal: AbortSignal
|
|
181
|
+
): Promise<Response> {
|
|
182
|
+
const host = checkUrl(url, network)
|
|
183
|
+
const addresses = await resolvePublic(host, network, signal)
|
|
184
|
+
|
|
185
|
+
// A name often has several addresses, and a server listens on only some of them: `localhost`
|
|
186
|
+
// resolves to `::1` first, where a dev server bound to 127.0.0.1 refuses. All were checked.
|
|
187
|
+
let refused: unknown
|
|
188
|
+
for (const address of addresses) {
|
|
189
|
+
try {
|
|
190
|
+
return await fetch(pinnedUrl(url, address), {
|
|
191
|
+
method: "GET",
|
|
192
|
+
redirect: "manual",
|
|
193
|
+
signal,
|
|
194
|
+
headers: { host: url.host, "user-agent": USER_AGENT, accept: ACCEPT },
|
|
195
|
+
...(url.protocol === "https:" && !isIP(host) ? { tls: { serverName: host } } : {}),
|
|
196
|
+
|
|
197
|
+
// A proxy from the environment is the computer's own way out; on a server it would be a
|
|
198
|
+
// path the checks above do not see. Loopback is this machine, never the proxy's.
|
|
199
|
+
...(network === "server" || LOOPBACK.check(address, family(address))
|
|
200
|
+
? { proxy: false }
|
|
201
|
+
: {}),
|
|
202
|
+
})
|
|
203
|
+
} catch (error) {
|
|
204
|
+
if (signal.aborted || !isRefused(error)) throw unreachable(url, error)
|
|
205
|
+
refused = error
|
|
206
|
+
}
|
|
207
|
+
}
|
|
208
|
+
|
|
209
|
+
throw unreachable(url, refused)
|
|
210
|
+
}
|
|
211
|
+
|
|
212
|
+
/** Refuse what is not a plain web URL; return its host, without the brackets of an IPv6 literal. */
|
|
213
|
+
function checkUrl(url: URL, network: WebFetchNetwork): string {
|
|
214
|
+
const refused = refusalBeforeDns(url, network)
|
|
215
|
+
if (refused) {
|
|
216
|
+
const blocked = refused.code === "blocked" ? "Blocked: " : ""
|
|
217
|
+
throw new WebFetchError(refused.code, `${blocked}${refused.reason}`)
|
|
218
|
+
}
|
|
219
|
+
|
|
220
|
+
return hostOf(url)
|
|
221
|
+
}
|
|
222
|
+
|
|
223
|
+
/**
|
|
224
|
+
* Why a URL is refused before DNS is asked, if it is: its scheme, its credentials, a cloud
|
|
225
|
+
* metadata name, or an address written in it that the network does not reach.
|
|
226
|
+
*/
|
|
227
|
+
function refusalBeforeDns(
|
|
228
|
+
url: URL,
|
|
229
|
+
network: WebFetchNetwork
|
|
230
|
+
): { readonly code: WebFetchErrorCode; readonly reason: string } | undefined {
|
|
231
|
+
if (url.protocol !== "http:" && url.protocol !== "https:") {
|
|
232
|
+
return {
|
|
233
|
+
code: "unsupported-url",
|
|
234
|
+
reason: `Only http and https URLs can be fetched, not ${url.protocol}.`,
|
|
235
|
+
}
|
|
236
|
+
}
|
|
237
|
+
|
|
238
|
+
if (url.username || url.password) {
|
|
239
|
+
return {
|
|
240
|
+
code: "unsupported-url",
|
|
241
|
+
reason: "URLs with a username or password are not fetched; remove the credentials.",
|
|
242
|
+
}
|
|
243
|
+
}
|
|
244
|
+
|
|
245
|
+
const host = hostOf(url)
|
|
246
|
+
|
|
247
|
+
if (METADATA_HOSTNAMES.has(host.replace(/\.$/, ""))) {
|
|
248
|
+
return { code: "blocked", reason: `'${host}' is a cloud metadata service.` }
|
|
249
|
+
}
|
|
250
|
+
|
|
251
|
+
if (isIP(host) && isBlocked(host, network)) {
|
|
252
|
+
return { code: "blocked", reason: privateAddress(host, host, network) }
|
|
253
|
+
}
|
|
254
|
+
|
|
255
|
+
return undefined
|
|
256
|
+
}
|
|
257
|
+
|
|
258
|
+
function hostOf(url: URL): string {
|
|
259
|
+
return url.hostname.startsWith("[") ? url.hostname.slice(1, -1) : url.hostname
|
|
260
|
+
}
|
|
261
|
+
|
|
262
|
+
function privateAddress(host: string, address: string, network: WebFetchNetwork): string {
|
|
263
|
+
const reach =
|
|
264
|
+
network === "computer" ? "public addresses and this computer's loopback" : "public addresses"
|
|
265
|
+
return `'${host}' is a private address (${address}). Only ${reach} can be fetched.`
|
|
266
|
+
}
|
|
267
|
+
|
|
268
|
+
async function resolvePublic(
|
|
269
|
+
host: string,
|
|
270
|
+
network: WebFetchNetwork,
|
|
271
|
+
signal: AbortSignal
|
|
272
|
+
): Promise<string[]> {
|
|
273
|
+
const addresses = isIP(host) ? [host] : await lookup(host, signal)
|
|
274
|
+
|
|
275
|
+
const blocked = addresses.find((address) => isBlocked(address, network))
|
|
276
|
+
if (blocked !== undefined) {
|
|
277
|
+
throw new WebFetchError("blocked", `Blocked: ${privateAddress(host, blocked, network)}`)
|
|
278
|
+
}
|
|
279
|
+
|
|
280
|
+
return addresses
|
|
281
|
+
}
|
|
282
|
+
|
|
283
|
+
async function lookup(host: string, signal: AbortSignal): Promise<string[]> {
|
|
284
|
+
try {
|
|
285
|
+
// `dns.lookup` cannot be cancelled; stop waiting for it instead.
|
|
286
|
+
const answers = await waitForAbort(dns.lookup(host, { all: true, verbatim: true }), signal)
|
|
287
|
+
if (answers.length > 0) return answers.map((answer) => answer.address)
|
|
288
|
+
} catch (error) {
|
|
289
|
+
if (signal.aborted) throw error
|
|
290
|
+
throw new WebFetchError("unresolved", `Could not resolve '${host}' (${errorCode(error)}).`, {
|
|
291
|
+
cause: error,
|
|
292
|
+
})
|
|
293
|
+
}
|
|
294
|
+
|
|
295
|
+
throw new WebFetchError("unresolved", `Could not resolve '${host}': no address.`)
|
|
296
|
+
}
|
|
297
|
+
|
|
298
|
+
function isBlocked(address: string, network: WebFetchNetwork): boolean {
|
|
299
|
+
// A NAT64 or 6to4 address goes through a gateway: its loopback is never the computer's own.
|
|
300
|
+
const embedded = embeddedIPv4(address)
|
|
301
|
+
if (embedded) return isBlocked(embedded.address, embedded.mapped ? network : "server")
|
|
302
|
+
|
|
303
|
+
const type = family(address)
|
|
304
|
+
if (LOOPBACK.check(address, type)) return network === "server"
|
|
305
|
+
|
|
306
|
+
return PRIVATE.check(address, type)
|
|
307
|
+
}
|
|
308
|
+
|
|
309
|
+
/**
|
|
310
|
+
* The IPv4 address an IPv6 one stands for: mapped (`::ffff:0:0/96`), NAT64 (`64:ff9b::/96`) or
|
|
311
|
+
* 6to4 (`2002::/16`). Each reaches that IPv4 host, so it is judged as one.
|
|
312
|
+
*/
|
|
313
|
+
function embeddedIPv4(
|
|
314
|
+
address: string
|
|
315
|
+
): { readonly address: string; readonly mapped: boolean } | undefined {
|
|
316
|
+
if (isIP(address) !== 6) return undefined
|
|
317
|
+
|
|
318
|
+
const words = ipv6Words(address)
|
|
319
|
+
const zero = (from: number, to: number) => words.slice(from, to).every((word) => word === 0)
|
|
320
|
+
|
|
321
|
+
if (zero(0, 5) && words[5] === 0xffff) {
|
|
322
|
+
return { address: ipv4(words[6], words[7]), mapped: true }
|
|
323
|
+
}
|
|
324
|
+
if (words[0] === 0x64 && words[1] === 0xff9b && zero(2, 6)) {
|
|
325
|
+
return { address: ipv4(words[6], words[7]), mapped: false }
|
|
326
|
+
}
|
|
327
|
+
if (words[0] === 0x2002) return { address: ipv4(words[1], words[2]), mapped: false }
|
|
328
|
+
|
|
329
|
+
return undefined
|
|
330
|
+
}
|
|
331
|
+
|
|
332
|
+
/** The eight 16-bit words of a valid IPv6 address. */
|
|
333
|
+
function ipv6Words(address: string): number[] {
|
|
334
|
+
let text = address.split("%")[0] ?? ""
|
|
335
|
+
|
|
336
|
+
// A dotted IPv4 tail (`::ffff:10.0.0.1`) is the last two words.
|
|
337
|
+
const tail = text.slice(text.lastIndexOf(":") + 1)
|
|
338
|
+
if (tail.includes(".")) {
|
|
339
|
+
const [a = 0, b = 0, c = 0, d = 0] = tail.split(".").map(Number)
|
|
340
|
+
text = `${text.slice(0, -tail.length)}${((a << 8) | b).toString(16)}:${((c << 8) | d).toString(16)}`
|
|
341
|
+
}
|
|
342
|
+
|
|
343
|
+
const [head = "", rest] = text.split("::")
|
|
344
|
+
const parse = (part: string) =>
|
|
345
|
+
part ? part.split(":").map((word) => Number.parseInt(word, 16)) : []
|
|
346
|
+
const left = parse(head)
|
|
347
|
+
const right = parse(rest ?? "")
|
|
348
|
+
|
|
349
|
+
return [...left, ...new Array<number>(8 - left.length - right.length).fill(0), ...right]
|
|
350
|
+
}
|
|
351
|
+
|
|
352
|
+
function ipv4(high = 0, low = 0): string {
|
|
353
|
+
return [high >> 8, high & 0xff, low >> 8, low & 0xff].join(".")
|
|
354
|
+
}
|
|
355
|
+
|
|
356
|
+
function pinnedUrl(url: URL, address: string): URL {
|
|
357
|
+
const pinned = new URL(url.href)
|
|
358
|
+
pinned.hostname = family(address) === "ipv6" ? `[${address}]` : address
|
|
359
|
+
return pinned
|
|
360
|
+
}
|
|
361
|
+
|
|
362
|
+
function redirectLocation(response: Response, base: URL): URL | undefined {
|
|
363
|
+
const location = response.headers.get("location")
|
|
364
|
+
if (!REDIRECT_STATUSES.has(response.status) || !location) return undefined
|
|
365
|
+
|
|
366
|
+
return URL.canParse(location, base) ? new URL(location, base) : undefined
|
|
367
|
+
}
|
|
368
|
+
|
|
369
|
+
/** The same host name, on the same scheme or upgraded from http to https. */
|
|
370
|
+
function isSameHost(from: URL, to: URL): boolean {
|
|
371
|
+
if (to.hostname !== from.hostname) return false
|
|
372
|
+
return to.protocol === from.protocol || (from.protocol === "http:" && to.protocol === "https:")
|
|
373
|
+
}
|
|
374
|
+
|
|
375
|
+
async function readPage(url: URL, response: Response, maxBytes: number): Promise<GuardedPage> {
|
|
376
|
+
const contentType = response.headers.get("content-type") ?? ""
|
|
377
|
+
const mediaType = contentType.split(";")[0]?.trim().toLowerCase()
|
|
378
|
+
const charset = /;\s*charset\s*=\s*"?([^";\s]+)/i.exec(contentType)?.[1]?.toLowerCase()
|
|
379
|
+
|
|
380
|
+
const chunks: Uint8Array[] = []
|
|
381
|
+
let size = 0
|
|
382
|
+
let truncated = false
|
|
383
|
+
|
|
384
|
+
// Read one chunk past the cap at most: a page exactly `maxBytes` long is not truncated.
|
|
385
|
+
const reader = response.body?.getReader()
|
|
386
|
+
while (reader && size <= maxBytes) {
|
|
387
|
+
const { done, value } = await reader.read()
|
|
388
|
+
if (done) break
|
|
389
|
+
chunks.push(value)
|
|
390
|
+
size += value.byteLength
|
|
391
|
+
}
|
|
392
|
+
|
|
393
|
+
if (reader && size > maxBytes) {
|
|
394
|
+
truncated = true
|
|
395
|
+
await reader.cancel()
|
|
396
|
+
}
|
|
397
|
+
|
|
398
|
+
return {
|
|
399
|
+
url: url.href,
|
|
400
|
+
status: response.status,
|
|
401
|
+
...(mediaType ? { contentType: mediaType } : {}),
|
|
402
|
+
...(charset ? { charset } : {}),
|
|
403
|
+
body: concat(chunks, Math.min(size, maxBytes)),
|
|
404
|
+
truncated,
|
|
405
|
+
}
|
|
406
|
+
}
|
|
407
|
+
|
|
408
|
+
function concat(chunks: readonly Uint8Array[], length: number): Uint8Array {
|
|
409
|
+
const body = new Uint8Array(length)
|
|
410
|
+
let offset = 0
|
|
411
|
+
|
|
412
|
+
for (const chunk of chunks) {
|
|
413
|
+
if (offset >= length) break
|
|
414
|
+
const part = chunk.subarray(0, length - offset)
|
|
415
|
+
body.set(part, offset)
|
|
416
|
+
offset += part.byteLength
|
|
417
|
+
}
|
|
418
|
+
|
|
419
|
+
return body
|
|
420
|
+
}
|
|
421
|
+
|
|
422
|
+
function unreachable(url: URL, error: unknown): WebFetchError {
|
|
423
|
+
return new WebFetchError("unreachable", `Could not fetch ${url.href} (${errorCode(error)}).`, {
|
|
424
|
+
cause: error,
|
|
425
|
+
})
|
|
426
|
+
}
|
|
427
|
+
|
|
428
|
+
function isRefused(error: unknown): boolean {
|
|
429
|
+
return ["ECONNREFUSED", "EHOSTUNREACH", "ENETUNREACH", "EADDRNOTAVAIL"].includes(errorCode(error))
|
|
430
|
+
}
|
|
431
|
+
|
|
432
|
+
function errorCode(error: unknown): string {
|
|
433
|
+
if (typeof error === "object" && error !== null && "code" in error) {
|
|
434
|
+
if (typeof error.code === "string") return error.code
|
|
435
|
+
}
|
|
436
|
+
return error instanceof Error ? error.message : String(error)
|
|
437
|
+
}
|
|
438
|
+
|
|
439
|
+
function family(address: string): "ipv4" | "ipv6" {
|
|
440
|
+
return isIP(address) === 6 ? "ipv6" : "ipv4"
|
|
441
|
+
}
|
|
442
|
+
|
|
443
|
+
function blockList(ranges: readonly string[]): BlockList {
|
|
444
|
+
const list = new BlockList()
|
|
445
|
+
|
|
446
|
+
for (const range of ranges) {
|
|
447
|
+
const [address = "", bits = ""] = range.split("/")
|
|
448
|
+
list.addSubnet(address, Number(bits), family(address))
|
|
449
|
+
}
|
|
450
|
+
|
|
451
|
+
return list
|
|
452
|
+
}
|