@qilin-agent/web-fetch-http 0.0.0-stage → 3.1.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/lib/index.js ADDED
@@ -0,0 +1,681 @@
1
+ import z from "@qilin-agent/schemastery";
2
+ import { WebError } from "@qilin-agent/web";
3
+ import { deadline, timeoutOf } from "@qilin-agent/timeout";
4
+ import { proxyRouteFor } from "@qilin-agent/http-proxy";
5
+ import { lookup } from "node:dns/promises";
6
+ import { isIP } from "node:net";
7
+ import ipaddr from "ipaddr.js";
8
+ //#region lib/types/network.js
9
+ /**
10
+ * Public-network resolution and address-pinned HTTP transport for `web-fetch-http`.
11
+ * One DNS answer set is validated before Undici receives it through a custom lookup,
12
+ * so the connection cannot resolve the hostname again to a private address.
13
+ *
14
+ * @module @qilin-agent/web-fetch-http/network
15
+ */
16
+ /** RFC 6052 prefix lengths that may carry an IPv4 destination through NAT64. */
17
+ const RFC6052_PREFIX_LENGTHS = [
18
+ 32,
19
+ 40,
20
+ 48,
21
+ 56,
22
+ 64,
23
+ 96
24
+ ];
25
+ const IPV4ONLY_DISCOVERY_HOST = "ipv4only.arpa";
26
+ const IPV4ONLY_SENTINELS = new Set(["192.0.0.170", "192.0.0.171"]);
27
+ /**
28
+ * Return whether an address is globally reachable unicast. IPv4-mapped IPv6 is
29
+ * classified by its embedded IPv4 address; transition and translation prefixes
30
+ * remain blocked because their eventual IPv4 destination cannot be pinned here.
31
+ *
32
+ * @param input - textual IPv4 or IPv6 address.
33
+ * @returns true only for a public unicast destination.
34
+ */
35
+ function isPublicIpAddress(input) {
36
+ let parsed;
37
+ try {
38
+ parsed = ipaddr.parse(stripIpv6Brackets(input));
39
+ } catch {
40
+ return false;
41
+ }
42
+ if (parsed instanceof ipaddr.IPv4) return parsed.range() === "unicast";
43
+ if (parsed.isIPv4MappedAddress()) return parsed.toIPv4Address().range() === "unicast";
44
+ return parsed.range() === "unicast";
45
+ }
46
+ /**
47
+ * Resolve a hostname once and reject the complete answer set if any destination
48
+ * is not public. The returned addresses are the only ones the transport may use.
49
+ *
50
+ * @param hostname - URL hostname, including brackets when it is an IPv6 literal.
51
+ * @param signal - aborts the wait for system resolution; an in-flight OS lookup may finish unused.
52
+ * @param resolver - lookup implementation, overridden only by focused tests.
53
+ * @returns the validated, non-empty address set.
54
+ */
55
+ async function resolvePublicAddresses(hostname, signal, resolver = lookup) {
56
+ const unbracketed = stripIpv6Brackets(hostname);
57
+ const literalFamily = isIP(unbracketed);
58
+ const resolved = literalFamily === 0 ? await raceWithSignal(resolver(unbracketed, {
59
+ all: true,
60
+ order: "verbatim"
61
+ }), signal) : [{
62
+ address: unbracketed,
63
+ family: literalFamily
64
+ }];
65
+ if (resolved.length === 0) throw new WebError(`hostname "${hostname}" resolved to no addresses`, "WEB_PROVIDER_ERROR");
66
+ const nat64Prefixes = resolved.some((entry) => entry.family === 6 && isIP(entry.address) === 6) ? await discoverNat64Prefixes(signal, resolver) : [];
67
+ const addresses = [];
68
+ for (const entry of resolved) {
69
+ if (entry.family !== 4 && entry.family !== 6 || isIP(entry.address) !== entry.family) throw new WebError(`hostname "${hostname}" resolved to an invalid IP address`, "WEB_PROVIDER_ERROR");
70
+ if (!isPublicIpAddress(entry.address)) throw new WebError(`URL hostname "${hostname}" resolves to a non-public IP address`, "WEB_BLOCKED_URL");
71
+ const translatedIpv4 = translatedIpv4Address(entry.address, nat64Prefixes);
72
+ if (translatedIpv4 !== void 0 && !isPublicIpAddress(translatedIpv4)) throw new WebError(`URL hostname "${hostname}" resolves through NAT64 to a non-public IPv4 address`, "WEB_BLOCKED_URL");
73
+ addresses.push({
74
+ address: entry.address,
75
+ family: entry.family
76
+ });
77
+ }
78
+ return addresses;
79
+ }
80
+ /** Discover the active DNS64 prefix set using RFC 7050's reserved hostname. */
81
+ async function discoverNat64Prefixes(signal, resolver) {
82
+ const discovered = await raceWithSignal(resolver(IPV4ONLY_DISCOVERY_HOST, {
83
+ all: true,
84
+ order: "verbatim"
85
+ }), signal);
86
+ const prefixes = [];
87
+ const seen = /* @__PURE__ */ new Set();
88
+ for (const entry of discovered) {
89
+ if (entry.family !== 6 || isIP(entry.address) !== 6) continue;
90
+ const bytes = ipaddr.parse(entry.address).toByteArray();
91
+ for (const length of RFC6052_PREFIX_LENGTHS) {
92
+ const embedded = embeddedIpv4Address(bytes, length);
93
+ if (embedded === void 0 || !IPV4ONLY_SENTINELS.has(embedded)) continue;
94
+ const prefixBytes = bytes.slice(0, length / 8);
95
+ const key = `${String(length)}:${prefixBytes.join(".")}`;
96
+ if (seen.has(key)) continue;
97
+ seen.add(key);
98
+ prefixes.push({
99
+ bytes: prefixBytes,
100
+ length
101
+ });
102
+ }
103
+ }
104
+ return prefixes;
105
+ }
106
+ /** Return the RFC 6052-embedded IPv4 address when an IPv6 address matches a discovered prefix. */
107
+ function translatedIpv4Address(input, prefixes) {
108
+ if (isIP(input) !== 6) return void 0;
109
+ const bytes = ipaddr.parse(input).toByteArray();
110
+ for (const prefix of prefixes) {
111
+ if (!prefix.bytes.every((byte, index) => bytes[index] === byte)) continue;
112
+ const embedded = embeddedIpv4Address(bytes, prefix.length);
113
+ if (embedded !== void 0) return embedded;
114
+ }
115
+ }
116
+ /** Extract one IPv4 address from an RFC 6052 IPv6 layout. */
117
+ function embeddedIpv4Address(bytes, prefixLength) {
118
+ if (prefixLength === 96) return bytes.slice(12, 16).join(".");
119
+ if (bytes[8] !== 0) return void 0;
120
+ const prefixBytes = prefixLength / 8;
121
+ const beforeReservedOctet = 8 - prefixBytes;
122
+ return [...bytes.slice(prefixBytes, prefixBytes + beforeReservedOctet), ...bytes.slice(9, 13 - beforeReservedOctet)].join(".");
123
+ }
124
+ /**
125
+ * Whether a hostname is an IP literal that {@link resolvePublicAddresses} would refuse.
126
+ *
127
+ * A proxied hop skips those checks because the proxy resolves the origin, but a literal needs no
128
+ * resolution: the address is already stated, and handing it to a proxy running on this machine
129
+ * would reach exactly the loopback or private service the checks exist to keep out of reach.
130
+ *
131
+ * @param hostname - a URL's hostname, bracketed or not.
132
+ * @returns true when the host is a literal address no request may be sent to.
133
+ */
134
+ function isNonPublicIpLiteral(hostname) {
135
+ const unbracketed = stripIpv6Brackets(hostname);
136
+ return isIP(unbracketed) !== 0 && !isPublicIpAddress(unbracketed);
137
+ }
138
+ /**
139
+ * Fetch through an agent whose lookup callback returns only the already validated address set. The
140
+ * URL hostname remains intact for HTTP Host and TLS SNI.
141
+ *
142
+ * The agent is this request's own because the address set is: pinning is how this package refuses a
143
+ * DNS answer that changes between validation and connection, and it may not apply process-wide —
144
+ * an operator-configured MCP server or model endpoint on loopback is a supported destination, and
145
+ * only the URLs this tool fetches are the model's to choose.
146
+ *
147
+ * @param url - validated HTTP(S) URL the policy does not route through a proxy.
148
+ * @param addresses - public addresses returned by {@link resolvePublicAddresses}.
149
+ * @param headers - request headers.
150
+ * @param signal - request and body-read cancellation signal.
151
+ * @returns a response plus the disposer its consumer must call.
152
+ */
153
+ async function requestPinned(url, addresses, headers, signal) {
154
+ const { Agent, fetch } = await import("undici");
155
+ const dispatcher = new Agent({
156
+ autoSelectFamily: true,
157
+ connect: { lookup: createPinnedLookup(addresses) }
158
+ });
159
+ try {
160
+ return {
161
+ response: await fetch(url, {
162
+ method: "GET",
163
+ redirect: "manual",
164
+ headers,
165
+ signal,
166
+ dispatcher
167
+ }),
168
+ close: async () => {
169
+ await dispatcher.close();
170
+ }
171
+ };
172
+ } catch (error) {
173
+ await dispatcher.close();
174
+ throw error;
175
+ }
176
+ }
177
+ /**
178
+ * Fetch through the dispatcher the proxy policy already installed, letting the proxy resolve the
179
+ * origin.
180
+ *
181
+ * No address set is pinned because none exists to pin: the proxy performs the lookup, and a
182
+ * connection pinned to a locally resolved address would reach the origin directly and defeat the
183
+ * proxy. The dispatcher is the process-wide one, so hops share its connection pool and no caller
184
+ * closes it.
185
+ *
186
+ * @param dispatcher - the route's dispatcher, from `proxyRouteFor`.
187
+ * @param url - validated HTTP(S) URL the policy routes through a proxy.
188
+ * @param headers - request headers.
189
+ * @param signal - request and body-read cancellation signal.
190
+ * @returns a response plus a disposer that releases nothing, so both paths close alike.
191
+ */
192
+ async function requestVia(dispatcher, url, headers, signal) {
193
+ const { fetch } = await import("undici");
194
+ return {
195
+ response: await fetch(url, {
196
+ method: "GET",
197
+ redirect: "manual",
198
+ headers,
199
+ signal,
200
+ dispatcher
201
+ }),
202
+ close: () => Promise.resolve()
203
+ };
204
+ }
205
+ /** Production network operations kept as an object so provider tests can replace resolution only. */
206
+ const publicHttpNetwork = {
207
+ resolve: resolvePublicAddresses,
208
+ request: requestPinned,
209
+ requestVia
210
+ };
211
+ /**
212
+ * Build the connector lookup that serves a fixed validated answer set.
213
+ *
214
+ * @param addresses - public addresses retained from the preceding resolution.
215
+ * @returns a Node-compatible lookup callback that performs no network resolution.
216
+ */
217
+ function createPinnedLookup(addresses) {
218
+ return (hostname, options, callback) => {
219
+ const family = typeof options.family === "number" ? options.family : options.family === "IPv4" ? 4 : options.family === "IPv6" ? 6 : 0;
220
+ const eligible = family === 0 ? addresses : addresses.filter((address) => address.family === family);
221
+ const selected = eligible[0];
222
+ if (selected === void 0) {
223
+ callback(Object.assign(/* @__PURE__ */ new Error(`no validated address for ${hostname} in family ${family}`), {
224
+ code: "ENOTFOUND",
225
+ hostname
226
+ }), options.all === true ? [] : "", family);
227
+ return;
228
+ }
229
+ if (options.all === true) {
230
+ callback(null, eligible.map((address) => ({ ...address })));
231
+ return;
232
+ }
233
+ callback(null, selected.address, selected.family);
234
+ };
235
+ }
236
+ /** Race a non-cancellable OS lookup without letting it delay tool cancellation. */
237
+ function raceWithSignal(promise, signal) {
238
+ const abortError = () => new Error("web fetch aborted during hostname resolution", { cause: signal.reason });
239
+ if (signal.aborted) return Promise.reject(abortError());
240
+ return new Promise((resolve, reject) => {
241
+ const abort = () => {
242
+ reject(abortError());
243
+ };
244
+ signal.addEventListener("abort", abort, { once: true });
245
+ promise.then(resolve, reject).finally(() => {
246
+ signal.removeEventListener("abort", abort);
247
+ });
248
+ });
249
+ }
250
+ /** WHATWG URL retains brackets around IPv6 hostnames; IP parsers do not. */
251
+ function stripIpv6Brackets(hostname) {
252
+ return hostname.startsWith("[") && hostname.endsWith("]") ? hostname.slice(1, -1) : hostname;
253
+ }
254
+ //#endregion
255
+ //#region lib/types/policy.js
256
+ /**
257
+ * URL validation and content-type classification for the local HTTP(S) fetch
258
+ * provider — the pure, network-free half. The provider's `fetch()` composes
259
+ * these with transport (redirect following, byte caps, decoding).
260
+ *
261
+ * @module @qilin-agent/web-fetch-http/policy
262
+ */
263
+ /** Maximum accepted request URL length enforced by the public fetch provider. */
264
+ const WEB_FETCH_MAX_URL_LENGTH = 2048;
265
+ /**
266
+ * Parse a request URL and enforce network-independent transport restrictions:
267
+ * HTTP(S) only and no embedded credentials. The provider applies this before
268
+ * resolving a destination.
269
+ *
270
+ * @param input - the raw URL string from the fetch request.
271
+ * @returns the parsed `URL`.
272
+ */
273
+ function parseFetchUrl(input) {
274
+ let url;
275
+ try {
276
+ url = new URL(input);
277
+ } catch (error) {
278
+ throw new WebError(`invalid URL: ${input}`, "WEB_INVALID_URL", { cause: error });
279
+ }
280
+ if (url.protocol !== "http:" && url.protocol !== "https:") throw new WebError(`unsupported URL scheme "${url.protocol}" (only http and https are allowed)`, "WEB_INVALID_URL");
281
+ if (url.username.length > 0 || url.password.length > 0) throw new WebError("credentials in URLs are not allowed", "WEB_BLOCKED_URL");
282
+ return url;
283
+ }
284
+ /**
285
+ * Validate a request URL against the provider's complete pre-network policy:
286
+ * bounded length plus the restrictions enforced by {@link parseFetchUrl}.
287
+ * Public-address resolution and connection pinning run after this check.
288
+ *
289
+ * @param input - the raw URL string from the fetch request.
290
+ * @returns the parsed `URL`.
291
+ */
292
+ function validateFetchUrl(input) {
293
+ if (input.length > 2048) throw new WebError(`URL exceeds the maximum length of ${WEB_FETCH_MAX_URL_LENGTH}`, "WEB_INVALID_URL");
294
+ return parseFetchUrl(input);
295
+ }
296
+ /**
297
+ * Two URLs are same-origin when scheme, hostname, and port match. A redirect
298
+ * that crosses origins is refused so each new origin requires a fresh tool call
299
+ * and public-address validation.
300
+ *
301
+ * @param a - one of the two URLs to compare.
302
+ * @param b - the other URL to compare.
303
+ * @returns true when `a` and `b` share scheme, hostname, and port.
304
+ */
305
+ function isSameOrigin(a, b) {
306
+ return a.protocol === b.protocol && a.hostname === b.hostname && a.port === b.port;
307
+ }
308
+ /**
309
+ * Classify a response `Content-Type` into a decodable body kind, or `undefined`
310
+ * for an unsupported (e.g. binary) type. `text/html` and `application/xhtml+xml`
311
+ * are `html`; other `text/*` plus a few structured text types are `text`.
312
+ *
313
+ * @param contentType - the raw `Content-Type` header, or `null` when the
314
+ * response carries none (unsupported).
315
+ * @returns the decodable kind, or `undefined` for an unsupported type.
316
+ */
317
+ function classifyContentType(contentType) {
318
+ const mime = (contentType ?? "").replace(/;.*$/s, "").trim().toLowerCase();
319
+ if (mime === "text/html" || mime === "application/xhtml+xml") return "html";
320
+ if (mime.startsWith("text/")) return "text";
321
+ if (mime === "application/json" || mime === "application/xml" || mime.endsWith("+json") || mime.endsWith("+xml")) return "text";
322
+ }
323
+ /**
324
+ * Extract the `charset` parameter from a response `Content-Type`, lower-cased,
325
+ * or `undefined` when absent. The provider feeds this label to `TextDecoder`
326
+ * so a non-UTF-8 response is decoded with its declared encoding rather than
327
+ * silently mangled into replacement characters.
328
+ *
329
+ * @param contentType - the raw `Content-Type` header, or `null` when the
330
+ * response carries none.
331
+ * @returns the lower-cased charset label, or `undefined` when none is declared.
332
+ */
333
+ function parseCharset(contentType) {
334
+ return /;\s*charset\s*=\s*"?([^";]+)"?/i.exec(contentType ?? "")?.[1]?.trim().toLowerCase();
335
+ }
336
+ /**
337
+ * Build a `TextDecoder` for the declared charset, falling back to UTF-8 when
338
+ * none is declared. Throws {@link WebError} `WEB_UNSUPPORTED_CONTENT_TYPE` when
339
+ * the label is present but not a charset `TextDecoder` recognizes — better to
340
+ * fail loudly than return mojibake.
341
+ *
342
+ * @param charset - the declared charset label (from {@link parseCharset}), or
343
+ * `undefined` to default to UTF-8.
344
+ * @returns a decoder for the declared (or defaulted) encoding.
345
+ */
346
+ function decoderForCharset(charset) {
347
+ if (charset === void 0) return new TextDecoder("utf-8");
348
+ try {
349
+ return new TextDecoder(charset);
350
+ } catch (error) {
351
+ throw new WebError(`unsupported charset "${charset}"`, "WEB_UNSUPPORTED_CONTENT_TYPE", { cause: error });
352
+ }
353
+ }
354
+ //#endregion
355
+ //#region lib/types/provider.js
356
+ /**
357
+ * Safe HTTP(S) retrieval for `ctx.web`: validates and pins public IP destinations, follows
358
+ * only same-origin redirects, enforces time and size limits, classifies and decodes text,
359
+ * and leaves presentation to `@qilin-agent/tool-web`. Requests carry no browser cookies
360
+ * or ambient credentials.
361
+ * @module @qilin-agent/web-fetch-http/provider
362
+ */
363
+ var __addDisposableResource = function(env, value, async) {
364
+ if (value !== null && value !== void 0) {
365
+ if (typeof value !== "object" && typeof value !== "function") throw new TypeError("Object expected.");
366
+ var dispose, inner;
367
+ if (async) {
368
+ if (!Symbol.asyncDispose) throw new TypeError("Symbol.asyncDispose is not defined.");
369
+ dispose = value[Symbol.asyncDispose];
370
+ }
371
+ if (dispose === void 0) {
372
+ if (!Symbol.dispose) throw new TypeError("Symbol.dispose is not defined.");
373
+ dispose = value[Symbol.dispose];
374
+ if (async) inner = dispose;
375
+ }
376
+ if (typeof dispose !== "function") throw new TypeError("Object not disposable.");
377
+ if (inner) dispose = function() {
378
+ try {
379
+ inner.call(this);
380
+ } catch (e) {
381
+ return Promise.reject(e);
382
+ }
383
+ };
384
+ env.stack.push({
385
+ value,
386
+ dispose,
387
+ async
388
+ });
389
+ } else if (async) env.stack.push({ async: true });
390
+ return value;
391
+ };
392
+ var __disposeResources = (function(SuppressedError) {
393
+ return function(env) {
394
+ function fail(e) {
395
+ env.error = env.hasError ? new SuppressedError(e, env.error, "An error was suppressed during disposal.") : e;
396
+ env.hasError = true;
397
+ }
398
+ var r, s = 0;
399
+ function next() {
400
+ while (r = env.stack.pop()) try {
401
+ if (!r.async && s === 1) return s = 0, env.stack.push(r), Promise.resolve().then(next);
402
+ if (r.dispose) {
403
+ var result = r.dispose.call(r.value);
404
+ if (r.async) return s |= 2, Promise.resolve(result).then(next, function(e) {
405
+ fail(e);
406
+ return next();
407
+ });
408
+ } else s |= 1;
409
+ } catch (e) {
410
+ fail(e);
411
+ }
412
+ if (s === 1) return env.hasError ? Promise.reject(env.error) : Promise.resolve();
413
+ if (env.hasError) throw env.error;
414
+ }
415
+ return next();
416
+ };
417
+ })(typeof SuppressedError === "function" ? SuppressedError : function(error, suppressed, message) {
418
+ var e = new Error(message);
419
+ return e.name = "SuppressedError", e.error = error, e.suppressed = suppressed, e;
420
+ });
421
+ /** Stable id this provider registers under. */
422
+ const LOCAL_FETCH_PROVIDER_ID = "http";
423
+ /** The anonymous public HTTP(S) fetch provider. */
424
+ var HttpFetchProvider = class {
425
+ limits;
426
+ resolveAddresses;
427
+ id = LOCAL_FETCH_PROVIDER_ID;
428
+ /**
429
+ * @param limits - resolved transport and response limits.
430
+ * @param resolveAddresses - resolver that rejects non-public destinations before returning.
431
+ */
432
+ constructor(limits, resolveAddresses = publicHttpNetwork.resolve) {
433
+ this.limits = limits;
434
+ this.resolveAddresses = resolveAddresses;
435
+ }
436
+ /** No credentials to check — an anonymous public fetcher is always usable. */
437
+ available() {
438
+ return true;
439
+ }
440
+ async fetch(request, signal) {
441
+ const env_1 = {
442
+ stack: [],
443
+ error: void 0,
444
+ hasError: false
445
+ };
446
+ try {
447
+ if (signal?.aborted) throw new WebError("web fetch aborted", "WEB_ABORTED");
448
+ const d = __addDisposableResource(env_1, deadline(signal, this.limits.timeoutMs, "WEB_FETCH_TIMEOUT"), false);
449
+ return await this.followAndRead(request.url, d.signal);
450
+ } catch (e_1) {
451
+ env_1.error = e_1;
452
+ env_1.hasError = true;
453
+ } finally {
454
+ __disposeResources(env_1);
455
+ }
456
+ }
457
+ /** Follow same-origin redirects up to the hop cap, then read the final response. */
458
+ async followAndRead(initialUrl, signal) {
459
+ let currentUrl = validateFetchUrl(initialUrl);
460
+ let redirectsFollowed = 0;
461
+ for (;;) {
462
+ const request = await this.requestOnce(currentUrl, signal);
463
+ const { response } = request;
464
+ try {
465
+ if (isRedirectStatus(response.status)) {
466
+ if (redirectsFollowed >= this.limits.maxRedirects) {
467
+ await response.body?.cancel();
468
+ throw new WebError(`exceeded the maximum of ${this.limits.maxRedirects} redirects`, "WEB_REDIRECT_BLOCKED");
469
+ }
470
+ const location = response.headers.get("location");
471
+ if (location === null) {
472
+ await response.body?.cancel();
473
+ throw new WebError(`redirect response (HTTP ${response.status}) without a Location header`, "WEB_PROVIDER_ERROR");
474
+ }
475
+ const target = resolveRedirect(location, currentUrl);
476
+ let validatedTarget;
477
+ try {
478
+ validatedTarget = validateFetchUrl(target.toString());
479
+ if (!isSameOrigin(validatedTarget, currentUrl)) throw new WebError(`cross-origin redirect to ${validatedTarget.origin} is not followed automatically; retry against that URL directly`, "WEB_REDIRECT_BLOCKED");
480
+ } catch (error) {
481
+ await response.body?.cancel();
482
+ throw error;
483
+ }
484
+ await response.body?.cancel();
485
+ currentUrl = validatedTarget;
486
+ redirectsFollowed++;
487
+ continue;
488
+ }
489
+ return await this.readBody(response, currentUrl, signal);
490
+ } finally {
491
+ await request.close();
492
+ }
493
+ }
494
+ }
495
+ async requestOnce(url, signal) {
496
+ const headers = {
497
+ "user-agent": this.limits.userAgent,
498
+ "accept": "text/html,application/xhtml+xml,text/*;q=0.9,application/json;q=0.8"
499
+ };
500
+ try {
501
+ const route = proxyRouteFor(url);
502
+ if (route.proxied && !isNonPublicIpLiteral(url.hostname)) return await publicHttpNetwork.requestVia(route.dispatcher, url, headers, signal);
503
+ const addresses = await this.resolveAddresses(url.hostname, signal);
504
+ return await publicHttpNetwork.request(url, addresses, headers, signal);
505
+ } catch (error) {
506
+ if (error instanceof WebError) throw error;
507
+ throw translateAbortOrNetwork(error, signal);
508
+ }
509
+ }
510
+ /** Read, byte-cap, classify, and decode the final response body. */
511
+ async readBody(response, finalUrl, signal) {
512
+ const contentType = response.headers.get("content-type");
513
+ const kind = classifyContentType(contentType);
514
+ if (kind === void 0) {
515
+ await response.body?.cancel();
516
+ throw new WebError(`unsupported content type "${contentType ?? "unknown"}"`, "WEB_UNSUPPORTED_CONTENT_TYPE");
517
+ }
518
+ let decoder;
519
+ try {
520
+ decoder = decoderForCharset(parseCharset(contentType));
521
+ } catch (error) {
522
+ await response.body?.cancel();
523
+ throw error;
524
+ }
525
+ const { bytes, truncatedByBytes } = await this.readCapped(response, signal);
526
+ const decoded = decoder.decode(bytes);
527
+ const truncatedByChars = decoded.length > this.limits.maxBodyChars;
528
+ const content = truncatedByChars ? decoded.slice(0, this.limits.maxBodyChars) : decoded;
529
+ const body = kind === "html" ? {
530
+ kind: "html",
531
+ content
532
+ } : {
533
+ kind: "text",
534
+ content
535
+ };
536
+ return {
537
+ url: finalUrl.toString(),
538
+ statusCode: response.status,
539
+ body,
540
+ truncated: truncatedByBytes || truncatedByChars
541
+ };
542
+ }
543
+ /**
544
+ * Read the response stream up to `maxResponseBytes`. A `Content-Length` over
545
+ * the cap rejects immediately with `WEB_FETCH_TOO_LARGE`; a stream that grows
546
+ * past the cap is cut short (`truncatedByBytes`) rather than rejected, so a
547
+ * server that under-reports still yields a bounded usable body.
548
+ */
549
+ async readCapped(response, signal) {
550
+ const declared = response.headers.get("content-length");
551
+ if (declared !== null) {
552
+ const length = Number(declared);
553
+ if (Number.isFinite(length) && length > this.limits.maxResponseBytes) {
554
+ await response.body?.cancel();
555
+ throw new WebError(`response exceeds the maximum of ${this.limits.maxResponseBytes} bytes`, "WEB_FETCH_TOO_LARGE");
556
+ }
557
+ }
558
+ /* v8 ignore next -- a 2xx Response from fetch always exposes a body stream; the null guard is defensive. */
559
+ if (response.body === null) return {
560
+ bytes: new Uint8Array(0),
561
+ truncatedByBytes: false
562
+ };
563
+ const chunks = [];
564
+ let total = 0;
565
+ let truncatedByBytes = false;
566
+ const reader = response.body.getReader();
567
+ try {
568
+ for (;;) {
569
+ const { done, value } = await reader.read();
570
+ if (done) break;
571
+ const remaining = this.limits.maxResponseBytes - total;
572
+ if (value.byteLength > remaining) {
573
+ chunks.push(value.subarray(0, remaining));
574
+ total += remaining;
575
+ truncatedByBytes = true;
576
+ break;
577
+ }
578
+ chunks.push(value);
579
+ total += value.byteLength;
580
+ }
581
+ } catch (error) {
582
+ /* v8 ignore next -- mid-stream read fault needs a network drop after headers; translate path covered by request-phase tests. */
583
+ throw translateAbortOrNetwork(error, signal);
584
+ } finally {
585
+ /* v8 ignore next 4 -- cancel() after a completed/broken read settles without rejecting; unobserved best-effort cleanup. */
586
+ await reader.cancel().catch(() => {});
587
+ }
588
+ const bytes = new Uint8Array(total);
589
+ let offset = 0;
590
+ for (const chunk of chunks) {
591
+ bytes.set(chunk, offset);
592
+ offset += chunk.byteLength;
593
+ }
594
+ return {
595
+ bytes,
596
+ truncatedByBytes
597
+ };
598
+ }
599
+ };
600
+ /** HTTP redirect status codes that carry a `Location`. */
601
+ function isRedirectStatus(status) {
602
+ return status === 301 || status === 302 || status === 303 || status === 307 || status === 308;
603
+ }
604
+ /** Resolve a (possibly relative) `Location` against the current URL. */
605
+ function resolveRedirect(location, base) {
606
+ try {
607
+ return new URL(location, base);
608
+ } catch (error) {
609
+ /* v8 ignore next 2 -- URL resolution against a valid absolute base effectively never throws; defensive guard. */
610
+ throw new WebError(`invalid redirect Location "${location}"`, "WEB_PROVIDER_ERROR", { cause: error });
611
+ }
612
+ }
613
+ /**
614
+ * Translate a thrown fetch/stream error into a `WebError`, classified by the
615
+ * deadline signal rather than the thrown value (which differs by phase: the
616
+ * request-phase `fetch` rejects with the abort reason, while the read-phase
617
+ * reader surfaces a bare `AbortError`). `timeoutOf(signal, 'WEB_FETCH_TIMEOUT')`
618
+ * recovering OUR reason means our timeout fired (`WEB_FETCH_TIMEOUT`); any other
619
+ * abort — an upstream cancel, or a foreign/outer deadline's timeout under
620
+ * nesting — is `WEB_ABORTED`; a throw with the signal NOT aborted is a
621
+ * transport/network failure (`WEB_PROVIDER_ERROR`).
622
+ */
623
+ function translateAbortOrNetwork(error, signal) {
624
+ const timeout = timeoutOf(signal, "WEB_FETCH_TIMEOUT");
625
+ if (timeout !== void 0) return new WebError("web fetch timed out", "WEB_FETCH_TIMEOUT", { cause: timeout });
626
+ if (signal.aborted) return new WebError("web fetch aborted", "WEB_ABORTED", { cause: error });
627
+ return new WebError(`web fetch failed: ${String(error)}`, "WEB_PROVIDER_ERROR", { cause: error });
628
+ }
629
+ //#endregion
630
+ //#region lib/types/index.js
631
+ /**
632
+ * Anonymous public HTTP(S) `WebFetchProvider` plugin. It contributes to the
633
+ * `ctx.web` registry without owning the service.
634
+ *
635
+ * @module @qilin-agent/web-fetch-http
636
+ */
637
+ const MAX_NODE_TIMER_DELAY_MS = 2147483647;
638
+ /** Default `User-Agent`: an explicit product agent, never a browser disguise. */
639
+ const DEFAULT_USER_AGENT = "deepseek-harness/0.0.1 (+https://github.com/deepseek-ai)";
640
+ /** Cordis plugin name used by loader diagnostics. */
641
+ const name = "web-fetch-http";
642
+ /** The web seam this provider registers into. */
643
+ const inject = ["web"];
644
+ const Config = z.object({
645
+ maxResponseBytes: z.number().default(5e6),
646
+ maxBodyChars: z.number().default(1e5),
647
+ timeoutMs: z.number().default(3e4),
648
+ maxRedirects: z.number().default(5),
649
+ userAgent: z.string().default(DEFAULT_USER_AGENT)
650
+ });
651
+ /** A resource limit (byte/char/length/timeout cap) must be a positive finite number. */
652
+ function assertPositiveFinite(name, value) {
653
+ if (!Number.isFinite(value) || value <= 0) throw new Error(`web-fetch-http: ${name} must be a positive finite number`);
654
+ }
655
+ /** Node coerces larger timer delays to 1 ms, so reject them at configuration time. */
656
+ function assertTimeoutMs(value) {
657
+ assertPositiveFinite("timeoutMs", value);
658
+ if (value > MAX_NODE_TIMER_DELAY_MS) throw new Error(`web-fetch-http: timeoutMs must be no greater than ${MAX_NODE_TIMER_DELAY_MS}`);
659
+ }
660
+ /** The redirect hop cap must be a non-negative integer (0 follows no redirects). */
661
+ function assertNonNegativeInteger(name, value) {
662
+ if (!Number.isInteger(value) || value < 0) throw new Error(`web-fetch-http: ${name} must be a non-negative integer`);
663
+ }
664
+ /** Register the local HTTP(S) fetch provider with `ctx.web`. */
665
+ function apply(ctx, config) {
666
+ const resolved = config;
667
+ assertPositiveFinite("maxResponseBytes", resolved.maxResponseBytes);
668
+ assertPositiveFinite("maxBodyChars", resolved.maxBodyChars);
669
+ assertTimeoutMs(resolved.timeoutMs);
670
+ assertNonNegativeInteger("maxRedirects", resolved.maxRedirects);
671
+ const limits = {
672
+ maxResponseBytes: resolved.maxResponseBytes,
673
+ maxBodyChars: resolved.maxBodyChars,
674
+ timeoutMs: resolved.timeoutMs,
675
+ maxRedirects: resolved.maxRedirects,
676
+ userAgent: resolved.userAgent
677
+ };
678
+ ctx.web.registerFetchProvider(new HttpFetchProvider(limits));
679
+ }
680
+ //#endregion
681
+ export { Config, DEFAULT_USER_AGENT, HttpFetchProvider, LOCAL_FETCH_PROVIDER_ID, apply, inject, name };