@queryrev/ai-readiness 0.0.0-stage → 1.0.0

package/dist/chunks/fetch-text-NUUnM91O.js

@@ -0,0 +1,866 @@
+import { isIP, isIPv4, isIPv6 } from "node:net";
+import { Transform, pipeline } from "node:stream";
+import { createBrotliDecompress, createGunzip, createInflate } from "node:zlib";
+import { Agent, buildConnector, request } from "undici";
+import { lookup } from "node:dns/promises";
+// src/net/address.ts
+var IPV4_REFUSED = [
+	{
+		base: ip4("168.63.129.16"),
+		bits: 32,
+		reason: "cloud_metadata"
+	},
+	{
+		base: ip4("0.0.0.0"),
+		bits: 8,
+		reason: "unspecified"
+	},
+	{
+		base: ip4("10.0.0.0"),
+		bits: 8,
+		reason: "private"
+	},
+	{
+		base: ip4("100.64.0.0"),
+		bits: 10,
+		reason: "shared"
+	},
+	{
+		base: ip4("127.0.0.0"),
+		bits: 8,
+		reason: "loopback"
+	},
+	{
+		base: ip4("169.254.0.0"),
+		bits: 16,
+		reason: "link_local"
+	},
+	{
+		base: ip4("172.16.0.0"),
+		bits: 12,
+		reason: "private"
+	},
+	{
+		base: ip4("192.0.0.0"),
+		bits: 24,
+		reason: "reserved"
+	},
+	{
+		base: ip4("192.0.2.0"),
+		bits: 24,
+		reason: "documentation"
+	},
+	{
+		base: ip4("192.31.196.0"),
+		bits: 24,
+		reason: "reserved"
+	},
+	{
+		base: ip4("192.52.193.0"),
+		bits: 24,
+		reason: "reserved"
+	},
+	{
+		base: ip4("192.88.99.0"),
+		bits: 24,
+		reason: "reserved"
+	},
+	{
+		base: ip4("192.168.0.0"),
+		bits: 16,
+		reason: "private"
+	},
+	{
+		base: ip4("192.175.48.0"),
+		bits: 24,
+		reason: "reserved"
+	},
+	{
+		base: ip4("198.18.0.0"),
+		bits: 15,
+		reason: "reserved"
+	},
+	{
+		base: ip4("198.51.100.0"),
+		bits: 24,
+		reason: "documentation"
+	},
+	{
+		base: ip4("203.0.113.0"),
+		bits: 24,
+		reason: "documentation"
+	},
+	{
+		base: ip4("224.0.0.0"),
+		bits: 4,
+		reason: "multicast"
+	},
+	{
+		base: ip4("255.255.255.255"),
+		bits: 32,
+		reason: "broadcast"
+	},
+	{
+		base: ip4("240.0.0.0"),
+		bits: 4,
+		reason: "reserved"
+	}
+];
+function ip4(dotted) {
+	const parts = dotted.split(".").map(Number);
+	return (parts[0] << 24 >>> 0) + (parts[1] << 16) + (parts[2] << 8) + parts[3] >>> 0;
+}
+function inRange4(value, range) {
+	if (range.bits === 0) return true;
+	const mask = range.bits === 32 ? 4294967295 : -1 << 32 - range.bits >>> 0;
+	return (value & mask) >>> 0 === (range.base & mask) >>> 0;
+}
+function classify4(address) {
+	const value = ip4(address);
+	for (const range of IPV4_REFUSED) if (inRange4(value, range)) return {
+		allowed: false,
+		reason: range.reason
+	};
+	return {
+		allowed: true,
+		family: 4
+	};
+}
+/** The eight 16-bit groups of an IPv6 address (with any dotted IPv4 tail folded in). */
+function groups6(address) {
+	let text = address.toLowerCase();
+	const zone = text.indexOf("%");
+	if (zone !== -1) text = text.slice(0, zone);
+	const lastColon = text.lastIndexOf(":");
+	const tail = text.slice(lastColon + 1);
+	if (tail.includes(".")) {
+		if (!isIPv4(tail)) return null;
+		const v = ip4(tail);
+		text = `${text.slice(0, lastColon + 1)}${(v >>> 16).toString(16)}:${(v & 65535).toString(16)}`;
+	}
+	const halves = text.split("::");
+	if (halves.length > 2) return null;
+	const parse = (part) => part === "" ? [] : part.split(":").map((group) => parseInt(group, 16));
+	const head = parse(halves[0]);
+	const rest = halves.length === 2 ? parse(halves[1]) : [];
+	const missing = 8 - head.length - rest.length;
+	if (halves.length === 1 ? missing !== 0 : missing < 1) return null;
+	const all = [
+		...head,
+		...new Array(halves.length === 2 ? missing : 0).fill(0),
+		...rest
+	];
+	if (all.length !== 8 || all.some((g) => !Number.isInteger(g) || g < 0 || g > 65535)) return null;
+	return all;
+}
+function classify6(address) {
+	if (address.includes("%")) return {
+		allowed: false,
+		reason: "link_local"
+	};
+	const g = groups6(address);
+	if (g === null) return {
+		allowed: false,
+		reason: "invalid"
+	};
+	const [g0, g1, g2, g3, g4, g5] = g;
+	const firstSixZero = g0 === 0 && g1 === 0 && g2 === 0 && g3 === 0 && g4 === 0;
+	if (firstSixZero && g5 === 0) {
+		if (g[6] === 0 && g[7] === 0) return {
+			allowed: false,
+			reason: "unspecified"
+		};
+		if (g[6] === 0 && g[7] === 1) return {
+			allowed: false,
+			reason: "loopback"
+		};
+		return {
+			allowed: false,
+			reason: "ipv4_embedded"
+		};
+	}
+	if (firstSixZero && g5 === 65535) return {
+		allowed: false,
+		reason: "ipv4_embedded"
+	};
+	if (g0 === 100 && g1 === 65435) return {
+		allowed: false,
+		reason: "ipv4_embedded"
+	};
+	if ((g0 & 65280) === 65280) return {
+		allowed: false,
+		reason: "multicast"
+	};
+	if ((g0 & 65472) === 65152) return {
+		allowed: false,
+		reason: "link_local"
+	};
+	if ((g0 & 65024) === 64512) return {
+		allowed: false,
+		reason: "private"
+	};
+	if ((g0 & 57344) !== 8192) return {
+		allowed: false,
+		reason: "not_global"
+	};
+	if (g0 === 8194) return {
+		allowed: false,
+		reason: "ipv4_embedded"
+	};
+	if (g0 === 8193 && g1 === 0) return {
+		allowed: false,
+		reason: "ipv4_embedded"
+	};
+	if (g0 === 8193 && g1 === 3512) return {
+		allowed: false,
+		reason: "documentation"
+	};
+	if (g0 === 16383 && (g1 & 61440) === 0) return {
+		allowed: false,
+		reason: "documentation"
+	};
+	if (g0 === 8193 && g1 < 512) return {
+		allowed: false,
+		reason: "reserved"
+	};
+	return {
+		allowed: true,
+		family: 6
+	};
+}
+/**
+* Classifies one IP address as the network guard sees it. Accepts the forms `dns.lookup`
+* returns: a canonical dotted quad, or an IPv6 address with or without brackets.
+*/
+function classifyAddress(address) {
+	const bare = address.startsWith("[") && address.endsWith("]") ? address.slice(1, -1) : address;
+	if (isIPv4(bare)) return classify4(bare);
+	if (isIPv6(bare.split("%")[0] ?? "")) return classify6(bare);
+	return {
+		allowed: false,
+		reason: "invalid"
+	};
+}
+// src/net/guard.ts
+/**
+* The network guard of a crawler. A crawler fetches URLs that come from the sites it reads
+* (links, sitemaps, redirects), so a URL can point anywhere, including at the machine it runs
+* on or its network: the cloud metadata address, a database on a private network, a service on
+* the loopback. The guard closes that door at three points:
+*
+* 1. Before a request, the URL itself: scheme `http` or `https`, port 80 or 443, no user
+*    name or password, and an IP-literal host must be an allowed address. The URL is parsed by
+*    the WHATWG parser, which turns decimal, octal, hexadecimal and shorthand IPv4 forms
+*    (`2130706433`, `0177.0.0.1`, `0x7f.1`, `127.1`) into the dotted quad before the check.
+* 2. At connection time, in the socket connector: the host name is resolved once, every
+*    address returned must be allowed (one refused address refuses the host), and the socket
+*    is opened to the address that was checked. Nothing resolves the name a second time, so a
+*    DNS answer that changes between check and connect (DNS rebinding) cannot slip through.
+* 3. Every redirect hop is a new request, so 1 and 2 apply to each hop (done by the fetcher that uses this guard).
+*
+* The guard is not configurable in production code: `createNetworkGuard` always uses the
+* strict rules. Tests reach a local fixture server through `createFixtureNetworkGuard` in the
+* package's testing entry point (`@queryrev/ai-readiness/testing`), which allows exactly one loopback address and one port.
+*/
+/** Ports the crawler connects to. */
+var CRAWL_PORTS = Object.freeze([80, 443]);
+/** Time allowed for one host-name lookup; past it the host counts as `dns_failed`. */
+var DNS_TIMEOUT_MS = 5e3;
+/** The operating system's resolver, as a connection would use it. */
+var systemResolver = async (hostname) => {
+	return (await lookup(hostname, {
+		all: true,
+		order: "verbatim"
+	})).map((answer) => answer.address);
+};
+/** A request the guard refused, or a host name that did not resolve. The message is a code. */
+var NetworkGuardError = class extends Error {
+	code;
+	/** For `refused_address`: why the address is not allowed. */
+	reason;
+	constructor(code, reason = null) {
+		super(reason === null ? `NetworkGuard: ${code}` : `NetworkGuard: ${code} (${reason})`);
+		this.name = "NetworkGuardError";
+		this.code = code;
+		this.reason = reason;
+	}
+};
+var STRICT_RULES = Object.freeze({
+	ports: CRAWL_PORTS,
+	classify: classifyAddress
+});
+function defaultPort(protocol) {
+	return protocol === "https:" ? 443 : 80;
+}
+function stripBrackets$1(host) {
+	return host.startsWith("[") && host.endsWith("]") ? host.slice(1, -1) : host;
+}
+function guardWithRules(rules, resolver, dnsTimeoutMs = DNS_TIMEOUT_MS) {
+	const checkAddress = (address) => {
+		const verdict = rules.classify(address);
+		if (!verdict.allowed) throw new NetworkGuardError("refused_address", verdict.reason);
+	};
+	const checkPort = (port) => {
+		if (!rules.ports.includes(port)) throw new NetworkGuardError("refused_port");
+	};
+	return Object.freeze({
+		checkUrl(url) {
+			if (url.protocol !== "http:" && url.protocol !== "https:") throw new NetworkGuardError("refused_scheme");
+			if (url.username !== "" || url.password !== "") throw new NetworkGuardError("refused_scheme");
+			const host = stripBrackets$1(url.hostname);
+			if (isIP(host) !== 0) checkAddress(host);
+			checkPort(url.port === "" ? defaultPort(url.protocol) : Number(url.port));
+		},
+		async vet(hostname, port) {
+			checkPort(port);
+			const host = stripBrackets$1(hostname);
+			if (host === "") throw new NetworkGuardError("dns_failed");
+			if (isIP(host) !== 0) {
+				checkAddress(host);
+				return host;
+			}
+			let addresses;
+			let timer;
+			try {
+				addresses = await Promise.race([resolver(host), new Promise((_, reject) => {
+					timer = setTimeout(() => reject(/* @__PURE__ */ new Error("dns_timeout")), dnsTimeoutMs);
+				})]);
+			} catch {
+				throw new NetworkGuardError("dns_failed");
+			} finally {
+				clearTimeout(timer);
+			}
+			if (addresses.length === 0) throw new NetworkGuardError("dns_failed");
+			for (const address of addresses) checkAddress(address);
+			return stripBrackets$1(addresses[0]);
+		}
+	});
+}
+/** The production guard: ports 80 and 443, public addresses only. */
+function createNetworkGuard(options = {}) {
+	return guardWithRules(STRICT_RULES, options.resolver ?? systemResolver, options.dnsTimeoutMs ?? 5e3);
+}
+/**
+* An undici socket connector that vets every new connection through the guard and connects to
+* the vetted address. The `Host` header and the TLS server name still come from the URL (undici
+* derives both from the origin, not from the address it connects to), so certificates are
+* verified against the site's name.
+*/
+function guardedConnector(guard, connectTimeoutMs, testCa) {
+	const connect = buildConnector({
+		timeout: connectTimeoutMs,
+		allowH2: false,
+		...testCa === void 0 ? {} : { ca: testCa }
+	});
+	return (options, callback) => {
+		const port = options.port === "" ? defaultPort(options.protocol) : Number(options.port);
+		guard.vet(options.hostname, port).then((address) => connect({
+			...options,
+			hostname: address
+		}, callback), (error) => callback(error instanceof Error ? error : new NetworkGuardError("refused_address"), null));
+	};
+}
+// src/net/fetch-text.ts
+/**
+* `fetchText`: the one way this library reads a file from a site. The library is run by
+* strangers against URLs they type, and a site's answer (a redirect, a sitemap entry, an
+* llms.txt link) can point anywhere, so every request is held to these rules:
+*
+* - The URL is parsed by the WHATWG parser. Only `http` and `https`, only ports 80 and 443, no
+*   user name or password, a host name (an IP-literal host is refused, in every spelling the
+*   parser accepts), at most 253 characters.
+* - The connection goes through the network guard: the name is resolved once, every address
+*   must be public (private, loopback, link-local, cloud metadata and the other special ranges
+*   are refused), and the socket is opened to the address that was checked, so a DNS answer
+*   that changes between check and connect cannot slip through.
+* - At most 3 redirects, each checked again as a new request; the final URL and host are
+*   reported.
+* - 5 seconds to connect, 10 seconds a request (network time of the whole redirect chain), and
+*   the caller's overall budget through `signal`.
+* - A size cap per kind of file, counted after decompression; a longer body is cut there and
+*   reported as cut.
+* - GET only, no cookies, no credentials, no proxy from the environment; the body is read as
+*   UTF-8 text.
+* - Politeness per host: one request a second, `Crawl-delay` up to 10 seconds, and no further
+*   request to a host that answered 429 or 503.
+*
+* Nothing a site does makes it throw: every failure is an `outcome` on the result.
+*/
+/** Who the library says it is. */
+var AI_READINESS_USER_AGENT = "Mozilla/5.0 (compatible; QueryRev-AIReadiness/1.0; +https://queryrev.com/bot)";
+/** Size caps in bytes, counted after decompression. */
+var FETCH_LIMITS = Object.freeze({
+	robots: 512e3,
+	llms: 1048576,
+	page: 2097152,
+	sitemap: 5242880
+});
+/** Redirects followed after the first request. */
+var MAX_REDIRECTS = 3;
+/** Longest host name accepted, in characters (as the parser writes it, punycode for IDNs). */
+var MAX_HOST_LENGTH = 253;
+/** Time allowed to open a connection (TCP, and TLS for https). */
+var CONNECT_TIMEOUT_MS = 5e3;
+/** Time allowed for one `fetchText` call, redirects included; waits for politeness excluded. */
+var REQUEST_TIMEOUT_MS = 1e4;
+/** Overall budgets a caller passes through `signal`: a site check, and llms.txt generation. */
+var CHECK_BUDGET_MS = 3e4;
+var GENERATE_BUDGET_MS = 12e4;
+/** Politeness: the smallest gap between two request starts to one host. */
+var HOST_INTERVAL_MS = 1e3;
+/** `Crawl-delay` is honoured up to this many seconds. */
+var MAX_CRAWL_DELAY_SECONDS = 10;
+/**
+* Compressed bytes read before decoding stops, above the decoded cap. Compression never makes
+* real text much larger, so this only bounds a stream that would never decode to the cap.
+*/
+var WIRE_SLACK_BYTES = 65536;
+var REDIRECT_STATUSES = /* @__PURE__ */ new Set([
+	301,
+	302,
+	303,
+	307,
+	308
+]);
+var ACCEPT = Object.freeze({
+	robots: "text/plain, */*;q=0.5",
+	llms: "text/markdown, text/plain;q=0.9, */*;q=0.5",
+	page: "text/html, application/xhtml+xml;q=0.9, */*;q=0.5",
+	sitemap: "application/xml, text/xml;q=0.9, */*;q=0.5"
+});
+/** Response headers passed on. Everything else, cookies first, is dropped. */
+var KEPT_HEADERS = Object.freeze([
+	"content-type",
+	"content-language",
+	"last-modified",
+	"etag",
+	"x-robots-tag",
+	"retry-after"
+]);
+/**
+* The production fetcher, with the strict guard: public addresses only. There is no option that
+* loosens it; the private-network opt-in for a command-line user lives in the `./cli` entry
+* point only.
+*
+* Use one fetcher per check (or per generation run) and call `close()` after: the fetcher
+* holds that run's connections and its politeness state.
+*/
+function createTextFetcher(options = {}) {
+	const resolver = options.resolver;
+	return buildTextFetcher({ guard: createNetworkGuard(resolver === void 0 ? {} : { resolver }) });
+}
+/** A host as politeness and reports key it: lower case, without one trailing dot. */
+function hostKey(hostname) {
+	const lower = hostname.toLowerCase();
+	return lower.endsWith(".") ? lower.slice(0, -1) : lower;
+}
+function stripBrackets(host) {
+	return host.startsWith("[") && host.endsWith("]") ? host.slice(1, -1) : host;
+}
+/** The URL rules applied before the guard. Exported for its tests only. */
+function checkTextUrl(input) {
+	let url;
+	try {
+		url = typeof input === "string" ? new URL(input) : input;
+	} catch {
+		return { refusal: "invalid_url" };
+	}
+	if (url.protocol !== "http:" && url.protocol !== "https:") return { refusal: "scheme" };
+	if (url.username !== "" || url.password !== "") return { refusal: "credentials" };
+	const host = stripBrackets(url.hostname);
+	if (host === "") return { refusal: "invalid_url" };
+	if (isIP(host) !== 0) return { refusal: "ip_literal" };
+	if (url.hostname.length > 253) return { refusal: "host_too_long" };
+	return { url };
+}
+function headerValues(headers, name) {
+	const value = headers[name];
+	if (value === void 0) return [];
+	return Array.isArray(value) ? value : [value];
+}
+function keptHeaders(headers) {
+	const kept = {};
+	for (const name of KEPT_HEADERS) {
+		const values = headerValues(headers, name);
+		if (values.length > 0) kept[name] = Object.freeze(values);
+	}
+	return kept;
+}
+function mediaType(headers) {
+	const value = headerValues(headers, "content-type")[0];
+	if (value === void 0) return null;
+	const type = value.split(";")[0].trim().toLowerCase();
+	return type === "" ? null : type;
+}
+function decoderFor(encoding) {
+	switch (encoding) {
+		case "":
+		case "identity": return null;
+		case "gzip":
+		case "x-gzip": return createGunzip();
+		case "deflate": return createInflate();
+		case "br": return createBrotliDecompress();
+		default: return "unknown";
+	}
+}
+function discard(body) {
+	body.on("error", () => {});
+	body.destroy();
+}
+function isZlibError(error) {
+	return error instanceof Error && typeof error.code === "string" && error.code.startsWith("Z_");
+}
+/**
+* Reads a body up to `limit` decoded bytes. Decoding stops at the cap, so a small compressed
+* bomb cannot grow past it, and the compressed stream itself is bounded too.
+*/
+async function readLimited(body, encodingHeader, limit) {
+	const encodings = encodingHeader.split(",").map((part) => part.trim().toLowerCase()).filter(Boolean);
+	const decoder = encodings.length > 1 ? "unknown" : decoderFor(encodings[0] ?? "");
+	if (decoder === "unknown") {
+		discard(body);
+		return {
+			body: null,
+			truncated: false,
+			decodeFailed: true
+		};
+	}
+	let wireTruncated = false;
+	let source;
+	if (decoder === null) source = body;
+	else {
+		let wireBytes = 0;
+		const counter = new Transform({ transform(chunk, _encoding, callback) {
+			wireBytes += chunk.length;
+			if (wireBytes > limit + WIRE_SLACK_BYTES) {
+				wireTruncated = true;
+				callback(/* @__PURE__ */ new Error("wire_limit"));
+				return;
+			}
+			callback(null, chunk);
+		} });
+		pipeline(body, counter, decoder, () => {});
+		source = decoder;
+	}
+	const chunks = [];
+	let kept = 0;
+	let truncated = false;
+	let decodeFailed = false;
+	try {
+		for await (const chunk of source) {
+			const room = limit - kept;
+			if (chunk.length > room) {
+				chunks.push(chunk.subarray(0, room));
+				kept += room;
+				truncated = true;
+				break;
+			}
+			chunks.push(chunk);
+			kept += chunk.length;
+		}
+	} catch (error) {
+		if (wireTruncated) truncated = true;
+		else if (decoder !== null && isZlibError(error)) decodeFailed = true;
+		else throw error;
+	} finally {
+		discard(body);
+		if (decoder !== null) decoder.destroy();
+	}
+	if (wireTruncated) truncated = true;
+	if (decodeFailed) return {
+		body: null,
+		truncated: false,
+		decodeFailed: true
+	};
+	return {
+		body: Buffer.concat(chunks, kept),
+		truncated,
+		decodeFailed: false
+	};
+}
+function errorChain(error) {
+	const chain = [];
+	let current = error;
+	while (current !== void 0 && current !== null && chain.length < 5) {
+		chain.push(current);
+		current = current instanceof Error ? current.cause : void 0;
+	}
+	return chain;
+}
+var TIMEOUT_CODES = /* @__PURE__ */ new Set([
+	"UND_ERR_HEADERS_TIMEOUT",
+	"UND_ERR_BODY_TIMEOUT",
+	"UND_ERR_CONNECT_TIMEOUT",
+	"ETIMEDOUT"
+]);
+function classifyError(error, budget) {
+	for (const item of errorChain(error)) {
+		if (!(item instanceof NetworkGuardError)) continue;
+		if (item.code === "refused_address") return {
+			outcome: "refused_address",
+			refusal: item.reason
+		};
+		if (item.code === "refused_port") return {
+			outcome: "refused_url",
+			refusal: "port"
+		};
+		if (item.code === "refused_scheme") return {
+			outcome: "refused_url",
+			refusal: "scheme"
+		};
+		return {
+			outcome: "dns_failed",
+			refusal: null
+		};
+	}
+	if (budget?.aborted === true) return {
+		outcome: "budget_exceeded",
+		refusal: null
+	};
+	for (const item of errorChain(error)) {
+		if (!(item instanceof Error)) continue;
+		const code = item.code;
+		if (item.name === "TimeoutError" || item.name === "AbortError") return {
+			outcome: "timeout",
+			refusal: null
+		};
+		if (code !== void 0 && TIMEOUT_CODES.has(code)) return {
+			outcome: "timeout",
+			refusal: null
+		};
+	}
+	return {
+		outcome: "network_error",
+		refusal: null
+	};
+}
+/** Waits `ms`, or rejects when the signal aborts first. */
+function wait(ms, signal) {
+	if (ms <= 0) return signal?.aborted === true ? Promise.reject(/* @__PURE__ */ new Error("aborted")) : Promise.resolve();
+	return new Promise((resolve, reject) => {
+		if (signal?.aborted === true) {
+			reject(/* @__PURE__ */ new Error("aborted"));
+			return;
+		}
+		const timer = setTimeout(() => {
+			signal?.removeEventListener("abort", onAbort);
+			resolve();
+		}, ms);
+		const onAbort = () => {
+			clearTimeout(timer);
+			reject(/* @__PURE__ */ new Error("aborted"));
+		};
+		signal?.addEventListener("abort", onAbort, { once: true });
+	});
+}
+function remember(map, key, value) {
+	map.delete(key);
+	map.set(key, value);
+	if (map.size > 1e3) map.delete(map.keys().next().value);
+}
+/**
+* One request a second per host, `Crawl-delay` up to 10 s, stop after 429 or 503. The state is
+* bounded: start slots already in the past are dropped, and each table keeps at most
+* MAX_TRACKED_HOSTS hosts (a check reads a handful; only a site that redirects to thousands of
+* hosts reaches the bound). Exported for its tests only.
+*/
+var HostPacing = class {
+	baseIntervalMs;
+	now;
+	nextAt = /* @__PURE__ */ new Map();
+	interval = /* @__PURE__ */ new Map();
+	stopped = /* @__PURE__ */ new Map();
+	constructor(baseIntervalMs, now = Date.now) {
+		this.baseIntervalMs = baseIntervalMs;
+		this.now = now;
+	}
+	/** Entries held per table, for tests of the bound. */
+	get sizes() {
+		return {
+			nextAt: this.nextAt.size,
+			interval: this.interval.size,
+			stopped: this.stopped.size
+		};
+	}
+	intervalFor(host) {
+		return this.interval.get(hostKey(host)) ?? this.baseIntervalMs;
+	}
+	setCrawlDelay(host, seconds) {
+		if (seconds === null || !Number.isFinite(seconds) || !(seconds > 0)) return;
+		const key = hostKey(host);
+		const delayMs = Math.min(seconds, 10) * 1e3;
+		const current = this.intervalFor(key);
+		if (delayMs <= current) return;
+		remember(this.interval, key, delayMs);
+		const next = this.nextAt.get(key);
+		if (next !== void 0) this.nextAt.set(key, next + delayMs - current);
+	}
+	stop(host) {
+		remember(this.stopped, hostKey(host), true);
+	}
+	isStopped(host) {
+		return this.stopped.has(hostKey(host));
+	}
+	/** Reserves the next start slot for a host synchronously, then waits for it. */
+	async acquire(host, signal) {
+		const key = hostKey(host);
+		const now = this.now();
+		for (const [other, at] of this.nextAt) if (at <= now) this.nextAt.delete(other);
+		const at = Math.max(now, this.nextAt.get(key) ?? 0);
+		remember(this.nextAt, key, at + this.intervalFor(key));
+		await wait(at - now, signal);
+	}
+};
+/** The fetcher with the given guard and timings. Production reaches it only with the defaults. */
+function buildTextFetcher(config) {
+	const connectTimeoutMs = config.connectTimeoutMs ?? 5e3;
+	const requestTimeoutMs = config.requestTimeoutMs ?? 1e4;
+	const guard = config.guard;
+	const pacing = new HostPacing(config.hostIntervalMs ?? 1e3);
+	const agent = new Agent({
+		connect: guardedConnector(guard, connectTimeoutMs, config.testCa),
+		connections: 1,
+		pipelining: 1,
+		headersTimeout: requestTimeoutMs,
+		bodyTimeout: requestTimeoutMs
+	});
+	async function fetchText(start, options) {
+		if (!Object.hasOwn(FETCH_LIMITS, options.kind)) throw new TypeError("fetchText: unknown kind");
+		const cap = FETCH_LIMITS[options.kind];
+		const asked = options.maxBytes;
+		const limitBytes = typeof asked === "number" && Number.isFinite(asked) && asked >= 0 ? Math.min(Math.floor(asked), cap) : cap;
+		const budget = options.signal;
+		const redirects = [];
+		const seen = /* @__PURE__ */ new Set();
+		let requests = 0;
+		let spentMs = 0;
+		let currentHref = start;
+		const finish = (outcome, refusal, extra = {}) => {
+			let finalHost = "";
+			try {
+				finalHost = hostKey(new URL(currentHref).hostname);
+			} catch {
+				finalHost = "";
+			}
+			return Object.freeze({
+				outcome,
+				refusal,
+				requestedUrl: start,
+				finalUrl: currentHref,
+				finalHost,
+				redirects: Object.freeze([...redirects]),
+				status: null,
+				contentType: null,
+				headers: Object.freeze({}),
+				text: null,
+				rawBody: null,
+				bytes: 0,
+				limitBytes,
+				truncated: false,
+				decodeFailed: false,
+				requests,
+				elapsedMs: spentMs,
+				...extra
+			});
+		};
+		let current;
+		for (;;) {
+			const checked = checkTextUrl(currentHref);
+			if ("refusal" in checked) return finish("refused_url", checked.refusal);
+			current = checked.url;
+			currentHref = current.href;
+			try {
+				guard.checkUrl(current);
+			} catch (error) {
+				const { outcome, refusal } = classifyError(error, budget);
+				return finish(outcome, refusal);
+			}
+			seen.add(current.href);
+			if (pacing.isStopped(current.hostname)) return finish("host_stopped", null);
+			try {
+				await pacing.acquire(current.hostname, budget);
+			} catch {
+				return finish("budget_exceeded", null);
+			}
+			const remainingMs = requestTimeoutMs - spentMs;
+			if (remainingMs <= 0) return finish("timeout", null);
+			const timeout = AbortSignal.timeout(Math.ceil(remainingMs));
+			const signal = budget === void 0 ? timeout : AbortSignal.any([budget, timeout]);
+			const started = performance.now();
+			requests += 1;
+			let response;
+			try {
+				response = await request(current, {
+					dispatcher: agent,
+					method: "GET",
+					headers: {
+						"user-agent": AI_READINESS_USER_AGENT,
+						accept: ACCEPT[options.kind],
+						"accept-encoding": "gzip, deflate, br"
+					},
+					signal
+				});
+			} catch (error) {
+				spentMs += performance.now() - started;
+				const { outcome, refusal } = classifyError(error, budget);
+				return finish(outcome, refusal);
+			}
+			const status = response.statusCode;
+			if (status === 429 || status === 503) pacing.stop(current.hostname);
+			const location = headerValues(response.headers, "location")[0];
+			if (REDIRECT_STATUSES.has(status) && location !== void 0) {
+				await response.body.dump({ limit: 65536 }).catch(() => {});
+				spentMs += performance.now() - started;
+				let target;
+				try {
+					target = new URL(location, current);
+				} catch {
+					target = null;
+				}
+				redirects.push(Object.freeze({
+					url: current.href,
+					status,
+					location: target?.href ?? null
+				}));
+				if (target === null) return finish("invalid_redirect", null, { status });
+				if (seen.has(target.href)) return finish("redirect_loop", null, { status });
+				if (redirects.length > 3) return finish("too_many_redirects", null, { status });
+				if (options.redirectAllowed !== void 0 && !options.redirectAllowed(target)) return finish("redirect_refused", null, { status });
+				currentHref = target.href;
+				continue;
+			}
+			const contentType = mediaType(response.headers);
+			let read;
+			try {
+				const encoding = headerValues(response.headers, "content-encoding").join(",");
+				read = await readLimited(response.body, encoding, limitBytes);
+			} catch (error) {
+				discard(response.body);
+				spentMs += performance.now() - started;
+				const { outcome, refusal } = classifyError(error, budget);
+				return finish(outcome, refusal, {
+					status,
+					contentType
+				});
+			}
+			spentMs += performance.now() - started;
+			return finish("response", null, {
+				status,
+				contentType,
+				headers: Object.freeze(keptHeaders(response.headers)),
+				text: read.body === null ? null : new TextDecoder("utf-8").decode(read.body, { stream: read.truncated }),
+				rawBody: read.body === null ? null : new Uint8Array(read.body),
+				bytes: read.body?.length ?? 0,
+				truncated: read.truncated,
+				decodeFailed: read.decodeFailed
+			});
+		}
+	}
+	return Object.freeze({
+		fetchText,
+		setCrawlDelay: (host, seconds) => pacing.setCrawlDelay(host, seconds),
+		isStopped: (host) => pacing.isStopped(host),
+		close: () => agent.close()
+	});
+}
+export { classifyAddress as C, systemResolver as S, NetworkGuardError as _, GENERATE_BUDGET_MS as a, guardWithRules as b, MAX_CRAWL_DELAY_SECONDS as c, REQUEST_TIMEOUT_MS as d, buildTextFetcher as f, DNS_TIMEOUT_MS as g, CRAWL_PORTS as h, FETCH_LIMITS as i, MAX_HOST_LENGTH as l, createTextFetcher as m, CHECK_BUDGET_MS as n, HOST_INTERVAL_MS as o, checkTextUrl as p, CONNECT_TIMEOUT_MS as r, KEPT_HEADERS as s, AI_READINESS_USER_AGENT as t, MAX_REDIRECTS as u, STRICT_RULES as v, guardedConnector as x, createNetworkGuard as y };