@queryrev/ai-readiness 0.0.0-stage → 1.0.1

package/dist/llms/findings.d.ts

@@ -0,0 +1,7 @@
+import type { Finding, FindingLevel } from '../facts/types.js';
+import type { LlmsReport } from './index.js';
+/** The finding ids this module can return. */
+export declare const LLMS_FINDING_IDS: readonly string[];
+export declare function llmsFinding(id: string, level: FindingLevel, message: string, snippet?: string): Finding;
+/** The findings for one validated answer. */
+export declare function llmsFindings(report: LlmsReport): Finding[];

package/dist/llms/index.d.ts

@@ -0,0 +1,88 @@
+/** llms.txt checks: telling the file from a page, and its structure (llmstxt.org). */
+/** Whether a body is an HTML document (a site that answers every address with a page). */
+export declare function looksLikeHtml(contentType: string | null, body: string): boolean;
+/**
+ * A robots.txt served at another address (a site that answers every path with it): a
+ * `User-agent` line and no Markdown link. It is not an llms.txt, whatever its first line.
+ */
+export declare function looksLikeRobotsTxt(body: string): boolean;
+/**
+ * The structure of an llms.txt, as llmstxt.org describes the file: a Markdown H1 with the
+ * site's name first (required), then a blockquote summary, then sections under H2 headings
+ * that list links (`- [name](url): notes`).
+ *
+ * - `summary`: a blockquote line (`> …`) before the first H2.
+ * - `sections`: H2 headings with at least one linked list item under them.
+ * - `links`: Markdown links anywhere in the file (images not counted).
+ * - `structured`: an H1 and at least one such section.
+ */
+export interface LlmsStructure {
+    readonly bytes: number;
+    readonly summary: boolean;
+    readonly sections: number;
+    readonly links: number;
+    readonly structured: boolean;
+    /** The link targets as written, in order. */
+    readonly targets: readonly string[];
+}
+/**
+ * Characters of a file read for its structure. An llms.txt is meant to be a short map; a
+ * longer file is checked on its start, so the work stays small whatever a site serves.
+ */
+export declare const LLMS_STRUCTURE_MAX_CHARS: number;
+export declare function llmsStructure(body: string): LlmsStructure;
+/**
+ * Whether an answer is an llms.txt: a 200, a text type that is not HTML, not a robots.txt
+ * served at that address, and a body whose first non-empty line is a Markdown H1 (`# Name`),
+ * which is the one part llmstxt.org requires.
+ */
+export declare function looksLikeLlmsTxt(status: number | null, contentType: string | null, body: string): boolean;
+/** Why an answer is or is not an llms.txt. */
+export type LlmsVerdict = 
+/** An llms.txt: a 200 with a text type, not a page or a robots.txt, starting with an H1. */
+'llms_txt'
+/** No answer arrived (refused, unreachable, timed out). */
+ | 'no_answer'
+/** 404 or 410: there is no file. */
+ | 'not_found'
+/** Another status than 200. */
+ | 'status'
+/** A 200 with a type that is not text. */
+ | 'not_text'
+/** A 200 that is an HTML page (a site that answers every address with a page). */
+ | 'html'
+/** A 200 that is a robots.txt served at this address. */
+ | 'robots'
+/** A 200 that is text but does not start with a Markdown H1. */
+ | 'no_h1';
+/** The validation of one answer at `/llms.txt`. */
+export interface LlmsReport {
+    /** Whether the answer is an llms.txt (the soft-404 rule: a 200 alone is not enough). */
+    readonly present: boolean;
+    readonly verdict: LlmsVerdict;
+    readonly status: number | null;
+    readonly contentType: string | null;
+    /** Bytes of the body read (after decompression). */
+    readonly bytes: number;
+    /** The body went over the size cap and was cut there. */
+    readonly truncated: boolean;
+    /** The first H1's text, as written, cut to 200 characters; null without one. */
+    readonly h1: string | null;
+    /** H1 headings outside code blocks; llmstxt.org asks for exactly one. */
+    readonly h1Count: number;
+    /** The structure, when the answer is an llms.txt. */
+    readonly structure: LlmsStructure | null;
+}
+/** Characters of an H1 kept in a report. */
+export declare const LLMS_H1_MAX = 200;
+/**
+ * Validates an answer at `/llms.txt` against the llmstxt.org format, after the soft-404 rule:
+ * an answer that is not a 200, is HTML, is a robots.txt, or does not start with an H1 is not
+ * an llms.txt, whatever its status. Pure: pass what was fetched.
+ */
+export declare function validateLlmsTxt(answer: {
+    readonly status: number | null;
+    readonly contentType: string | null;
+    readonly body: string | null;
+    readonly truncated: boolean;
+}): LlmsReport;

package/dist/llms/links.d.ts

@@ -0,0 +1,46 @@
+import type { Finding } from '../facts/types.js';
+import { type TextFetcher } from '../net/fetch-text.js';
+/** Links checked in one run. */
+export declare const LLMS_LINKS_MAX = 20;
+/** Bytes of each linked page that are read. */
+export declare const LLMS_LINK_BODY_BYTES: number;
+export type LlmsLinkOutcome = 
+/** A 2xx answer (after at most 3 redirects on the same site). */
+'ok'
+/** 404 or 410. */
+ | 'missing'
+/** Another 4xx or 5xx answer. */
+ | 'error_status'
+/** No answer, a refused address, a redirect off the site, or a loop. */
+ | 'unreachable'
+/** The robots.txt of the link's own origin keeps this reader from it, so it was not requested. */
+ | 'not_allowed'
+/** The robots.txt of the link's own origin could not be read, so the link was not requested. */
+ | 'robots_unreadable'
+/** The time budget ran out before the link was asked. */
+ | 'not_checked';
+export interface LlmsLinkResult {
+    /** The link as an absolute address. */
+    readonly url: string;
+    readonly outcome: LlmsLinkOutcome;
+    readonly status: number | null;
+}
+export interface LlmsLinksReport {
+    /** Links in the file that were not checked because they point at another site or are not web addresses. */
+    readonly skipped: number;
+    /** Links in the file beyond the first 20 that were left unchecked. */
+    readonly beyondLimit: number;
+    /** False when robots.txt could not be read, so no link was requested. */
+    readonly robotsRead: boolean;
+    readonly results: readonly LlmsLinkResult[];
+}
+export interface CheckLlmsLinksOptions {
+    readonly fetcher: TextFetcher;
+    /** The site's address; relative links are resolved against it, and only its site is checked. */
+    readonly base: string;
+    readonly signal?: AbortSignal;
+    readonly registrableDomain?: (host: string) => string;
+}
+export declare function checkLlmsLinks(targets: readonly string[], options: CheckLlmsLinksOptions): Promise<LlmsLinksReport>;
+/** The findings of a link check. One finding for the set; the evidence lists what is wrong. */
+export declare function llmsLinkFindings(report: LlmsLinksReport): Finding[];

package/dist/net/address.d.ts

@@ -0,0 +1,26 @@
+/**
+ * Which IP addresses a crawler may connect to (the network guard).
+ *
+ * The rule is an allow-list, not a block-list: an IPv4 address is allowed only when it is in
+ * none of the special-purpose ranges of the IANA registry, and an IPv6 address only when it is
+ * global unicast (`2000::/3`) outside the special-purpose blocks inside that range. Anything
+ * this module cannot parse as a canonical IPv4 dotted quad or an IPv6 address is refused, so a
+ * resolver answer in an odd form (`0177.0.0.1`, `2130706433`) fails closed.
+ *
+ * IPv6 forms that carry an IPv4 address (IPv4-mapped `::ffff:a.b.c.d`, IPv4-compatible
+ * `::a.b.c.d`, NAT64 `64:ff9b::/96`, 6to4 `2002::/16`, Teredo `2001::/32`) are refused outright:
+ * a site does not need them to be crawled, and each is a way to reach an internal IPv4 address.
+ */
+export type AddressRefusal = 'invalid' | 'unspecified' | 'loopback' | 'private' | 'link_local' | 'shared' | 'reserved' | 'documentation' | 'multicast' | 'broadcast' | 'ipv4_embedded' | 'not_global' | 'cloud_metadata';
+export type AddressClass = {
+    readonly allowed: true;
+    readonly family: 4 | 6;
+} | {
+    readonly allowed: false;
+    readonly reason: AddressRefusal;
+};
+/**
+ * Classifies one IP address as the network guard sees it. Accepts the forms `dns.lookup`
+ * returns: a canonical dotted quad, or an IPv6 address with or without brackets.
+ */
+export declare function classifyAddress(address: string): AddressClass;

package/dist/net/charset.d.ts

@@ -0,0 +1,20 @@
+/**
+ * Reading a page's bytes as text. One decoder for the library and for any program that uses it,
+ * so a page in windows-1251, Shift_JIS, GBK or EUC-KR is counted the same everywhere. The order
+ * is the one browsers use, shortened: a byte order mark; the `charset` of the Content-Type
+ * header; a `<meta charset>` or `<meta http-equiv="Content-Type">` in the first 2 KiB; else
+ * UTF-8. Labels are the WHATWG Encoding Standard's; a label it does not know is skipped.
+ */
+export interface DecodedText {
+    readonly text: string;
+    /** The encoding the bytes were read as (a WHATWG name, such as `windows-1251`). */
+    readonly charset: string;
+}
+/** The character set of a page by the rule above. */
+export declare function detectCharset(bytes: Uint8Array, contentTypeHeader: string | null): string;
+/**
+ * The bytes of an HTML page as text. `contentTypeHeader` is the raw `Content-Type` header, with
+ * its parameters. A body cut at a size cap (`truncated`) loses no character to the cut: an
+ * incomplete one at the end is held back.
+ */
+export declare function decodeHtml(bytes: Uint8Array, contentTypeHeader: string | null, truncated?: boolean): DecodedText;

package/dist/net/cli.d.ts

@@ -0,0 +1,13 @@
+import { type AddressClass } from './address.js';
+import { type TextFetcher, type TextFetcherOptions } from './fetch-text.js';
+import { type HostResolver, type NetworkGuard } from './guard.js';
+/** The address rule of the private-network opt-in. */
+export declare function classifyPrivateNetworkAddress(address: string): AddressClass;
+/** The guard of the opt-in: ports 80 and 443, the address rule above. */
+export declare function createPrivateNetworkGuard(resolver?: HostResolver): NetworkGuard;
+export interface CliTextFetcherOptions extends TextFetcherOptions {
+    /** Set only from the user's explicit command-line flag. Off by default. */
+    readonly allowPrivateNetwork?: boolean;
+}
+/** The CLI's fetcher: strict unless the user asked for the private network. One per run. */
+export declare function createCliTextFetcher(options?: CliTextFetcherOptions): TextFetcher;

package/dist/net/fetch-text.d.ts

@@ -0,0 +1,191 @@
+import type { AddressRefusal } from './address.js';
+import { type HostResolver, type NetworkGuard } from './guard.js';
+/**
+ * `fetchText`: the one way this library reads a file from a site. The library is run by
+ * strangers against URLs they type, and a site's answer (a redirect, a sitemap entry, an
+ * llms.txt link) can point anywhere, so every request is held to these rules:
+ *
+ * - The URL is parsed by the WHATWG parser. Only `http` and `https`, only ports 80 and 443, no
+ *   user name or password, a host name (an IP-literal host is refused, in every spelling the
+ *   parser accepts), at most 253 characters.
+ * - The connection goes through the network guard: the name is resolved once, every address
+ *   must be public (private, loopback, link-local, cloud metadata and the other special ranges
+ *   are refused), and the socket is opened to the address that was checked, so a DNS answer
+ *   that changes between check and connect cannot slip through.
+ * - At most 3 redirects, each checked again as a new request; the final URL and host are
+ *   reported.
+ * - 5 seconds to connect, 10 seconds a request (network time of the whole redirect chain), and
+ *   the caller's overall budget through `signal`.
+ * - A size cap per kind of file, counted after decompression; a longer body is cut there and
+ *   reported as cut.
+ * - GET only, no cookies, no credentials, no proxy from the environment; the body is read as
+ *   UTF-8 text.
+ * - Politeness per host: one request a second, `Crawl-delay` up to 10 seconds, and no further
+ *   request to a host that answered 429 or 503.
+ *
+ * Nothing a site does makes it throw: every failure is an `outcome` on the result.
+ */
+/** Who the library says it is. */
+export declare const AI_READINESS_USER_AGENT = "Mozilla/5.0 (compatible; QueryRev-AIReadiness/1.0; +https://queryrev.com/bot)";
+/** The kinds of file the library reads, each with its own size cap. */
+export type FetchKind = 'robots' | 'llms' | 'page' | 'sitemap';
+/** Size caps in bytes, counted after decompression. */
+export declare const FETCH_LIMITS: Readonly<Record<FetchKind, number>>;
+/** Redirects followed after the first request. */
+export declare const MAX_REDIRECTS = 3;
+/** Longest host name accepted, in characters (as the parser writes it, punycode for IDNs). */
+export declare const MAX_HOST_LENGTH = 253;
+/** Time allowed to open a connection (TCP, and TLS for https). */
+export declare const CONNECT_TIMEOUT_MS = 5000;
+/** Time allowed for one `fetchText` call, redirects included; waits for politeness excluded. */
+export declare const REQUEST_TIMEOUT_MS = 10000;
+/** Overall budgets a caller passes through `signal`: a site check, and llms.txt generation. */
+export declare const CHECK_BUDGET_MS = 30000;
+export declare const GENERATE_BUDGET_MS = 120000;
+/** Politeness: the smallest gap between two request starts to one host. */
+export declare const HOST_INTERVAL_MS = 1000;
+/** `Crawl-delay` is honoured up to this many seconds. */
+export declare const MAX_CRAWL_DELAY_SECONDS = 10;
+/** Response headers passed on. Everything else, cookies first, is dropped. */
+export declare const KEPT_HEADERS: readonly ["content-type", "content-language", "last-modified", "etag", "x-robots-tag", "retry-after"];
+export type KeptHeader = (typeof KEPT_HEADERS)[number];
+/** Why a URL was refused before any connection. */
+export type UrlRefusal = 'invalid_url' | 'scheme' | 'credentials' | 'ip_literal' | 'host_too_long' | 'port';
+export type TextFetchOutcome = 
+/** An HTTP answer that is not a followed redirect (any status). */
+'response'
+/** The URL (or a redirect target) broke a URL rule; `refusal` says which. */
+ | 'refused_url'
+/** The host resolved to an address that is not allowed; `refusal` says why. */
+ | 'refused_address' | 'dns_failed' | 'too_many_redirects' | 'redirect_loop'
+/** A redirect without a usable `Location`. */
+ | 'invalid_redirect'
+/** A redirect the caller's `redirectAllowed` refused; its target was not contacted. */
+ | 'redirect_refused'
+/** The connection or the request ran out of its time. */
+ | 'timeout'
+/** The caller's overall budget (`signal`) ran out. */
+ | 'budget_exceeded'
+/** The host answered 429 or 503 earlier; no further request is sent to it. */
+ | 'host_stopped' | 'network_error';
+export interface RedirectHop {
+    readonly url: string;
+    readonly status: number;
+    /** Where it pointed, resolved against `url`; null when `Location` was unusable. */
+    readonly location: string | null;
+}
+export interface TextFetchResult {
+    readonly outcome: TextFetchOutcome;
+    readonly refusal: UrlRefusal | AddressRefusal | null;
+    readonly requestedUrl: string;
+    /** The last URL requested or refused (the requested URL when there were no redirects). */
+    readonly finalUrl: string;
+    /** Host of `finalUrl`; empty when the URL did not parse. */
+    readonly finalHost: string;
+    readonly redirects: readonly RedirectHop[];
+    /** Status of the last answer; null when none arrived. */
+    readonly status: number | null;
+    /** Media type without parameters, lower case; null when absent. */
+    readonly contentType: string | null;
+    readonly headers: Readonly<Partial<Record<KeptHeader, readonly string[]>>>;
+    /** The body as UTF-8 text, up to the cap; null when no body was read or it did not decode. */
+    readonly text: string | null;
+    /**
+     * The same decoded bytes, for a file that is not text (a gzip sitemap served as a file, not as
+     * a content encoding); null when no body was read.
+     */
+    readonly rawBody: Uint8Array | null;
+    /** Decoded bytes kept. */
+    readonly bytes: number;
+    /** The cap that applied. */
+    readonly limitBytes: number;
+    /** The body went over the cap and was cut there. */
+    readonly truncated: boolean;
+    /** The content encoding was unknown or the compressed stream was corrupt. */
+    readonly decodeFailed: boolean;
+    /** Requests attempted, a connection the guard refused included. */
+    readonly requests: number;
+    /** Network milliseconds of the whole call, waits for politeness excluded. */
+    readonly elapsedMs: number;
+}
+export interface FetchTextOptions {
+    readonly kind: FetchKind;
+    /** A lower cap than the kind's (for example to read only the start of a page). */
+    readonly maxBytes?: number;
+    /** The caller's overall budget. When it aborts, the call ends as `budget_exceeded`. */
+    readonly signal?: AbortSignal;
+    /**
+     * Where a redirect may lead, asked before each further hop (after the URL rules and before
+     * the guard). A hop it refuses ends the call as `redirect_refused`, and its target is never
+     * contacted. The guard still checks every hop it allows.
+     */
+    readonly redirectAllowed?: (target: URL) => boolean;
+}
+export interface TextFetcher {
+    fetchText(url: string, options: FetchTextOptions): Promise<TextFetchResult>;
+    /** Applies robots.txt `Crawl-delay` (seconds) for a host, capped at 10. Only ever slows. */
+    setCrawlDelay(host: string, seconds: number | null): void;
+    /** Whether a host answered 429 or 503 and gets no further request. */
+    isStopped(host: string): boolean;
+    close(): Promise<void>;
+}
+export interface TextFetcherOptions {
+    /** The resolver; the operating system's by default. Every answer is still checked. */
+    readonly resolver?: HostResolver;
+}
+/** Settings only the testing entry point may change (fixture port, short timeouts). */
+export interface TextFetcherConfig {
+    readonly guard: NetworkGuard;
+    readonly connectTimeoutMs?: number;
+    readonly requestTimeoutMs?: number;
+    readonly hostIntervalMs?: number;
+    /** Extra trusted CA certificate; never set in production. */
+    readonly testCa?: string;
+}
+/**
+ * The production fetcher, with the strict guard: public addresses only. There is no option that
+ * loosens it; the private-network opt-in for a command-line user lives in the `./cli` entry
+ * point only.
+ *
+ * Use one fetcher per check (or per generation run) and call `close()` after: the fetcher
+ * holds that run's connections and its politeness state.
+ */
+export declare function createTextFetcher(options?: TextFetcherOptions): TextFetcher;
+/** A host as politeness and reports key it: lower case, without one trailing dot. */
+export declare function hostKey(hostname: string): string;
+/** The URL rules applied before the guard. Exported for its tests only. */
+export declare function checkTextUrl(input: string | URL): {
+    url: URL;
+} | {
+    refusal: UrlRefusal;
+};
+/** Hosts whose politeness state is kept; past it the oldest entry is dropped. */
+export declare const MAX_TRACKED_HOSTS = 1000;
+/**
+ * One request a second per host, `Crawl-delay` up to 10 s, stop after 429 or 503. The state is
+ * bounded: start slots already in the past are dropped, and each table keeps at most
+ * MAX_TRACKED_HOSTS hosts (a check reads a handful; only a site that redirects to thousands of
+ * hosts reaches the bound). Exported for its tests only.
+ */
+export declare class HostPacing {
+    private readonly baseIntervalMs;
+    private readonly now;
+    private readonly nextAt;
+    private readonly interval;
+    private readonly stopped;
+    constructor(baseIntervalMs: number, now?: () => number);
+    /** Entries held per table, for tests of the bound. */
+    get sizes(): {
+        readonly nextAt: number;
+        readonly interval: number;
+        readonly stopped: number;
+    };
+    intervalFor(host: string): number;
+    setCrawlDelay(host: string, seconds: number | null): void;
+    stop(host: string): void;
+    isStopped(host: string): boolean;
+    /** Reserves the next start slot for a host synchronously, then waits for it. */
+    acquire(host: string, signal: AbortSignal | undefined): Promise<void>;
+}
+/** The fetcher with the given guard and timings. Production reaches it only with the defaults. */
+export declare function buildTextFetcher(config: TextFetcherConfig): TextFetcher;

package/dist/net/guard.d.ts

@@ -0,0 +1,71 @@
+import { buildConnector } from 'undici';
+import { type AddressClass, type AddressRefusal } from './address.js';
+/**
+ * The network guard of a crawler. A crawler fetches URLs that come from the sites it reads
+ * (links, sitemaps, redirects), so a URL can point anywhere, including at the machine it runs
+ * on or its network: the cloud metadata address, a database on a private network, a service on
+ * the loopback. The guard closes that door at three points:
+ *
+ * 1. Before a request, the URL itself: scheme `http` or `https`, port 80 or 443, no user
+ *    name or password, and an IP-literal host must be an allowed address. The URL is parsed by
+ *    the WHATWG parser, which turns decimal, octal, hexadecimal and shorthand IPv4 forms
+ *    (`2130706433`, `0177.0.0.1`, `0x7f.1`, `127.1`) into the dotted quad before the check.
+ * 2. At connection time, in the socket connector: the host name is resolved once, every
+ *    address returned must be allowed (one refused address refuses the host), and the socket
+ *    is opened to the address that was checked. Nothing resolves the name a second time, so a
+ *    DNS answer that changes between check and connect (DNS rebinding) cannot slip through.
+ * 3. Every redirect hop is a new request, so 1 and 2 apply to each hop (done by the fetcher that uses this guard).
+ *
+ * The guard is not configurable in production code: `createNetworkGuard` always uses the
+ * strict rules. Tests reach a local fixture server through `createFixtureNetworkGuard` in the
+ * package's testing entry point (`@queryrev/ai-readiness/testing`), which allows exactly one loopback address and one port.
+ */
+/** Ports the crawler connects to. */
+export declare const CRAWL_PORTS: readonly number[];
+/** Time allowed for one host-name lookup; past it the host counts as `dns_failed`. */
+export declare const DNS_TIMEOUT_MS = 5000;
+/** Resolves a host name to its IP addresses (all of them, as `dns.lookup` with `all`). */
+export type HostResolver = (hostname: string) => Promise<readonly string[]>;
+/** The operating system's resolver, as a connection would use it. */
+export declare const systemResolver: HostResolver;
+export type GuardErrorCode = 'refused_scheme' | 'refused_port' | 'refused_address' | 'dns_failed';
+/** A request the guard refused, or a host name that did not resolve. The message is a code. */
+export declare class NetworkGuardError extends Error {
+    readonly code: GuardErrorCode;
+    /** For `refused_address`: why the address is not allowed. */
+    readonly reason: AddressRefusal | null;
+    constructor(code: GuardErrorCode, reason?: AddressRefusal | null);
+}
+export interface NetworkGuard {
+    /** Checks a URL before it is requested. Throws NetworkGuardError when it is refused. */
+    checkUrl(url: URL): void;
+    /**
+     * Resolves a host (a name or an IP literal, brackets allowed) and checks every address.
+     * Returns the address to connect to. Throws NetworkGuardError.
+     */
+    vet(hostname: string, port: number): Promise<string>;
+}
+/** The rules a guard applies. Not exported from the package: production uses the strict set. */
+export interface GuardRules {
+    readonly ports: readonly number[];
+    readonly classify: (address: string) => AddressClass;
+}
+export declare const STRICT_RULES: GuardRules;
+export declare function guardWithRules(rules: GuardRules, resolver: HostResolver, dnsTimeoutMs?: number): NetworkGuard;
+/** The production guard: ports 80 and 443, public addresses only. */
+export declare function createNetworkGuard(options?: {
+    readonly resolver?: HostResolver;
+    readonly dnsTimeoutMs?: number;
+}): NetworkGuard;
+/**
+ * An undici socket connector that vets every new connection through the guard and connects to
+ * the vetted address. The `Host` header and the TLS server name still come from the URL (undici
+ * derives both from the origin, not from the address it connects to), so certificates are
+ * verified against the site's name.
+ */
+export declare function guardedConnector(guard: NetworkGuard, connectTimeoutMs: number, 
+/**
+ * Extra trusted CA certificates. Only the testing entry point passes one (the fixture's
+ * self-signed certificate); production trusts Node's default CA store and nothing else.
+ */
+testCa?: string): buildConnector.connector;

package/dist/net/testing/fixture-fetcher.d.ts

@@ -0,0 +1,15 @@
+import { type TextFetcher } from '../fetch-text.js';
+import type { HostResolver } from '../guard.js';
+/**
+ * A text fetcher for tests against a local fixture server: the fixture guard (127.0.0.1 on one
+ * port only) and, unless a test shortens them, the production timings and politeness.
+ */
+export declare function createFixtureTextFetcher(options: {
+    readonly port: number;
+    readonly resolver: HostResolver;
+    readonly connectTimeoutMs?: number;
+    readonly requestTimeoutMs?: number;
+    readonly hostIntervalMs?: number;
+    /** PEM of a fixture's self-signed certificate, to trust in this fetcher only. */
+    readonly ca?: string;
+}): TextFetcher;

package/dist/net/testing/fixture-guard.d.ts

@@ -0,0 +1,20 @@
+import { type HostResolver, type NetworkGuard } from '../guard.js';
+/**
+ * A network guard for tests against the local fixture site. It allows exactly one address,
+ * 127.0.0.1, on exactly one port, the fixture's. Every other address is refused: with the
+ * strict rule's reason when the strict rules refuse it, and as `not_global` when they would
+ * allow it, so a test can never reach the public internet. Ports other than the fixture's are
+ * refused, 80 and 443 included.
+ *
+ * This lives in the testing entry point only; production code has no way to build a guard
+ * with other rules.
+ */
+export declare const FIXTURE_ADDRESS = "127.0.0.1";
+export declare function createFixtureNetworkGuard(options: {
+    readonly port: number;
+    readonly resolver: HostResolver;
+}): NetworkGuard;
+/** A resolver from a fixed table; unknown names fail as `ENOTFOUND` would. Counts lookups. */
+export declare function tableResolver(table: Readonly<Record<string, readonly string[]>>): HostResolver & {
+    readonly lookups: string[];
+};

package/dist/net/testing/index.d.ts

@@ -0,0 +1,7 @@
+/**
+ * Test helpers: a guard that allows one loopback address and one port, a resolver from a fixed
+ * table, and a text fetcher built on them. For tests of code that fetches from a local fixture
+ * server only.
+ */
+export { createFixtureNetworkGuard, FIXTURE_ADDRESS, tableResolver } from './fixture-guard.js';
+export { createFixtureTextFetcher } from './fixture-fetcher.js';

package/dist/net-cli.js

@@ -0,0 +1 @@
+export { t as classifyPrivateNetworkAddress, n as createCliTextFetcher, r as createPrivateNetworkGuard } from "./chunks/cli-BK65ic84.js";

package/dist/robots/index.d.ts

@@ -0,0 +1,67 @@
+/**
+ * robots.txt (RFC 9309). Parsing and matching are pure functions of text.
+ *
+ * Matching: the group whose user-agent equals the product token (case-insensitive) applies,
+ * all such groups merged; otherwise the `*` groups merged; otherwise nothing is disallowed.
+ * The longest matching rule wins, `allow` wins a tie. `*` matches any run of characters and a
+ * final `$` anchors the end. `/robots.txt` itself is always allowed.
+ */
+/** Bytes of a robots.txt that are parsed (RFC 9309 asks for at least 500 KiB). */
+export declare const ROBOTS_MAX_BYTES: number;
+/** A robots.txt answer is used for at most this long (RFC 9309, section 2.4). */
+export declare const ROBOTS_MAX_AGE_MS: number;
+/** Redirects followed for robots.txt (RFC 9309 asks for at least five). */
+export declare const ROBOTS_MAX_REDIRECTS = 5;
+export interface RobotsRule {
+    readonly allow: boolean;
+    /** Pattern as written, percent-encoding normalised. */
+    readonly pattern: string;
+    /** The rule's line in the file, counted from 1. */
+    readonly line: number;
+}
+export interface RobotsGroup {
+    /** Product tokens of the group, lower case; `*` for the catch-all. */
+    readonly agents: readonly string[];
+    readonly rules: readonly RobotsRule[];
+    /** `Crawl-delay` in seconds when given; not part of the standard. */
+    readonly crawlDelay: number | null;
+    /** The line of the group's first `User-agent` line, counted from 1. */
+    readonly line: number;
+}
+export interface RobotsTxt {
+    readonly groups: readonly RobotsGroup[];
+    /** `Sitemap:` lines, absolute URLs as written (resolved against the robots.txt URL). */
+    readonly sitemaps: readonly string[];
+}
+/**
+ * Parses a robots.txt body. Never throws; unknown lines are ignored. A group is a run of
+ * user-agent lines and the rules after it; only an Allow or Disallow line ends the run, so a
+ * Sitemap, Crawl-delay or unknown line between two user-agent lines does not split the group.
+ */
+export declare function parseRobots(body: string, robotsUrl?: string): RobotsTxt;
+/**
+ * Which rule decides a path for a product token, and through which group.
+ *
+ * - `via`: `named` when a group names the token, `catch_all` when the `*` group applies, `none`
+ *   when no group applies (everything is allowed).
+ * - `group`: the first applying group's line and agents (several groups that name the token
+ *   are merged; the one that holds the deciding rule is given when a rule decides).
+ * - `rule`: the deciding rule, or null when no rule matches (allowed).
+ */
+export interface RuleMatch {
+    readonly allowed: boolean;
+    readonly via: 'named' | 'catch_all' | 'none';
+    readonly group: {
+        readonly line: number;
+        readonly agents: readonly string[];
+    } | null;
+    readonly rule: RobotsRule | null;
+}
+export declare function matchRobots(robots: RobotsTxt, productToken: string, pathAndQuery: string): RuleMatch;
+/**
+ * Whether robots.txt allows a URL path (path plus `?query`, as sent in the request line) for
+ * a product token.
+ */
+export declare function isAllowedByRobots(robots: RobotsTxt, productToken: string, pathAndQuery: string): boolean;
+/** `Crawl-delay` for a product token, in seconds; null when none is set. */
+export declare function crawlDelayFor(robots: RobotsTxt, productToken: string): number | null;

package/dist/run.js

@@ -0,0 +1 @@
+export { t as CLI_VERSION, n as runCli } from "./chunks/main-Bd0uzoHe.js";

package/dist/testing.js

@@ -0,0 +1,58 @@
+import { C as classifyAddress, b as guardWithRules, f as buildTextFetcher } from "./chunks/fetch-text-NUUnM91O.js";
+// src/net/testing/fixture-guard.ts
+/**
+* A network guard for tests against the local fixture site. It allows exactly one address,
+* 127.0.0.1, on exactly one port, the fixture's. Every other address is refused: with the
+* strict rule's reason when the strict rules refuse it, and as `not_global` when they would
+* allow it, so a test can never reach the public internet. Ports other than the fixture's are
+* refused, 80 and 443 included.
+*
+* This lives in the testing entry point only; production code has no way to build a guard
+* with other rules.
+*/
+var FIXTURE_ADDRESS = "127.0.0.1";
+function createFixtureNetworkGuard(options) {
+	return guardWithRules({
+		ports: [options.port],
+		classify: (address) => {
+			if (address === "127.0.0.1") return {
+				allowed: true,
+				family: 4
+			};
+			const strict = classifyAddress(address);
+			return strict.allowed ? {
+				allowed: false,
+				reason: "not_global"
+			} : strict;
+		}
+	}, options.resolver);
+}
+/** A resolver from a fixed table; unknown names fail as `ENOTFOUND` would. Counts lookups. */
+function tableResolver(table) {
+	const lookups = [];
+	const resolve = (hostname) => {
+		lookups.push(hostname);
+		const answer = table[hostname.toLowerCase()];
+		if (answer === void 0) return Promise.reject(/* @__PURE__ */ new Error("ENOTFOUND"));
+		return Promise.resolve(answer);
+	};
+	return Object.assign(resolve, { lookups });
+}
+// src/net/testing/fixture-fetcher.ts
+/**
+* A text fetcher for tests against a local fixture server: the fixture guard (127.0.0.1 on one
+* port only) and, unless a test shortens them, the production timings and politeness.
+*/
+function createFixtureTextFetcher(options) {
+	return buildTextFetcher({
+		guard: createFixtureNetworkGuard({
+			port: options.port,
+			resolver: options.resolver
+		}),
+		...options.connectTimeoutMs === void 0 ? {} : { connectTimeoutMs: options.connectTimeoutMs },
+		...options.requestTimeoutMs === void 0 ? {} : { requestTimeoutMs: options.requestTimeoutMs },
+		...options.hostIntervalMs === void 0 ? {} : { hostIntervalMs: options.hostIntervalMs },
+		...options.ca === void 0 ? {} : { testCa: options.ca }
+	});
+}
+export { FIXTURE_ADDRESS, createFixtureNetworkGuard, createFixtureTextFetcher, tableResolver };